Laravel Queue Dashboard Best Practices – Grafana, Prometheus & Custom Metrics

// app/Console/Kernel.php
$schedule->command('horizon:snapshot')->everyFiveMinutes();

use Laravel\Horizon\Contracts\MetricsRepository;

$metrics = app(MetricsRepository::class);

// Get throughput for a job class (jobs per minute)
$throughput = $metrics->throughputForJob(App\Jobs\SendEmail::class);

// Get average runtime in milliseconds
$runtime = $metrics->runtimeForJob(App\Jobs\SendEmail::class);

// Get queue throughput and wait time
$queueThroughput = $metrics->throughputForQueue('emails');
$waitTime        = $metrics->waitTimeFor('redis', 'emails');  // seconds

// app/Console/Commands/CollectQueueMetrics.php
class CollectQueueMetrics extends Command
{
    protected $signature   = 'metrics:queue-collect';
    protected $description = 'Collect queue metrics and push to time-series DB';

    public function handle(): void
    {
        $queues = ['default', 'emails', 'payments', 'notifications'];

        foreach ($queues as $queue) {
            // Pending job count
            $pending = DB::table('jobs')
                ->where('queue', $queue)
                ->count();

            // Jobs reserved (currently being processed)
            $reserved = DB::table('jobs')
                ->where('queue', $queue)
                ->whereNotNull('reserved_at')
                ->count();

            // Failed in last 5 minutes
            $failed = DB::table('failed_jobs')
                ->where('queue', $queue)
                ->where('failed_at', '>=', now()->subMinutes(5))
                ->count();

            // Push to your metrics system
            Metrics::gauge("queue.pending.{$queue}", $pending);
            Metrics::gauge("queue.reserved.{$queue}", $reserved);
            Metrics::counter("queue.failed.{$queue}", $failed);
        }
    }
}

// Schedule every minute
$schedule->command('metrics:queue-collect')->everyMinute()->withoutOverlapping();

// app/Jobs/Middleware/RecordMetrics.php
class RecordMetrics
{
    public function handle(object $job, \Closure $next): void
    {
        $jobClass = class_basename($job);
        $start    = microtime(true);
        $status   = 'success';

        try {
            $next($job);
        } catch (\Throwable $e) {
            $status = 'failure';
            throw $e;
        } finally {
            $duration = (microtime(true) - $start) * 1000;  // ms

            // Push to StatsD, Prometheus pushgateway, Datadog, etc.
            Metrics::histogram('queue.job.duration', $duration, [
                'job'    => $jobClass,
                'queue'  => $job->queue ?? 'default',
                'status' => $status,
            ]);
        }
    }
}

composer require promphp/prometheus_client_php

// In your metrics collector command:
use Prometheus\CollectorRegistry;
use Prometheus\Storage\Redis as PrometheusRedis;

$registry = new CollectorRegistry(new PrometheusRedis(['host' => 'redis']));

$pendingGauge = $registry->getOrRegisterGauge(
    'laravel',
    'queue_pending_jobs',
    'Number of pending jobs per queue',
    ['queue']
);
$pendingGauge->set($pending, [$queue]);

// Push to Prometheus Pushgateway
$pushGateway = new \PrometheusPushGateway\PushGateway('pushgateway:9091');
$pushGateway->pushAdd($registry, 'laravel_queue', ['instance' => gethostname()]);

// routes/web.php — Prometheus scrape endpoint
Route::get('/metrics', function () {
    $registry = app(CollectorRegistry::class);
    $renderer = new \Prometheus\RenderTextFormat();

    return response($renderer->render($registry->getMetricFamilySamples()), 200, [
        'Content-Type' => \Prometheus\RenderTextFormat::MIME_TYPE,
    ]);
})->middleware('auth.prometheus');  // IP whitelist for Prometheus scraper only

# PromQL
laravel_queue_pending_jobs{queue="emails"}
laravel_queue_pending_jobs{queue="payments"}

# Alert threshold: > 500 pending = warning, > 2000 = critical

# Jobs processed per minute, grouped by job class
rate(laravel_queue_job_duration_count[5m]) * 60

# Color code: green > 100/min, yellow > 50/min, red < 10/min

# P50, P95, P99 job duration
histogram_quantile(0.50, rate(laravel_queue_job_duration_bucket[5m]))
histogram_quantile(0.95, rate(laravel_queue_job_duration_bucket[5m]))
histogram_quantile(0.99, rate(laravel_queue_job_duration_bucket[5m]))

# P99 > 5s = investigate slow jobs

# Failure rate as percentage
rate(laravel_queue_job_duration_count{status="failure"}[5m])
/
rate(laravel_queue_job_duration_count[5m])
* 100

# Alert: > 5% failure rate = warning, > 20% = critical

# Not from Prometheus — pull from database in a separate panel
# Use Grafana MySQL/PostgreSQL datasource:
SELECT
  DATE_FORMAT(failed_at, '%H:%i') as time,
  queue,
  SUBSTRING_INDEX(exception, '\\n', 1) as error,
  COUNT(*) as count
FROM failed_jobs
WHERE failed_at >= NOW() - INTERVAL 1 HOUR
GROUP BY queue, error
ORDER BY count DESC
LIMIT 20;

# Grafana Alert Rule — fires only if condition is true for 10+ minutes
# This prevents alerts from transient bursts that clear quickly
WHEN avg() OF query(laravel_queue_pending_jobs{queue="payments"}, 10m) > 200
FOR 10m
SEVERITY: critical
MESSAGE: "Payments queue backlog: {{ value }} jobs pending for 10+ minutes"

# Alert when failure rate > 10% for 5 minutes
WHEN avg() OF (failure_rate_query) > 10
FOR 5m
SEVERITY: warning
MESSAGE: "Queue failure rate {{ value }}% on {{ queue }}"

# Alert when throughput drops > 80% below 1-hour average
# This catches worker crashes even when the queue depth is still low
WHEN (current_throughput / avg_throughput_1h) < 0.2
FOR 3m
SEVERITY: critical
MESSAGE: "Queue throughput dropped to {{ value }} jobs/min — workers may be down"

# Alert when P95 job duration is 3x higher than the daily average
# Catches memory leaks, slow queries, and dependency degradation
WHEN p95_duration > (daily_avg_p95 * 3)
FOR 5m
SEVERITY: warning
MESSAGE: "Job {{ job_class }} P95 duration {{ value }}ms — 3x above normal"