ai-gateway-go/internal/outbox/worker.go

package outbox

import (
	"context"
	"errors"
	"log/slog"
	"math"
	"time"
)

type WorkerConfig struct {
	WorkerID     string
	BatchSize    int
	PollInterval time.Duration
	Lease        time.Duration
	MaxAttempts  int
	MaxBackoff   time.Duration
}

type Worker struct {
	store     *Store
	publisher *RedisPublisher
	config    WorkerConfig
	logger    *slog.Logger
}

func NewWorker(store *Store, publisher *RedisPublisher, config WorkerConfig, logger *slog.Logger) *Worker {
	return &Worker{store: store, publisher: publisher, config: config, logger: logger}
}

func (w *Worker) Run(ctx context.Context) error {
	for {
		processed, err := w.runBatch(ctx)
		if err != nil && ctx.Err() == nil && w.logger != nil {
			w.logger.Error("outbox batch failed", "error", err)
		}
		if ctx.Err() != nil {
			return nil
		}
		if err == nil && processed > 0 {
			continue
		}
		timer := time.NewTimer(w.config.PollInterval)
		select {
		case <-ctx.Done():
			timer.Stop()
			return nil
		case <-timer.C:
		}
	}
}

func (w *Worker) runBatch(ctx context.Context) (int, error) {
	// Gate on the downstream publisher before claiming anything. Claiming
	// increments each event's attempt counter, so claiming during a Redis
	// outage would burn every queued event's delivery budget and dead-letter
	// the whole queue the moment the budget ran out — even though the events
	// themselves were never at fault. When Redis is unreachable we back off
	// instead, leaving events untouched in PostgreSQL until it recovers.
	if err := w.publisher.Ping(ctx); err != nil {
		return 0, err
	}
	events, err := w.store.Claim(ctx, w.config.WorkerID, w.config.BatchSize, w.config.Lease)
	if err != nil {
		return 0, err
	}
	var batchErr error
	for _, event := range events {
		result, publishErr := w.publisher.Publish(ctx, event)
		if publishErr == nil {
			if err := w.store.MarkProcessed(ctx, event.EventID, w.config.WorkerID, result.StreamID); err != nil {
				batchErr = errors.Join(batchErr, err)
			}
			continue
		}
		delay := retryDelay(event.Attempts, w.config.MaxBackoff)
		if err := w.store.MarkFailed(ctx, event, w.config.WorkerID, publishErr, w.config.MaxAttempts, delay); err != nil {
			batchErr = errors.Join(batchErr, err)
		}
	}
	return len(events), batchErr
}

func retryDelay(attempt int, maximum time.Duration) time.Duration {
	seconds := math.Pow(2, float64(max(attempt-1, 0)))
	delay := time.Duration(seconds * float64(time.Second))
	if delay > maximum {
		return maximum
	}
	return delay
}