Migrate task queue from in-memory to BullMQ + Redis
Publish Image / publish (push) Successful in 36s

Replace the volatile in-memory Map/Set queue with BullMQ backed by Redis
for persistence across restarts, automatic retry with exponential backoff,
and non-blocking workspace cleanup.

- Two queues: workspace-create (concurrency-limited) and workspace-cleanup
  (with polling instead of sleep-based stop→delete)
- Active workspaces tracked in Redis hash for dedup across restarts
- Stale sweep every 10 minutes catches orphaned workspaces
- Graceful shutdown on SIGTERM/SIGINT
- Replace node-cron with BullMQ repeatable jobs

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Daniel Samson
2026-04-12 15:40:12 +01:00
co-authored by Claude Opus 4.6
parent a6ff7b7fb5
commit 13e96e9356
16 changed files with 710 additions and 180 deletions
+12
View File
@@ -25,6 +25,14 @@ export interface Config {
callbackUrl: string;
/** Optional webhook secret for verifying Gitea signatures */
webhookSecret?: string;
/** Redis connection URL (e.g. redis://redis:6379) */
redisUrl: string;
/** Max concurrent workspace creations (default: 2) */
queueConcurrency: number;
/** Max concurrent cleanup jobs (default: 5) */
cleanupConcurrency: number;
/** Minutes before a workspace is considered stale (default: 120) */
staleWorkspaceMinutes: number;
}
function required(name: string): string {
@@ -50,5 +58,9 @@ export function loadConfig(): Config {
giteaUrl: process.env.GITEA_URL || "https://gitea.samson.media",
callbackUrl: required("CALLBACK_URL"),
webhookSecret: process.env.WEBHOOK_SECRET,
redisUrl: required("REDIS_URL"),
queueConcurrency: parseInt(process.env.QUEUE_CONCURRENCY || "2", 10),
cleanupConcurrency: parseInt(process.env.CLEANUP_CONCURRENCY || "5", 10),
staleWorkspaceMinutes: parseInt(process.env.STALE_WORKSPACE_MINUTES || "120", 10),
};
}