Reconcile queue state on startup, remove age-based staleness check
Publish Image / publish (push) Successful in 20s

On boot, query Coder for running workspaces and re-adopt them into the
active queue. This prevents 409 conflicts after restarts and ensures
callbacks still work for in-flight tasks.

On 409: only delete stopped/failed workspaces. Running workspaces are
adopted — never killed based on age.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Daniel Samson
2026-04-12 14:01:56 +01:00
co-authored by Claude Opus 4.6
parent 36c80458e0
commit ec0b012586
3 changed files with 69 additions and 12 deletions
+5
View File
@@ -84,6 +84,11 @@ if (maintenanceRepos.length > 0) {
// Start // Start
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Reconcile queue state with Coder before accepting traffic
queue.reconcile().catch((err) =>
console.error("[startup] reconcile failed:", err),
);
serve({ fetch: app.fetch, port: config.port }, (info) => { serve({ fetch: app.fetch, port: config.port }, (info) => {
console.log(`SDLC Orchestrator listening on :${info.port} (concurrency: ${concurrency})`); console.log(`SDLC Orchestrator listening on :${info.port} (concurrency: ${concurrency})`);
}); });
+28
View File
@@ -35,6 +35,34 @@ export class TaskQueue {
this.concurrency = concurrency; this.concurrency = concurrency;
} }
/**
* Reconcile in-memory state with Coder on startup.
* Re-adopts any running workspaces so callbacks and dedup work correctly.
*/
async reconcile(): Promise<void> {
const workspaces = await this.coder.listWorkspaces();
const runningStatuses = ["starting", "running", "started"];
for (const ws of workspaces) {
if (!runningStatuses.includes(ws.latestBuildStatus)) continue;
// Only adopt workspaces that match our naming pattern: {taskType}-{repo}-{issue}
const parts = ws.name.match(/^(.+?)-(.+?)-(\d+)$/);
if (!parts) continue;
this.active.set(ws.name, {
workspaceId: ws.id,
workspaceName: ws.name,
startedAt: new Date(), // approximate — we don't know the real start time
});
console.log(`[queue] reconciled: adopted workspace "${ws.name}" (status: ${ws.latestBuildStatus})`);
}
if (this.active.size > 0) {
console.log(`[queue] reconcile complete: ${this.active.size} active workspace(s) adopted`);
}
}
/** Build a dedup key for a task. */ /** Build a dedup key for a task. */
static key(task: TaskRequest): string { static key(task: TaskRequest): string {
return `${task.taskType}-${task.giteaRepo}-${task.issueNumber}`; return `${task.taskType}-${task.giteaRepo}-${task.issueNumber}`;
+36 -12
View File
@@ -51,33 +51,27 @@ export class CoderClient {
); );
// Handle 409 conflict — workspace with this name already exists. // Handle 409 conflict — workspace with this name already exists.
// This happens when the orchestrator restarts and loses in-memory state. // This can happen if the orchestrator restarted and lost in-memory state.
// Only delete if the workspace is old (>30 min) or stopped/failed.
// If it's young and running, it may be mid-task — skip to avoid data loss.
if (res.status === 409) { if (res.status === 409) {
const existing = await this.findWorkspaceByName(name); const existing = await this.findWorkspaceByName(name);
if (!existing) { if (!existing) {
throw new Error(`Coder 409 but workspace "${name}" not found — possible race condition`); throw new Error(`Coder 409 but workspace "${name}" not found — possible race condition`);
} }
const ageMs = Date.now() - new Date(existing.createdAt).getTime();
const ageMin = Math.round(ageMs / 60_000);
const stoppedStatuses = ["stopped", "failed", "canceled", "deleted"]; const stoppedStatuses = ["stopped", "failed", "canceled", "deleted"];
const isStopped = stoppedStatuses.includes(existing.latestBuildStatus); if (stoppedStatuses.includes(existing.latestBuildStatus)) {
const isStale = ageMin > 30; // Workspace is done — safe to replace
if (isStopped || isStale) {
console.log( console.log(
`[coder] workspace "${name}" already exists (age: ${ageMin}m, status: ${existing.latestBuildStatus}) — deleting and retrying`, `[coder] workspace "${name}" exists but ${existing.latestBuildStatus} — deleting and retrying`,
); );
await this.deleteWorkspace(existing.id); await this.deleteWorkspace(existing.id);
await new Promise((resolve) => setTimeout(resolve, 5000)); await new Promise((resolve) => setTimeout(resolve, 5000));
return this.createWorkspace(task); return this.createWorkspace(task);
} }
// Workspace is young and still running — don't kill it // Workspace is still running — adopt it, don't kill it
console.log( console.log(
`[coder] workspace "${name}" is active (age: ${ageMin}m, status: ${existing.latestBuildStatus}) — skipping creation`, `[coder] workspace "${name}" is still ${existing.latestBuildStatus} — adopting existing workspace`,
); );
return { id: existing.id, name: existing.name }; return { id: existing.id, name: existing.name };
} }
@@ -127,6 +121,36 @@ export class CoderClient {
} }
} }
/**
* List all workspaces owned by the authenticated user.
* Used at startup to reconcile in-memory state with Coder.
*/
async listWorkspaces(): Promise<
Array<{ id: string; name: string; latestBuildStatus: string }>
> {
const res = await fetch(
`${this.baseUrl}/api/v2/workspaces?q=owner:me`,
{
headers: { "Coder-Session-Token": this.token },
},
);
if (!res.ok) return [];
const data = (await res.json()) as {
workspaces: Array<{
id: string;
name: string;
latest_build: { status: string };
}>;
};
return (data.workspaces ?? []).map((w) => ({
id: w.id,
name: w.name,
latestBuildStatus: w.latest_build?.status ?? "unknown",
}));
}
async findWorkspaceByName(name: string): Promise<{ async findWorkspaceByName(name: string): Promise<{
id: string; id: string;
name: string; name: string;