Compare commits

..
2 Commits
Author SHA1 Message Date
Daniel SamsonandClaude Opus 4.6 ec0b012586 Reconcile queue state on startup, remove age-based staleness check
Publish Image / publish (push) Successful in 20s
On boot, query Coder for running workspaces and re-adopt them into the
active queue. This prevents 409 conflicts after restarts and ensures
callbacks still work for in-flight tasks.

On 409: only delete stopped/failed workspaces. Running workspaces are
adopted — never killed based on age.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-04-12 14:01:56 +01:00
Daniel SamsonandClaude Opus 4.6 36c80458e0 Safe 409 handling: check workspace age/status before deleting
Publish Image / publish (push) Successful in 20s
Instead of blindly deleting on 409 conflict, now checks:
- Stopped/failed → safe to delete and recreate
- Running but >30 min old → stale, delete and recreate
- Running and <30 min old → adopt existing workspace (mid-task)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-04-12 13:59:59 +01:00
3 changed files with 105 additions and 10 deletions
+5
View File
@@ -84,6 +84,11 @@ if (maintenanceRepos.length > 0) {
// Start // Start
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Reconcile queue state with Coder before accepting traffic
queue.reconcile().catch((err) =>
console.error("[startup] reconcile failed:", err),
);
serve({ fetch: app.fetch, port: config.port }, (info) => { serve({ fetch: app.fetch, port: config.port }, (info) => {
console.log(`SDLC Orchestrator listening on :${info.port} (concurrency: ${concurrency})`); console.log(`SDLC Orchestrator listening on :${info.port} (concurrency: ${concurrency})`);
}); });
+28
View File
@@ -35,6 +35,34 @@ export class TaskQueue {
this.concurrency = concurrency; this.concurrency = concurrency;
} }
/**
* Reconcile in-memory state with Coder on startup.
* Re-adopts any running workspaces so callbacks and dedup work correctly.
*/
async reconcile(): Promise<void> {
const workspaces = await this.coder.listWorkspaces();
const runningStatuses = ["starting", "running", "started"];
for (const ws of workspaces) {
if (!runningStatuses.includes(ws.latestBuildStatus)) continue;
// Only adopt workspaces that match our naming pattern: {taskType}-{repo}-{issue}
const parts = ws.name.match(/^(.+?)-(.+?)-(\d+)$/);
if (!parts) continue;
this.active.set(ws.name, {
workspaceId: ws.id,
workspaceName: ws.name,
startedAt: new Date(), // approximate — we don't know the real start time
});
console.log(`[queue] reconciled: adopted workspace "${ws.name}" (status: ${ws.latestBuildStatus})`);
}
if (this.active.size > 0) {
console.log(`[queue] reconcile complete: ${this.active.size} active workspace(s) adopted`);
}
}
/** Build a dedup key for a task. */ /** Build a dedup key for a task. */
static key(task: TaskRequest): string { static key(task: TaskRequest): string {
return `${task.taskType}-${task.giteaRepo}-${task.issueNumber}`; return `${task.taskType}-${task.giteaRepo}-${task.issueNumber}`;
+72 -10
View File
@@ -50,16 +50,30 @@ export class CoderClient {
}, },
); );
// Handle 409 conflict — stale workspace from a previous orchestrator instance // Handle 409 conflict — workspace with this name already exists.
// This can happen if the orchestrator restarted and lost in-memory state.
if (res.status === 409) { if (res.status === 409) {
console.log(`[coder] workspace "${name}" already exists — deleting stale workspace and retrying`);
const existing = await this.findWorkspaceByName(name); const existing = await this.findWorkspaceByName(name);
if (existing) { if (!existing) {
await this.deleteWorkspace(existing.id); throw new Error(`Coder 409 but workspace "${name}" not found — possible race condition`);
// Wait for deletion to propagate
await new Promise((resolve) => setTimeout(resolve, 5000));
} }
return this.createWorkspace(task);
const stoppedStatuses = ["stopped", "failed", "canceled", "deleted"];
if (stoppedStatuses.includes(existing.latestBuildStatus)) {
// Workspace is done — safe to replace
console.log(
`[coder] workspace "${name}" exists but ${existing.latestBuildStatus} — deleting and retrying`,
);
await this.deleteWorkspace(existing.id);
await new Promise((resolve) => setTimeout(resolve, 5000));
return this.createWorkspace(task);
}
// Workspace is still running — adopt it, don't kill it
console.log(
`[coder] workspace "${name}" is still ${existing.latestBuildStatus} — adopting existing workspace`,
);
return { id: existing.id, name: existing.name };
} }
if (!res.ok) { if (!res.ok) {
@@ -107,7 +121,42 @@ export class CoderClient {
} }
} }
async findWorkspaceByName(name: string): Promise<{ id: string } | null> { /**
* List all workspaces owned by the authenticated user.
* Used at startup to reconcile in-memory state with Coder.
*/
async listWorkspaces(): Promise<
Array<{ id: string; name: string; latestBuildStatus: string }>
> {
const res = await fetch(
`${this.baseUrl}/api/v2/workspaces?q=owner:me`,
{
headers: { "Coder-Session-Token": this.token },
},
);
if (!res.ok) return [];
const data = (await res.json()) as {
workspaces: Array<{
id: string;
name: string;
latest_build: { status: string };
}>;
};
return (data.workspaces ?? []).map((w) => ({
id: w.id,
name: w.name,
latestBuildStatus: w.latest_build?.status ?? "unknown",
}));
}
async findWorkspaceByName(name: string): Promise<{
id: string;
name: string;
createdAt: string;
latestBuildStatus: string;
} | null> {
const res = await fetch( const res = await fetch(
`${this.baseUrl}/api/v2/workspaces?q=name:${encodeURIComponent(name)}`, `${this.baseUrl}/api/v2/workspaces?q=name:${encodeURIComponent(name)}`,
{ {
@@ -119,8 +168,21 @@ export class CoderClient {
if (!res.ok) return null; if (!res.ok) return null;
const data = (await res.json()) as { workspaces: Array<{ id: string; name: string }> }; const data = (await res.json()) as {
workspaces: Array<{
id: string;
name: string;
created_at: string;
latest_build: { status: string };
}>;
};
const match = data.workspaces?.find((w) => w.name === name); const match = data.workspaces?.find((w) => w.name === name);
return match ? { id: match.id } : null; if (!match) return null;
return {
id: match.id,
name: match.name,
createdAt: match.created_at,
latestBuildStatus: match.latest_build?.status ?? "unknown",
};
} }
} }