Reconcile queue state on startup, remove age-based staleness check
Publish Image / publish (push) Successful in 20s
Publish Image / publish (push) Successful in 20s
On boot, query Coder for running workspaces and re-adopt them into the active queue. This prevents 409 conflicts after restarts and ensures callbacks still work for in-flight tasks. On 409: only delete stopped/failed workspaces. Running workspaces are adopted — never killed based on age. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
36c80458e0
commit
ec0b012586
@@ -84,6 +84,11 @@ if (maintenanceRepos.length > 0) {
|
||||
// Start
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Reconcile queue state with Coder before accepting traffic
|
||||
queue.reconcile().catch((err) =>
|
||||
console.error("[startup] reconcile failed:", err),
|
||||
);
|
||||
|
||||
serve({ fetch: app.fetch, port: config.port }, (info) => {
|
||||
console.log(`SDLC Orchestrator listening on :${info.port} (concurrency: ${concurrency})`);
|
||||
});
|
||||
|
||||
@@ -35,6 +35,34 @@ export class TaskQueue {
|
||||
this.concurrency = concurrency;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconcile in-memory state with Coder on startup.
|
||||
* Re-adopts any running workspaces so callbacks and dedup work correctly.
|
||||
*/
|
||||
async reconcile(): Promise<void> {
|
||||
const workspaces = await this.coder.listWorkspaces();
|
||||
const runningStatuses = ["starting", "running", "started"];
|
||||
|
||||
for (const ws of workspaces) {
|
||||
if (!runningStatuses.includes(ws.latestBuildStatus)) continue;
|
||||
|
||||
// Only adopt workspaces that match our naming pattern: {taskType}-{repo}-{issue}
|
||||
const parts = ws.name.match(/^(.+?)-(.+?)-(\d+)$/);
|
||||
if (!parts) continue;
|
||||
|
||||
this.active.set(ws.name, {
|
||||
workspaceId: ws.id,
|
||||
workspaceName: ws.name,
|
||||
startedAt: new Date(), // approximate — we don't know the real start time
|
||||
});
|
||||
console.log(`[queue] reconciled: adopted workspace "${ws.name}" (status: ${ws.latestBuildStatus})`);
|
||||
}
|
||||
|
||||
if (this.active.size > 0) {
|
||||
console.log(`[queue] reconcile complete: ${this.active.size} active workspace(s) adopted`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Build a dedup key for a task. */
|
||||
static key(task: TaskRequest): string {
|
||||
return `${task.taskType}-${task.giteaRepo}-${task.issueNumber}`;
|
||||
|
||||
+36
-12
@@ -51,33 +51,27 @@ export class CoderClient {
|
||||
);
|
||||
|
||||
// Handle 409 conflict — workspace with this name already exists.
|
||||
// This happens when the orchestrator restarts and loses in-memory state.
|
||||
// Only delete if the workspace is old (>30 min) or stopped/failed.
|
||||
// If it's young and running, it may be mid-task — skip to avoid data loss.
|
||||
// This can happen if the orchestrator restarted and lost in-memory state.
|
||||
if (res.status === 409) {
|
||||
const existing = await this.findWorkspaceByName(name);
|
||||
if (!existing) {
|
||||
throw new Error(`Coder 409 but workspace "${name}" not found — possible race condition`);
|
||||
}
|
||||
|
||||
const ageMs = Date.now() - new Date(existing.createdAt).getTime();
|
||||
const ageMin = Math.round(ageMs / 60_000);
|
||||
const stoppedStatuses = ["stopped", "failed", "canceled", "deleted"];
|
||||
const isStopped = stoppedStatuses.includes(existing.latestBuildStatus);
|
||||
const isStale = ageMin > 30;
|
||||
|
||||
if (isStopped || isStale) {
|
||||
if (stoppedStatuses.includes(existing.latestBuildStatus)) {
|
||||
// Workspace is done — safe to replace
|
||||
console.log(
|
||||
`[coder] workspace "${name}" already exists (age: ${ageMin}m, status: ${existing.latestBuildStatus}) — deleting and retrying`,
|
||||
`[coder] workspace "${name}" exists but ${existing.latestBuildStatus} — deleting and retrying`,
|
||||
);
|
||||
await this.deleteWorkspace(existing.id);
|
||||
await new Promise((resolve) => setTimeout(resolve, 5000));
|
||||
return this.createWorkspace(task);
|
||||
}
|
||||
|
||||
// Workspace is young and still running — don't kill it
|
||||
// Workspace is still running — adopt it, don't kill it
|
||||
console.log(
|
||||
`[coder] workspace "${name}" is active (age: ${ageMin}m, status: ${existing.latestBuildStatus}) — skipping creation`,
|
||||
`[coder] workspace "${name}" is still ${existing.latestBuildStatus} — adopting existing workspace`,
|
||||
);
|
||||
return { id: existing.id, name: existing.name };
|
||||
}
|
||||
@@ -127,6 +121,36 @@ export class CoderClient {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* List all workspaces owned by the authenticated user.
|
||||
* Used at startup to reconcile in-memory state with Coder.
|
||||
*/
|
||||
async listWorkspaces(): Promise<
|
||||
Array<{ id: string; name: string; latestBuildStatus: string }>
|
||||
> {
|
||||
const res = await fetch(
|
||||
`${this.baseUrl}/api/v2/workspaces?q=owner:me`,
|
||||
{
|
||||
headers: { "Coder-Session-Token": this.token },
|
||||
},
|
||||
);
|
||||
|
||||
if (!res.ok) return [];
|
||||
|
||||
const data = (await res.json()) as {
|
||||
workspaces: Array<{
|
||||
id: string;
|
||||
name: string;
|
||||
latest_build: { status: string };
|
||||
}>;
|
||||
};
|
||||
return (data.workspaces ?? []).map((w) => ({
|
||||
id: w.id,
|
||||
name: w.name,
|
||||
latestBuildStatus: w.latest_build?.status ?? "unknown",
|
||||
}));
|
||||
}
|
||||
|
||||
async findWorkspaceByName(name: string): Promise<{
|
||||
id: string;
|
||||
name: string;
|
||||
|
||||
Reference in New Issue
Block a user