Reconcile queue state on startup, remove age-based staleness check
Publish Image / publish (push) Successful in 20s
Publish Image / publish (push) Successful in 20s
On boot, query Coder for running workspaces and re-adopt them into the active queue. This prevents 409 conflicts after restarts and ensures callbacks still work for in-flight tasks. On 409: only delete stopped/failed workspaces. Running workspaces are adopted — never killed based on age. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
36c80458e0
commit
ec0b012586
@@ -84,6 +84,11 @@ if (maintenanceRepos.length > 0) {
|
|||||||
// Start
|
// Start
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
// Reconcile queue state with Coder before accepting traffic
|
||||||
|
queue.reconcile().catch((err) =>
|
||||||
|
console.error("[startup] reconcile failed:", err),
|
||||||
|
);
|
||||||
|
|
||||||
serve({ fetch: app.fetch, port: config.port }, (info) => {
|
serve({ fetch: app.fetch, port: config.port }, (info) => {
|
||||||
console.log(`SDLC Orchestrator listening on :${info.port} (concurrency: ${concurrency})`);
|
console.log(`SDLC Orchestrator listening on :${info.port} (concurrency: ${concurrency})`);
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -35,6 +35,34 @@ export class TaskQueue {
|
|||||||
this.concurrency = concurrency;
|
this.concurrency = concurrency;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reconcile in-memory state with Coder on startup.
|
||||||
|
* Re-adopts any running workspaces so callbacks and dedup work correctly.
|
||||||
|
*/
|
||||||
|
async reconcile(): Promise<void> {
|
||||||
|
const workspaces = await this.coder.listWorkspaces();
|
||||||
|
const runningStatuses = ["starting", "running", "started"];
|
||||||
|
|
||||||
|
for (const ws of workspaces) {
|
||||||
|
if (!runningStatuses.includes(ws.latestBuildStatus)) continue;
|
||||||
|
|
||||||
|
// Only adopt workspaces that match our naming pattern: {taskType}-{repo}-{issue}
|
||||||
|
const parts = ws.name.match(/^(.+?)-(.+?)-(\d+)$/);
|
||||||
|
if (!parts) continue;
|
||||||
|
|
||||||
|
this.active.set(ws.name, {
|
||||||
|
workspaceId: ws.id,
|
||||||
|
workspaceName: ws.name,
|
||||||
|
startedAt: new Date(), // approximate — we don't know the real start time
|
||||||
|
});
|
||||||
|
console.log(`[queue] reconciled: adopted workspace "${ws.name}" (status: ${ws.latestBuildStatus})`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.active.size > 0) {
|
||||||
|
console.log(`[queue] reconcile complete: ${this.active.size} active workspace(s) adopted`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/** Build a dedup key for a task. */
|
/** Build a dedup key for a task. */
|
||||||
static key(task: TaskRequest): string {
|
static key(task: TaskRequest): string {
|
||||||
return `${task.taskType}-${task.giteaRepo}-${task.issueNumber}`;
|
return `${task.taskType}-${task.giteaRepo}-${task.issueNumber}`;
|
||||||
|
|||||||
+36
-12
@@ -51,33 +51,27 @@ export class CoderClient {
|
|||||||
);
|
);
|
||||||
|
|
||||||
// Handle 409 conflict — workspace with this name already exists.
|
// Handle 409 conflict — workspace with this name already exists.
|
||||||
// This happens when the orchestrator restarts and loses in-memory state.
|
// This can happen if the orchestrator restarted and lost in-memory state.
|
||||||
// Only delete if the workspace is old (>30 min) or stopped/failed.
|
|
||||||
// If it's young and running, it may be mid-task — skip to avoid data loss.
|
|
||||||
if (res.status === 409) {
|
if (res.status === 409) {
|
||||||
const existing = await this.findWorkspaceByName(name);
|
const existing = await this.findWorkspaceByName(name);
|
||||||
if (!existing) {
|
if (!existing) {
|
||||||
throw new Error(`Coder 409 but workspace "${name}" not found — possible race condition`);
|
throw new Error(`Coder 409 but workspace "${name}" not found — possible race condition`);
|
||||||
}
|
}
|
||||||
|
|
||||||
const ageMs = Date.now() - new Date(existing.createdAt).getTime();
|
|
||||||
const ageMin = Math.round(ageMs / 60_000);
|
|
||||||
const stoppedStatuses = ["stopped", "failed", "canceled", "deleted"];
|
const stoppedStatuses = ["stopped", "failed", "canceled", "deleted"];
|
||||||
const isStopped = stoppedStatuses.includes(existing.latestBuildStatus);
|
if (stoppedStatuses.includes(existing.latestBuildStatus)) {
|
||||||
const isStale = ageMin > 30;
|
// Workspace is done — safe to replace
|
||||||
|
|
||||||
if (isStopped || isStale) {
|
|
||||||
console.log(
|
console.log(
|
||||||
`[coder] workspace "${name}" already exists (age: ${ageMin}m, status: ${existing.latestBuildStatus}) — deleting and retrying`,
|
`[coder] workspace "${name}" exists but ${existing.latestBuildStatus} — deleting and retrying`,
|
||||||
);
|
);
|
||||||
await this.deleteWorkspace(existing.id);
|
await this.deleteWorkspace(existing.id);
|
||||||
await new Promise((resolve) => setTimeout(resolve, 5000));
|
await new Promise((resolve) => setTimeout(resolve, 5000));
|
||||||
return this.createWorkspace(task);
|
return this.createWorkspace(task);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Workspace is young and still running — don't kill it
|
// Workspace is still running — adopt it, don't kill it
|
||||||
console.log(
|
console.log(
|
||||||
`[coder] workspace "${name}" is active (age: ${ageMin}m, status: ${existing.latestBuildStatus}) — skipping creation`,
|
`[coder] workspace "${name}" is still ${existing.latestBuildStatus} — adopting existing workspace`,
|
||||||
);
|
);
|
||||||
return { id: existing.id, name: existing.name };
|
return { id: existing.id, name: existing.name };
|
||||||
}
|
}
|
||||||
@@ -127,6 +121,36 @@ export class CoderClient {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* List all workspaces owned by the authenticated user.
|
||||||
|
* Used at startup to reconcile in-memory state with Coder.
|
||||||
|
*/
|
||||||
|
async listWorkspaces(): Promise<
|
||||||
|
Array<{ id: string; name: string; latestBuildStatus: string }>
|
||||||
|
> {
|
||||||
|
const res = await fetch(
|
||||||
|
`${this.baseUrl}/api/v2/workspaces?q=owner:me`,
|
||||||
|
{
|
||||||
|
headers: { "Coder-Session-Token": this.token },
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!res.ok) return [];
|
||||||
|
|
||||||
|
const data = (await res.json()) as {
|
||||||
|
workspaces: Array<{
|
||||||
|
id: string;
|
||||||
|
name: string;
|
||||||
|
latest_build: { status: string };
|
||||||
|
}>;
|
||||||
|
};
|
||||||
|
return (data.workspaces ?? []).map((w) => ({
|
||||||
|
id: w.id,
|
||||||
|
name: w.name,
|
||||||
|
latestBuildStatus: w.latest_build?.status ?? "unknown",
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
async findWorkspaceByName(name: string): Promise<{
|
async findWorkspaceByName(name: string): Promise<{
|
||||||
id: string;
|
id: string;
|
||||||
name: string;
|
name: string;
|
||||||
|
|||||||
Reference in New Issue
Block a user