Compare commits

...
4 Commits
Author SHA1 Message Date
Daniel SamsonandClaude Opus 4.6 ec0b012586 Reconcile queue state on startup, remove age-based staleness check
Publish Image / publish (push) Successful in 20s
On boot, query Coder for running workspaces and re-adopt them into the
active queue. This prevents 409 conflicts after restarts and ensures
callbacks still work for in-flight tasks.

On 409: only delete stopped/failed workspaces. Running workspaces are
adopted — never killed based on age.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-04-12 14:01:56 +01:00
Daniel SamsonandClaude Opus 4.6 36c80458e0 Safe 409 handling: check workspace age/status before deleting
Publish Image / publish (push) Successful in 20s
Instead of blindly deleting on 409 conflict, now checks:
- Stopped/failed → safe to delete and recreate
- Running but >30 min old → stale, delete and recreate
- Running and <30 min old → adopt existing workspace (mid-task)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-04-12 13:59:59 +01:00
Daniel SamsonandClaude Opus 4.6 06cf4d1b9e Handle stale workspaces: 409 conflict auto-cleanup + verbose logging
Publish Image / publish (push) Successful in 20s
When the orchestrator restarts, it loses in-memory queue state. If a
stale workspace still exists, createWorkspace now auto-deletes it and
retries instead of failing with 409. Also adds --verbose to Claude Code
invocation for better task log debugging.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-04-12 13:58:08 +01:00
Daniel SamsonandClaude Opus 4.6 4f1d306fdf Fix ImagePullBackOff: use multi-arch base image tag
The tag `ubuntu-arm64` doesn't exist on Docker Hub. Switch to `latest`
which includes arm64 in its multi-arch manifest.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-04-12 13:44:44 +01:00
4 changed files with 184 additions and 6 deletions
+74 -3
View File
@@ -442,6 +442,25 @@ resource "coder_agent" "main" {
web_terminal = true web_terminal = true
} }
metadata {
display_name = "Task Status"
key = "task_status"
script = <<-EOS
if [ -f ~/task-output.log ]; then
if grep -q "Task completed" ~/task-output.log 2>/dev/null; then
echo "✅ Complete"
else
echo "⏳ Running ($(wc -l < ~/task-output.log) lines)"
fi
elif [ -n "$TASK_TYPE" ]; then
echo "⏳ Starting..."
else
echo "Interactive"
fi
EOS
interval = 10
}
env = { env = {
ANTHROPIC_API_KEY = data.coder_parameter.anthropic_api_key.value ANTHROPIC_API_KEY = data.coder_parameter.anthropic_api_key.value
GITHUB_TOKEN = data.coder_external_auth.github.access_token GITHUB_TOKEN = data.coder_external_auth.github.access_token
@@ -516,6 +535,41 @@ resource "coder_agent" "main" {
npm run db:migrate:local npm run db:migrate:local
fi fi
# --- Start log viewer on port 13338 ---
if [ -n "$TASK_TYPE" ]; then
touch ~/task-output.log
cat > ~/log-server.sh << 'LOGEOF'
#!/bin/bash
while true; do
{
echo "HTTP/1.1 200 OK"
echo "Content-Type: text/html; charset=utf-8"
echo "Connection: close"
echo ""
echo "<html><head><title>Task Log</title>"
echo "<meta http-equiv='refresh' content='5'>"
echo "<style>body{background:#1e1e1e;color:#d4d4d4;font-family:monospace;font-size:13px;padding:16px;white-space:pre-wrap;}"
echo "h2{color:#569cd6;margin:0 0 8px}.meta{color:#6a9955;margin-bottom:16px;display:block}</style></head><body>"
echo "<h2>$TASK_TYPE #$ISSUE_NUMBER — $(hostname)</h2>"
if [ -f ~/task-output.log ]; then
LINES=$(wc -l < ~/task-output.log)
if grep -q "Task completed" ~/task-output.log 2>/dev/null; then
echo "<span class='meta'>Status: ✅ Complete ($LINES lines)</span>"
else
echo "<span class='meta'>Status: ⏳ Running ($LINES lines) — auto-refreshing every 5s</span>"
fi
sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g' ~/task-output.log
else
echo "Waiting for task to start..."
fi
echo "</body></html>"
} | nc -l -p 13338 -q 1 2>/dev/null || true
done
LOGEOF
chmod +x ~/log-server.sh
nohup bash ~/log-server.sh &>/dev/null &
fi
# --- Automated task execution --- # --- Automated task execution ---
if [ -n "$TASK_TYPE" ] && [ -d ~/project ]; then if [ -n "$TASK_TYPE" ] && [ -d ~/project ]; then
cd ~/project cd ~/project
@@ -525,8 +579,10 @@ resource "coder_agent" "main" {
ARGS="$DEPLOY_ENV" ARGS="$DEPLOY_ENV"
fi fi
# Run Claude Code in non-interactive mode # Run Claude Code in non-interactive mode with tool access
claude --print --dangerously-skip-permissions "/project:$TASK_TYPE $ARGS" 2>&1 | tee ~/task-output.log claude -p --dangerously-skip-permissions --verbose "/project:$TASK_TYPE $ARGS" 2>&1 | tee ~/task-output.log
EXIT_CODE=$?
echo "Claude exited with code: $EXIT_CODE" | tee -a ~/task-output.log
echo "Task completed. Output saved to ~/task-output.log" echo "Task completed. Output saved to ~/task-output.log"
@@ -540,6 +596,21 @@ resource "coder_agent" "main" {
EOT EOT
} }
resource "coder_app" "task_log" {
agent_id = coder_agent.main.id
slug = "task-log"
display_name = "Task Log"
icon = "/icon/document.svg"
url = "http://localhost:13338"
share = "owner"
healthcheck {
url = "http://localhost:13338"
interval = 10
threshold = 3
}
}
# ============================================================================= # =============================================================================
# Main Workspace Deployment # Main Workspace Deployment
# ============================================================================= # =============================================================================
@@ -620,7 +691,7 @@ resource "kubernetes_deployment_v1" "workspace" {
container { container {
name = "coder-agent" name = "coder-agent"
image = "codercom/enterprise-base:ubuntu-arm64" image = "codercom/enterprise-base:latest"
command = ["sh", "-c", coder_agent.main.init_script] command = ["sh", "-c", coder_agent.main.init_script]
+5
View File
@@ -84,6 +84,11 @@ if (maintenanceRepos.length > 0) {
// Start // Start
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// Reconcile queue state with Coder before accepting traffic
queue.reconcile().catch((err) =>
console.error("[startup] reconcile failed:", err),
);
serve({ fetch: app.fetch, port: config.port }, (info) => { serve({ fetch: app.fetch, port: config.port }, (info) => {
console.log(`SDLC Orchestrator listening on :${info.port} (concurrency: ${concurrency})`); console.log(`SDLC Orchestrator listening on :${info.port} (concurrency: ${concurrency})`);
}); });
+28
View File
@@ -35,6 +35,34 @@ export class TaskQueue {
this.concurrency = concurrency; this.concurrency = concurrency;
} }
/**
* Reconcile in-memory state with Coder on startup.
* Re-adopts any running workspaces so callbacks and dedup work correctly.
*/
async reconcile(): Promise<void> {
const workspaces = await this.coder.listWorkspaces();
const runningStatuses = ["starting", "running", "started"];
for (const ws of workspaces) {
if (!runningStatuses.includes(ws.latestBuildStatus)) continue;
// Only adopt workspaces that match our naming pattern: {taskType}-{repo}-{issue}
const parts = ws.name.match(/^(.+?)-(.+?)-(\d+)$/);
if (!parts) continue;
this.active.set(ws.name, {
workspaceId: ws.id,
workspaceName: ws.name,
startedAt: new Date(), // approximate — we don't know the real start time
});
console.log(`[queue] reconciled: adopted workspace "${ws.name}" (status: ${ws.latestBuildStatus})`);
}
if (this.active.size > 0) {
console.log(`[queue] reconcile complete: ${this.active.size} active workspace(s) adopted`);
}
}
/** Build a dedup key for a task. */ /** Build a dedup key for a task. */
static key(task: TaskRequest): string { static key(task: TaskRequest): string {
return `${task.taskType}-${task.giteaRepo}-${task.issueNumber}`; return `${task.taskType}-${task.giteaRepo}-${task.issueNumber}`;
+77 -3
View File
@@ -50,6 +50,32 @@ export class CoderClient {
}, },
); );
// Handle 409 conflict — workspace with this name already exists.
// This can happen if the orchestrator restarted and lost in-memory state.
if (res.status === 409) {
const existing = await this.findWorkspaceByName(name);
if (!existing) {
throw new Error(`Coder 409 but workspace "${name}" not found — possible race condition`);
}
const stoppedStatuses = ["stopped", "failed", "canceled", "deleted"];
if (stoppedStatuses.includes(existing.latestBuildStatus)) {
// Workspace is done — safe to replace
console.log(
`[coder] workspace "${name}" exists but ${existing.latestBuildStatus} — deleting and retrying`,
);
await this.deleteWorkspace(existing.id);
await new Promise((resolve) => setTimeout(resolve, 5000));
return this.createWorkspace(task);
}
// Workspace is still running — adopt it, don't kill it
console.log(
`[coder] workspace "${name}" is still ${existing.latestBuildStatus} — adopting existing workspace`,
);
return { id: existing.id, name: existing.name };
}
if (!res.ok) { if (!res.ok) {
const text = await res.text(); const text = await res.text();
throw new Error(`Coder API error ${res.status}: ${text}`); throw new Error(`Coder API error ${res.status}: ${text}`);
@@ -95,7 +121,42 @@ export class CoderClient {
} }
} }
async findWorkspaceByName(name: string): Promise<{ id: string } | null> { /**
* List all workspaces owned by the authenticated user.
* Used at startup to reconcile in-memory state with Coder.
*/
async listWorkspaces(): Promise<
Array<{ id: string; name: string; latestBuildStatus: string }>
> {
const res = await fetch(
`${this.baseUrl}/api/v2/workspaces?q=owner:me`,
{
headers: { "Coder-Session-Token": this.token },
},
);
if (!res.ok) return [];
const data = (await res.json()) as {
workspaces: Array<{
id: string;
name: string;
latest_build: { status: string };
}>;
};
return (data.workspaces ?? []).map((w) => ({
id: w.id,
name: w.name,
latestBuildStatus: w.latest_build?.status ?? "unknown",
}));
}
async findWorkspaceByName(name: string): Promise<{
id: string;
name: string;
createdAt: string;
latestBuildStatus: string;
} | null> {
const res = await fetch( const res = await fetch(
`${this.baseUrl}/api/v2/workspaces?q=name:${encodeURIComponent(name)}`, `${this.baseUrl}/api/v2/workspaces?q=name:${encodeURIComponent(name)}`,
{ {
@@ -107,8 +168,21 @@ export class CoderClient {
if (!res.ok) return null; if (!res.ok) return null;
const data = (await res.json()) as { workspaces: Array<{ id: string; name: string }> }; const data = (await res.json()) as {
workspaces: Array<{
id: string;
name: string;
created_at: string;
latest_build: { status: string };
}>;
};
const match = data.workspaces?.find((w) => w.name === name); const match = data.workspaces?.find((w) => w.name === name);
return match ? { id: match.id } : null; if (!match) return null;
return {
id: match.id,
name: match.name,
createdAt: match.created_at,
latestBuildStatus: match.latest_build?.status ?? "unknown",
};
} }
} }