Safe 409 handling: check workspace age/status before deleting

Instead of blindly deleting on 409 conflict, now checks:
- Stopped/failed → safe to delete and recreate
- Running but >30 min old → stale, delete and recreate
- Running and <30 min old → adopt existing workspace (mid-task)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

coder/cloudflare-worker/main.tf

@@ -208,6 +208,15 @@ data "coder_parameter" "anthropic_api_key" {
   mutable      = false
 }
 
+data "coder_parameter" "callback_url" {
+  name         = "callback_url"
+  display_name = "Task Complete Callback URL"
+  description  = "URL to POST when task finishes (set by orchestrator)"
+  type         = "string"
+  default      = ""
+  mutable      = false
+}
+
 data "coder_parameter" "deploy_env" {
   name         = "deploy_env"
   display_name = "Deploy Environment"
@@ -433,6 +442,25 @@ resource "coder_agent" "main" {
     web_terminal           = true
   }
 
+  metadata {
+    display_name = "Task Status"
+    key          = "task_status"
+    script       = <<-EOS
+      if [ -f ~/task-output.log ]; then
+        if grep -q "Task completed" ~/task-output.log 2>/dev/null; then
+          echo "✅ Complete"
+        else
+          echo "⏳ Running ($(wc -l < ~/task-output.log) lines)"
+        fi
+      elif [ -n "$TASK_TYPE" ]; then
+        echo "⏳ Starting..."
+      else
+        echo "Interactive"
+      fi
+    EOS
+    interval     = 10
+  }
+
   env = {
     ANTHROPIC_API_KEY = data.coder_parameter.anthropic_api_key.value
     GITHUB_TOKEN      = data.coder_external_auth.github.access_token
@@ -443,6 +471,7 @@ resource "coder_agent" "main" {
     ISSUE_NUMBER      = data.coder_parameter.issue_number.value
     REPO_CLONE_URL    = data.coder_parameter.repo_clone_url.value
     DEPLOY_ENV        = data.coder_parameter.deploy_env.value
+    CALLBACK_URL      = data.coder_parameter.callback_url.value
   }
 
   startup_script = <<-EOT
@@ -506,6 +535,41 @@ resource "coder_agent" "main" {
       npm run db:migrate:local
     fi
 
+    # --- Start log viewer on port 13338 ---
+    if [ -n "$TASK_TYPE" ]; then
+      touch ~/task-output.log
+      cat > ~/log-server.sh << 'LOGEOF'
+#!/bin/bash
+while true; do
+  {
+    echo "HTTP/1.1 200 OK"
+    echo "Content-Type: text/html; charset=utf-8"
+    echo "Connection: close"
+    echo ""
+    echo "<html><head><title>Task Log</title>"
+    echo "<meta http-equiv='refresh' content='5'>"
+    echo "<style>body{background:#1e1e1e;color:#d4d4d4;font-family:monospace;font-size:13px;padding:16px;white-space:pre-wrap;}"
+    echo "h2{color:#569cd6;margin:0 0 8px}.meta{color:#6a9955;margin-bottom:16px;display:block}</style></head><body>"
+    echo "<h2>$TASK_TYPE #$ISSUE_NUMBER — $(hostname)</h2>"
+    if [ -f ~/task-output.log ]; then
+      LINES=$(wc -l < ~/task-output.log)
+      if grep -q "Task completed" ~/task-output.log 2>/dev/null; then
+        echo "<span class='meta'>Status: ✅ Complete ($LINES lines)</span>"
+      else
+        echo "<span class='meta'>Status: ⏳ Running ($LINES lines) — auto-refreshing every 5s</span>"
+      fi
+      sed 's/&/\&amp;/g; s/</\&lt;/g; s/>/\&gt;/g' ~/task-output.log
+    else
+      echo "Waiting for task to start..."
+    fi
+    echo "</body></html>"
+  } | nc -l -p 13338 -q 1 2>/dev/null || true
+done
+LOGEOF
+      chmod +x ~/log-server.sh
+      nohup bash ~/log-server.sh &>/dev/null &
+    fi
+
     # --- Automated task execution ---
     if [ -n "$TASK_TYPE" ] && [ -d ~/project ]; then
       cd ~/project
@@ -515,14 +579,38 @@ resource "coder_agent" "main" {
         ARGS="$DEPLOY_ENV"
       fi
 
-      # Run Claude Code in non-interactive mode
+      # Run Claude Code in non-interactive mode with tool access
-      claude --print --dangerously-skip-permissions "/project:$TASK_TYPE $ARGS" 2>&1 | tee ~/task-output.log
+      claude -p --dangerously-skip-permissions --verbose "/project:$TASK_TYPE $ARGS" 2>&1 | tee ~/task-output.log
+      EXIT_CODE=$?
+      echo "Claude exited with code: $EXIT_CODE" | tee -a ~/task-output.log
 
       echo "Task completed. Output saved to ~/task-output.log"
+
+      # Notify orchestrator that the task is done
+      if [ -n "$CALLBACK_URL" ]; then
+        curl -s -X POST -H "Content-Type: application/json" \
+          -d '{"status":"complete"}' \
+          "$CALLBACK_URL" || echo "Callback failed (non-fatal)"
+      fi
     fi
   EOT
 }
 
+resource "coder_app" "task_log" {
+  agent_id     = coder_agent.main.id
+  slug         = "task-log"
+  display_name = "Task Log"
+  icon         = "/icon/document.svg"
+  url          = "http://localhost:13338"
+  share        = "owner"
+
+  healthcheck {
+    url       = "http://localhost:13338"
+    interval  = 10
+    threshold = 3
+  }
+}
+
 # =============================================================================
 # Main Workspace Deployment
 # =============================================================================
@@ -603,7 +691,7 @@ resource "kubernetes_deployment_v1" "workspace" {
 
         container {
           name  = "coder-agent"
-          image = "codercom/enterprise-base:ubuntu-arm64"
+          image = "codercom/enterprise-base:latest"
 
           command = ["sh", "-c", coder_agent.main.init_script]
 
@@ -657,6 +745,11 @@ resource "kubernetes_deployment_v1" "workspace" {
             value = data.coder_parameter.deploy_env.value
           }
 
+          env {
+            name  = "CALLBACK_URL"
+            value = data.coder_parameter.callback_url.value
+          }
+
           resources {
             requests = {
               cpu    = "${tonumber(data.coder_parameter.cpu.value) * 500}m"

src/config.ts

@@ -19,6 +19,8 @@ export interface Config {
   botReviewUsername: string;
   /** Gitea base URL (e.g. https://gitea.samson.media) */
   giteaUrl: string;
+  /** Base URL for task-complete callbacks (e.g. https://sdlc.samson.media) */
+  callbackUrl: string;
   /** Optional webhook secret for verifying Gitea signatures */
   webhookSecret?: string;
 }
@@ -43,6 +45,7 @@ export function loadConfig(): Config {
     botDevUsername: process.env.BOT_DEV_USERNAME || "claude-dev",
     botReviewUsername: process.env.BOT_REVIEW_USERNAME || "claude-review",
     giteaUrl: process.env.GITEA_URL || "https://gitea.samson.media",
+    callbackUrl: required("CALLBACK_URL"),
     webhookSecret: process.env.WEBHOOK_SECRET,
   };
 }

src/index.ts

@@ -49,6 +49,17 @@ app.post("/webhook/gitea-pr-review", prReview(config, queue, giteaClient));
 app.post("/webhook/gitea-pr-review-rework", prRework(config, queue));
 app.post("/webhook/gitea-release", release(config, queue));
 
+// ---------------------------------------------------------------------------
+// Task-complete callback — workspaces call this when done
+// ---------------------------------------------------------------------------
+
+app.post("/webhook/task-complete/:name", async (c) => {
+  const name = c.req.param("name");
+  console.log(`[callback] task-complete received for: ${name}`);
+  const found = await queue.taskComplete(name);
+  return c.json({ ok: true, found });
+});
+
 // ---------------------------------------------------------------------------
 // Scheduled maintenance cron (Monday 9:00 AM)
 // ---------------------------------------------------------------------------

src/queue.ts

@@ -7,16 +7,24 @@ interface QueuedTask {
   comment?: { useReviewAccount: boolean; body: string };
 }
 
+interface ActiveWorkspace {
+  workspaceId: string;
+  workspaceName: string;
+  startedAt: Date;
+}
+
 /**
- * Task queue with deduplication and concurrency control.
+ * Task queue with deduplication, concurrency control, and workspace lifecycle.
  *
  * - Tasks are keyed by `{taskType}-{repo}-{issue}` for dedup
- * - If a task with the same key is already pending or running, new requests are dropped
+ * - If a task with the same key is pending, running, or has an active workspace, new requests are dropped
  * - Concurrency is configurable (defaults to 2)
+ * - Tracks active workspaces and stops them on task-complete callback
  */
 export class TaskQueue {
   private pending = new Map<string, QueuedTask>();
   private running = new Set<string>();
+  private active = new Map<string, ActiveWorkspace>();
   private concurrency: number;
   private coder: CoderClient;
   private gitea: GiteaClient;
@@ -46,6 +54,11 @@ export class TaskQueue {
   ): boolean {
     const key = TaskQueue.key(task);
 
+    if (this.active.has(key)) {
+      console.log(`[queue] dropped (active workspace): ${key}`);
+      return false;
+    }
+
     if (this.running.has(key)) {
       console.log(`[queue] dropped (running): ${key}`);
       return false;
@@ -58,16 +71,50 @@ export class TaskQueue {
 
     this.pending.set(key, { task, comment });
     console.log(
-      `[queue] enqueued: ${key} (pending: ${this.pending.size}, running: ${this.running.size}/${this.concurrency})`,
+      `[queue] enqueued: ${key} (pending: ${this.pending.size}, running: ${this.running.size}/${this.concurrency}, active: ${this.active.size})`,
     );
 
     this.drain();
     return true;
   }
 
+  /**
+   * Handle task-complete callback from a workspace.
+   * Stops the workspace via Coder API and removes it from active tracking.
+   */
+  async taskComplete(workspaceName: string): Promise<boolean> {
+    const entry = this.active.get(workspaceName);
+    if (!entry) {
+      console.log(`[queue] task-complete for unknown workspace: ${workspaceName}`);
+      return false;
+    }
+
+    const elapsed = Math.round(
+      (Date.now() - entry.startedAt.getTime()) / 1000,
+    );
+    console.log(
+      `[queue] task complete: ${workspaceName} (ran for ${elapsed}s) — stopping workspace`,
+    );
+
+    this.active.delete(workspaceName);
+
+    try {
+      await this.coder.deleteWorkspace(entry.workspaceId);
+      console.log(`[queue] workspace deleted: ${workspaceName}`);
+    } catch (err) {
+      console.error(`[queue] failed to delete workspace ${workspaceName}:`, err);
+    }
+
+    // Drain in case pending tasks were waiting for capacity
+    this.drain();
+
+    return true;
+  }
+
   /** Process queued tasks up to the concurrency limit. */
   private drain(): void {
-    while (this.running.size < this.concurrency && this.pending.size > 0) {
+    const totalInFlight = this.running.size + this.active.size;
+    while (totalInFlight + this.pending.size > 0 && this.running.size + this.active.size < this.concurrency && this.pending.size > 0) {
       const [key, entry] = this.pending.entries().next().value!;
       this.pending.delete(key);
       this.running.add(key);
@@ -81,7 +128,14 @@ export class TaskQueue {
     try {
       console.log(`[queue] processing: ${key}`);
       const workspace = await this.coder.createWorkspace(task);
-      console.log(`[queue] workspace created: ${workspace.name}`);
+      console.log(`[queue] workspace created: ${workspace.name} (id: ${workspace.id})`);
+
+      // Move from running → active (workspace is now alive, waiting for callback)
+      this.active.set(key, {
+        workspaceId: workspace.id,
+        workspaceName: workspace.name,
+        startedAt: new Date(),
+      });
 
       if (comment) {
         await this.gitea.commentOnIssue(
@@ -96,15 +150,29 @@ export class TaskQueue {
       console.error(`[queue] failed: ${key}`, err);
     } finally {
       this.running.delete(key);
-      this.drain();
+      // Don't drain here — active workspaces count toward concurrency
     }
   }
 
   /** Current queue status for health/debug endpoints. */
-  status(): { pending: string[]; running: string[]; concurrency: number } {
+  status(): {
+    pending: string[];
+    running: string[];
+    active: Record<string, { workspaceId: string; elapsed: number }>;
+    concurrency: number;
+  } {
+    const activeMap: Record<string, { workspaceId: string; elapsed: number }> = {};
+    for (const [key, entry] of this.active) {
+      activeMap[key] = {
+        workspaceId: entry.workspaceId,
+        elapsed: Math.round((Date.now() - entry.startedAt.getTime()) / 1000),
+      };
+    }
+
     return {
       pending: [...this.pending.keys()],
       running: [...this.running.keys()],
+      active: activeMap,
       concurrency: this.concurrency,
     };
   }

src/services/coder.ts

@@ -7,12 +7,14 @@ export class CoderClient {
   private token: string;
   private templateId: string;
   private anthropicApiKey: string;
+  private callbackUrl: string;
 
   constructor(config: Config) {
     this.baseUrl = config.coderUrl.replace(/\/$/, "");
     this.token = config.coderToken;
     this.templateId = config.coderTemplateId;
     this.anthropicApiKey = config.anthropicApiKey;
+    this.callbackUrl = config.callbackUrl;
   }
 
   async createWorkspace(task: TaskRequest): Promise<{ id: string; name: string }> {
@@ -29,6 +31,7 @@ export class CoderClient {
         { name: "gitea_org", value: task.giteaOrg },
         { name: "gitea_repo", value: task.giteaRepo },
         { name: "anthropic_api_key", value: this.anthropicApiKey },
+        { name: "callback_url", value: `${this.callbackUrl}/webhook/task-complete/${name}` },
         ...(task.deployEnv
           ? [{ name: "deploy_env", value: task.deployEnv }]
           : []),
@@ -47,6 +50,38 @@ export class CoderClient {
       },
     );
 
+    // Handle 409 conflict — workspace with this name already exists.
+    // This happens when the orchestrator restarts and loses in-memory state.
+    // Only delete if the workspace is old (>30 min) or stopped/failed.
+    // If it's young and running, it may be mid-task — skip to avoid data loss.
+    if (res.status === 409) {
+      const existing = await this.findWorkspaceByName(name);
+      if (!existing) {
+        throw new Error(`Coder 409 but workspace "${name}" not found — possible race condition`);
+      }
+
+      const ageMs = Date.now() - new Date(existing.createdAt).getTime();
+      const ageMin = Math.round(ageMs / 60_000);
+      const stoppedStatuses = ["stopped", "failed", "canceled", "deleted"];
+      const isStopped = stoppedStatuses.includes(existing.latestBuildStatus);
+      const isStale = ageMin > 30;
+
+      if (isStopped || isStale) {
+        console.log(
+          `[coder] workspace "${name}" already exists (age: ${ageMin}m, status: ${existing.latestBuildStatus}) — deleting and retrying`,
+        );
+        await this.deleteWorkspace(existing.id);
+        await new Promise((resolve) => setTimeout(resolve, 5000));
+        return this.createWorkspace(task);
+      }
+
+      // Workspace is young and still running — don't kill it
+      console.log(
+        `[coder] workspace "${name}" is active (age: ${ageMin}m, status: ${existing.latestBuildStatus}) — skipping creation`,
+      );
+      return { id: existing.id, name: existing.name };
+    }
+
     if (!res.ok) {
       const text = await res.text();
       throw new Error(`Coder API error ${res.status}: ${text}`);
@@ -55,4 +90,75 @@ export class CoderClient {
     const data = (await res.json()) as { id: string; name: string };
     return { id: data.id, name: data.name };
   }
+
+  async stopWorkspace(workspaceId: string): Promise<void> {
+    const res = await fetch(
+      `${this.baseUrl}/api/v2/workspaces/${workspaceId}/builds`,
+      {
+        method: "POST",
+        headers: {
+          "Content-Type": "application/json",
+          "Coder-Session-Token": this.token,
+        },
+        body: JSON.stringify({ transition: "stop" }),
+      },
+    );
+
+    if (!res.ok) {
+      const text = await res.text();
+      throw new Error(`Coder stop error ${res.status}: ${text}`);
+    }
+  }
+
+  async deleteWorkspace(workspaceId: string): Promise<void> {
+    const res = await fetch(
+      `${this.baseUrl}/api/v2/workspaces/${workspaceId}`,
+      {
+        method: "DELETE",
+        headers: {
+          "Coder-Session-Token": this.token,
+        },
+      },
+    );
+
+    if (!res.ok) {
+      const text = await res.text();
+      throw new Error(`Coder delete error ${res.status}: ${text}`);
+    }
+  }
+
+  async findWorkspaceByName(name: string): Promise<{
+    id: string;
+    name: string;
+    createdAt: string;
+    latestBuildStatus: string;
+  } | null> {
+    const res = await fetch(
+      `${this.baseUrl}/api/v2/workspaces?q=name:${encodeURIComponent(name)}`,
+      {
+        headers: {
+          "Coder-Session-Token": this.token,
+        },
+      },
+    );
+
+    if (!res.ok) return null;
+
+    const data = (await res.json()) as {
+      workspaces: Array<{
+        id: string;
+        name: string;
+        created_at: string;
+        latest_build: { status: string };
+      }>;
+    };
+    const match = data.workspaces?.find((w) => w.name === name);
+    if (!match) return null;
+    return {
+      id: match.id,
+      name: match.name,
+      createdAt: match.created_at,
+      latestBuildStatus: match.latest_build?.status ?? "unknown",
+    };
+  }
 }