Skip to main content

max / makenotwork

CI pipeline: WAM-triggered CI with auto-revert on failure Revised architecture: post-receive hook creates a WAM ci-trigger ticket, astra's ci-watcher polls for triggers and runs CI. Avoids Tailscale SSH ACL issues between Hetzner and astra. - ci-on-push.sh: pulls latest, runs full test suite, auto-reverts and pushes revert if tests fail (enforces no-regressions rule) - ci-watcher.sh: polls WAM every 30s for ci-trigger tickets, claims and runs CI, closes trigger on completion - post-receive hook: creates WAM ticket on main push (replaces SSH) - cicd.md: updated roadmap with TUI monitor and staging steps Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Author: Max J. <87768334+MaxJMath@users.noreply.github.com> · 2026-04-25 20:42 UTC
Commit: 0d2944286fa74ae12c2b9d6097a72f316bacf601
Parent: cfb7e9e
3 files changed, +169 insertions, -58 deletions
M server/deploy/ci-on-push.sh +100 -51
@@ -2,9 +2,11 @@
2 2 # CI wrapper for push-triggered builds on astra.
3 3 # Called by the post-receive hook on Hetzner via SSH.
4 4 #
5 - # Pulls latest code, runs the CI suite, and reports results to WAM.
5 + # Pulls latest code, runs the CI suite, reports results to WAM.
6 + # If tests fail on main, auto-reverts the offending commit(s) and pushes
7 + # the revert — enforcing the no-regressions rule from operations.md.
6 8 #
7 - # Location on astra: /home/max/staging/ci-on-push.sh
9 + # Location on astra: /home/max/mnw-ci/server/deploy/ci-on-push.sh
8 10 # Usage: ci-on-push.sh [branch_name]
9 11
10 12 set -uo pipefail
@@ -12,87 +14,134 @@ set -uo pipefail
12 14 export PATH="$HOME/.cargo/bin:$PATH"
13 15
14 16 BRANCH="${1:-main}"
15 - STAGING_DIR="$HOME/staging"
16 - SERVER_DIR="$STAGING_DIR/server"
17 + REPO_DIR="$HOME/mnw-ci"
18 + SERVER_DIR="$REPO_DIR/server"
17 19 WAM_URL="${WAM_URL:-http://100.120.174.96:7890}"
18 20 START_TIME=$(date +%s)
19 - LOG_FILE="$STAGING_DIR/ci-latest.log"
21 + LOG_FILE="$REPO_DIR/ci-latest.log"
20 22
21 23 echo "=== CI triggered for branch: $BRANCH ==="
22 24 echo "Started: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
23 25
24 26 # Pull latest code
25 - cd "$STAGING_DIR" || exit 1
27 + cd "$REPO_DIR" || exit 1
28 + echo "[pull] Fetching latest..."
29 + git fetch origin "$BRANCH" 2>&1
26 30
27 - if [ -d ".git" ]; then
28 - echo "[pull] Fetching latest..."
29 - git fetch origin "$BRANCH" 2>&1
30 - git reset --hard "origin/$BRANCH" 2>&1
31 - else
32 - echo "[pull] No git repo in staging dir, skipping pull"
33 - fi
31 + # Record what we're about to test
32 + OLD_HEAD=$(git rev-parse HEAD 2>/dev/null || echo "none")
33 + git reset --hard "origin/$BRANCH" 2>&1
34 + NEW_HEAD=$(git rev-parse HEAD)
35 + COMMIT_MSG=$(git log --oneline -1)
36 +
37 + echo "[pull] $OLD_HEAD -> $NEW_HEAD"
38 + echo "[pull] $COMMIT_MSG"
39 +
40 + # Run CI from the server directory
41 + cd "$SERVER_DIR" || exit 1
34 42
35 - # Run CI (capture output for WAM ticket)
36 - cd "$SERVER_DIR" 2>/dev/null || cd "$STAGING_DIR" || exit 1
43 + export SQLX_OFFLINE=true
44 + export TEST_DATABASE_URL="${TEST_DATABASE_URL:-postgres:///postgres}"
45 + export RUST_TEST_THREADS="${RUST_TEST_THREADS:-8}"
46 + export CARGO_INCREMENTAL=0
47 + export RUST_BACKTRACE=1
37 48
38 49 echo "[ci] Running CI suite..."
39 - CI_OUTPUT=$("$STAGING_DIR/run-ci.sh" 2>&1) || true
50 + CI_OUTPUT=$("$REPO_DIR/server/deploy/run-ci.sh" 2>&1) || true
40 51 CI_EXIT=$?
41 52
42 53 END_TIME=$(date +%s)
43 54 DURATION=$(( END_TIME - START_TIME ))
44 55
45 56 # Extract summary from CI output
46 - SUMMARY=$(echo "$CI_OUTPUT" | grep -A 50 "CI Summary" | tail -n +2)
47 57 PASS_COUNT=$(echo "$CI_OUTPUT" | grep -c "^ PASS" || true)
48 58 FAIL_COUNT=$(echo "$CI_OUTPUT" | grep -c "^ FAIL" || true)
49 59
50 60 # Save full log
51 61 echo "$CI_OUTPUT" > "$LOG_FILE"
52 62
53 - # Determine ticket priority and title
63 + # --- Handle results ---
64 +
65 + wam_ticket() {
66 + local title="$1" body="$2" priority="$3" source_ref="$4"
67 + local escaped_title escaped_body
68 + escaped_title=$(echo "$title" | python3 -c 'import sys,json; print(json.dumps(sys.stdin.read().strip()))' 2>/dev/null || echo "\"$title\"")
69 + escaped_body=$(echo "$body" | python3 -c 'import sys,json; print(json.dumps(sys.stdin.read()))' 2>/dev/null || echo '""')
70 + curl -sf -X POST "$WAM_URL/tickets" \
71 + -H "Content-Type: application/json" \
72 + -d "{\"title\": $escaped_title, \"body\": $escaped_body, \"priority\": \"$priority\", \"source\": \"ci\", \"source_ref\": \"$source_ref\"}" \
73 + >/dev/null 2>&1 || echo "[warn] Failed to create WAM ticket"
74 + }
75 +
54 76 if [ $CI_EXIT -eq 0 ]; then
55 - PRIORITY="low"
77 + # --- CI PASSED ---
56 78 TITLE="CI passed: $BRANCH ($PASS_COUNT steps, ${DURATION}s)"
57 - STATUS_LINE="All steps passed."
79 + BODY="Commit: $COMMIT_MSG
80 + Duration: ${DURATION}s
81 + Steps passed: $PASS_COUNT
82 +
83 + All steps passed."
84 +
85 + wam_ticket "$TITLE" "$BODY" "low" "$NEW_HEAD"
86 + echo ""
87 + echo "=== CI PASSED ($PASS_COUNT steps, ${DURATION}s) ==="
58 88 else
59 - PRIORITY="high"
60 - TITLE="CI failed: $BRANCH ($FAIL_COUNT step(s) failed, ${DURATION}s)"
61 - # Extract failed step names
89 + # --- CI FAILED — AUTO-REVERT ---
62 90 FAILED_STEPS=$(echo "$CI_OUTPUT" | grep "^ FAIL" | sed 's/^ FAIL / - /')
63 - STATUS_LINE="Failed steps:\n$FAILED_STEPS"
64 - fi
65 91
66 - # Build ticket body (truncate to avoid huge payloads)
67 - BODY=$(cat <<TICKET
68 - Branch: $BRANCH
92 + echo ""
93 + echo "=== CI FAILED — REVERTING ==="
94 + echo ""
95 +
96 + cd "$REPO_DIR" || exit 1
97 +
98 + # Count how many new commits since old head
99 + if [ "$OLD_HEAD" = "none" ] || [ "$OLD_HEAD" = "$NEW_HEAD" ]; then
100 + # Can't determine what to revert (first run or no change)
101 + REVERT_STATUS="could not determine commits to revert"
102 + else
103 + NEW_COMMITS=$(git rev-list "$OLD_HEAD..$NEW_HEAD" --count 2>/dev/null || echo "0")
104 +
105 + if [ "$NEW_COMMITS" -eq 1 ]; then
106 + # Single commit — revert it
107 + git revert --no-edit HEAD 2>&1
108 + REVERT_STATUS="reverted 1 commit ($NEW_HEAD)"
109 + elif [ "$NEW_COMMITS" -gt 1 ]; then
110 + # Multiple commits — revert the range
111 + git revert --no-edit "$OLD_HEAD..$NEW_HEAD" 2>&1
112 + REVERT_STATUS="reverted $NEW_COMMITS commits ($OLD_HEAD..$NEW_HEAD)"
113 + else
114 + REVERT_STATUS="no new commits to revert"
115 + fi
116 +
117 + # Push the revert back to origin
118 + if git push origin "$BRANCH" 2>&1; then
119 + REVERT_STATUS="$REVERT_STATUS — pushed to origin"
120 + else
121 + REVERT_STATUS="$REVERT_STATUS — PUSH FAILED (manual intervention needed)"
122 + fi
123 + fi
124 +
125 + TITLE="CI FAILED + REVERTED: $BRANCH ($FAIL_COUNT step(s) failed)"
126 + BODY="Commit: $COMMIT_MSG
69 127 Duration: ${DURATION}s
70 128 Steps passed: $PASS_COUNT
71 129 Steps failed: $FAIL_COUNT
72 130
73 - $STATUS_LINE
131 + Failed steps:
132 + $FAILED_STEPS
133 +
134 + Revert: $REVERT_STATUS
135 +
136 + No-regressions rule enforced automatically.
137 + Fix the issue and re-push.
74 138
75 139 Last 30 lines of output:
76 - $(echo "$CI_OUTPUT" | tail -30)
77 - TICKET
78 - )
79 -
80 - # Create WAM ticket
81 - ESCAPED_BODY=$(echo "$BODY" | python3 -c 'import sys,json; print(json.dumps(sys.stdin.read()))' 2>/dev/null || echo '""')
82 -
83 - curl -sf -X POST "$WAM_URL/tickets" \
84 - -H "Content-Type: application/json" \
85 - -d "{
86 - \"title\": $(echo "$TITLE" | python3 -c 'import sys,json; print(json.dumps(sys.stdin.read().strip()))'),
87 - \"body\": $ESCAPED_BODY,
88 - \"priority\": \"$PRIORITY\",
89 - \"source\": \"ci\",
90 - \"source_ref\": \"$BRANCH\"
91 - }" >/dev/null 2>&1 || echo "[warn] Failed to create WAM ticket"
92 -
93 - echo ""
94 - echo "$SUMMARY"
95 - echo ""
96 - echo "Duration: ${DURATION}s"
97 - echo "WAM ticket: $PRIORITY"
140 + $(echo "$CI_OUTPUT" | tail -30)"
141 +
142 + wam_ticket "$TITLE" "$BODY" "critical" "$NEW_HEAD"
143 + echo "Revert: $REVERT_STATUS"
144 + echo ""
145 + fi
146 +
98 147 exit $CI_EXIT
A server/deploy/ci-watcher.sh +62
@@ -0,0 +1,62 @@
1 + #!/bin/bash
2 + # CI watcher for astra — polls WAM for ci-trigger tickets and runs CI.
3 + #
4 + # Runs as a background service on astra. Checks WAM every 30 seconds for
5 + # open tickets with source=ci-trigger. When found, claims the ticket
6 + # (marks it in-progress), runs CI, and updates the ticket with results.
7 + #
8 + # Location on astra: /home/max/mnw-ci/server/deploy/ci-watcher.sh
9 + # Systemd unit: ci-watcher.service
10 + #
11 + # Usage: ci-watcher.sh
12 +
13 + set -uo pipefail
14 +
15 + export PATH="$HOME/.cargo/bin:$PATH"
16 +
17 + WAM_URL="${WAM_URL:-http://100.120.174.96:7890}"
18 + REPO_DIR="$HOME/mnw-ci"
19 + POLL_INTERVAL=30
20 + CI_SCRIPT="$REPO_DIR/server/deploy/ci-on-push.sh"
21 +
22 + echo "CI watcher started (polling WAM every ${POLL_INTERVAL}s)"
23 +
24 + while true; do
25 + sleep "$POLL_INTERVAL"
26 +
27 + # Check for open ci-trigger tickets
28 + RESPONSE=$(curl -sf "$WAM_URL/tickets?source=ci-trigger&status=open" 2>/dev/null) || continue
29 + COUNT=$(echo "$RESPONSE" | python3 -c 'import sys,json; print(json.load(sys.stdin).get("count",0))' 2>/dev/null || echo "0")
30 +
31 + if [ "$COUNT" = "0" ] || [ -z "$COUNT" ]; then
32 + continue
33 + fi
34 +
35 + # Get the first trigger ticket
36 + TICKET_ID=$(echo "$RESPONSE" | python3 -c 'import sys,json; d=json.load(sys.stdin); print(d["data"][0]["id"])' 2>/dev/null) || continue
37 + COMMIT=$(echo "$RESPONSE" | python3 -c 'import sys,json; d=json.load(sys.stdin); print(d["data"][0].get("source_ref","unknown"))' 2>/dev/null || echo "unknown")
38 +
39 + echo ""
40 + echo "=== CI trigger found: $COMMIT ==="
41 +
42 + # Claim the ticket (mark in-progress so we don't pick it up again)
43 + curl -sf -X PATCH "$WAM_URL/tickets/$TICKET_ID" \
44 + -H "Content-Type: application/json" \
45 + -d '{"status": "in-progress"}' >/dev/null 2>&1
46 +
47 + # Run CI
48 + if "$CI_SCRIPT" main 2>&1; then
49 + # CI passed — close the trigger ticket
50 + curl -sf -X PATCH "$WAM_URL/tickets/$TICKET_ID" \
51 + -H "Content-Type: application/json" \
52 + -d '{"status": "closed"}' >/dev/null 2>&1
53 + else
54 + # CI failed — ci-on-push.sh already created a failure ticket and reverted.
55 + # Close the trigger ticket.
56 + curl -sf -X PATCH "$WAM_URL/tickets/$TICKET_ID" \
57 + -H "Content-Type: application/json" \
58 + -d '{"status": "closed"}' >/dev/null 2>&1
59 + fi
60 +
61 + echo "=== CI run complete ==="
62 + done
M server/deploy/post-receive-hook.sh +7 -7
@@ -7,9 +7,8 @@
7 7 #
8 8 # Install: copy to /opt/git/max/makenotwork.git/hooks/post-receive
9 9
10 - ASTRA_HOST="max@100.106.221.39"
11 - ASTRA_CI_SCRIPT="/home/max/staging/ci-on-push.sh"
12 10 BUILD_TOKEN="a11d2b9ff121a70e7391b36d00e812da6bba1dd91ed17d4b5e0f0dbb7f7f66cc"
11 + WAM_URL="http://127.0.0.1:7890"
13 12
14 13 while read oldrev newrev refname; do
15 14 case "$refname" in
@@ -27,11 +26,12 @@ while read oldrev newrev refname; do
27 26 >/dev/null 2>&1 &
28 27 ;;
29 28 refs/heads/main)
30 - # CI trigger: SSH to astra in background
31 - # Uses the git user's SSH key (set up during CI provisioning)
32 - ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 \
33 - "$ASTRA_HOST" "$ASTRA_CI_SCRIPT main" \
34 - </dev/null >/dev/null 2>&1 &
29 + # CI trigger: create a WAM ticket that astra's CI watcher picks up.
30 + # Avoids SSH ACL issues — both machines can reach WAM on localhost/tailnet.
31 + curl -sf -X POST "$WAM_URL/tickets" \
32 + -H "Content-Type: application/json" \
33 + -d "{\"title\": \"CI trigger: main pushed ($newrev)\", \"priority\": \"medium\", \"source\": \"ci-trigger\", \"source_ref\": \"$newrev\"}" \
34 + >/dev/null 2>&1 &
35 35 ;;
36 36 esac
37 37 done