CI pipeline: WAM-triggered CI with auto-revert on failure Revised architecture: post-receive hook creates a WAM ci-trigger ticket, astra's ci-watcher polls for triggers and runs CI. Avoids Tailscale SSH ACL issues between Hetzner and astra. - ci-on-push.sh: pulls latest, runs full test suite, auto-reverts and pushes revert if tests fail (enforces no-regressions rule) - ci-watcher.sh: polls WAM every 30s for ci-trigger tickets, claims and runs CI, closes trigger on completion - post-receive hook: creates WAM ticket on main push (replaces SSH) - cicd.md: updated roadmap with TUI monitor and staging steps
- Co-Authored-By
Author: Max J. <87768334+MaxJMath@users.noreply.github.com> - 2026-04-25 20:42 UTC
Commit:
0d2944286fa74ae12c2b9d6097a72f316bacf601Parent:
3 files changed,
+171 insertions,
-60 deletions
OldNewLine
@@ -2,9 +2,11 @@
2
2
# CI wrapper for push-triggered builds on astra.3
3
# Called by the post-receive hook on Hetzner via SSH.4
4
#5
# Pulls latest code, runs the CI suite, and reports results to WAM.5
# Pulls latest code, runs the CI suite, reports results to WAM.6
# If tests fail on main, auto-reverts the offending commit(s) and pushes7
# the revert — enforcing the no-regressions rule from operations.md.6
8
#7
# Location on astra: /home/max/staging/ci-on-push.sh9
# Location on astra: /home/max/mnw-ci/server/deploy/ci-on-push.sh8
10
# Usage: ci-on-push.sh [branch_name]9
11
10
12
set -uo pipefail@@ -12,87 +14,134 @@
12
14
export PATH="$HOME/.cargo/bin:$PATH"13
15
14
16
BRANCH="${1:-main}"15
STAGING_DIR="$HOME/staging"16
SERVER_DIR="$STAGING_DIR/server"17
REPO_DIR="$HOME/mnw-ci"18
SERVER_DIR="$REPO_DIR/server"17
19
WAM_URL="${WAM_URL:-http://100.120.174.96:7890}"18
20
START_TIME=$(date +%s)19
LOG_FILE="$STAGING_DIR/ci-latest.log"21
LOG_FILE="$REPO_DIR/ci-latest.log"20
22
21
23
echo "=== CI triggered for branch: $BRANCH ==="22
24
echo "Started: $(date -u +%Y-%m-%dT%H:%M:%SZ)"23
25
24
26
# Pull latest code25
cd "$STAGING_DIR" || exit 127
cd "$REPO_DIR" || exit 128
echo "[pull] Fetching latest..."29
git fetch origin "$BRANCH" 2>&126
30
27
if [ -d ".git" ]; then28
echo "[pull] Fetching latest..."29
git fetch origin "$BRANCH" 2>&130
git reset --hard "origin/$BRANCH" 2>&131
else32
echo "[pull] No git repo in staging dir, skipping pull"33
fi31
# Record what we're about to test32
OLD_HEAD=$(git rev-parse HEAD 2>/dev/null || echo "none")33
git reset --hard "origin/$BRANCH" 2>&134
NEW_HEAD=$(git rev-parse HEAD)35
COMMIT_MSG=$(git log --oneline -1)34
36
35
# Run CI (capture output for WAM ticket)36
cd "$SERVER_DIR" 2>/dev/null || cd "$STAGING_DIR" || exit 137
echo "[pull] $OLD_HEAD -> $NEW_HEAD"38
echo "[pull] $COMMIT_MSG"39
40
# Run CI from the server directory41
cd "$SERVER_DIR" || exit 142
43
export SQLX_OFFLINE=true44
export TEST_DATABASE_URL="${TEST_DATABASE_URL:-postgres:///postgres}"45
export RUST_TEST_THREADS="${RUST_TEST_THREADS:-8}"46
export CARGO_INCREMENTAL=047
export RUST_BACKTRACE=137
48
38
49
echo "[ci] Running CI suite..."39
CI_OUTPUT=$("$STAGING_DIR/run-ci.sh" 2>&1) || true50
CI_OUTPUT=$("$REPO_DIR/server/deploy/run-ci.sh" 2>&1) || true40
51
CI_EXIT=$?41
52
42
53
END_TIME=$(date +%s)43
54
DURATION=$(( END_TIME - START_TIME ))44
55
45
56
# Extract summary from CI output46
SUMMARY=$(echo "$CI_OUTPUT" | grep -A 50 "CI Summary" | tail -n +2)47
57
PASS_COUNT=$(echo "$CI_OUTPUT" | grep -c "^ PASS" || true)48
58
FAIL_COUNT=$(echo "$CI_OUTPUT" | grep -c "^ FAIL" || true)49
59
50
60
# Save full log51
61
echo "$CI_OUTPUT" > "$LOG_FILE"52
62
53
# Determine ticket priority and title54
if [ $CI_EXIT -eq 0 ]; then55
PRIORITY="low"56
TITLE="CI passed: $BRANCH ($PASS_COUNT steps, ${DURATION}s)"57
STATUS_LINE="All steps passed."58
else59
PRIORITY="high"60
TITLE="CI failed: $BRANCH ($FAIL_COUNT step(s) failed, ${DURATION}s)"61
# Extract failed step names62
FAILED_STEPS=$(echo "$CI_OUTPUT" | grep "^ FAIL" | sed 's/^ FAIL / - /')63
STATUS_LINE="Failed steps:\n$FAILED_STEPS"64
fi63
# --- Handle results ---65
64
66
# Build ticket body (truncate to avoid huge payloads)67
BODY=$(cat <<TICKET68
Branch: $BRANCH65
wam_ticket() {66
local title="$1" body="$2" priority="$3" source_ref="$4"67
local escaped_title escaped_body68
escaped_title=$(echo "$title" | python3 -c 'import sys,json; print(json.dumps(sys.stdin.read().strip()))' 2>/dev/null || echo "\"$title\"")69
escaped_body=$(echo "$body" | python3 -c 'import sys,json; print(json.dumps(sys.stdin.read()))' 2>/dev/null || echo '""')70
curl -sf -X POST "$WAM_URL/tickets" \71
-H "Content-Type: application/json" \72
-d "{\"title\": $escaped_title, \"body\": $escaped_body, \"priority\": \"$priority\", \"source\": \"ci\", \"source_ref\": \"$source_ref\"}" \73
>/dev/null 2>&1 || echo "[warn] Failed to create WAM ticket"74
}75
76
if [ $CI_EXIT -eq 0 ]; then77
# --- CI PASSED ---78
TITLE="CI passed: $BRANCH ($PASS_COUNT steps, ${DURATION}s)"79
BODY="Commit: $COMMIT_MSG80
Duration: ${DURATION}s81
Steps passed: $PASS_COUNT82
83
All steps passed."84
85
wam_ticket "$TITLE" "$BODY" "low" "$NEW_HEAD"86
echo ""87
echo "=== CI PASSED ($PASS_COUNT steps, ${DURATION}s) ==="88
else89
# --- CI FAILED — AUTO-REVERT ---90
FAILED_STEPS=$(echo "$CI_OUTPUT" | grep "^ FAIL" | sed 's/^ FAIL / - /')91
92
echo ""93
echo "=== CI FAILED — REVERTING ==="94
echo ""95
96
cd "$REPO_DIR" || exit 197
98
# Count how many new commits since old head99
if [ "$OLD_HEAD" = "none" ] || [ "$OLD_HEAD" = "$NEW_HEAD" ]; then100
# Can't determine what to revert (first run or no change)101
REVERT_STATUS="could not determine commits to revert"102
else103
NEW_COMMITS=$(git rev-list "$OLD_HEAD..$NEW_HEAD" --count 2>/dev/null || echo "0")104
105
if [ "$NEW_COMMITS" -eq 1 ]; then106
# Single commit — revert it107
git revert --no-edit HEAD 2>&1108
REVERT_STATUS="reverted 1 commit ($NEW_HEAD)"109
elif [ "$NEW_COMMITS" -gt 1 ]; then110
# Multiple commits — revert the range111
git revert --no-edit "$OLD_HEAD..$NEW_HEAD" 2>&1112
REVERT_STATUS="reverted $NEW_COMMITS commits ($OLD_HEAD..$NEW_HEAD)"113
else114
REVERT_STATUS="no new commits to revert"115
fi116
117
# Push the revert back to origin118
if git push origin "$BRANCH" 2>&1; then119
REVERT_STATUS="$REVERT_STATUS — pushed to origin"120
else121
REVERT_STATUS="$REVERT_STATUS — PUSH FAILED (manual intervention needed)"122
fi123
fi124
125
TITLE="CI FAILED + REVERTED: $BRANCH ($FAIL_COUNT step(s) failed)"126
BODY="Commit: $COMMIT_MSG69
127
Duration: ${DURATION}s70
128
Steps passed: $PASS_COUNT71
129
Steps failed: $FAIL_COUNT72
130
73
$STATUS_LINE131
Failed steps:132
$FAILED_STEPS133
134
Revert: $REVERT_STATUS135
136
No-regressions rule enforced automatically.137
Fix the issue and re-push.74
138
75
139
Last 30 lines of output:76
$(echo "$CI_OUTPUT" | tail -30)77
TICKET78
)140
$(echo "$CI_OUTPUT" | tail -30)"79
141
80
# Create WAM ticket81
ESCAPED_BODY=$(echo "$BODY" | python3 -c 'import sys,json; print(json.dumps(sys.stdin.read()))' 2>/dev/null || echo '""')142
wam_ticket "$TITLE" "$BODY" "critical" "$NEW_HEAD"143
echo "Revert: $REVERT_STATUS"144
echo ""145
fi82
146
83
curl -sf -X POST "$WAM_URL/tickets" \84
-H "Content-Type: application/json" \85
-d "{86
\"title\": $(echo "$TITLE" | python3 -c 'import sys,json; print(json.dumps(sys.stdin.read().strip()))'),87
\"body\": $ESCAPED_BODY,88
\"priority\": \"$PRIORITY\",89
\"source\": \"ci\",90
\"source_ref\": \"$BRANCH\"91
}" >/dev/null 2>&1 || echo "[warn] Failed to create WAM ticket"92
93
echo ""94
echo "$SUMMARY"95
echo ""96
echo "Duration: ${DURATION}s"97
echo "WAM ticket: $PRIORITY"98
147
exit $CI_EXITOldNewLine
@@ -7,9 +7,8 @@
7
7
#8
8
# Install: copy to /opt/git/max/makenotwork.git/hooks/post-receive9
9
10
ASTRA_HOST="max@100.106.221.39"11
ASTRA_CI_SCRIPT="/home/max/staging/ci-on-push.sh"12
10
BUILD_TOKEN="a11d2b9ff121a70e7391b36d00e812da6bba1dd91ed17d4b5e0f0dbb7f7f66cc"11
WAM_URL="http://127.0.0.1:7890"13
12
14
13
while read oldrev newrev refname; do15
14
case "$refname" in@@ -27,11 +26,12 @@
27
26
>/dev/null 2>&1 &28
27
;;29
28
refs/heads/main)30
# CI trigger: SSH to astra in background31
# Uses the git user's SSH key (set up during CI provisioning)32
ssh -o StrictHostKeyChecking=no -o ConnectTimeout=5 \33
"$ASTRA_HOST" "$ASTRA_CI_SCRIPT main" \34
</dev/null >/dev/null 2>&1 &29
# CI trigger: create a WAM ticket that astra's CI watcher picks up.30
# Avoids SSH ACL issues — both machines can reach WAM on localhost/tailnet.31
curl -sf -X POST "$WAM_URL/tickets" \32
-H "Content-Type: application/json" \33
-d "{\"title\": \"CI trigger: main pushed ($newrev)\", \"priority\": \"medium\", \"source\": \"ci-trigger\", \"source_ref\": \"$newrev\"}" \34
>/dev/null 2>&1 &35
35
;;36
36
esac37
37
doneOldNewLine
@@ -1,0 +1,62 @@
1
#!/bin/bash2
# CI watcher for astra — polls WAM for ci-trigger tickets and runs CI.3
#4
# Runs as a background service on astra. Checks WAM every 30 seconds for5
# open tickets with source=ci-trigger. When found, claims the ticket6
# (marks it in-progress), runs CI, and updates the ticket with results.7
#8
# Location on astra: /home/max/mnw-ci/server/deploy/ci-watcher.sh9
# Systemd unit: ci-watcher.service10
#11
# Usage: ci-watcher.sh12
13
set -uo pipefail14
15
export PATH="$HOME/.cargo/bin:$PATH"16
17
WAM_URL="${WAM_URL:-http://100.120.174.96:7890}"18
REPO_DIR="$HOME/mnw-ci"19
POLL_INTERVAL=3020
CI_SCRIPT="$REPO_DIR/server/deploy/ci-on-push.sh"21
22
echo "CI watcher started (polling WAM every ${POLL_INTERVAL}s)"23
24
while true; do25
sleep "$POLL_INTERVAL"26
27
# Check for open ci-trigger tickets28
RESPONSE=$(curl -sf "$WAM_URL/tickets?source=ci-trigger&status=open" 2>/dev/null) || continue29
COUNT=$(echo "$RESPONSE" | python3 -c 'import sys,json; print(json.load(sys.stdin).get("count",0))' 2>/dev/null || echo "0")30
31
if [ "$COUNT" = "0" ] || [ -z "$COUNT" ]; then32
continue33
fi34
35
# Get the first trigger ticket36
TICKET_ID=$(echo "$RESPONSE" | python3 -c 'import sys,json; d=json.load(sys.stdin); print(d["data"][0]["id"])' 2>/dev/null) || continue37
COMMIT=$(echo "$RESPONSE" | python3 -c 'import sys,json; d=json.load(sys.stdin); print(d["data"][0].get("source_ref","unknown"))' 2>/dev/null || echo "unknown")38
39
echo ""40
echo "=== CI trigger found: $COMMIT ==="41
42
# Claim the ticket (mark in-progress so we don't pick it up again)43
curl -sf -X PATCH "$WAM_URL/tickets/$TICKET_ID" \44
-H "Content-Type: application/json" \45
-d '{"status": "in-progress"}' >/dev/null 2>&146
47
# Run CI48
if "$CI_SCRIPT" main 2>&1; then49
# CI passed — close the trigger ticket50
curl -sf -X PATCH "$WAM_URL/tickets/$TICKET_ID" \51
-H "Content-Type: application/json" \52
-d '{"status": "closed"}' >/dev/null 2>&153
else54
# CI failed — ci-on-push.sh already created a failure ticket and reverted.55
# Close the trigger ticket.56
curl -sf -X PATCH "$WAM_URL/tickets/$TICKET_ID" \57
-H "Content-Type: application/json" \58
-d '{"status": "closed"}' >/dev/null 2>&159
fi60
61
echo "=== CI run complete ==="62
done