| 7 |
7 |
|
use serde::{Deserialize, Serialize};
|
| 8 |
8 |
|
use sqlx::Row;
|
| 9 |
9 |
|
|
|
10 |
+ |
mod promotion;
|
|
11 |
+ |
use promotion::{promote_inner, set_partial, clear_partial};
|
|
12 |
+ |
|
| 10 |
13 |
|
pub fn router(state: AppState) -> Router {
|
| 11 |
14 |
|
let prom = state.prom.clone();
|
| 12 |
15 |
|
let token = state.api_token.clone();
|
| 260 |
263 |
|
.map_err(|e| crate::error::Error::Other(anyhow::anyhow!("promote task failed to join: {e}")))?
|
| 261 |
264 |
|
}
|
| 262 |
265 |
|
|
| 263 |
|
- |
async fn promote_inner(
|
| 264 |
|
- |
s: AppState,
|
| 265 |
|
- |
tier: String,
|
| 266 |
|
- |
body: PromoteBody,
|
| 267 |
|
- |
) -> Result<Json<serde_json::Value>> {
|
| 268 |
|
- |
// Serialize the whole check -> deploy -> advance against any concurrent
|
| 269 |
|
- |
// promote/rollback (CF3). Held for the entire task.
|
| 270 |
|
- |
let _deploy_guard = s.deploy_lock.lock().await;
|
| 271 |
|
- |
let tier = crate::domain::TierId::new(tier);
|
| 272 |
|
- |
let idx = s.topo.tiers.iter().position(|t| t.name == tier)
|
| 273 |
|
- |
.ok_or(crate::error::Error::NotFound)?;
|
| 274 |
|
- |
if idx == 0 {
|
| 275 |
|
- |
return Err(crate::error::Error::GateBlocked(
|
| 276 |
|
- |
"cannot /promote to the first tier; use /rebuild".into(),
|
| 277 |
|
- |
));
|
| 278 |
|
- |
}
|
| 279 |
|
- |
let target = &s.topo.tiers[idx];
|
| 280 |
|
- |
let source = &s.topo.tiers[idx - 1];
|
| 281 |
|
- |
|
| 282 |
|
- |
// Resolve version: explicit if given, else the source tier's current.
|
| 283 |
|
- |
let version_str = match body.version {
|
| 284 |
|
- |
Some(v) => v,
|
| 285 |
|
- |
None => sqlx::query_scalar::<_, Option<String>>(
|
| 286 |
|
- |
"SELECT current_version FROM tier_state WHERE tier = ?",
|
| 287 |
|
- |
)
|
| 288 |
|
- |
.bind(&source.name)
|
| 289 |
|
- |
.fetch_optional(&s.pool).await
|
| 290 |
|
- |
.map_err(crate::error::Error::Db)?
|
| 291 |
|
- |
.flatten()
|
| 292 |
|
- |
.ok_or_else(|| crate::error::Error::GateBlocked(
|
| 293 |
|
- |
format!("no version specified and tier {} has no current_version", source.name),
|
| 294 |
|
- |
))?,
|
| 295 |
|
- |
};
|
| 296 |
|
- |
let version = crate::domain::Version::parse(&version_str)
|
| 297 |
|
- |
.map_err(|e| crate::error::Error::Other(anyhow::anyhow!(e)))?;
|
| 298 |
|
- |
|
| 299 |
|
- |
// 1. Predecessor must have all of its configured gates satisfied for this
|
| 300 |
|
- |
// version (with optional hotfix override that skips burn_in). Evaluated
|
| 301 |
|
- |
// against the topology gate list, so a gate that never ran blocks the
|
| 302 |
|
- |
// promote instead of being treated as green.
|
| 303 |
|
- |
let pending = unsatisfied_gates(&s.pool, &source.name, &source.gates, &version_str, body.hotfix).await?;
|
| 304 |
|
- |
if !pending.is_empty() {
|
| 305 |
|
- |
return Err(crate::error::Error::GateBlocked(format!(
|
| 306 |
|
- |
"{} gate(s) not satisfied on tier {}: {}",
|
| 307 |
|
- |
pending.len(),
|
| 308 |
|
- |
source.name,
|
| 309 |
|
- |
pending.join(", "),
|
| 310 |
|
- |
)));
|
| 311 |
|
- |
}
|
| 312 |
|
- |
|
| 313 |
|
- |
// 2. Look up the artifact for this version.
|
| 314 |
|
- |
let bin: Option<(String,)> = sqlx::query_as(
|
| 315 |
|
- |
"SELECT artifact_path FROM versions WHERE version = ?",
|
| 316 |
|
- |
)
|
| 317 |
|
- |
.bind(&version)
|
| 318 |
|
- |
.fetch_optional(&s.pool)
|
| 319 |
|
- |
.await
|
| 320 |
|
- |
.map_err(crate::error::Error::Db)?;
|
| 321 |
|
- |
let Some((bin,)) = bin else {
|
| 322 |
|
- |
return Err(crate::error::Error::NotFound);
|
| 323 |
|
- |
};
|
| 324 |
|
- |
let bin_path = std::path::PathBuf::from(bin);
|
| 325 |
|
- |
// `artifact_path` is the primary binary; the staged release dir is its parent.
|
| 326 |
|
- |
let staged_dir = bin_path.parent()
|
| 327 |
|
- |
.ok_or_else(|| crate::error::Error::Other(anyhow::anyhow!("artifact_path has no parent")))?
|
| 328 |
|
- |
.to_path_buf();
|
| 329 |
|
- |
|
| 330 |
|
- |
// The version this tier was running before this promote — the rollback
|
| 331 |
|
- |
// target if a canary node fails partway through a multi-node rollout.
|
| 332 |
|
- |
let prev_version: Option<String> = sqlx::query_scalar(
|
| 333 |
|
- |
"SELECT current_version FROM tier_state WHERE tier = ?",
|
| 334 |
|
- |
)
|
| 335 |
|
- |
.bind(&target.name)
|
| 336 |
|
- |
.fetch_optional(&s.pool).await.map_err(crate::error::Error::Db)?.flatten();
|
| 337 |
|
- |
|
| 338 |
|
- |
// 3. Deploy to each node. Sequential canary is the only policy
|
| 339 |
|
- |
// implemented in v0; parallel is a one-line change once we trust the
|
| 340 |
|
- |
// sequential path. Track the nodes already flipped to the new version so
|
| 341 |
|
- |
// a mid-rollout failure can roll them back (canary rollback).
|
| 342 |
|
- |
let mut deployed: Vec<&crate::topology::Node> = Vec::new();
|
| 343 |
|
- |
for node in &target.nodes {
|
| 344 |
|
- |
let started = chrono::Utc::now().to_rfc3339();
|
| 345 |
|
- |
crate::events::emit(&s.events, crate::events::Event::DeployStart {
|
| 346 |
|
- |
tier: target.name.clone(), node: node.name.clone(), version: version.clone(),
|
| 347 |
|
- |
});
|
| 348 |
|
- |
let executor = s.executors.get(&node.name).cloned()
|
| 349 |
|
- |
.unwrap_or_else(|| crate::state::build_executor(node));
|
| 350 |
|
- |
let result = crate::deploy::deploy_node(executor.as_ref(), node, &version_str, &staged_dir, s.cfg.primary_bin()).await;
|
| 351 |
|
- |
let finished = chrono::Utc::now().to_rfc3339();
|
| 352 |
|
- |
let (outcome_obj, err_for_propagation) = match result {
|
| 353 |
|
- |
Ok(_) => (crate::outcome::DeployOutcome::ok(), None),
|
| 354 |
|
- |
Err(e) => {
|
| 355 |
|
- |
let msg = format!("{e:#}");
|
| 356 |
|
- |
let kind = crate::classify::classify_deploy_error(&msg);
|
| 357 |
|
- |
(crate::outcome::DeployOutcome::failed(kind), Some(e))
|
| 358 |
|
- |
}
|
| 359 |
|
- |
};
|
| 360 |
|
- |
let outcome_json = serde_json::to_string(&outcome_obj)
|
| 361 |
|
- |
.unwrap_or_else(|e| format!("{{\"_serialize_error\":{e:?}}}"));
|
| 362 |
|
- |
sqlx::query(
|
| 363 |
|
- |
"INSERT INTO deploys (version, tier, node, started_at, finished_at, outcome, outcome_json, hotfix, reset_burn_in)
|
| 364 |
|
- |
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
| 365 |
|
- |
)
|
| 366 |
|
- |
.bind(&version).bind(&target.name).bind(&node.name)
|
| 367 |
|
- |
.bind(&started).bind(&finished).bind(outcome_obj.status_str())
|
| 368 |
|
- |
.bind(&outcome_json)
|
| 369 |
|
- |
.bind(body.hotfix as i64).bind(body.reset_burn_in as i64)
|
| 370 |
|
- |
.execute(&s.pool).await.map_err(crate::error::Error::Db)?;
|
| 371 |
|
- |
if let Some(e) = err_for_propagation {
|
| 372 |
|
- |
let crate::outcome::DeployStatus::Failed { failure } = outcome_obj.status else {
|
| 373 |
|
- |
unreachable!("err_for_propagation is Some iff status is Failed");
|
| 374 |
|
- |
};
|
| 375 |
|
- |
tracing::error!(
|
| 376 |
|
- |
tier = %target.name, node = %node.name, version = %version,
|
| 377 |
|
- |
failure = failure.summary(),
|
| 378 |
|
- |
"deploy failed; current symlink left intact, tier_state not advanced"
|
| 379 |
|
- |
);
|
| 380 |
|
- |
crate::events::emit(&s.events, crate::events::Event::DeployFailed {
|
| 381 |
|
- |
tier: target.name.clone(), node: node.name.clone(),
|
| 382 |
|
- |
version: version.clone(), failure,
|
| 383 |
|
- |
});
|
| 384 |
|
- |
|
| 385 |
|
- |
// Canary rollback: restore every node this promote touched — the
|
| 386 |
|
- |
// ones already flipped to the new version AND this failed node
|
| 387 |
|
- |
// (whose state is indeterminate: the symlink swap may have landed
|
| 388 |
|
- |
// before the restart failed) — back to the tier's prior version, so
|
| 389 |
|
- |
// the fleet is left consistent on `prev` rather than split-brain.
|
| 390 |
|
- |
// Nodes after this one were never touched and stay on `prev`.
|
| 391 |
|
- |
deployed.push(node);
|
| 392 |
|
- |
let touched = deployed.len();
|
| 393 |
|
- |
match prev_version.as_deref() {
|
| 394 |
|
- |
Some(prev) => {
|
| 395 |
|
- |
let restored = rollback_deployed_nodes(&s, &target.name, &deployed, prev).await;
|
| 396 |
|
- |
tracing::warn!(
|
| 397 |
|
- |
tier = %target.name, restored, of = touched,
|
| 398 |
|
- |
from = %version, to = prev,
|
| 399 |
|
- |
"canary failed mid-rollout; rolled touched nodes back to the previous version",
|
| 400 |
|
- |
);
|
| 401 |
|
- |
if restored > 0
|
| 402 |
|
- |
&& let Ok(prev_v) = crate::domain::Version::parse(prev)
|
| 403 |
|
- |
{
|
| 404 |
|
- |
crate::events::emit(&s.events, crate::events::Event::Rollback {
|
| 405 |
|
- |
tier: target.name.clone(), from: version.clone(), to: prev_v,
|
| 406 |
|
- |
});
|
| 407 |
|
- |
}
|
| 408 |
|
- |
// If every touched node was restored, the tier is consistent on
|
| 409 |
|
- |
// `prev` — clear any stale flag. Otherwise it is genuinely
|
| 410 |
|
- |
// split-brain: record exactly how, for /state.
|
| 411 |
|
- |
if restored == touched {
|
| 412 |
|
- |
clear_partial(&s, &target.name).await;
|
| 413 |
|
- |
} else {
|
| 414 |
|
- |
set_partial(&s, &target.name, &format!(
|
| 415 |
|
- |
"canary rollback incomplete: {restored}/{touched} nodes restored to {prev}; \
|
| 416 |
|
- |
{} may still be on {version} — manual check needed",
|
| 417 |
|
- |
touched - restored,
|
| 418 |
|
- |
)).await;
|
| 419 |
|
- |
}
|
| 420 |
|
- |
}
|
| 421 |
|
- |
None => {
|
| 422 |
|
- |
tracing::error!(
|
| 423 |
|
- |
tier = %target.name, count = touched, version = %version,
|
| 424 |
|
- |
"canary failed on a first deploy (no previous version to restore to); \
|
| 425 |
|
- |
touched nodes remain on the new version — manual cleanup needed",
|
| 426 |
|
- |
);
|
| 427 |
|
- |
set_partial(&s, &target.name, &format!(
|
| 428 |
|
- |
"first-deploy canary failed: {touched} node(s) left on {version}, \
|
| 429 |
|
- |
no prior version to restore — manual cleanup needed",
|
| 430 |
|
- |
)).await;
|
| 431 |
|
- |
}
|
| 432 |
|
- |
}
|
| 433 |
|
- |
return Err(crate::error::Error::Other(e));
|
| 434 |
|
- |
}
|
| 435 |
|
- |
deployed.push(node);
|
| 436 |
|
- |
crate::events::emit(&s.events, crate::events::Event::DeployOk {
|
| 437 |
|
- |
tier: target.name.clone(), node: node.name.clone(), version: version.clone(),
|
| 438 |
|
- |
});
|
| 439 |
|
- |
}
|
| 440 |
|
- |
|
| 441 |
|
- |
// 3b. Run this tier's post-deploy gates (node_health) against the freshly
|
| 442 |
|
- |
// deployed nodes and record their outcomes. These rows are the evidence
|
| 443 |
|
- |
// the NEXT promote (this tier -> the following one) checks via
|
| 444 |
|
- |
// `unsatisfied_gates`. Before CF1, only the host tier ran gates, so
|
| 445 |
|
- |
// A/B/C had no evidence and promotion waved through; node_health now
|
| 446 |
|
- |
// proves the deployed nodes are serving (Run-2 SERIOUS-3: boot_smoke
|
| 447 |
|
- |
// used to re-run the staged binary locally and proved nothing about the
|
| 448 |
|
- |
// node). burn_in / manual_confirm are not run here — they are evaluated
|
| 449 |
|
- |
// live / by the operator at the next promote. A failed gate does not
|
| 450 |
|
- |
// unwind this deploy (the artifact is already live on the tier); it
|
| 451 |
|
- |
// blocks the next promote, which is the fail-closed behavior we want.
|
| 452 |
|
- |
let post_deploy: Vec<crate::topology::Gate> =
|
| 453 |
|
- |
target.gates.iter().filter(|g| g.runs_post_deploy()).cloned().collect();
|
| 454 |
|
- |
if !post_deploy.is_empty() {
|
| 455 |
|
- |
// node_health probes each node the deploy just shipped to, over the same
|
| 456 |
|
- |
// executor the deploy used. Build the probe set from the tier's nodes and
|
| 457 |
|
- |
// the startup executor map; a node missing an executor (shouldn't happen
|
| 458 |
|
- |
// — both come from the same topology) is skipped, and an empty set makes
|
| 459 |
|
- |
// node_health Blocked (fail closed).
|
| 460 |
|
- |
let nodes: Vec<crate::gates::NodeProbe> = target
|
| 461 |
|
- |
.nodes
|
| 462 |
|
- |
.iter()
|
| 463 |
|
- |
.filter_map(|n| {
|
| 464 |
|
- |
s.executors.get(&n.name).map(|exec| crate::gates::NodeProbe {
|
| 465 |
|
- |
node: n.name.clone(),
|
| 466 |
|
- |
service: n.service_name.clone(),
|
| 467 |
|
- |
health_url: n.health_url.clone(),
|
| 468 |
|
- |
executor: exec.clone(),
|
| 469 |
|
- |
})
|
| 470 |
|
- |
})
|
| 471 |
|
- |
.collect();
|
| 472 |
|
- |
let ctx = crate::gates::GateCtx {
|
| 473 |
|
- |
pool: s.pool.clone(),
|
| 474 |
|
- |
cfg: s.cfg.clone(),
|
| 475 |
|
- |
tier: target.name.clone(),
|
| 476 |
|
- |
version: version.clone(),
|
| 477 |
|
- |
// No worktree at promote time; node_health works over executors, not
|
| 478 |
|
- |
// a checkout.
|
| 479 |
|
- |
worktree: std::path::PathBuf::new(),
|
| 480 |
|
- |
events: s.events.clone(),
|
| 481 |
|
- |
nodes,
|
| 482 |
|
- |
};
|
| 483 |
|
- |
match crate::gates::run_all(&ctx, &post_deploy).await {
|
| 484 |
|
- |
Ok(true) => {}
|
| 485 |
|
- |
Ok(false) => tracing::warn!(
|
| 486 |
|
- |
tier = %target.name, version = %version,
|
| 487 |
|
- |
"post-deploy gate(s) failed; tier advanced but promotion to the next tier will be blocked",
|
| 488 |
|
- |
),
|
| 489 |
|
- |
Err(e) => tracing::error!(
|
| 490 |
|
- |
tier = %target.name, version = %version, error = %e,
|
| 491 |
|
- |
"post-deploy gate execution errored; promotion to the next tier will be blocked",
|
| 492 |
|
- |
),
|
| 493 |
|
- |
}
|
| 494 |
|
- |
}
|
| 495 |
|
- |
|
| 496 |
|
- |
// 4. Advance tier_state through the single sealed forward-advance op (atomic
|
| 497 |
|
- |
// self-referential UPDATE; no read-modify-write to lose under concurrency,
|
| 498 |
|
- |
// CF3). We hold deploy_lock for this whole handler, so the advance is
|
| 499 |
|
- |
// serialized against rollback and the host build path's advance.
|
| 500 |
|
- |
// reset_burn_in on the *source* tier nulls its clock only when the operator
|
| 501 |
|
- |
// explicitly asked.
|
| 502 |
|
- |
crate::runs::advance_tier(&s.pool, target.name.as_str(), &version)
|
| 503 |
|
- |
.await
|
| 504 |
|
- |
.map_err(crate::error::Error::Db)?;
|
| 505 |
|
- |
|
| 506 |
|
- |
if body.reset_burn_in {
|
| 507 |
|
- |
sqlx::query("UPDATE tier_state SET burn_in_started_at = NULL WHERE tier = ?")
|
| 508 |
|
- |
.bind(&source.name)
|
| 509 |
|
- |
.execute(&s.pool).await.map_err(crate::error::Error::Db)?;
|
| 510 |
|
- |
}
|
| 511 |
|
- |
|
| 512 |
|
- |
// A clean full rollout to every node clears any prior partial flag on this tier.
|
| 513 |
|
- |
clear_partial(&s, &target.name).await;
|
| 514 |
|
- |
|
| 515 |
|
- |
crate::events::emit(&s.events, crate::events::Event::PromoteComplete {
|
| 516 |
|
- |
tier: target.name.clone(), version: version.clone(),
|
| 517 |
|
- |
});
|
| 518 |
|
- |
metrics::counter!("sando_promotes_total", "tier" => target.name.to_string()).increment(1);
|
| 519 |
|
- |
tracing::info!(
|
| 520 |
|
- |
version = %version, tier = %target.name,
|
| 521 |
|
- |
hotfix = body.hotfix, reset_burn_in = body.reset_burn_in,
|
| 522 |
|
- |
"promote complete",
|
| 523 |
|
- |
);
|
| 524 |
|
- |
|
| 525 |
|
- |
Ok(Json(serde_json::json!({
|
| 526 |
|
- |
"tier": target.name,
|
| 527 |
|
- |
"version": version,
|
| 528 |
|
- |
"nodes_deployed": target.nodes.iter().map(|n| n.name.clone()).collect::<Vec<_>>(),
|
| 529 |
|
- |
})))
|
| 530 |
|
- |
}
|
| 531 |
|
- |
|
| 532 |
|
- |
/// After a canary node fails mid-promote, restore the nodes already flipped to
|
| 533 |
|
- |
/// the new version back to `prev_version`, leaving the tier consistent (all on
|
| 534 |
|
- |
/// the old version) rather than split-brain. Best-effort: every node is
|
| 535 |
|
- |
/// attempted; a per-node failure is logged but never propagated (the promote is
|
| 536 |
|
- |
/// already failing). Returns how many nodes were successfully restored. Returns
|
| 537 |
|
- |
/// 0 (with an error log) when the previous version has no recorded artifact to
|
| 538 |
|
- |
/// roll back to.
|
| 539 |
|
- |
async fn rollback_deployed_nodes(
|
| 540 |
|
- |
s: &AppState,
|
| 541 |
|
- |
tier: &crate::domain::TierId,
|
| 542 |
|
- |
nodes: &[&crate::topology::Node],
|
| 543 |
|
- |
prev_version: &str,
|
| 544 |
|
- |
) -> usize {
|
| 545 |
|
- |
let bin: Option<(String,)> = match sqlx::query_as(
|
| 546 |
|
- |
"SELECT artifact_path FROM versions WHERE version = ?",
|
| 547 |
|
- |
)
|
| 548 |
|
- |
.bind(prev_version)
|
| 549 |
|
- |
.fetch_optional(&s.pool)
|
| 550 |
|
- |
.await
|
| 551 |
|
- |
{
|
| 552 |
|
- |
Ok(b) => b,
|
| 553 |
|
- |
Err(e) => {
|
| 554 |
|
- |
tracing::error!(tier = %tier, prev = prev_version, error = %e,
|
| 555 |
|
- |
"canary rollback: looking up the previous artifact failed; nodes left on the new version");
|
| 556 |
|
- |
return 0;
|
| 557 |
|
- |
}
|
| 558 |
|
- |
};
|
| 559 |
|
- |
let Some((bin,)) = bin else {
|
| 560 |
|
- |
tracing::error!(tier = %tier, prev = prev_version, nodes = nodes.len(),
|
| 561 |
|
- |
"canary rollback: previous version has no artifact_path; nodes left on the new version");
|
| 562 |
|
- |
return 0;
|
| 563 |
|
- |
};
|
| 564 |
|
- |
let Some(staged_dir) = std::path::PathBuf::from(&bin).parent().map(|p| p.to_path_buf()) else {
|
| 565 |
|
- |
tracing::error!(tier = %tier, prev = prev_version,
|
| 566 |
|
- |
"canary rollback: previous artifact_path has no parent dir; nodes left on the new version");
|
| 567 |
|
- |
return 0;
|
| 568 |
|
- |
};
|
| 569 |
|
- |
|
| 570 |
|
- |
let mut restored = 0usize;
|
| 571 |
|
- |
for node in nodes {
|
| 572 |
|
- |
let executor = s.executors.get(&node.name).cloned()
|
| 573 |
|
- |
.unwrap_or_else(|| crate::state::build_executor(node));
|
| 574 |
|
- |
match crate::deploy::deploy_node(executor.as_ref(), node, prev_version, &staged_dir, s.cfg.primary_bin()).await {
|
| 575 |
|
- |
Ok(_) => {
|
| 576 |
|
- |
restored += 1;
|
| 577 |
|
- |
tracing::warn!(tier = %tier, node = %node.name, version = prev_version,
|
| 578 |
|
- |
"canary rollback: node restored to the previous version");
|
| 579 |
|
- |
}
|
| 580 |
|
- |
Err(e) => tracing::error!(
|
| 581 |
|
- |
tier = %tier, node = %node.name, version = prev_version, error = %format!("{e:#}"),
|
| 582 |
|
- |
"canary rollback FAILED for node; it remains on the new version — manual intervention needed",
|
| 583 |
|
- |
),
|
| 584 |
|
- |
}
|
| 585 |
|
- |
}
|
| 586 |
|
- |
restored
|
| 587 |
|
- |
}
|
| 588 |
|
- |
|
| 589 |
|
- |
/// Flag a tier as left in a partial / mixed-version state, with a human-readable
|
| 590 |
|
- |
/// reason surfaced through `/state` and the TUI. Best-effort: a failure to record
|
| 591 |
|
- |
/// the flag is logged, never propagated — the caller is already on an error path
|
| 592 |
|
- |
/// and the worse outcome is to mask the original failure with a bookkeeping one.
|
| 593 |
|
- |
async fn set_partial(s: &AppState, tier: &crate::domain::TierId, reason: &str) {
|
| 594 |
|
- |
if let Err(e) = sqlx::query("UPDATE tier_state SET partial_reason = ? WHERE tier = ?")
|
| 595 |
|
- |
.bind(reason)
|
| 596 |
|
- |
.bind(tier)
|
| 597 |
|
- |
.execute(&s.pool)
|
| 598 |
|
- |
.await
|
| 599 |
|
- |
{
|
| 600 |
|
- |
tracing::error!(tier = %tier, reason, error = %e,
|
| 601 |
|
- |
"failed to record tier partial state; the fleet may be inconsistent without a /state flag");
|
| 602 |
|
- |
}
|
| 603 |
|
- |
metrics::gauge!("sando_tier_partial", "tier" => tier.to_string()).set(1.0);
|
| 604 |
|
- |
}
|
| 605 |
|
- |
|
| 606 |
|
- |
/// Clear a tier's partial flag after a clean full promote or rollback. Errors are
|
| 607 |
|
- |
/// logged but not propagated: the deploy itself succeeded, and a stale flag is a
|
| 608 |
|
- |
/// visible nuisance, not a safety regression (the operator sees a partial marker
|
| 609 |
|
- |
/// on a tier that is actually fine, and re-checks).
|
| 610 |
|
- |
async fn clear_partial(s: &AppState, tier: &crate::domain::TierId) {
|
| 611 |
|
- |
if let Err(e) = sqlx::query("UPDATE tier_state SET partial_reason = NULL WHERE tier = ?")
|
| 612 |
|
- |
.bind(tier)
|
| 613 |
|
- |
.execute(&s.pool)
|
| 614 |
|
- |
.await
|
| 615 |
|
- |
{
|
| 616 |
|
- |
tracing::warn!(tier = %tier, error = %e, "failed to clear tier partial flag");
|
| 617 |
|
- |
}
|
| 618 |
|
- |
metrics::gauge!("sando_tier_partial", "tier" => tier.to_string()).set(0.0);
|
| 619 |
|
- |
}
|
| 620 |
|
- |
|
| 621 |
|
- |
/// Returns the kinds of `tier`'s *configured* gates that are not satisfied for
|
| 622 |
|
- |
/// `version`. `hotfix` suppresses the `burn_in` requirement only.
|
| 623 |
|
- |
///
|
| 624 |
|
- |
/// Fail-closed against the topology gate list (the CF1 fix). The previous
|
| 625 |
|
- |
/// version inspected only existing `gate_runs` rows, so a configured gate that
|
| 626 |
|
- |
/// had *never run* produced no row and was invisibly treated as green — letting
|
| 627 |
|
- |
/// a promote wave through with zero evidence (it shipped 0.9.5 to prod with
|
| 628 |
|
- |
/// tier A's `boot_smoke` never recorded). Now every configured gate must show
|
| 629 |
|
- |
/// positive evidence:
|
| 630 |
|
- |
/// - `burn_in` is evaluated live against the tier's clock (a stored `blocked`
|
| 631 |
|
- |
/// row would otherwise never flip to passed as time elapses);
|
| 632 |
|
- |
/// - every other kind requires a `passed` row for (tier, version) — a missing
|
| 633 |
|
- |
/// or non-passed latest row counts as unsatisfied.
|
| 634 |
|
- |
async fn unsatisfied_gates(
|
| 635 |
|
- |
pool: &sqlx::SqlitePool,
|
| 636 |
|
- |
tier: &crate::domain::TierId,
|
| 637 |
|
- |
gates: &[crate::topology::Gate],
|
| 638 |
|
- |
version: &str,
|
| 639 |
|
- |
hotfix: bool,
|
| 640 |
|
- |
) -> std::result::Result<Vec<String>, crate::error::Error> {
|
| 641 |
|
- |
use crate::topology::Gate;
|
| 642 |
|
- |
let mut bad = Vec::new();
|
| 643 |
|
- |
for gate in gates {
|
| 644 |
|
- |
let kind = gate.kind();
|
| 645 |
|
- |
match gate {
|
| 646 |
|
- |
Gate::BurnIn { hours } => {
|
| 647 |
|
- |
if hotfix {
|
| 648 |
|
- |
continue;
|
| 649 |
|
- |
}
|
| 650 |
|
- |
let ok = crate::gates::burn_in_satisfied(pool, tier, *hours)
|
| 651 |
|
- |
.await
|
| 652 |
|
- |
.map_err(crate::error::Error::Other)?;
|
| 653 |
|
- |
if !ok {
|
| 654 |
|
- |
bad.push(kind.as_str().to_string());
|
| 655 |
|
- |
}
|
| 656 |
|
- |
}
|
| 657 |
|
- |
Gate::ManualConfirm => {
|
| 658 |
|
- |
// A confirmation must be *fresh*: recorded at or after the
|
| 659 |
|
- |
// version's current landing on this tier (tier_state
|
| 660 |
|
- |
// .burn_in_started_at, the per-deploy clock). Without this a
|
| 661 |
|
- |
// confirmation row survives a rollback + rollback-forward and
|
| 662 |
|
- |
// waves a re-deploy of the same version through with no fresh
|
| 663 |
|
- |
// operator sign-off — weaker than burn_in, which is clock-based.
|
| 664 |
|
- |
// No baseline (NULL) => fail closed: require a fresh confirm.
|
| 665 |
|
- |
let confirmed_at: Option<String> = sqlx::query_scalar(
|
| 666 |
|
- |
"SELECT finished_at FROM gate_runs
|
| 667 |
|
- |
WHERE tier = ?1 AND version = ?2 AND gate_kind = 'manual_confirm' AND status = 'passed'
|
| 668 |
|
- |
ORDER BY id DESC LIMIT 1",
|
| 669 |
|
- |
)
|
| 670 |
|
- |
.bind(tier.as_str())
|
| 671 |
|
- |
.bind(version)
|
| 672 |
|
- |
.fetch_optional(pool)
|
| 673 |
|
- |
.await
|
| 674 |
|
- |
.map_err(crate::error::Error::Db)?
|
| 675 |
|
- |
.flatten();
|
| 676 |
|
- |
let landed_at: Option<String> = sqlx::query_scalar(
|
| 677 |
|
- |
"SELECT burn_in_started_at FROM tier_state WHERE tier = ?",
|
| 678 |
|
- |
)
|
| 679 |
|
- |
.bind(tier.as_str())
|
| 680 |
|
- |
.fetch_optional(pool)
|
| 681 |
|
- |
.await
|
| 682 |
|
- |
.map_err(crate::error::Error::Db)?
|
| 683 |
|
- |
.flatten();
|
| 684 |
|
- |
let fresh = match (confirmed_at, landed_at) {
|
| 685 |
|
- |
(Some(c), Some(l)) => {
|
| 686 |
|
- |
match (
|
| 687 |
|
- |
chrono::DateTime::parse_from_rfc3339(&c),
|
| 688 |
|
- |
chrono::DateTime::parse_from_rfc3339(&l),
|
| 689 |
|
- |
) {
|
| 690 |
|
- |
(Ok(cd), Ok(ld)) => cd >= ld,
|
| 691 |
|
- |
_ => false, // unparseable timestamp -> fail closed
|
| 692 |
|
- |
}
|
| 693 |
|
- |
}
|
| 694 |
|
- |
_ => false,
|
| 695 |
|
- |
};
|
| 696 |
|
- |
if !fresh {
|
| 697 |
|
- |
bad.push(kind.as_str().to_string());
|
| 698 |
|
- |
}
|
| 699 |
|
- |
}
|
| 700 |
|
- |
// Build/post-deploy gates that leave a `gate_runs` row: the latest
|
| 701 |
|
- |
// row for this (tier, version, kind) must be `passed`. Listed
|
| 702 |
|
- |
// explicitly (no `_` catch-all) so adding a new `Gate` variant is a
|
| 703 |
|
- |
// compile error here until its promotion semantics are decided —
|
| 704 |
|
- |
// a transient-`blocked` kind silently falling into "needs a passed
|
| 705 |
|
- |
// row" would be permanently unsatisfiable.
|
| 706 |
|
- |
Gate::CargoTest | Gate::MigrationDryRun | Gate::BootSmoke | Gate::NodeHealth => {
|
| 707 |
|
- |
// Latest row for this configured gate kind; NULL/missing/any
|
| 708 |
|
- |
// non-'passed' status all count as unsatisfied (fail closed).
|
| 709 |
|
- |
let status: Option<String> = sqlx::query_scalar(
|
| 710 |
|
- |
"SELECT status FROM gate_runs
|
| 711 |
|
- |
WHERE tier = ?1 AND version = ?2 AND gate_kind = ?3
|
| 712 |
|
- |
ORDER BY id DESC LIMIT 1",
|
| 713 |
|
- |
)
|
| 714 |
|
- |
.bind(tier.as_str())
|
| 715 |
|
- |
.bind(version)
|
| 716 |
|
- |
.bind(kind.as_str())
|
| 717 |
|
- |
.fetch_optional(pool)
|
| 718 |
|
- |
.await
|
| 719 |
|
- |
.map_err(crate::error::Error::Db)?
|
| 720 |
|
- |
.flatten();
|
| 721 |
|
- |
if status.as_deref() != Some("passed") {
|
| 722 |
|
- |
bad.push(kind.as_str().to_string());
|
| 723 |
|
- |
}
|
| 724 |
|
- |
}
|
| 725 |
|
- |
}
|
| 726 |
|
- |
}
|
| 727 |
|
- |
Ok(bad)
|
| 728 |
|
- |
}
|
| 729 |
|
- |
|
| 730 |
266 |
|
async fn rollback(
|
| 731 |
267 |
|
State(s): State<AppState>,
|
| 732 |
268 |
|
Path(tier): Path<String>,
|
| 1186 |
722 |
|
#[cfg(test)]
|
| 1187 |
723 |
|
mod tests {
|
| 1188 |
724 |
|
use super::*;
|
|
725 |
+ |
use super::promotion::{rollback_deployed_nodes, unsatisfied_gates};
|
| 1189 |
726 |
|
use crate::config::Config;
|
| 1190 |
727 |
|
use crate::topology::{BackupConfig, CanaryPolicy, Gate, Node, RepoConfig, Tier, Topology};
|
| 1191 |
728 |
|
use axum::body::Body;
|