max / synckit
- Co-Authored-By
- Claude Opus 5 (1M context) <noreply@anthropic.com>
16 files changed,
+1219 insertions,
-34 deletions
| @@ -1,6 +1,6 @@ | |||
| 1 | 1 | [package] | |
| 2 | 2 | name = "synckit-client" | |
| 3 | - | version = "0.8.0" | |
| 3 | + | version = "0.8.1" | |
| 4 | 4 | edition = "2024" | |
| 5 | 5 | license = "MIT" | |
| 6 | 6 | description = "SyncKit client SDK with end-to-end encryption" |
| @@ -129,9 +129,17 @@ | |||
| 129 | 129 | the bare row data: | |
| 130 | 130 | ||
| 131 | 131 | ```json | |
| 132 | - | { "__skver": 2, "__skhlc": { "wall_ms": …, "counter": …, "node": "…" }, "data": <row|null> } | |
| 132 | + | { "__skver": 2, "__skhlc": { "wall_ms": …, "counter": …, "node": "…" }, "data": <row|null>, | |
| 133 | + | "__sksv": 4 } | |
| 133 | 134 | ``` | |
| 134 | 135 | ||
| 136 | + | `__sksv` is the sender's `SyncSchema` storage version, present only when the app | |
| 137 | + | declared one (see "The storage-version gate"). It rides inside the sealed | |
| 138 | + | envelope so the server never learns which schema an account is on, and it was | |
| 139 | + | added *within* v2 rather than as a new `__skver`: a reader of this format | |
| 140 | + | addresses `__skhlc` and `data` by name and ignores anything else, so an unstamped | |
| 141 | + | build reads a stamped envelope exactly as it always did. | |
| 142 | + | ||
| 135 | 143 | Encrypting the envelope (rather than just the row) is what carries the hybrid | |
| 136 | 144 | logical clock (HLC) inside the E2E ciphertext, so the server can order entries by | |
| 137 | 145 | `seq` but never sees or orders by the clock. A Delete has `data: null` but still | |
| @@ -564,6 +572,90 @@ | |||
| 564 | 572 | snapshot, and apply all read one `columns` list, the drift class is eliminated by | |
| 565 | 573 | construction and the round-trip test each app carries today is deleted. | |
| 566 | 574 | ||
| 575 | + | ## The storage-version gate | |
| 576 | + | ||
| 577 | + | When a storage change is breaking, clients **refuse to sync rather than | |
| 578 | + | degrade**, and the local migration runs first. A client that syncs everything it | |
| 579 | + | recognises and ignores the rest looks like it worked; the user learns about the | |
| 580 | + | gap later, from missing data, rather than immediately from a refusal. | |
| 581 | + | ||
| 582 | + | What moves the number is the `SyncSchema` manifest, not migrations. There is no | |
| 583 | + | numbered migration ledger to count, and keying on one would over-refuse: local | |
| 584 | + | DDL (a new index, a table absent from every manifest, `sync_conflict_stash`) | |
| 585 | + | changes nothing that crosses the wire. | |
| 586 | + | ||
| 587 | + | | change | fires the gate | | |
| 588 | + | |---|---| | |
| 589 | + | | frontend / UI change | no | | |
| 590 | + | | local-only table or index | no | | |
| 591 | + | | new synced table | yes | | |
| 592 | + | | new synced column | yes | | |
| 593 | + | | changed `pk` or `RowIdScheme` | yes | | |
| 594 | + | | changed `SyncMode` / `DeleteMode` | yes | | |
| 595 | + | ||
| 596 | + | ### A declared integer, enforced by a derived fingerprint | |
| 597 | + | ||
| 598 | + | ```rust | |
| 599 | + | let schema = SyncSchema::new(tables).storage_version(4); | |
| 600 | + | ||
| 601 | + | // Committed alongside the manifest, in the app's own source. | |
| 602 | + | const LEDGER: &[LedgerEntry] = &[ | |
| 603 | + | LedgerEntry { version: 4, fingerprint: "9f2c1a…" }, | |
| 604 | + | ]; | |
| 605 | + | ||
| 606 | + | #[test] | |
| 607 | + | fn manifest_matches_its_declared_storage_version() { | |
| 608 | + | schema().check_ledger(LEDGER).unwrap(); | |
| 609 | + | } | |
| 610 | + | ``` | |
| 611 | + | ||
| 612 | + | `SyncSchema::wire_manifest()` renders one line per table, in declared order, over | |
| 613 | + | the wire-visible facts only: table name, `emitted_columns()` (which *is* the wire | |
| 614 | + | projection), `pk`, `RowIdScheme`, `SyncMode`, `DeleteMode`. `fingerprint()` is its | |
| 615 | + | SHA-256. Declaration order is preserved rather than sorted, because it is the | |
| 616 | + | foreign-key apply order and therefore observable. | |
| 617 | + | ||
| 618 | + | An integer alone relies on memory; a fingerprint alone has no ordering, so two | |
| 619 | + | clients could tell they differ and not which needs the update. Together the number | |
| 620 | + | cannot drift from what it describes, and `check_ledger` fails a manifest edit that | |
| 621 | + | forgot the bump, printing the manifest so a diff names what moved. | |
| 622 | + | ||
| 623 | + | ### Two comparisons | |
| 624 | + | ||
| 625 | + | **This device against its own store.** `sync_state.storage_version` records the | |
| 626 | + | version the store was last shaped by. `SyncStore::sync_now` checks it before it | |
| 627 | + | registers a device or reads a row, so a refusal has touched nothing. A store that | |
| 628 | + | has never been stamped adopts the declared version instead of refusing: it was | |
| 629 | + | written before the gate existed, not by a version we disagree with. After its own | |
| 630 | + | local migration an app calls `SyncStore::stamp_storage_version()`, which is the | |
| 631 | + | claim that the store now matches the manifest. | |
| 632 | + | ||
| 633 | + | **This device against a peer.** Clients share one changelog, so a peer on another | |
| 634 | + | manifest is the case the policy exists for. Every pushed change seals `__sksv`, | |
| 635 | + | and the pull loop checks each entry before anything is applied and before the | |
| 636 | + | cursor moves, so a refused page is still on the server when both sides agree. | |
| 637 | + | ||
| 638 | + | Comparison is equality, not a floor. An "additive" change is still breaking for | |
| 639 | + | the older peer two ways: it pulls rows for a table it does not know and either | |
| 640 | + | errors or advances its cursor past them, and if it re-pushes a row whose new | |
| 641 | + | column it does not project, LWW wipes that column on the newer client. | |
| 642 | + | ||
| 643 | + | ``` | |
| 644 | + | mine < theirs -> "Update this device." | |
| 645 | + | mine > theirs -> "Another device is out of date." | |
| 646 | + | ``` | |
| 647 | + | ||
| 648 | + | Both arrive as `SyncKitError::StorageVersion(StorageVersionRefusal)`, so an app | |
| 649 | + | can show the upgrade message rather than a generic sync failure. | |
| 650 | + | ||
| 651 | + | ### Adoption | |
| 652 | + | ||
| 653 | + | The gate is off until a manifest declares a version, which is what every app | |
| 654 | + | predating it keeps. An app that declares none pushes byte-identical envelopes and | |
| 655 | + | its stores are never stamped. Per-collection version negotiation and a | |
| 656 | + | compatibility floor are both deliberately not built: each puts protocol work in | |
| 657 | + | front of every feature that adds a record type. | |
| 658 | + | ||
| 567 | 659 | ## The engine and its database seam | |
| 568 | 660 | ||
| 569 | 661 | The one real portability constraint: GO and BB drive SQLite through an async |
| @@ -963,6 +963,7 @@ | |||
| 963 | 963 | let mut entry = make_entry(table, row_id, op, ts); | |
| 964 | 964 | entry.hlc = Hlc::from_legacy(ts.timestamp_millis(), DeviceId::new(device_id)); | |
| 965 | 965 | PulledChange { | |
| 966 | + | storage_version: None, | |
| 966 | 967 | entry, | |
| 967 | 968 | device_id: DeviceId::new(device_id), | |
| 968 | 969 | seq, |
| @@ -93,6 +93,75 @@ | |||
| 93 | 93 | #[cfg(feature = "store")] | |
| 94 | 94 | #[error("Database error: {0}")] | |
| 95 | 95 | Database(String), | |
| 96 | + | ||
| 97 | + | /// The sync refused because the two sides are on different storage versions. | |
| 98 | + | /// | |
| 99 | + | /// A breaking storage change means clients refuse to sync rather than | |
| 100 | + | /// degrade: nothing was written, no records were dropped, and no manifest was | |
| 101 | + | /// written back. Match on this to show the upgrade message | |
| 102 | + | /// ([`StorageVersionRefusal::message`]) rather than a generic sync failure. | |
| 103 | + | #[error("{0}")] | |
| 104 | + | StorageVersion(StorageVersionRefusal), | |
| 105 | + | } | |
| 106 | + | ||
| 107 | + | /// Which side supplied the version this build disagreed with. | |
| 108 | + | #[derive(Debug, Clone, Copy, PartialEq, Eq)] | |
| 109 | + | pub enum VersionSource { | |
| 110 | + | /// This device's own store, stamped by whichever build last migrated it. | |
| 111 | + | LocalStore, | |
| 112 | + | /// A peer, read off a change it pushed to the shared changelog. | |
| 113 | + | Peer, | |
| 114 | + | } | |
| 115 | + | ||
| 116 | + | /// The detail behind [`SyncKitError::StorageVersion`]. | |
| 117 | + | #[derive(Debug, Clone, Copy, PartialEq, Eq)] | |
| 118 | + | pub struct StorageVersionRefusal { | |
| 119 | + | /// The storage version this build's `SyncSchema` declares. | |
| 120 | + | pub mine: u32, | |
| 121 | + | /// The storage version the other side is at. | |
| 122 | + | pub theirs: u32, | |
| 123 | + | /// Where `theirs` was read from. | |
| 124 | + | pub source: VersionSource, | |
| 125 | + | } | |
| 126 | + | ||
| 127 | + | impl StorageVersionRefusal { | |
| 128 | + | /// The message to show a user, chosen by which side is behind. | |
| 129 | + | /// | |
| 130 | + | /// Ordering is the whole reason the version is an integer rather than only a | |
| 131 | + | /// fingerprint: a hash tells two clients they differ, not which of them needs | |
| 132 | + | /// the update. | |
| 133 | + | pub fn message(&self) -> &'static str { | |
| 134 | + | match (self.source, self.mine < self.theirs) { | |
| 135 | + | (VersionSource::LocalStore, true) => "This store uses a newer format. Update to sync.", | |
| 136 | + | (VersionSource::LocalStore, false) => { | |
| 137 | + | "This store has not been migrated to the current format yet." | |
| 138 | + | } | |
| 139 | + | (VersionSource::Peer, true) => "Update this device.", | |
| 140 | + | (VersionSource::Peer, false) => "Another device is out of date.", | |
| 141 | + | } | |
| 142 | + | } | |
| 143 | + | ||
| 144 | + | /// Whether this build is the older side. | |
| 145 | + | pub fn local_is_older(&self) -> bool { | |
| 146 | + | self.mine < self.theirs | |
| 147 | + | } | |
| 148 | + | } | |
| 149 | + | ||
| 150 | + | impl std::fmt::Display for StorageVersionRefusal { | |
| 151 | + | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { | |
| 152 | + | let side = match self.source { | |
| 153 | + | VersionSource::LocalStore => "local store", | |
| 154 | + | VersionSource::Peer => "peer", | |
| 155 | + | }; | |
| 156 | + | write!( | |
| 157 | + | f, | |
| 158 | + | "{} (this build declares storage version {}, the {side} is at {}); \ | |
| 159 | + | nothing was written", | |
| 160 | + | self.message(), | |
| 161 | + | self.mine, | |
| 162 | + | self.theirs | |
| 163 | + | ) | |
| 164 | + | } | |
| 96 | 165 | } | |
| 97 | 166 | ||
| 98 | 167 | #[cfg(feature = "keychain")] |
| @@ -589,6 +589,13 @@ | |||
| 589 | 589 | pub device_id: DeviceId, | |
| 590 | 590 | /// Server sequence number (total ordering). | |
| 591 | 591 | pub seq: i64, | |
| 592 | + | /// The storage version the originating device sealed into this change, or | |
| 593 | + | /// `None` from a build that predates the stamp or declares no version. | |
| 594 | + | /// | |
| 595 | + | /// Read by the pull gate before anything is applied: clients share one | |
| 596 | + | /// changelog, so a peer on a different manifest is refused rather than | |
| 597 | + | /// half-applied. See `store::version`. | |
| 598 | + | pub storage_version: Option<u32>, | |
| 592 | 599 | } | |
| 593 | 600 | ||
| 594 | 601 | // ── Keys ── |
| @@ -347,9 +347,10 @@ | |||
| 347 | 347 | ) -> Result<i64> { | |
| 348 | 348 | let token = self.require_token()?; | |
| 349 | 349 | let group_str = group_id.to_string(); | |
| 350 | + | let storage_version = self.storage_version(); | |
| 350 | 351 | let wire_changes = changes | |
| 351 | 352 | .into_iter() | |
| 352 | - | .map(|c| Self::encrypt_group_change_with_key(&group_str, c, gck)) | |
| 353 | + | .map(|c| Self::encrypt_group_change_with_key(&group_str, c, gck, storage_version)) | |
| 353 | 354 | .collect::<Result<Vec<_>>>()?; | |
| 354 | 355 | ||
| 355 | 356 | let body = Bytes::from(serde_json::to_vec(&WirePushRequest { | |
| @@ -524,7 +525,7 @@ | |||
| 524 | 525 | let mut e = insert("tasks", "r1", serde_json::json!({ "title": "shared" })); | |
| 525 | 526 | e.hlc = hlc; | |
| 526 | 527 | ||
| 527 | - | let wire = SyncKitClient::encrypt_group_change_with_key("grp-1", e, &gck).unwrap(); | |
| 528 | + | let wire = SyncKitClient::encrypt_group_change_with_key("grp-1", e, &gck, None).unwrap(); | |
| 528 | 529 | let pulled = | |
| 529 | 530 | SyncKitClient::decrypt_group_change_to_pulled("grp-1", to_pull(wire, device, 3), &gck) | |
| 530 | 531 | .unwrap(); | |
| @@ -545,6 +546,7 @@ | |||
| 545 | 546 | "grp-A", | |
| 546 | 547 | insert("t", "r", serde_json::json!(1)), | |
| 547 | 548 | &gck, | |
| 549 | + | None, | |
| 548 | 550 | ) | |
| 549 | 551 | .unwrap(); | |
| 550 | 552 | // Same GCK, but a different group id in the AAD: the open fails closed. | |
| @@ -565,6 +567,7 @@ | |||
| 565 | 567 | "grp-1", | |
| 566 | 568 | insert("t", "r", serde_json::json!(1)), | |
| 567 | 569 | &gck, | |
| 570 | + | None, | |
| 568 | 571 | ) | |
| 569 | 572 | .unwrap(); | |
| 570 | 573 | let err = SyncKitClient::decrypt_group_change_to_pulled( |
| @@ -291,7 +291,7 @@ | |||
| 291 | 291 | #[cfg(test)] | |
| 292 | 292 | pub(super) fn encrypt_change(&self, entry: ChangeEntry) -> Result<WireChangeEntry> { | |
| 293 | 293 | let master_key = self.require_master_key()?; | |
| 294 | - | Self::encrypt_change_with_key(entry, &master_key) | |
| 294 | + | Self::encrypt_change_with_key(entry, &master_key, self.storage_version()) | |
| 295 | 295 | } | |
| 296 | 296 | ||
| 297 | 297 | /// Decrypt a pulled legacy entry that has no encrypted payload (a pre-HLC | |
| @@ -319,8 +319,24 @@ | |||
| 319 | 319 | /// The `__skver` tag is a positive, explicit version marker. A reader | |
| 320 | 320 | /// dispatches on it (see [`WireVersion`]) rather than structurally guessing, | |
| 321 | 321 | /// so a future format bump is rejected loudly instead of silently misread. | |
| 322 | - | fn hlc_envelope(hlc: &Hlc, data: Option<&serde_json::Value>) -> serde_json::Value { | |
| 323 | - | serde_json::json!({ "__skver": ENVELOPE_VERSION, "__skhlc": hlc, "data": data }) | |
| 322 | + | /// `__sksv` carries the sender's `SyncSchema` storage version when the app | |
| 323 | + | /// declared one. It rides *inside* the sealed envelope rather than on the | |
| 324 | + | /// wire entry so the server never learns which schema an account is on, and | |
| 325 | + | /// it is added within v2 rather than bumping `__skver`: a reader of this | |
| 326 | + | /// format addresses `__skhlc` and `data` by name and ignores anything else, | |
| 327 | + | /// so an unstamped build reads a stamped envelope exactly as it always did. | |
| 328 | + | /// Omitted entirely when undeclared, leaving the bytes unchanged. | |
| 329 | + | fn hlc_envelope( | |
| 330 | + | hlc: &Hlc, | |
| 331 | + | data: Option<&serde_json::Value>, | |
| 332 | + | storage_version: Option<u32>, | |
| 333 | + | ) -> serde_json::Value { | |
| 334 | + | let mut envelope = | |
| 335 | + | serde_json::json!({ "__skver": ENVELOPE_VERSION, "__skhlc": hlc, "data": data }); | |
| 336 | + | if let (Some(v), Some(obj)) = (storage_version, envelope.as_object_mut()) { | |
| 337 | + | obj.insert("__sksv".into(), serde_json::json!(v)); | |
| 338 | + | } | |
| 339 | + | envelope | |
| 324 | 340 | } | |
| 325 | 341 | ||
| 326 | 342 | /// Split a decrypted payload back into `(hlc, data)`. | |
| @@ -338,8 +354,15 @@ | |||
| 338 | 354 | decrypted: serde_json::Value, | |
| 339 | 355 | node: crate::ids::DeviceId, | |
| 340 | 356 | timestamp_ms: i64, | |
| 341 | - | ) -> Result<(Hlc, Option<serde_json::Value>)> { | |
| 357 | + | ) -> Result<(Hlc, Option<serde_json::Value>, Option<u32>)> { | |
| 342 | 358 | if let Some(obj) = decrypted.as_object() { | |
| 359 | + | // Absent, null, or not an integer all read as "no stamp": the field is | |
| 360 | + | // optional by construction, so a malformed one must not fail a change | |
| 361 | + | // that is otherwise fine. The gate treats an absent stamp as passing. | |
| 362 | + | let storage_version = obj | |
| 363 | + | .get("__sksv") | |
| 364 | + | .and_then(serde_json::Value::as_u64) | |
| 365 | + | .and_then(|v| u32::try_from(v).ok()); | |
| 343 | 366 | if let Some(tag) = obj.get("__skver") { | |
| 344 | 367 | // Explicit version present: dispatch, rejecting unknown loudly. | |
| 345 | 368 | let tag = tag.as_u64().ok_or_else(|| { | |
| @@ -354,7 +377,7 @@ | |||
| 354 | 377 | SyncKitError::Crypto("v2 envelope missing __skhlc".into()) | |
| 355 | 378 | })?; | |
| 356 | 379 | let data = obj.get("data").cloned().filter(|v| !v.is_null()); | |
| 357 | - | Ok((hlc, data)) | |
| 380 | + | Ok((hlc, data, storage_version)) | |
| 358 | 381 | } | |
| 359 | 382 | }; | |
| 360 | 383 | } | |
| @@ -364,11 +387,11 @@ | |||
| 364 | 387 | .and_then(|v| serde_json::from_value::<Hlc>(v.clone()).ok()) | |
| 365 | 388 | { | |
| 366 | 389 | let data = obj.get("data").cloned().filter(|v| !v.is_null()); | |
| 367 | - | return Ok((hlc, data)); | |
| 390 | + | return Ok((hlc, data, storage_version)); | |
| 368 | 391 | } | |
| 369 | 392 | } | |
| 370 | 393 | // Legacy bare-row payload: the decrypted value is the row data itself. | |
| 371 | - | Ok((Hlc::from_legacy(timestamp_ms, node), Some(decrypted))) | |
| 394 | + | Ok((Hlc::from_legacy(timestamp_ms, node), Some(decrypted), None)) | |
| 372 | 395 | } | |
| 373 | 396 | ||
| 374 | 397 | /// Encrypt with a pre-loaded key. Used by `push()` to avoid per-entry lock | |
| @@ -379,9 +402,10 @@ | |||
| 379 | 402 | pub(super) fn encrypt_change_with_key( | |
| 380 | 403 | entry: ChangeEntry, | |
| 381 | 404 | master_key: &[u8; 32], | |
| 405 | + | storage_version: Option<u32>, | |
| 382 | 406 | ) -> Result<WireChangeEntry> { | |
| 383 | 407 | let ctx = crypto::AeadContext::entry(&entry.table, &entry.row_id); | |
| 384 | - | let envelope = Self::hlc_envelope(&entry.hlc, entry.data.as_ref()); | |
| 408 | + | let envelope = Self::hlc_envelope(&entry.hlc, entry.data.as_ref(), storage_version); | |
| 385 | 409 | let encrypted_data = Some(crypto::encrypt_json_aad(&envelope, master_key, &ctx)?); | |
| 386 | 410 | ||
| 387 | 411 | Ok(WireChangeEntry { | |
| @@ -413,11 +437,12 @@ | |||
| 413 | 437 | ) -> Result<crate::types::PulledChange> { | |
| 414 | 438 | let device_id = entry.device_id; | |
| 415 | 439 | let seq = entry.seq; | |
| 416 | - | let decrypted = Self::decrypt_change_with_key(entry, master_key)?; | |
| 440 | + | let (decrypted, storage_version) = Self::decrypt_change_parts(entry, master_key)?; | |
| 417 | 441 | Ok(crate::types::PulledChange { | |
| 418 | 442 | entry: decrypted, | |
| 419 | 443 | device_id, | |
| 420 | 444 | seq, | |
| 445 | + | storage_version, | |
| 421 | 446 | }) | |
| 422 | 447 | } | |
| 423 | 448 | ||
| @@ -460,9 +485,10 @@ | |||
| 460 | 485 | group_id: &str, | |
| 461 | 486 | entry: ChangeEntry, | |
| 462 | 487 | gck: &[u8; 32], | |
| 488 | + | storage_version: Option<u32>, | |
| 463 | 489 | ) -> Result<WireChangeEntry> { | |
| 464 | 490 | let ctx = crypto::AeadContext::group_entry(group_id, &entry.table, &entry.row_id); | |
| 465 | - | let envelope = Self::hlc_envelope(&entry.hlc, entry.data.as_ref()); | |
| 491 | + | let envelope = Self::hlc_envelope(&entry.hlc, entry.data.as_ref(), storage_version); | |
| 466 | 492 | let encrypted_data = Some(crypto::encrypt_json_aad(&envelope, gck, &ctx)?); | |
| 467 | 493 | ||
| 468 | 494 | Ok(WireChangeEntry { | |
| @@ -487,7 +513,7 @@ | |||
| 487 | 513 | let device_id = entry.device_id; | |
| 488 | 514 | let seq = entry.seq; | |
| 489 | 515 | let ctx = crypto::AeadContext::group_entry(group_id, &entry.table, &entry.row_id); | |
| 490 | - | let (hlc, data) = match entry.data { | |
| 516 | + | let (hlc, data, storage_version) = match entry.data { | |
| 491 | 517 | Some(ref value) => { | |
| 492 | 518 | let decrypted = crypto::decrypt_json_aad(value, gck, &ctx)?; | |
| 493 | 519 | Self::split_hlc_envelope(decrypted, device_id, entry.timestamp.timestamp_millis())? | |
| @@ -495,6 +521,7 @@ | |||
| 495 | 521 | None => ( | |
| 496 | 522 | Hlc::from_legacy(entry.timestamp.timestamp_millis(), device_id), | |
| 497 | 523 | None, | |
| 524 | + | None, | |
| 498 | 525 | ), | |
| 499 | 526 | }; | |
| 500 | 527 | Ok(crate::types::PulledChange { | |
| @@ -509,6 +536,7 @@ | |||
| 509 | 536 | }, | |
| 510 | 537 | device_id, | |
| 511 | 538 | seq, | |
| 539 | + | storage_version, | |
| 512 | 540 | }) | |
| 513 | 541 | } | |
| 514 | 542 | ||
| @@ -517,8 +545,19 @@ | |||
| 517 | 545 | entry: PullChangeEntry, | |
| 518 | 546 | master_key: &[u8; 32], | |
| 519 | 547 | ) -> Result<ChangeEntry> { | |
| 548 | + | Self::decrypt_change_parts(entry, master_key).map(|(entry, _)| entry) | |
| 549 | + | } | |
| 550 | + | ||
| 551 | + | /// Decrypt, keeping the sender's storage stamp alongside the entry. | |
| 552 | + | /// | |
| 553 | + | /// The stamp is envelope metadata rather than part of the change, so it stops | |
| 554 | + | /// here unless a caller asks for it; only the pull gate does. | |
| 555 | + | fn decrypt_change_parts( | |
| 556 | + | entry: PullChangeEntry, | |
| 557 | + | master_key: &[u8; 32], | |
| 558 | + | ) -> Result<(ChangeEntry, Option<u32>)> { | |
| 520 | 559 | let ctx = crypto::AeadContext::entry(&entry.table, &entry.row_id); | |
| 521 | - | let (hlc, data) = match entry.data { | |
| 560 | + | let (hlc, data, storage_version) = match entry.data { | |
| 522 | 561 | Some(ref value) => { | |
| 523 | 562 | let decrypted = crypto::decrypt_json_aad(value, master_key, &ctx)?; | |
| 524 | 563 | Self::split_hlc_envelope( | |
| @@ -530,18 +569,22 @@ | |||
| 530 | 569 | None => ( | |
| 531 | 570 | Hlc::from_legacy(entry.timestamp.timestamp_millis(), entry.device_id), | |
| 532 | 571 | None, | |
| 572 | + | None, | |
| 533 | 573 | ), | |
| 534 | 574 | }; | |
| 535 | 575 | ||
| 536 | - | Ok(ChangeEntry { | |
| 537 | - | table: entry.table, | |
| 538 | - | op: entry.op, | |
| 539 | - | row_id: entry.row_id, | |
| 540 | - | hlc, | |
| 541 | - | timestamp: entry.timestamp, | |
| 542 | - | data, | |
| 543 | - | extra: serde_json::Map::default(), | |
| 544 | - | }) | |
| 576 | + | Ok(( | |
| 577 | + | ChangeEntry { | |
| 578 | + | table: entry.table, | |
| 579 | + | op: entry.op, | |
| 580 | + | row_id: entry.row_id, | |
| 581 | + | hlc, | |
| 582 | + | timestamp: entry.timestamp, | |
| 583 | + | data, | |
| 584 | + | extra: serde_json::Map::default(), | |
| 585 | + | }, | |
| 586 | + | storage_version, | |
| 587 | + | )) | |
| 545 | 588 | } | |
| 546 | 589 | } | |
| 547 | 590 | ||
| @@ -711,23 +754,108 @@ | |||
| 711 | 754 | ||
| 712 | 755 | // v2 envelope: explicit __skver, parsed by version. | |
| 713 | 756 | let v2 = serde_json::json!({ "__skver": 2, "__skhlc": hlc, "data": {"k": "v"} }); | |
| 714 | - | let (got, data) = SyncKitClient::split_hlc_envelope(v2, node, 0).unwrap(); | |
| 757 | + | let (got, data, _) = SyncKitClient::split_hlc_envelope(v2, node, 0).unwrap(); | |
| 715 | 758 | assert_eq!(got, hlc); | |
| 716 | 759 | assert_eq!(data, Some(serde_json::json!({"k": "v"}))); | |
| 717 | 760 | ||
| 718 | 761 | // gen-1 envelope: __skhlc present, no version tag. | |
| 719 | 762 | let gen1 = serde_json::json!({ "__skhlc": hlc, "data": null }); | |
| 720 | - | let (got, data) = SyncKitClient::split_hlc_envelope(gen1, node, 0).unwrap(); | |
| 763 | + | let (got, data, _) = SyncKitClient::split_hlc_envelope(gen1, node, 0).unwrap(); | |
| 721 | 764 | assert_eq!(got, hlc); | |
| 722 | 765 | assert_eq!(data, None); | |
| 723 | 766 | ||
| 724 | 767 | // Bare legacy row: HLC synthesized from node + timestamp. | |
| 725 | 768 | let bare = serde_json::json!({ "title": "buy milk" }); | |
| 726 | - | let (got, data) = SyncKitClient::split_hlc_envelope(bare.clone(), node, 1234).unwrap(); | |
| 769 | + | let (got, data, _) = SyncKitClient::split_hlc_envelope(bare.clone(), node, 1234).unwrap(); | |
| 727 | 770 | assert_eq!(got, Hlc::from_legacy(1234, node)); | |
| 728 | 771 | assert_eq!(data, Some(bare)); | |
| 729 | 772 | } | |
| 730 | 773 | ||
| 774 | + | #[test] | |
| 775 | + | fn the_storage_stamp_rides_inside_the_sealed_envelope_and_survives_a_round_trip() { | |
| 776 | + | let node = DeviceId::new(Uuid::from_u128(1)); | |
| 777 | + | let hlc = Hlc { | |
| 778 | + | wall_ms: 5, | |
| 779 | + | counter: 2, | |
| 780 | + | node, | |
| 781 | + | }; | |
| 782 | + | ||
| 783 | + | let stamped = | |
| 784 | + | SyncKitClient::hlc_envelope(&hlc, Some(&serde_json::json!({"k": "v"})), Some(4)); | |
| 785 | + | assert_eq!(stamped["__sksv"], serde_json::json!(4)); | |
| 786 | + | let (got, data, version) = SyncKitClient::split_hlc_envelope(stamped, node, 0).unwrap(); | |
| 787 | + | assert_eq!(got, hlc); | |
| 788 | + | assert_eq!(data, Some(serde_json::json!({"k": "v"}))); | |
| 789 | + | assert_eq!(version, Some(4)); | |
| 790 | + | ||
| 791 | + | // A Delete carries no row payload and still carries the stamp, which is | |
| 792 | + | // what lets the gate see a peer whose only pending change is a delete. | |
| 793 | + | let delete = SyncKitClient::hlc_envelope(&hlc, None, Some(4)); | |
| 794 | + | let (_, data, version) = SyncKitClient::split_hlc_envelope(delete, node, 0).unwrap(); | |
| 795 | + | assert_eq!(data, None); | |
| 796 | + | assert_eq!(version, Some(4)); | |
| 797 | + | } | |
| 798 | + | ||
| 799 | + | #[test] | |
| 800 | + | fn an_undeclared_version_leaves_the_sealed_bytes_exactly_as_they_were() { | |
| 801 | + | let node = DeviceId::new(Uuid::from_u128(1)); | |
| 802 | + | let hlc = Hlc { | |
| 803 | + | wall_ms: 5, | |
| 804 | + | counter: 2, | |
| 805 | + | node, | |
| 806 | + | }; | |
| 807 | + | let plain = SyncKitClient::hlc_envelope(&hlc, None, None); | |
| 808 | + | assert!(plain.get("__sksv").is_none()); | |
| 809 | + | assert_eq!( | |
| 810 | + | plain, | |
| 811 | + | serde_json::json!({ "__skver": 2, "__skhlc": hlc, "data": null }), | |
| 812 | + | "an app that declares no version must push byte-identical envelopes" | |
| 813 | + | ); | |
| 814 | + | } | |
| 815 | + | ||
| 816 | + | /// The stamp is added *within* v2 rather than as a new `__skver`, so a reader | |
| 817 | + | /// that predates it addresses `__skhlc` and `data` by name and is unaffected. | |
| 818 | + | #[test] | |
| 819 | + | fn a_stamped_envelope_still_reads_as_an_ordinary_v2_envelope() { | |
| 820 | + | let node = DeviceId::new(Uuid::from_u128(1)); | |
| 821 | + | let hlc = Hlc { | |
| 822 | + | wall_ms: 9, | |
| 823 | + | counter: 1, | |
| 824 | + | node, | |
| 825 | + | }; | |
| 826 | + | let stamped = SyncKitClient::hlc_envelope(&hlc, Some(&serde_json::json!(7)), Some(12)); | |
| 827 | + | assert_eq!(stamped["__skver"], serde_json::json!(2)); | |
| 828 | + | assert_eq!( | |
| 829 | + | serde_json::from_value::<Hlc>(stamped["__skhlc"].clone()).unwrap(), | |
| 830 | + | hlc | |
| 831 | + | ); | |
| 832 | + | assert_eq!(stamped["data"], serde_json::json!(7)); | |
| 833 | + | } | |
| 834 | + | ||
| 835 | + | /// Optional by construction: a malformed stamp must not fail a change that is | |
| 836 | + | /// otherwise fine. | |
| 837 | + | #[test] | |
| 838 | + | fn a_malformed_stamp_reads_as_no_stamp() { | |
| 839 | + | let node = DeviceId::new(Uuid::from_u128(1)); | |
| 840 | + | let hlc = Hlc { | |
| 841 | + | wall_ms: 1, | |
| 842 | + | counter: 0, | |
| 843 | + | node, | |
| 844 | + | }; | |
| 845 | + | for bad in [ | |
| 846 | + | serde_json::json!("four"), | |
| 847 | + | serde_json::json!(-1), | |
| 848 | + | serde_json::json!(null), | |
| 849 | + | serde_json::json!(u64::from(u32::MAX) + 1), | |
| 850 | + | ] { | |
| 851 | + | let env = serde_json::json!({ | |
| 852 | + | "__skver": 2, "__skhlc": hlc, "data": null, "__sksv": bad | |
| 853 | + | }); | |
| 854 | + | let (_, _, version) = SyncKitClient::split_hlc_envelope(env, node, 0).unwrap(); | |
| 855 | + | assert_eq!(version, None, "bad stamp {bad} should read as absent"); | |
| 856 | + | } | |
| 857 | + | } | |
| 858 | + | ||
| 731 | 859 | #[test] | |
| 732 | 860 | fn split_envelope_rejects_unknown_version_loudly() { | |
| 733 | 861 | // The X2 hazard: a future envelope version must error, not silently |
| @@ -420,6 +420,12 @@ | |||
| 420 | 420 | /// which is the behaviour this client always had. See | |
| 421 | 421 | /// [`resume`](crate::client::resume). | |
| 422 | 422 | resume_store: RwLock<Option<Arc<dyn resume::BlobResumeStore>>>, | |
| 423 | + | /// The storage version sealed into every pushed change, so a peer can refuse | |
| 424 | + | /// a shared changelog it does not fully understand. | |
| 425 | + | /// | |
| 426 | + | /// `None` unless something declares one (the `SyncStore` engine does, from | |
| 427 | + | /// its `SyncSchema`), which leaves the wire format exactly as it was. | |
| 428 | + | storage_version: RwLock<Option<u32>>, | |
| 423 | 429 | } | |
| 424 | 430 | ||
| 425 | 431 | impl SyncKitClient { | |
| @@ -479,6 +485,7 @@ | |||
| 479 | 485 | pending_key: RwLock::new(None), | |
| 480 | 486 | gck_cache: RwLock::new(std::collections::HashMap::new()), | |
| 481 | 487 | resume_store: RwLock::new(None), | |
| 488 | + | storage_version: RwLock::new(None), | |
| 482 | 489 | } | |
| 483 | 490 | } | |
| 484 | 491 | ||
| @@ -501,6 +508,7 @@ | |||
| 501 | 508 | pending_key: RwLock::new(None), | |
| 502 | 509 | gck_cache: RwLock::new(std::collections::HashMap::new()), | |
| 503 | 510 | resume_store: RwLock::new(None), | |
| 511 | + | storage_version: RwLock::new(None), | |
| 504 | 512 | } | |
| 505 | 513 | } | |
| 506 | 514 | ||
| @@ -518,6 +526,23 @@ | |||
| 518 | 526 | *self.resume_store.write() = Some(store); | |
| 519 | 527 | } | |
| 520 | 528 | ||
| 529 | + | /// Declare the `SyncSchema` storage version to seal into every change this | |
| 530 | + | /// client pushes, so a peer pulling it can refuse a manifest it does not | |
| 531 | + | /// share. | |
| 532 | + | /// | |
| 533 | + | /// The `SyncStore` engine calls this from its schema when the store is built, | |
| 534 | + | /// so an app driving `SyncStore` gets the stamp without asking. `None` (the | |
| 535 | + | /// default, and what an app that declares no version keeps) leaves the sealed | |
| 536 | + | /// bytes exactly as they were. | |
| 537 | + | pub fn set_storage_version(&self, version: Option<u32>) { | |
| 538 | + | *self.storage_version.write() = version; | |
| 539 | + | } | |
| 540 | + | ||
| 541 | + | /// The storage version sealed into pushed changes, if any. | |
| 542 | + | pub fn storage_version(&self) -> Option<u32> { | |
| 543 | + | *self.storage_version.read() | |
| 544 | + | } | |
| 545 | + | ||
| 521 | 546 | /// The installed resume store, if any. | |
| 522 | 547 | pub(crate) fn resume_store(&self) -> Option<Arc<dyn resume::BlobResumeStore>> { | |
| 523 | 548 | self.resume_store.read().clone() |
| @@ -87,9 +87,10 @@ | |||
| 87 | 87 | self.require_master_key()? | |
| 88 | 88 | }; | |
| 89 | 89 | let master_key: &[u8; 32] = &key_holder; | |
| 90 | + | let storage_version = self.storage_version(); | |
| 90 | 91 | let wire_changes = changes | |
| 91 | 92 | .into_iter() | |
| 92 | - | .map(|c| Self::encrypt_change_with_key(c, master_key)) | |
| 93 | + | .map(|c| Self::encrypt_change_with_key(c, master_key, storage_version)) | |
| 93 | 94 | .collect::<Result<Vec<_>>>()?; | |
| 94 | 95 | ||
| 95 | 96 | let body = Bytes::from(serde_json::to_vec(&WirePushRequest { |
| @@ -173,6 +173,10 @@ | |||
| 173 | 173 | entry, | |
| 174 | 174 | device_id, | |
| 175 | 175 | seq, | |
| 176 | + | // A held entry already passed the peer storage gate on the pull that | |
| 177 | + | // held it, and the hold stores the change rather than the envelope, so | |
| 178 | + | // there is no stamp to re-read and nothing left to re-check. | |
| 179 | + | storage_version: None, | |
| 176 | 180 | }); | |
| 177 | 181 | } | |
| 178 | 182 | Ok(out) | |
| @@ -425,6 +429,7 @@ | |||
| 425 | 429 | ||
| 426 | 430 | fn pulled(table: &str, row_id: &str, seq: i64) -> PulledChange { | |
| 427 | 431 | PulledChange { | |
| 432 | + | storage_version: None, | |
| 428 | 433 | entry: ChangeEntry { | |
| 429 | 434 | table: table.into(), | |
| 430 | 435 | op: ChangeOp::Insert, |
| @@ -144,7 +144,15 @@ | |||
| 144 | 144 | } | |
| 145 | 145 | ||
| 146 | 146 | /// Finish building. | |
| 147 | - | pub fn build(self) -> SyncStore<C> { | |
| 147 | + | pub fn build(self) -> SyncStore<C> | |
| 148 | + | where | |
| 149 | + | C: SyncTransport, | |
| 150 | + | { | |
| 151 | + | // The transport seals this into every pushed change, so a peer can gate | |
| 152 | + | // on it. Declared once here rather than per push: the schema cannot | |
| 153 | + | // change under a built store. | |
| 154 | + | self.client | |
| 155 | + | .set_storage_version(self.schema.declared_storage_version()); | |
| 148 | 156 | SyncStore { | |
| 149 | 157 | db: self.db, | |
| 150 | 158 | client: Arc::new(self.client), | |
| @@ -162,6 +170,14 @@ | |||
| 162 | 170 | /// snapshot if needed, push local changes, pull and apply remote changes, sync | |
| 163 | 171 | /// blobs (if a policy is set), then clean up and stamp the last-sync time. | |
| 164 | 172 | pub async fn sync_now(&self) -> Result<SyncOutcome> { | |
| 173 | + | // The local gate runs before anything else touches the network: migration | |
| 174 | + | // is not a thing the sync does, it is a thing that has to have already | |
| 175 | + | // happened for the sync to be allowed. A refusal here has registered no | |
| 176 | + | // device, read no row and written nothing. | |
| 177 | + | let schema = self.schema.clone(); | |
| 178 | + | self.blocking(move |conn| super::version::enforce_local(conn, &schema)) | |
| 179 | + | .await?; | |
| 180 | + | ||
| 165 | 181 | // Defensive: a database written by an older client might carry a stale flag. | |
| 166 | 182 | self.blocking(clear_applying_remote).await?; | |
| 167 | 183 | ||
| @@ -200,6 +216,29 @@ | |||
| 200 | 216 | }) | |
| 201 | 217 | } | |
| 202 | 218 | ||
| 219 | + | /// Stamp the store with this schema's declared storage version. | |
| 220 | + | /// | |
| 221 | + | /// The app calls this at the end of its own local migration. The stamp is the | |
| 222 | + | /// claim that the store now matches the manifest, so it belongs after the | |
| 223 | + | /// migration has finished and, where the app can manage it, in the same | |
| 224 | + | /// transaction: writing it early is what would make a half-migrated store | |
| 225 | + | /// observable to a peer. | |
| 226 | + | /// | |
| 227 | + | /// A no-op when the manifest declares no version. | |
| 228 | + | pub async fn stamp_storage_version(&self) -> Result<()> { | |
| 229 | + | let Some(version) = self.schema.declared_storage_version() else { | |
| 230 | + | return Ok(()); | |
| 231 | + | }; | |
| 232 | + | self.blocking(move |conn| super::version::stamp_version(conn, version)) | |
| 233 | + | .await | |
| 234 | + | } | |
| 235 | + | ||
| 236 | + | /// The storage version currently stamped on this store, or `None` if it has | |
| 237 | + | /// never been stamped. | |
| 238 | + | pub async fn stored_storage_version(&self) -> Result<Option<u32>> { | |
| 239 | + | self.blocking(super::version::stored_version).await | |
| 240 | + | } | |
| 241 | + | ||
| 203 | 242 | /// Count unpushed local changes. | |
| 204 | 243 | pub async fn pending_changes(&self) -> Result<i64> { | |
| 205 | 244 | self.blocking(count_pending_changes).await | |
| @@ -486,6 +525,7 @@ | |||
| 486 | 525 | .enumerate() | |
| 487 | 526 | .filter(|(i, _)| (*i as i64 + 1) > cursor) | |
| 488 | 527 | .map(|(i, (d, e))| PulledChange { | |
| 528 | + | storage_version: None, | |
| 489 | 529 | entry: e.clone(), | |
| 490 | 530 | device_id: *d, | |
| 491 | 531 | seq: i as i64 + 1, | |
| @@ -536,6 +576,7 @@ | |||
| 536 | 576 | .enumerate() | |
| 537 | 577 | .filter(|(i, (gid, _, _))| *gid == group_id && (*i as i64 + 1) > cursor) | |
| 538 | 578 | .map(|(i, (_, dev, entry))| PulledChange { | |
| 579 | + | storage_version: None, | |
| 539 | 580 | entry: entry.clone(), | |
| 540 | 581 | device_id: *dev, | |
| 541 | 582 | seq: i as i64 + 1, |
| @@ -621,6 +621,7 @@ | |||
| 621 | 621 | .find(|e| e.row_id == id) | |
| 622 | 622 | .unwrap(); | |
| 623 | 623 | PulledChange { | |
| 624 | + | storage_version: None, | |
| 624 | 625 | entry, | |
| 625 | 626 | device_id: node, | |
| 626 | 627 | seq, | |
| @@ -843,6 +844,7 @@ | |||
| 843 | 844 | .find(|e| e.op == ChangeOp::Delete) | |
| 844 | 845 | .unwrap(); | |
| 845 | 846 | let pulled = PulledChange { | |
| 847 | + | storage_version: None, | |
| 846 | 848 | entry: del, | |
| 847 | 849 | device_id: bn, | |
| 848 | 850 | seq: 2, | |
| @@ -961,6 +963,7 @@ | |||
| 961 | 963 | |(row, dev, wall_off, counter, payload)| { | |
| 962 | 964 | let node = node(u128::from(dev)); | |
| 963 | 965 | PulledChange { | |
| 966 | + | storage_version: None, | |
| 964 | 967 | entry: ChangeEntry { | |
| 965 | 968 | table: "note".into(), | |
| 966 | 969 | op: ChangeOp::Update, | |
| @@ -1170,6 +1173,7 @@ | |||
| 1170 | 1173 | /// A remote change with an explicit HLC and payload, as a peer would send it. | |
| 1171 | 1174 | fn remote_change(from: DeviceId, id: &str, name: &str, hlc: Hlc, seq: i64) -> PulledChange { | |
| 1172 | 1175 | PulledChange { | |
| 1176 | + | storage_version: None, | |
| 1173 | 1177 | entry: ChangeEntry { | |
| 1174 | 1178 | table: "note".to_string(), | |
| 1175 | 1179 | op: ChangeOp::Update, | |
| @@ -1499,6 +1503,7 @@ | |||
| 1499 | 1503 | /// A remote change for the card table with an explicit payload and wall clock. | |
| 1500 | 1504 | fn card_change(from: DeviceId, wall_ms: i64, data: serde_json::Value) -> PulledChange { | |
| 1501 | 1505 | PulledChange { | |
| 1506 | + | storage_version: None, | |
| 1502 | 1507 | entry: ChangeEntry { | |
| 1503 | 1508 | table: "card".into(), | |
| 1504 | 1509 | op: ChangeOp::Update, | |
| @@ -1852,6 +1857,7 @@ | |||
| 1852 | 1857 | ||
| 1853 | 1858 | fn job_change(from: DeviceId, wall_ms: i64, data: serde_json::Value) -> PulledChange { | |
| 1854 | 1859 | PulledChange { | |
| 1860 | + | storage_version: None, | |
| 1855 | 1861 | entry: ChangeEntry { | |
| 1856 | 1862 | table: "job".into(), | |
| 1857 | 1863 | op: ChangeOp::Update, |
| @@ -10,9 +10,9 @@ | |||
| 10 | 10 | //! The modules: schema types ([`schema`]), trigger/migration generation | |
| 11 | 11 | //! ([`migrate`]), the DB seam + bookkeeping state ([`db`]), the FK-ordered apply | |
| 12 | 12 | //! engine ([`apply`]), the HLC ledger + conflict dispatch ([`hlc`]), the | |
| 13 | - | //! push/pull loops + lifecycle ([`sync`]), the blob engine ([`blob`]), and the | |
| 14 | - | //! [`SyncStore`](facade::SyncStore) facade + scheduler ([`facade`], | |
| 15 | - | //! [`scheduler`]). | |
| 13 | + | //! push/pull loops + lifecycle ([`sync`]), the blob engine ([`blob`]), the | |
| 14 | + | //! storage-version gate ([`version`]), and the [`SyncStore`](facade::SyncStore) | |
| 15 | + | //! facade + scheduler ([`facade`], [`scheduler`]). | |
| 16 | 16 | //! | |
| 17 | 17 | //! Start at [`SyncStore`](facade::SyncStore): declare a [`SyncSchema`](schema), | |
| 18 | 18 | //! build a store, and call `sync_now` or `spawn_scheduler`. | |
| @@ -32,6 +32,7 @@ | |||
| 32 | 32 | pub(crate) mod snapshot; | |
| 33 | 33 | pub(crate) mod stash; | |
| 34 | 34 | pub mod sync; | |
| 35 | + | pub mod version; | |
| 35 | 36 | ||
| 36 | 37 | pub use apply::{ApplyOutcome, Unapplied, apply_remote_changes}; | |
| 37 | 38 | pub use blob::{ | |
| @@ -51,9 +52,13 @@ | |||
| 51 | 52 | }; | |
| 52 | 53 | pub use resume::SqliteResumeStore; | |
| 53 | 54 | pub use scheduler::{NoopObserver, SyncObserver, SyncState}; | |
| 54 | - | pub use schema::{ConflictStrategy, DeleteMode, RowIdScheme, SyncMode, SyncSchema, SyncTable}; | |
| 55 | + | pub use schema::{ | |
| 56 | + | ConflictStrategy, DeleteMode, LedgerEntry, LedgerMismatch, RowIdScheme, SyncMode, SyncSchema, | |
| 57 | + | SyncTable, | |
| 58 | + | }; | |
| 55 | 59 | pub use sync::{ | |
| 56 | 60 | PUSH_BATCH_LIMIT, PullOutcome, SyncScope, SyncTransport, cleanup_changelog, | |
| 57 | 61 | create_initial_snapshot, enforce_changelog_retention, ensure_device_registered, pull_changes, | |
| 58 | 62 | pull_scope, push_changes, push_scope, | |
| 59 | 63 | }; | |
| 64 | + | pub use version::{STORAGE_VERSION_KEY, check_peer, enforce_local, stamp_version, stored_version}; |
| @@ -11,6 +11,10 @@ | |||
| 11 | 11 | //! See `docs/architecture.md` ("The `SyncStore` higher-level helper") for the | |
| 12 | 12 | //! full design and worked GoingsOn / audiofiles / Balanced Breakfast manifests. | |
| 13 | 13 | ||
| 14 | + | use std::fmt::Write; | |
| 15 | + | ||
| 16 | + | use sha2::{Digest, Sha256}; | |
| 17 | + | ||
| 14 | 18 | /// How the engine resolves concurrent edits to the same row. | |
| 15 | 19 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] | |
| 16 | 20 | pub enum ConflictStrategy { | |
| @@ -300,6 +304,33 @@ | |||
| 300 | 304 | self | |
| 301 | 305 | } | |
| 302 | 306 | ||
| 307 | + | /// This table's line in [`SyncSchema::wire_manifest`]: the wire-visible | |
| 308 | + | /// facts about it, and nothing else. | |
| 309 | + | pub(crate) fn wire_line(&self) -> String { | |
| 310 | + | let row_id = match self.row_id { | |
| 311 | + | RowIdScheme::PrimaryKey => "pk", | |
| 312 | + | RowIdScheme::Hashed => "hashed", | |
| 313 | + | }; | |
| 314 | + | let mode = match self.mode { | |
| 315 | + | SyncMode::Full => "full", | |
| 316 | + | // The `set` columns are not repeated here: `emitted_columns` below | |
| 317 | + | // already carries them, and the discriminant is what distinguishes a | |
| 318 | + | // partial update from a full upsert over the same projection. | |
| 319 | + | SyncMode::PartialUpdate { .. } => "partial", | |
| 320 | + | }; | |
| 321 | + | let deletes = match self.deletes { | |
| 322 | + | DeleteMode::Hard => "hard".to_string(), | |
| 323 | + | DeleteMode::Ignore => "ignore".to_string(), | |
| 324 | + | DeleteMode::Tombstone { column } => format!("tombstone:{column}"), | |
| 325 | + | }; | |
| 326 | + | format!( | |
| 327 | + | "{} pk={} row_id={row_id} mode={mode} deletes={deletes} cols={}", | |
| 328 | + | self.name, | |
| 329 | + | self.pk.join(","), | |
| 330 | + | self.emitted_columns().join(","), | |
| 331 | + | ) | |
| 332 | + | } | |
| 333 | + | ||
| 303 | 334 | /// The columns emitted into `sync_changelog.data` for an INSERT/UPDATE: | |
| 304 | 335 | /// every whitelisted column for `Full`, or the primary key plus the `set` | |
| 305 | 336 | /// columns (deduped, PK first) for `PartialUpdate`. Public so a consumer can | |
| @@ -330,6 +361,7 @@ | |||
| 330 | 361 | pub struct SyncSchema { | |
| 331 | 362 | pub(crate) tables: Vec<SyncTable>, | |
| 332 | 363 | pub(crate) conflict: ConflictStrategy, | |
| 364 | + | pub(crate) storage_version: Option<u32>, | |
| 333 | 365 | } | |
| 334 | 366 | ||
| 335 | 367 | impl SyncSchema { | |
| @@ -339,6 +371,7 @@ | |||
| 339 | 371 | Self { | |
| 340 | 372 | tables, | |
| 341 | 373 | conflict: ConflictStrategy::HybridLogicalClock, | |
| 374 | + | storage_version: None, | |
| 342 | 375 | } | |
| 343 | 376 | } | |
| 344 | 377 | ||
| @@ -364,4 +397,416 @@ | |||
| 364 | 397 | pub(crate) fn any_hashed(&self) -> bool { | |
| 365 | 398 | self.tables.iter().any(|t| t.row_id == RowIdScheme::Hashed) | |
| 366 | 399 | } | |
| 400 | + | ||
| 401 | + | /// Declare the storage version this manifest is at. | |
| 402 | + | /// | |
| 403 | + | /// The number a client compares to decide whether it may sync at all. When a | |
| 404 | + | /// storage change is breaking, clients refuse to sync rather than degrade: | |
| 405 | + | /// the local migration happens first, on the device, and sync resumes once | |
| 406 | + | /// both sides are on the new shape. A client meeting a store it does not | |
| 407 | + | /// fully understand does not sync partially, does not skip the records it | |
| 408 | + | /// cannot read, and does not write back a manifest it only half understands. | |
| 409 | + | /// | |
| 410 | + | /// **What moves the number is this manifest, not migrations.** Local-only DDL | |
| 411 | + | /// (a new index, a table absent from every [`SyncSchema`]) changes nothing | |
| 412 | + | /// that crosses the wire and must not fire the gate. What does: | |
| 413 | + | /// | |
| 414 | + | /// | change | fires | | |
| 415 | + | /// |---|---| | |
| 416 | + | /// | frontend / UI change | no | | |
| 417 | + | /// | local-only table or index | no | | |
| 418 | + | /// | new synced table | yes | | |
| 419 | + | /// | new synced column | yes | | |
| 420 | + | /// | changed `pk` or [`RowIdScheme`] | yes | | |
| 421 | + | /// | changed [`SyncMode`] / [`DeleteMode`] | yes | | |
| 422 | + | /// | |
| 423 | + | /// Leaving it undeclared leaves the gate off, which is what an app that has | |
| 424 | + | /// not adopted this yet gets. Declaring it is the opt-in, and | |
| 425 | + | /// [`check_ledger`](Self::check_ledger) is what stops a later manifest edit | |
| 426 | + | /// from forgetting the bump. | |
| 427 | + | #[must_use] | |
| 428 | + | pub fn storage_version(mut self, version: u32) -> Self { | |
| 429 | + | self.storage_version = Some(version); | |
| 430 | + | self | |
| 431 | + | } | |
| 432 | + | ||
| 433 | + | /// The declared storage version, or `None` if this manifest has not adopted | |
| 434 | + | /// the gate. | |
| 435 | + | pub fn declared_storage_version(&self) -> Option<u32> { | |
| 436 | + | self.storage_version | |
| 437 | + | } | |
| 438 | + | ||
| 439 | + | /// The canonical, wire-visible rendering of this manifest: one line per | |
| 440 | + | /// table, in declared order. | |
| 441 | + | /// | |
| 442 | + | /// Only what a peer can observe is in it, table names, [`emitted_columns`] | |
| 443 | + | /// (which *is* the wire projection), `pk`, [`RowIdScheme`], [`SyncMode`] and | |
| 444 | + | /// [`DeleteMode`]. Local-only policy is deliberately absent, so changing how | |
| 445 | + | /// this device resolves a conflict or which local column carries group | |
| 446 | + | /// provenance does not lock two devices apart. | |
| 447 | + | /// | |
| 448 | + | /// **Declaration order is preserved rather than sorted**, because it is not | |
| 449 | + | /// merely presentation: it is the foreign-key order the engine upserts in and | |
| 450 | + | /// deletes in reverse of, so reordering the tables reorders what a peer | |
| 451 | + | /// observes. | |
| 452 | + | /// | |
| 453 | + | /// [`emitted_columns`]: SyncTable::emitted_columns | |
| 454 | + | pub fn wire_manifest(&self) -> String { | |
| 455 | + | let mut out = String::new(); | |
| 456 | + | for t in &self.tables { | |
| 457 | + | out.push_str(&t.wire_line()); | |
| 458 | + | out.push('\n'); | |
| 459 | + | } | |
| 460 | + | out | |
| 461 | + | } | |
| 462 | + | ||
| 463 | + | /// SHA-256 of [`wire_manifest`](Self::wire_manifest), lowercase hex. | |
| 464 | + | /// | |
| 465 | + | /// Stable across compilations and across machines: it hashes declared | |
| 466 | + | /// `&'static` data, nothing derived from the build. | |
| 467 | + | pub fn fingerprint(&self) -> String { | |
| 468 | + | let mut hasher = Sha256::new(); | |
| 469 | + | hasher.update(self.wire_manifest().as_bytes()); | |
| 470 | + | let digest = hasher.finalize(); | |
| 471 | + | let mut hex = String::with_capacity(64); | |
| 472 | + | for byte in digest { | |
| 473 | + | write!(hex, "{byte:02x}").expect("fmt::Write for String is infallible"); | |
| 474 | + | } | |
| 475 | + | hex | |
| 476 | + | } | |
| 477 | + | ||
| 478 | + | /// Check this manifest against the committed version-to-fingerprint ledger. | |
| 479 | + | /// | |
| 480 | + | /// The mechanism that makes forgetting the bump impossible. An app commits | |
| 481 | + | /// one [`LedgerEntry`] per storage version it has ever shipped and calls this | |
| 482 | + | /// from a test; editing the manifest without bumping fails that test, and the | |
| 483 | + | /// failure carries the current wire manifest so a diff against the previous | |
| 484 | + | /// commit names what moved. | |
| 485 | + | /// | |
| 486 | + | /// A declared integer alone would rely on memory, and a fingerprint alone has | |
| 487 | + | /// no ordering, so a client could tell two manifests differ and not which is | |
| 488 | + | /// newer. Both together give an ordered number that cannot drift from what it | |
| 489 | + | /// describes. | |
| 490 | + | /// | |
| 491 | + | /// ```no_run | |
| 492 | + | /// # use synckit_client::store::{LedgerEntry, SyncSchema}; | |
| 493 | + | /// # fn schema() -> SyncSchema { SyncSchema::new(vec![]) } | |
| 494 | + | /// const LEDGER: &[LedgerEntry] = &[ | |
| 495 | + | /// LedgerEntry { version: 1, fingerprint: "9f2c1a..." }, | |
| 496 | + | /// ]; | |
| 497 | + | /// | |
| 498 | + | /// #[test] | |
| 499 | + | /// fn manifest_matches_its_declared_storage_version() { | |
| 500 | + | /// schema().check_ledger(LEDGER).unwrap(); | |
| 501 | + | /// } | |
| 502 | + | /// ``` | |
| 503 | + | pub fn check_ledger(&self, ledger: &[LedgerEntry]) -> Result<(), LedgerMismatch> { | |
| 504 | + | let current = self.fingerprint(); | |
| 505 | + | let Some(version) = self.storage_version else { | |
| 506 | + | return Err(LedgerMismatch::Undeclared { | |
| 507 | + | fingerprint: current, | |
| 508 | + | manifest: self.wire_manifest(), | |
| 509 | + | }); | |
| 510 | + | }; | |
| 511 | + | match ledger.iter().find(|e| e.version == version) { | |
| 512 | + | None => Err(LedgerMismatch::Unrecorded { | |
| 513 | + | version, | |
| 514 | + | fingerprint: current, | |
| 515 | + | manifest: self.wire_manifest(), | |
| 516 | + | }), | |
| 517 | + | Some(entry) if entry.fingerprint != current => Err(LedgerMismatch::Moved { | |
| 518 | + | version, | |
| 519 | + | recorded: entry.fingerprint, | |
| 520 | + | fingerprint: current, | |
| 521 | + | manifest: self.wire_manifest(), | |
| 522 | + | }), | |
| 523 | + | Some(_) => Ok(()), | |
| 524 | + | } | |
| 525 | + | } | |
| 526 | + | } | |
| 527 | + | ||
| 528 | + | /// One committed `(storage_version, fingerprint)` pair. | |
| 529 | + | /// | |
| 530 | + | /// The ledger is a plain `const` array in the app's own source: it is committed | |
| 531 | + | /// history, so it only ever grows a row. | |
| 532 | + | #[derive(Debug, Clone, Copy, PartialEq, Eq)] | |
| 533 | + | pub struct LedgerEntry { | |
| 534 | + | /// The storage version this row records. | |
| 535 | + | pub version: u32, | |
| 536 | + | /// [`SyncSchema::fingerprint`] as of that version, lowercase hex. | |
| 537 | + | pub fingerprint: &'static str, | |
| 538 | + | } | |
| 539 | + | ||
| 540 | + | /// Why [`SyncSchema::check_ledger`] refused. | |
| 541 | + | /// | |
| 542 | + | /// Each variant carries the current wire manifest, so the test output is the | |
| 543 | + | /// thing to diff rather than a hash to go looking for. | |
| 544 | + | #[derive(Debug, Clone, PartialEq, Eq)] | |
| 545 | + | pub enum LedgerMismatch { | |
| 546 | + | /// The manifest declares no [`storage_version`](SyncSchema::storage_version), | |
| 547 | + | /// so there is nothing to check it against. | |
| 548 | + | Undeclared { | |
| 549 | + | /// The manifest's current fingerprint. | |
| 550 | + | fingerprint: String, | |
| 551 | + | /// The current wire manifest. | |
| 552 | + | manifest: String, | |
| 553 | + | }, | |
| 554 | + | /// The declared version has no row in the ledger. | |
| 555 | + | Unrecorded { | |
| 556 | + | /// The declared storage version. | |
| 557 | + | version: u32, | |
| 558 | + | /// The manifest's current fingerprint. | |
| 559 | + | fingerprint: String, | |
| 560 | + | /// The current wire manifest. | |
| 561 | + | manifest: String, | |
| 562 | + | }, | |
| 563 | + | /// The declared version is in the ledger under a different fingerprint: the | |
| 564 | + | /// manifest moved and the version did not. | |
| 565 | + | Moved { | |
| 566 | + | /// The declared storage version. | |
| 567 | + | version: u32, | |
| 568 | + | /// The fingerprint the ledger committed for that version. | |
| 569 | + | recorded: &'static str, | |
| 570 | + | /// The manifest's current fingerprint. | |
| 571 | + | fingerprint: String, | |
| 572 | + | /// The current wire manifest. | |
| 573 | + | manifest: String, | |
| 574 | + | }, | |
| 575 | + | } | |
| 576 | + | ||
| 577 | + | impl std::fmt::Display for LedgerMismatch { | |
| 578 | + | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { | |
| 579 | + | match self { | |
| 580 | + | Self::Undeclared { | |
| 581 | + | fingerprint, | |
| 582 | + | manifest, | |
| 583 | + | } => write!( | |
| 584 | + | f, | |
| 585 | + | "this SyncSchema declares no storage_version. Add \ | |
| 586 | + | `.storage_version(1)` and a ledger row \ | |
| 587 | + | `LedgerEntry {{ version: 1, fingerprint: \"{fingerprint}\" }}`.\n\ | |
| 588 | + | Wire manifest:\n{manifest}" | |
| 589 | + | ), | |
| 590 | + | Self::Unrecorded { | |
| 591 | + | version, | |
| 592 | + | fingerprint, | |
| 593 | + | manifest, | |
| 594 | + | } => write!( | |
| 595 | + | f, | |
| 596 | + | "storage_version {version} has no ledger row. Add \ | |
| 597 | + | `LedgerEntry {{ version: {version}, fingerprint: \"{fingerprint}\" }}`.\n\ | |
| 598 | + | Wire manifest:\n{manifest}" | |
| 599 | + | ), | |
| 600 | + | Self::Moved { | |
| 601 | + | version, | |
| 602 | + | recorded, | |
| 603 | + | fingerprint, | |
| 604 | + | manifest, | |
| 605 | + | } => write!( | |
| 606 | + | f, | |
| 607 | + | "the wire manifest moved under storage_version {version}: the ledger \ | |
| 608 | + | records {recorded}, the manifest now fingerprints {fingerprint}. \ | |
| 609 | + | Bump storage_version to {} and add a ledger row; diff the manifest \ | |
| 610 | + | below against the previous commit to see what moved.\n\ | |
| 611 | + | Wire manifest:\n{manifest}", | |
| 612 | + | version + 1 | |
| 613 | + | ), | |
| 614 | + | } | |
| 615 | + | } | |
| 616 | + | } | |
| 617 | + | ||
| 618 | + | impl std::error::Error for LedgerMismatch {} | |
| 619 | + | ||
| 620 | + | #[cfg(test)] | |
| 621 | + | mod tests { | |
| 622 | + | use super::*; | |
| 623 | + | ||
| 624 | + | fn base() -> SyncSchema { | |
| 625 | + | SyncSchema::new(vec![ | |
| 626 | + | SyncTable::full("project", &["id", "name"]), | |
| 627 | + | SyncTable::full("task", &["id", "project_id", "title", "done"]), | |
| 628 | + | ]) | |
| 629 | + | .storage_version(1) | |
| 630 | + | } | |
| 631 | + | ||
| 632 | + | #[test] | |
| 633 | + | fn wire_manifest_is_one_line_per_table_in_declared_order() { | |
| 634 | + | let m = base().wire_manifest(); | |
| 635 | + | let lines: Vec<&str> = m.lines().collect(); | |
| 636 | + | assert_eq!( | |
| 637 | + | lines, | |
| 638 | + | vec![ | |
| 639 | + | "project pk=id row_id=pk mode=full deletes=hard cols=id,name", | |
| 640 | + | "task pk=id row_id=pk mode=full deletes=hard cols=id,project_id,title,done", | |
| 641 | + | ] | |
| 642 | + | ); | |
| 643 | + | } | |
| 644 | + | ||
| 645 | + | #[test] | |
| 646 | + | fn fingerprint_is_stable_across_calls() { | |
| 647 | + | assert_eq!(base().fingerprint(), base().fingerprint()); | |
| 648 | + | assert_eq!(base().fingerprint().len(), 64); | |
| 649 | + | } | |
| 650 | + | ||
| 651 | + | /// Each of the four rows the policy says fires the gate. | |
| 652 | + | #[test] | |
| 653 | + | fn every_ruled_wire_change_moves_the_fingerprint() { | |
| 654 | + | let before = base().fingerprint(); | |
| 655 | + | ||
| 656 | + | // A new synced table. | |
| 657 | + | let added = SyncSchema::new(vec![ | |
| 658 | + | SyncTable::full("project", &["id", "name"]), | |
| 659 | + | SyncTable::full("task", &["id", "project_id", "title", "done"]), | |
| 660 | + | SyncTable::full("label", &["id", "name"]), | |
| 661 | + | ]); | |
| 662 | + | assert_ne!(added.fingerprint(), before, "new synced table"); | |
| 663 | + | ||
| 664 | + | // A new synced column. | |
| 665 | + | let column = SyncSchema::new(vec![ | |
| 666 | + | SyncTable::full("project", &["id", "name", "archived"]), | |
| 667 | + | SyncTable::full("task", &["id", "project_id", "title", "done"]), | |
| 668 | + | ]); | |
| 669 | + | assert_ne!(column.fingerprint(), before, "new synced column"); | |
| 670 | + | ||
| 671 | + | // A changed primary key. | |
| 672 | + | let pk = SyncSchema::new(vec![ | |
| 673 | + | SyncTable::full("project", &["id", "name"]).pk(&["name"]), | |
| 674 | + | SyncTable::full("task", &["id", "project_id", "title", "done"]), | |
| 675 | + | ]); | |
| 676 | + | assert_ne!(pk.fingerprint(), before, "changed pk"); | |
| 677 | + | ||
| 678 | + | // A changed RowIdScheme. | |
| 679 | + | let hashed = SyncSchema::new(vec![ | |
| 680 | + | SyncTable::full("project", &["id", "name"]).hashed(), | |
| 681 | + | SyncTable::full("task", &["id", "project_id", "title", "done"]), | |
| 682 | + | ]); | |
| 683 | + | assert_ne!(hashed.fingerprint(), before, "changed RowIdScheme"); | |
| 684 | + | ||
| 685 | + | // A changed SyncMode, even where the projection is unchanged. | |
| 686 | + | let partial = SyncSchema::new(vec![ | |
| 687 | + | SyncTable::full("project", &["id", "name"]).partial_update(&["name"]), | |
| 688 | + | SyncTable::full("task", &["id", "project_id", "title", "done"]), | |
| 689 | + | ]); | |
| 690 | + | assert_ne!(partial.fingerprint(), before, "changed SyncMode"); | |
| 691 | + | assert_eq!( | |
| 692 | + | partial.tables()[0].emitted_columns(), | |
| 693 | + | base().tables()[0].emitted_columns(), | |
| 694 | + | "same bytes on the wire, different meaning: the mode is what moved" | |
| 695 | + | ); | |
| 696 | + | ||
| 697 | + | // A changed DeleteMode. | |
| 698 | + | let ignore = SyncSchema::new(vec![ | |
| 699 | + | SyncTable::full("project", &["id", "name"]).ignore_deletes(), | |
| 700 | + | SyncTable::full("task", &["id", "project_id", "title", "done"]), | |
| 701 | + | ]); | |
| 702 | + | assert_ne!(ignore.fingerprint(), before, "changed DeleteMode"); | |
| 703 | + | let tombstone = SyncSchema::new(vec![ | |
| 704 | + | SyncTable::full("project", &["id", "name"]).tombstone("deleted_at"), | |
| 705 | + | SyncTable::full("task", &["id", "project_id", "title", "done"]), | |
| 706 | + | ]); | |
| 707 | + | assert_ne!(tombstone.fingerprint(), before, "tombstone DeleteMode"); | |
| 708 | + | assert_ne!( | |
| 709 | + | tombstone.fingerprint(), | |
| 710 | + | ignore.fingerprint(), | |
| 711 | + | "the two non-hard delete modes are distinguishable" | |
| 712 | + | ); | |
| 713 | + | } | |
| 714 | + | ||
| 715 | + | /// Declaration order is the FK apply order, so it is wire-visible and the | |
| 716 | + | /// manifest is deliberately not sorted. | |
| 717 | + | #[test] | |
| 718 | + | fn reordering_tables_moves_the_fingerprint() { | |
| 719 | + | let reordered = SyncSchema::new(vec![ | |
| 720 | + | SyncTable::full("task", &["id", "project_id", "title", "done"]), | |
| 721 | + | SyncTable::full("project", &["id", "name"]), | |
| 722 | + | ]); | |
| 723 | + | assert_ne!(reordered.fingerprint(), base().fingerprint()); | |
| 724 | + | } | |
| 725 | + | ||
| 726 | + | /// The boundary the policy draws: local-only policy is not wire-visible, so | |
| 727 | + | /// changing it must not lock two devices apart. If one of these ever should | |
| 728 | + | /// fire the gate, this test is where the decision lands. | |
| 729 | + | #[test] | |
| 730 | + | fn local_only_policy_does_not_move_the_fingerprint() { | |
| 731 | + | let before = base().fingerprint(); | |
| 732 | + | for schema in [ | |
| 733 | + | SyncSchema::new(vec![ | |
| 734 | + | SyncTable::full("project", &["id", "name"]).preserve_local(&["name"]), | |
| 735 | + | SyncTable::full("task", &["id", "project_id", "title", "done"]), | |
| 736 | + | ]), | |
| 737 | + | SyncSchema::new(vec![ | |
| 738 | + | SyncTable::full("project", &["id", "name"]).field_merge(&[]), | |
| 739 | + | SyncTable::full("task", &["id", "project_id", "title", "done"]), | |
| 740 | + | ]), | |
| 741 | + | SyncSchema::new(vec![ | |
| 742 | + | SyncTable::full("project", &["id", "name"]).group_scoped("group_id"), | |
| 743 | + | SyncTable::full("task", &["id", "project_id", "title", "done"]), | |
| 744 | + | ]), | |
| 745 | + | base().conflict_strategy(ConflictStrategy::ServerOrder), | |
| 746 | + | ] { | |
| 747 | + | assert_eq!(schema.fingerprint(), before); | |
| 748 | + | } | |
| 749 | + | } | |
| 750 | + | ||
| 751 | + | #[test] | |
| 752 | + | fn the_declared_version_is_not_part_of_the_fingerprint() { | |
| 753 | + | // Otherwise a bump would move the thing it is meant to describe and the | |
| 754 | + | // ledger could never match. | |
| 755 | + | assert_eq!( | |
| 756 | + | base().storage_version(9).fingerprint(), | |
| 757 | + | base().fingerprint() | |
| 758 | + | ); | |
| 759 | + | } | |
| 760 | + | ||
| 761 | + | #[test] | |
| 762 | + | fn check_ledger_passes_when_the_manifest_matches_its_row() { | |
| 763 | + | let s = base(); | |
| 764 | + | let fp: &'static str = Box::leak(s.fingerprint().into_boxed_str()); | |
| 765 | + | let ledger = [LedgerEntry { | |
| 766 | + | version: 1, | |
| 767 | + | fingerprint: fp, | |
| 768 | + | }]; | |
| 769 | + | assert!(s.check_ledger(&ledger).is_ok()); | |
| 770 | + | } | |
| 771 | + | ||
| 772 | + | #[test] | |
| 773 | + | fn check_ledger_refuses_an_undeclared_manifest() { | |
| 774 | + | let s = SyncSchema::new(vec![SyncTable::full("note", &["id"])]); | |
| 775 | + | assert!(matches!( | |
| 776 | + | s.check_ledger(&[]), | |
| 777 | + | Err(LedgerMismatch::Undeclared { .. }) | |
| 778 | + | )); | |
| 779 | + | } | |
| 780 | + | ||
| 781 | + | #[test] | |
| 782 | + | fn check_ledger_refuses_a_version_with_no_row() { | |
| 783 | + | assert!(matches!( | |
| 784 | + | base().check_ledger(&[]), | |
| 785 | + | Err(LedgerMismatch::Unrecorded { version: 1, .. }) | |
| 786 | + | )); | |
| 787 | + | } | |
| 788 | + | ||
| 789 | + | /// The point of the whole mechanism: edit the manifest, forget the bump, fail. | |
| 790 | + | #[test] | |
| 791 | + | fn check_ledger_catches_a_manifest_edit_without_a_bump() { | |
| 792 | + | let shipped = base(); | |
| 793 | + | let fp: &'static str = Box::leak(shipped.fingerprint().into_boxed_str()); | |
| 794 | + | let ledger = [LedgerEntry { | |
| 795 | + | version: 1, | |
| 796 | + | fingerprint: fp, | |
| 797 | + | }]; | |
| 798 | + | ||
| 799 | + | let edited = SyncSchema::new(vec![ | |
| 800 | + | SyncTable::full("project", &["id", "name", "archived"]), | |
| 801 | + | SyncTable::full("task", &["id", "project_id", "title", "done"]), | |
| 802 | + | ]) | |
| 803 | + | .storage_version(1); | |
| 804 | + | ||
| 805 | + | let err = edited.check_ledger(&ledger).unwrap_err(); | |
| 806 | + | assert!(matches!(err, LedgerMismatch::Moved { version: 1, .. })); | |
| 807 | + | let msg = err.to_string(); | |
| 808 | + | assert!(msg.contains("Bump storage_version to 2"), "{msg}"); | |
| 809 | + | // The failure carries the manifest, so the diff names what moved. | |
| 810 | + | assert!(msg.contains("cols=id,name,archived"), "{msg}"); | |
| 811 | + | } | |
| 367 | 812 | } |
| @@ -91,6 +91,14 @@ | |||
| 91 | 91 | cursor: i64, | |
| 92 | 92 | ) -> impl Future<Output = Result<(Vec<PulledChange>, i64, bool)>> + Send; | |
| 93 | 93 | ||
| 94 | + | /// Declare the manifest storage version this device stamps onto every change | |
| 95 | + | /// it pushes. | |
| 96 | + | /// | |
| 97 | + | /// Defaults to a no-op: a transport that carries no stamp (a test double, a | |
| 98 | + | /// future minimal SDK) is simply one whose peers cannot gate on it, which is | |
| 99 | + | /// the same position every client was in before the gate existed. | |
| 100 | + | fn set_storage_version(&self, _version: Option<u32>) {} | |
| 101 | + | ||
| 94 | 102 | /// The groups this user belongs to, as `(group_id, current_gck_version)`, | |
| 95 | 103 | /// the scopes to sync beyond personal. Defaults to none, so a transport that | |
| 96 | 104 | /// does not support groups (a test double, a future minimal SDK) never syncs | |
| @@ -168,6 +176,10 @@ | |||
| 168 | 176 | SyncKitClient::pull_rich(self, device_id, cursor) | |
| 169 | 177 | } | |
| 170 | 178 | ||
| 179 | + | fn set_storage_version(&self, version: Option<u32>) { | |
| 180 | + | SyncKitClient::set_storage_version(self, version); | |
| 181 | + | } | |
| 182 | + | ||
| 171 | 183 | fn list_group_scopes(&self) -> impl Future<Output = Result<Vec<(GroupId, i32)>>> + Send { | |
| 172 | 184 | async move { | |
| 173 | 185 | Ok(self | |
| @@ -507,6 +519,14 @@ | |||
| 507 | 519 | } | |
| 508 | 520 | }; | |
| 509 | 521 | ||
| 522 | + | // The peer gate, before anything is applied and before the cursor moves. | |
| 523 | + | // A refusal here leaves the page on the server and the cursor where it | |
| 524 | + | // was, so the sync resumes from exactly this point once both sides agree. | |
| 525 | + | let mine = schema.declared_storage_version(); | |
| 526 | + | for change in &pulled { | |
| 527 | + | super::version::check_peer(mine, change.storage_version)?; | |
| 528 | + | } | |
| 529 | + | ||
| 510 | 530 | // An empty page still runs an apply when something is held, so a row | |
| 511 | 531 | // waiting on a parent that arrived through another path clears without | |
| 512 | 532 | // needing new remote traffic to carry it. It is still the last page. | |
| @@ -664,6 +684,11 @@ | |||
| 664 | 684 | struct FakeServer { | |
| 665 | 685 | log: Arc<Mutex<Vec<(DeviceId, ChangeEntry)>>>, | |
| 666 | 686 | group_log: Arc<Mutex<Vec<(GroupId, DeviceId, ChangeEntry)>>>, | |
| 687 | + | /// The storage version a pulled page appears to have been sealed under. | |
| 688 | + | /// Stands in for the `__sksv` the real transport reads out of the | |
| 689 | + | /// envelope; the fake log holds decrypted entries, so there is no | |
| 690 | + | /// envelope here to carry it. | |
| 691 | + | peer_version: Arc<Mutex<Option<u32>>>, | |
| 667 | 692 | } | |
| 668 | 693 | ||
| 669 | 694 | impl SyncTransport for FakeServer { | |
| @@ -717,6 +742,7 @@ | |||
| 717 | 742 | cursor: i64, | |
| 718 | 743 | ) -> impl Future<Output = Result<(Vec<PulledChange>, i64, bool)>> + Send { | |
| 719 | 744 | let log = self.log.clone(); | |
| 745 | + | let peer_version = *self.peer_version.lock().unwrap(); | |
| 720 | 746 | async move { | |
| 721 | 747 | let l = log.lock().unwrap(); | |
| 722 | 748 | let out: Vec<PulledChange> = l | |
| @@ -724,6 +750,7 @@ | |||
| 724 | 750 | .enumerate() | |
| 725 | 751 | .filter(|(i, _)| (*i as i64 + 1) > cursor) | |
| 726 | 752 | .map(|(i, (dev, entry))| PulledChange { | |
| 753 | + | storage_version: peer_version, | |
| 727 | 754 | entry: entry.clone(), | |
| 728 | 755 | device_id: *dev, | |
| 729 | 756 | seq: i as i64 + 1, | |
| @@ -817,6 +844,93 @@ | |||
| 817 | 844 | (db, DeviceId::new(uuid::Uuid::from_u128(n))) | |
| 818 | 845 | } | |
| 819 | 846 | ||
| 847 | + | /// The gate's end-to-end case: a peer on a different manifest is refused | |
| 848 | + | /// before anything is applied, and the cursor does not move, so the page is | |
| 849 | + | /// still there once both sides agree. | |
| 850 | + | #[tokio::test] | |
| 851 | + | async fn a_peer_on_another_storage_version_is_refused_and_nothing_lands() { | |
| 852 | + | let dir = tempdir(); | |
| 853 | + | let (writer, writer_node) = device(&dir.join("a.db"), 1); | |
| 854 | + | let (reader, reader_node) = device(&dir.join("b.db"), 2); | |
| 855 | + | let server = FakeServer::default(); | |
| 856 | + | let gated = schema().storage_version(4); | |
| 857 | + | ||
| 858 | + | edit(&writer, "n1", "from the newer device"); | |
| 859 | + | push_scope(&writer, &server, &gated, writer_node, SyncScope::Personal) | |
| 860 | + | .await | |
| 861 | + | .unwrap(); | |
| 862 | + | // The peer is a manifest ahead. | |
| 863 | + | *server.peer_version.lock().unwrap() = Some(5); | |
| 864 | + | ||
| 865 | + | let err = pull_scope(&reader, &server, &gated, reader_node, SyncScope::Personal) | |
| 866 | + | .await | |
| 867 | + | .unwrap_err(); | |
| 868 | + | let r = match err { | |
| 869 | + | SyncKitError::StorageVersion(r) => r, | |
| 870 | + | other => panic!("expected a storage-version refusal, got {other:?}"), | |
| 871 | + | }; | |
| 872 | + | assert_eq!((r.mine, r.theirs), (4, 5)); | |
| 873 | + | assert_eq!(r.message(), "Update this device."); | |
| 874 | + | ||
| 875 | + | // No partial write, no dropped records, and the cursor is where it was. | |
| 876 | + | let conn = reader.open().unwrap(); | |
| 877 | + | let rows: i64 = conn | |
| 878 | + | .query_row("SELECT COUNT(*) FROM note", [], |r| r.get(0)) | |
| 879 | + | .unwrap(); | |
| 880 | + | assert_eq!(rows, 0, "nothing was applied"); | |
| 881 | + | assert_eq!( | |
| 882 | + | get_scope_cursor(&conn, "").unwrap(), | |
| 883 | + | 0, | |
| 884 | + | "the cursor did not advance past a page that was never applied" | |
| 885 | + | ); | |
| 886 | + | ||
| 887 | + | // Once the reader catches up, the same page applies. | |
| 888 | + | let matched = schema().storage_version(5); | |
| 889 | + | pull_scope(&reader, &server, &matched, reader_node, SyncScope::Personal) | |
| 890 | + | .await | |
| 891 | + | .unwrap(); | |
| 892 | + | let name: String = conn | |
| 893 | + | .query_row("SELECT name FROM note WHERE id = 'n1'", [], |r| r.get(0)) | |
| 894 | + | .unwrap(); | |
| 895 | + | assert_eq!(name, "from the newer device"); | |
| 896 | + | } | |
| 897 | + | ||
| 898 | + | /// An app that has not adopted the gate must be entirely unaffected. | |
| 899 | + | #[tokio::test] | |
| 900 | + | async fn an_undeclared_manifest_pulls_a_stamped_page_as_before() { | |
| 901 | + | let dir = tempdir(); | |
| 902 | + | let (writer, writer_node) = device(&dir.join("a.db"), 1); | |
| 903 | + | let (reader, reader_node) = device(&dir.join("b.db"), 2); | |
| 904 | + | let server = FakeServer::default(); | |
| 905 | + | ||
| 906 | + | edit(&writer, "n1", "hello"); | |
| 907 | + | push_scope( | |
| 908 | + | &writer, | |
| 909 | + | &server, | |
| 910 | + | &schema(), | |
| 911 | + | writer_node, | |
| 912 | + | SyncScope::Personal, | |
| 913 | + | ) | |
| 914 | + | .await | |
| 915 | + | .unwrap(); | |
| 916 | + | *server.peer_version.lock().unwrap() = Some(9); | |
| 917 | + | ||
| 918 | + | pull_scope( | |
| 919 | + | &reader, | |
| 920 | + | &server, | |
| 921 | + | &schema(), | |
| 922 | + | reader_node, | |
| 923 | + | SyncScope::Personal, | |
| 924 | + | ) | |
| 925 | + | .await | |
| 926 | + | .unwrap(); | |
| 927 | + | let conn = reader.open().unwrap(); | |
| 928 | + | let name: String = conn | |
| 929 | + | .query_row("SELECT name FROM note WHERE id = 'n1'", [], |r| r.get(0)) | |
| 930 | + | .unwrap(); | |
| 931 | + | assert_eq!(name, "hello"); | |
| 932 | + | } | |
| 933 | + | ||
| 820 | 934 | fn edit(db: &DbSource, id: &str, name: &str) { | |
| 821 | 935 | let conn = db.open().unwrap(); | |
| 822 | 936 | conn.execute( |