max / makenotwork

perf: set-based sync-log compaction, off-runtime quota walk, bounded scan lookups ultra-fuzz Run 6 --deep (drive Performance A -> A+): - compact_all_sync_logs: one set-based DELETE ... USING (floors CTE) over every compactable (app, user) pair instead of a SELECT + serial per-pair DELETE on the shared pool inside the maintenance tick (the "SyncKit compaction at scale" cold spot). Never-pulled-device safety unchanged (HAVING MIN > 0 over all devices). - git push quota: dir_size_bytes recursive walk moved to spawn_blocking so a large account can't stall a runtime worker per push. - scan pipeline: MalwareBazaar + MetaDefender second-opinion lookups wrapped in a 10s timeout (SCAN_EXTERNAL_LOOKUP_TIMEOUT_SECS); a slow third party degrades to Skip (FailOpen) instead of holding a scan worker slot. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: Max Johnson <me@maxj.phd> · 2026-06-24 20:17 UTC

Commit: 9e62bf7aab5b21aed117a623ac5e5d014bac432f

Parent: 6fcfeee

4 files changed, +62 insertions, -30 deletions

M server/src/constants.rs +7

			@@ -235,6 +235,13 @@ pub const SCAN_SPOOL_SLACK_BYTES: u64 = 16 * 1024 * 1024; // 16 MiB
235	235		/// prefix is the right trade. Above this, YARA sees the prefix and logs the cap.
236	236		pub const SCAN_YARA_MAX_BYTES: usize = 512 * 1024 * 1024; // 512 MiB
237	237
	238	+	/// Per-call deadline for the optional external second-opinion lookups
	239	+	/// (MalwareBazaar, MetaDefender). They are FailOpen by design; bounding each
	240	+	/// await keeps a slow or unreachable third party from holding a scan worker slot
	241	+	/// indefinitely (ultra-fuzz Run 6 Performance). On timeout the layer reports
	242	+	/// Skip — the same shape as the disabled case — never blocking the file.
	243	+	pub const SCAN_EXTERNAL_LOOKUP_TIMEOUT_SECS: u64 = 10;
	244	+
238	245		// -- Caddy on-demand TLS --
239	246		// Caps concurrent cache-miss DB lookups in `/api/domains/caddy-ask`. Cache hits
240	247		// are unbounded (DashMap). At capacity, the handler returns 503 so Caddy retries

M server/src/db/synckit/keys.rs +21 -27

			@@ -203,37 +203,31 @@ pub async fn compact_all_sync_logs(pool: &PgPool, min_age_days: i64) -> Result<u
203	203		// and the `HAVING MIN > 0` skips the pair entirely until it catches up. A
204	204		// truly-abandoned never-pulled device is bounded by `prune_sync_log`'s
205	205		// retention backstop, not this cursor path.
206		-	let pairs: Vec<(SyncAppId, UserId, i64)> = sqlx::query_as(
	206	+	// One set-based DELETE over every compactable pair instead of a SELECT + a
	207	+	// serial DELETE per (app, user) pair on the shared pool (ultra-fuzz Run 6,
	208	+	// "SyncKit compaction at scale"). The `floors` CTE reproduces the per-pair
	209	+	// safe sequence; the never-pulled-device safety still rides on `HAVING MIN > 0`
	210	+	// over ALL devices, so a freshly-registered device at seq 0 pins its pair out
	211	+	// of compaction until it catches up.
	212	+	let result = sqlx::query(
207	213		r#"
208		-	SELECT app_id, user_id, MIN(last_pulled_seq) AS min_seq
209		-	FROM sync_devices
210		-	GROUP BY app_id, user_id
211		-	HAVING MIN(last_pulled_seq) > 0
	214	+	DELETE FROM sync_log sl
	215	+	USING (
	216	+	SELECT app_id, user_id, MIN(last_pulled_seq) AS safe_seq
	217	+	FROM sync_devices
	218	+	GROUP BY app_id, user_id
	219	+	HAVING MIN(last_pulled_seq) > 0
	220	+	) floors
	221	+	WHERE sl.app_id = floors.app_id
	222	+	AND sl.user_id = floors.user_id
	223	+	AND sl.seq <= floors.safe_seq
	224	+	AND sl.created_at < NOW() - make_interval(days => $1::int)
212	225		"#,
213	226		)
214		-	.fetch_all(pool)
	227	+	.bind(min_age_days)
	228	+	.execute(pool)
215	229		.await?;
216	230
217		-	let mut total_deleted: u64 = 0;
218		-	for (app_id, user_id, safe_seq) in pairs {
219		-	let result = sqlx::query(
220		-	r#"
221		-	DELETE FROM sync_log
222		-	WHERE app_id = $1 AND user_id = $2
223		-	AND seq <= $3
224		-	AND created_at < NOW() - make_interval(days => $4::int)
225		-	"#,
226		-	)
227		-	.bind(app_id)
228		-	.bind(user_id)
229		-	.bind(safe_seq)
230		-	.bind(min_age_days)
231		-	.execute(pool)
232		-	.await?;
233		-
234		-	total_deleted += result.rows_affected();
235		-	}
236		-
237		-	Ok(total_deleted)
	231	+	Ok(result.rows_affected())
238	232		}
239	233

M server/src/git_ssh.rs +6 -1

			@@ -123,7 +123,12 @@ async fn exec_git_operation(
123	123		// `receive.maxInputSize` cap it bounds total growth per account.
124	124		let git_root = git_repos_root();
125	125		let owner_dir = git_root.join(owner_username.as_ref());
126		-	let used = dir_size_bytes(&owner_dir);
	126	+	// The recursive directory walk is synchronous filesystem I/O; run it on
	127	+	// the blocking pool so a large account can't stall a runtime worker per
	128	+	// push (ultra-fuzz Run 6 Performance cold spot).
	129	+	let used = tokio::task::spawn_blocking(move \|\| dir_size_bytes(&owner_dir))
	130	+	.await
	131	+	.map_err(\|e\| anyhow::anyhow!("git quota check task failed: {e}"))?;
127	132		if used > crate::constants::GIT_USER_DISK_QUOTA_BYTES {
128	133		anyhow::bail!(
129	134		"push rejected: account git storage quota exceeded ({} of {} bytes used)",

M server/src/scanning/mod.rs +28 -2

			@@ -492,8 +492,22 @@ impl ScanPipeline {
492	492		layers.push(clamav_result);
493	493		layers.push(urlhaus_result);
494	494
	495	+	let lookup_timeout =
	496	+	std::time::Duration::from_secs(crate::constants::SCAN_EXTERNAL_LOOKUP_TIMEOUT_SECS);
	497	+
495	498		layers.push(if self.malwarebazaar_enabled {
496		-	hash_lookup::check_malwarebazaar(&sha256, self.abuse_ch_auth_key.as_deref()).await
	499	+	let fut = hash_lookup::check_malwarebazaar(&sha256, self.abuse_ch_auth_key.as_deref());
	500	+	match tokio::time::timeout(lookup_timeout, fut).await {
	501	+	Ok(layer) => layer,
	502	+	Err(_) => {
	503	+	tracing::warn!("MalwareBazaar lookup timed out; treating as Skip (FailOpen)");
	504	+	LayerResult {
	505	+	layer: "malwarebazaar",
	506	+	verdict: LayerVerdict::Skip,
	507	+	detail: Some("MalwareBazaar lookup timed out".to_string()),
	508	+	}
	509	+	}
	510	+	}
497	511		} else {
498	512		LayerResult {
499	513		layer: "malwarebazaar",
			@@ -503,7 +517,19 @@ impl ScanPipeline {
503	517		});
504	518
505	519		layers.push(if suspicion_present(&layers) {
506		-	metadefender::check_metadefender(&sha256, self.metadefender_api_key.as_deref()).await
	520	+	let fut =
	521	+	metadefender::check_metadefender(&sha256, self.metadefender_api_key.as_deref());
	522	+	match tokio::time::timeout(lookup_timeout, fut).await {
	523	+	Ok(layer) => layer,
	524	+	Err(_) => {
	525	+	tracing::warn!("MetaDefender lookup timed out; treating as Skip (FailOpen)");
	526	+	LayerResult {
	527	+	layer: "metadefender",
	528	+	verdict: LayerVerdict::Skip,
	529	+	detail: Some("MetaDefender lookup timed out".to_string()),
	530	+	}
	531	+	}
	532	+	}
507	533		} else {
508	534		LayerResult {
509	535		layer: "metadefender",