Skip to main content

max / makenotwork

15.6 KB · 451 lines History Blame Raw
1 //! Argument validation against the JSON Schema a tool publishes.
2 //!
3 //! Every [`Tool`](crate::Tool) declares an `input_schema()`. Before this
4 //! module existed, nothing checked incoming arguments against it: each tool
5 //! re-validated by hand, inconsistently, and a model that dropped a required
6 //! field got whatever ad-hoc message that tool happened to write. The agent
7 //! loop then flattened that message to prose, so the model guessed instead of
8 //! correcting. [`ToolRegistry::call`](crate::ToolRegistry::call) now validates
9 //! centrally and returns typed [`SlotError`]s the loop can feed back as a
10 //! repair turn.
11 //!
12 //! # Supported subset
13 //!
14 //! Deliberately not a conformant JSON Schema implementation. It covers the
15 //! keywords tool schemas actually use:
16 //!
17 //! - `type` — `object`, `string`, `number`, `integer`, `boolean`, `array`,
18 //! `null`, or an array of those.
19 //! - `properties` — per-field `type` and `enum` checks.
20 //! - `required` — the field must be present and non-null.
21 //! - `enum` — the value must equal one of the listed values.
22 //! - `additionalProperties: false` — unlisted fields are rejected.
23 //!
24 //! Anything else (`oneOf`, `allOf`, `$ref`, `format`, `pattern`,
25 //! `minimum`, nested sub-schemas beyond one level) is **not enforced**. That
26 //! is the intended failure mode: an unrecognised keyword means "not checked
27 //! here", never "rejected". Validation only ever fails closed on the subset it
28 //! understands, so adding a schema keyword can never silently start rejecting
29 //! calls that used to work.
30 //!
31 //! # Two deliberate deviations
32 //!
33 //! - **`null` counts as missing.** JSON Schema says `required` is satisfied by
34 //! a present key whatever its value, so `{"body": null}` is technically
35 //! valid. For a model-facing surface that is a distinction without a
36 //! difference: a tool that required a value did not get one. It is reported
37 //! as [`SlotProblem::Missing`].
38 //! - **Empty strings are valid.** `""` is a string. Whether a given tool
39 //! tolerates an empty one is app policy, not schema policy, so tools keep
40 //! their own emptiness checks.
41
42 use std::fmt;
43
44 use serde::{Deserialize, Serialize};
45 use serde_json::Value;
46
47 /// One thing wrong with one field, named so a model can correct it.
48 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
49 pub struct SlotError {
50 /// The offending field name.
51 pub field: String,
52 pub problem: SlotProblem,
53 }
54
55 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
56 #[serde(tag = "problem", rename_all = "snake_case")]
57 pub enum SlotProblem {
58 /// Declared `required` but absent (or explicitly `null`).
59 Missing {
60 /// Declared type, when the schema names one. Helps the model supply
61 /// the right shape on the retry rather than guessing twice.
62 expected: Option<String>,
63 },
64 /// Present, but not the declared type.
65 TypeMismatch { expected: String, actual: String },
66 /// Present and correctly typed, but outside the declared `enum`.
67 NotInEnum { allowed: Vec<String> },
68 /// Not in `properties`, and the schema sets `additionalProperties: false`.
69 Unexpected,
70 }
71
72 impl fmt::Display for SlotError {
73 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
74 match &self.problem {
75 SlotProblem::Missing { expected: Some(ty) } => {
76 write!(f, "missing required field `{}` ({ty})", self.field)
77 }
78 SlotProblem::Missing { expected: None } => {
79 write!(f, "missing required field `{}`", self.field)
80 }
81 SlotProblem::TypeMismatch { expected, actual } => {
82 write!(f, "field `{}` must be {expected}, got {actual}", self.field)
83 }
84 SlotProblem::NotInEnum { allowed } => write!(
85 f,
86 "field `{}` must be one of: {}",
87 self.field,
88 allowed.join(", ")
89 ),
90 SlotProblem::Unexpected => write!(f, "unexpected field `{}`", self.field),
91 }
92 }
93 }
94
95 /// Render a slot list as one model-facing line.
96 pub fn render_slots(slots: &[SlotError]) -> String {
97 slots
98 .iter()
99 .map(ToString::to_string)
100 .collect::<Vec<_>>()
101 .join("; ")
102 }
103
104 /// Validate `args` against `schema`, collecting every problem rather than
105 /// stopping at the first. A model correcting one field at a time across
106 /// several round trips is the exact waste this is meant to prevent.
107 ///
108 /// # Errors
109 ///
110 /// Returns every [`SlotError`] found. An empty error vector is never returned:
111 /// no problems means `Ok`.
112 pub fn validate(schema: &Value, args: &Value) -> Result<(), Vec<SlotError>> {
113 // A schema that isn't an object describes nothing we can check.
114 let Some(schema) = schema.as_object() else {
115 return Ok(());
116 };
117
118 // Only object schemas are meaningful for tool arguments. If the schema
119 // declares some other top-level type, we have nothing to say about it.
120 if let Some(ty) = schema.get("type").and_then(Value::as_str)
121 && ty != "object"
122 {
123 return Ok(());
124 }
125
126 let properties = schema.get("properties").and_then(Value::as_object);
127 let required = schema.get("required").and_then(Value::as_array);
128
129 // Nothing declared means nothing to enforce. Notably this accepts the
130 // empty `{"type":"object","properties":{}}` schema that `Refusal` and
131 // argument-less tools use.
132 if properties.is_none() && required.is_none() {
133 return Ok(());
134 }
135
136 let Some(args) = args.as_object() else {
137 // Callers that send a non-object where an object schema is declared
138 // get one clear error rather than a per-field pile.
139 return Err(vec![SlotError {
140 field: "(arguments)".to_string(),
141 problem: SlotProblem::TypeMismatch {
142 expected: "object".to_string(),
143 actual: type_name(args).to_string(),
144 },
145 }]);
146 };
147
148 let mut slots = Vec::new();
149
150 // Required fields. Absent or null both count as missing.
151 if let Some(required) = required {
152 for name in required.iter().filter_map(Value::as_str) {
153 if args.get(name).is_none_or(Value::is_null) {
154 slots.push(SlotError {
155 field: name.to_string(),
156 problem: SlotProblem::Missing {
157 expected: properties.and_then(|p| p.get(name)).and_then(declared_type),
158 },
159 });
160 }
161 }
162 }
163
164 // Present fields: type and enum. A field already reported missing is
165 // skipped so one mistake never produces two complaints.
166 if let Some(properties) = properties {
167 for (name, value) in args {
168 let Some(subschema) = properties.get(name) else {
169 continue;
170 };
171 if value.is_null() {
172 continue;
173 }
174 if let Some(expected) = declared_type(subschema)
175 && !type_matches(subschema, value)
176 {
177 slots.push(SlotError {
178 field: name.clone(),
179 problem: SlotProblem::TypeMismatch {
180 expected,
181 actual: type_name(value).to_string(),
182 },
183 });
184 continue;
185 }
186 if let Some(allowed) = subschema.get("enum").and_then(Value::as_array)
187 && !allowed.contains(value)
188 {
189 slots.push(SlotError {
190 field: name.clone(),
191 problem: SlotProblem::NotInEnum {
192 allowed: allowed.iter().map(render_value).collect(),
193 },
194 });
195 }
196 }
197
198 // Unknown fields, only when the schema explicitly closes the object.
199 if schema.get("additionalProperties") == Some(&Value::Bool(false)) {
200 for name in args.keys() {
201 if !properties.contains_key(name) {
202 slots.push(SlotError {
203 field: name.clone(),
204 problem: SlotProblem::Unexpected,
205 });
206 }
207 }
208 }
209 }
210
211 if slots.is_empty() { Ok(()) } else { Err(slots) }
212 }
213
214 /// The schema's declared type as a display string, if it names one.
215 fn declared_type(subschema: &Value) -> Option<String> {
216 match subschema.get("type")? {
217 Value::String(s) => Some(s.clone()),
218 // `"type": ["string", "null"]` — report the union as written.
219 Value::Array(types) => {
220 let names: Vec<_> = types
221 .iter()
222 .filter_map(Value::as_str)
223 .map(str::to_string)
224 .collect();
225 if names.is_empty() {
226 None
227 } else {
228 Some(names.join(" or "))
229 }
230 }
231 _ => None,
232 }
233 }
234
235 /// Whether `value` satisfies the subschema's `type`, which may be a union.
236 fn type_matches(subschema: &Value, value: &Value) -> bool {
237 match subschema.get("type") {
238 Some(Value::String(ty)) => matches_one(ty, value),
239 Some(Value::Array(types)) => types
240 .iter()
241 .filter_map(Value::as_str)
242 .any(|ty| matches_one(ty, value)),
243 // No declared type, or a type we don't understand: not our business.
244 _ => true,
245 }
246 }
247
248 fn matches_one(ty: &str, value: &Value) -> bool {
249 match ty {
250 "string" => value.is_string(),
251 "boolean" => value.is_boolean(),
252 "object" => value.is_object(),
253 "array" => value.is_array(),
254 "null" => value.is_null(),
255 "number" => value.is_number(),
256 // JSON has one number type; a whole-valued float is a valid integer.
257 // Models routinely emit `3.0` where a count is wanted, and rejecting
258 // that would be pedantry, not safety.
259 "integer" => value.as_i64().is_some() || value.as_f64().is_some_and(|f| f.fract() == 0.0),
260 // Unrecognised type keyword: fail open.
261 _ => true,
262 }
263 }
264
265 fn type_name(value: &Value) -> &'static str {
266 match value {
267 Value::Null => "null",
268 Value::Bool(_) => "boolean",
269 Value::Number(_) => "number",
270 Value::String(_) => "string",
271 Value::Array(_) => "array",
272 Value::Object(_) => "object",
273 }
274 }
275
276 fn render_value(value: &Value) -> String {
277 match value {
278 Value::String(s) => s.clone(),
279 other => other.to_string(),
280 }
281 }
282
283 #[cfg(test)]
284 mod tests {
285 use super::*;
286 use serde_json::json;
287
288 fn schema() -> Value {
289 json!({
290 "type": "object",
291 "properties": {
292 "slug": { "type": "string" },
293 "body": { "type": "string" },
294 "count": { "type": "integer" },
295 "mode": { "type": "string", "enum": ["fast", "slow"] }
296 },
297 "required": ["slug", "body"]
298 })
299 }
300
301 fn fields(err: &[SlotError]) -> Vec<&str> {
302 err.iter().map(|s| s.field.as_str()).collect()
303 }
304
305 #[test]
306 fn accepts_a_valid_object() {
307 assert!(validate(&schema(), &json!({ "slug": "a", "body": "b" })).is_ok());
308 }
309
310 #[test]
311 fn reports_every_missing_field_at_once() {
312 let err = validate(&schema(), &json!({})).unwrap_err();
313 assert_eq!(fields(&err), ["slug", "body"]);
314 }
315
316 #[test]
317 fn missing_carries_the_declared_type() {
318 let err = validate(&schema(), &json!({ "slug": "a" })).unwrap_err();
319 assert_eq!(
320 err[0].problem,
321 SlotProblem::Missing {
322 expected: Some("string".to_string())
323 }
324 );
325 assert_eq!(err[0].to_string(), "missing required field `body` (string)");
326 }
327
328 #[test]
329 fn explicit_null_counts_as_missing() {
330 let err = validate(&schema(), &json!({ "slug": "a", "body": null })).unwrap_err();
331 assert_eq!(fields(&err), ["body"]);
332 }
333
334 #[test]
335 fn empty_string_is_a_valid_string() {
336 // Emptiness is app policy, not schema policy.
337 assert!(validate(&schema(), &json!({ "slug": "", "body": "" })).is_ok());
338 }
339
340 #[test]
341 fn catches_a_type_mismatch() {
342 let err = validate(&schema(), &json!({ "slug": 7, "body": "b" })).unwrap_err();
343 assert_eq!(
344 err[0].problem,
345 SlotProblem::TypeMismatch {
346 expected: "string".to_string(),
347 actual: "number".to_string()
348 }
349 );
350 }
351
352 #[test]
353 fn a_missing_field_is_not_also_a_type_error() {
354 let err = validate(&schema(), &json!({ "slug": "a" })).unwrap_err();
355 assert_eq!(err.len(), 1);
356 }
357
358 #[test]
359 fn whole_valued_floats_satisfy_integer() {
360 let args = json!({ "slug": "a", "body": "b", "count": 3.0 });
361 assert!(validate(&schema(), &args).is_ok());
362 let err = validate(
363 &schema(),
364 &json!({ "slug": "a", "body": "b", "count": 3.5 }),
365 );
366 assert!(err.is_err());
367 }
368
369 #[test]
370 fn enforces_enum_membership() {
371 let args = json!({ "slug": "a", "body": "b", "mode": "sideways" });
372 let err = validate(&schema(), &args).unwrap_err();
373 assert_eq!(
374 err[0].problem,
375 SlotProblem::NotInEnum {
376 allowed: vec!["fast".to_string(), "slow".to_string()]
377 }
378 );
379 assert_eq!(
380 err[0].to_string(),
381 "field `mode` must be one of: fast, slow"
382 );
383 }
384
385 #[test]
386 fn unknown_fields_pass_unless_the_schema_closes_the_object() {
387 let args = json!({ "slug": "a", "body": "b", "extra": 1 });
388 assert!(validate(&schema(), &args).is_ok());
389
390 let mut closed = schema();
391 closed["additionalProperties"] = json!(false);
392 let err = validate(&closed, &args).unwrap_err();
393 assert_eq!(fields(&err), ["extra"]);
394 assert_eq!(err[0].to_string(), "unexpected field `extra`");
395 }
396
397 #[test]
398 fn non_object_arguments_produce_one_error() {
399 let err = validate(&schema(), &json!("just a string")).unwrap_err();
400 assert_eq!(err.len(), 1);
401 assert_eq!(err[0].field, "(arguments)");
402 }
403
404 #[test]
405 fn empty_and_absent_schemas_accept_anything() {
406 // The shape `Refusal` and argument-less tools publish.
407 let empty = json!({ "type": "object", "properties": {} });
408 assert!(validate(&empty, &json!({ "anything": 1 })).is_ok());
409 assert!(validate(&json!({}), &json!({ "anything": 1 })).is_ok());
410 assert!(validate(&json!(null), &json!({ "anything": 1 })).is_ok());
411 }
412
413 #[test]
414 fn unsupported_keywords_fail_open() {
415 // `oneOf` is not in the subset. It must not reject, and the `required`
416 // it sits beside must still be enforced.
417 let s = json!({
418 "type": "object",
419 "properties": { "a": { "type": "string", "pattern": "^z" } },
420 "required": ["a"],
421 "oneOf": [{ "required": ["impossible"] }]
422 });
423 assert!(validate(&s, &json!({ "a": "not-matching-pattern" })).is_ok());
424 assert!(validate(&s, &json!({})).is_err());
425 }
426
427 #[test]
428 fn union_types_are_reported_as_written() {
429 let s = json!({
430 "type": "object",
431 "properties": { "a": { "type": ["string", "null"] } },
432 "required": ["a"]
433 });
434 assert!(validate(&s, &json!({ "a": "x" })).is_ok());
435 let err = validate(&s, &json!({ "a": 1 })).unwrap_err();
436 assert_eq!(
437 err[0].to_string(),
438 "field `a` must be string or null, got number"
439 );
440 }
441
442 #[test]
443 fn renders_a_slot_list_as_one_line() {
444 let err = validate(&schema(), &json!({})).unwrap_err();
445 assert_eq!(
446 render_slots(&err),
447 "missing required field `slug` (string); missing required field `body` (string)"
448 );
449 }
450 }
451