max / everycycle

2.2 KB · 58 lines History Blame Raw

1	//! Client → daemon requests.
2
3	use core::time::Duration;
4
5	use crate::model::{ModelRef, Priority, PromptSpec, SamplingParams};
6
7	/// Opaque request identifier. Clients pick the value (typically a
8	/// monotonic counter or a UUID); the daemon echoes it on every
9	/// event for the request.
10	#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
11	pub struct RequestId(pub u64);
12
13	/// One client request envelope.
14	#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
15	pub struct ClientRequest {
16	pub id: RequestId,
17	pub kind: RequestKind,
18	}
19
20	/// The discriminated union of every operation the native API
21	/// supports. New variants are additive in semver-minor; removing
22	/// or changing a variant is semver-major.
23	#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
24	pub enum RequestKind {
25	/// Run an inference. Token stream returned as a sequence of
26	/// `ServerEvent::Tokens` events.
27	Inference(InferenceRequest),
28	/// Snapshot of fleet state. Returns one
29	/// `ServerEvent::FleetDescription` event.
30	DescribeFleet,
31	/// Ask the daemon to load a model into memory. Returns an
32	/// `Accepted` event followed by progress or an error.
33	LoadModel(ModelRef),
34	/// Ask the daemon to evict a model. Returns `Accepted` and a
35	/// terminal `Tokens { finished: true }` shaped event (empty
36	/// chunk) when complete; this lets clients treat completion as
37	/// the same primitive across all request kinds.
38	UnloadModel(ModelRef),
39	/// Cancel a previously-submitted request. The request being
40	/// cancelled receives a terminal event with `finished: true`
41	/// and a cancellation `ErrorKind`.
42	Cancel(RequestId),
43	}
44
45	/// Body of an `Inference` request.
46	#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
47	pub struct InferenceRequest {
48	pub model: ModelRef,
49	pub prompt: PromptSpec,
50	pub sampling: SamplingParams,
51	pub stream: bool,
52	pub priority: Priority,
53	/// Soft wall-clock deadline. The supervisor will not start
54	/// work it believes cannot finish inside the deadline; it does
55	/// not interrupt work already started. `None` means no deadline.
56	pub deadline: Option<Duration>,
57	}
58