Skip to main content

max / everycycle

9.1 KB · 257 lines History Blame Raw
1 //! The executor trait.
2 //!
3 //! An *executor* is a vendor-specific compute backend bound to one or
4 //! more devices. A1 ships with a single CUDA executor wrapping
5 //! `ggml-cuda` against one Tesla P40. A2 adds tensor-split across two
6 //! P40s under the same executor. A4 introduces a second executor
7 //! (AMD via `ggml-rocm`) and exercises the cross-vendor activation
8 //! handoff path — the same path that already exists in the trait
9 //! from day one, just with no second implementer to call it.
10 //!
11 //! The shape of this trait is load-bearing for the C-ABI module
12 //! boundary that lands in Thread I. Decisions taken here propagate
13 //! into the stable `everycycle-api` surface; revisit before adding a
14 //! method that cannot be flattened to a `repr(C)` call.
15
16 use everycycle_hal::PciBusAddress;
17
18 /// Vendor of the kernel collection this executor wraps. The
19 /// supervisor uses this for routing and for picking a same-vendor
20 /// fast path when one is available.
21 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
22 pub enum Vendor {
23 Cuda,
24 Rocm,
25 Vulkan,
26 Cpu,
27 }
28
29 /// Numeric type of the values in a tensor. Mirrors the precisions
30 /// the appraise probe measures; the runtime asks the executor only
31 /// for dtypes it knows the executor declared support for.
32 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
33 pub enum Dtype {
34 Fp32,
35 Fp16,
36 Bf16,
37 Fp8E4M3,
38 Fp8E5M2,
39 Int8,
40 Int4,
41 }
42
43 /// Logical tensor shape. Row-major; rank up to eight is enough for
44 /// every workload in the B-thread roadmap.
45 #[derive(Clone, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]
46 pub struct TensorShape {
47 pub dims: Vec<u64>,
48 }
49
50 impl TensorShape {
51 #[must_use]
52 pub fn element_count(&self) -> u64 {
53 self.dims.iter().product()
54 }
55 }
56
57 /// One device the executor is bound to. The bus address is the
58 /// stable identity; the kind tag is informational and matches the
59 /// HAL inventory.
60 #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
61 pub struct BoundDevice {
62 pub bus_address: PciBusAddress,
63 pub display_name: String,
64 }
65
66 /// Opaque handle to an activation tensor that lives inside one
67 /// executor's address space. The handle has meaning *only* to the
68 /// executor that produced it; passing it to a different executor is
69 /// a programming error. Cross-executor transfers go through the
70 /// portable form.
71 ///
72 /// Lifetime is reference-counted at the supervisor; the executor
73 /// must hold the underlying memory live until `release_activation`
74 /// is called.
75 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
76 pub struct ActivationHandle(pub u64);
77
78 /// Wire-portable activation. The supervisor uses this to bridge
79 /// between executors of different vendors and to checkpoint state
80 /// across crashes.
81 ///
82 /// The bytes are laid out in row-major order at the declared dtype.
83 /// Endianness is little. Quantized blocks (Q4_K, IQ4_XS, ...) are
84 /// not represented here — activations are dequantized to a real
85 /// dtype before export. Cross-vendor weight handoff is a different
86 /// problem and does not flow through this path.
87 #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
88 pub struct PortableActivation {
89 pub shape: TensorShape,
90 pub dtype: Dtype,
91 pub bytes: Vec<u8>,
92 }
93
94 /// A unit of compute the planner has assigned to one executor.
95 ///
96 /// Opaque at this layer: the planner emits a vendor-specific plan
97 /// the executor knows how to decode (e.g. a serialized ggml
98 /// computation graph fragment, or a CUDA-graph descriptor). The
99 /// supervisor never inspects this — it just routes it.
100 #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
101 pub struct SubGraphPlan {
102 pub vendor: Vendor,
103 pub payload: Vec<u8>,
104 }
105
106 /// Errors an executor call can return. Variants will grow with
107 /// experience; the closed set today is enough for A1.
108 #[derive(Debug)]
109 pub enum ExecutorError {
110 /// Requested dtype is not implemented on this executor.
111 UnsupportedDtype(Dtype),
112 /// Plan vendor does not match this executor's vendor.
113 VendorMismatch { plan: Vendor, executor: Vendor },
114 /// Allocation failed — usually VRAM exhaustion.
115 OutOfMemory,
116 /// Handle was issued by a different executor instance.
117 UnknownHandle(ActivationHandle),
118 /// Vendor SDK returned an error string. We pass it through
119 /// rather than try to enumerate every possible vendor error.
120 Backend(String),
121 }
122
123 impl core::fmt::Display for ExecutorError {
124 fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
125 match self {
126 Self::UnsupportedDtype(d) => write!(f, "executor does not support dtype {d:?}"),
127 Self::VendorMismatch { plan, executor } => {
128 write!(
129 f,
130 "plan vendor {plan:?} does not match executor vendor {executor:?}"
131 )
132 }
133 Self::OutOfMemory => f.write_str("executor allocation failed"),
134 Self::UnknownHandle(h) => write!(f, "unknown activation handle {h:?}"),
135 Self::Backend(s) => write!(f, "backend error: {s}"),
136 }
137 }
138 }
139
140 impl std::error::Error for ExecutorError {}
141
142 /// The vendor backend boundary.
143 ///
144 /// Every method is callable across threads; implementations must be
145 /// internally synchronized. The supervisor will issue concurrent
146 /// calls into one executor from multiple client tasks, and the
147 /// multi-client design (Thread C) assumes this from A1.
148 pub trait Executor: Send + Sync {
149 /// Vendor of the wrapped kernel collection.
150 fn vendor(&self) -> Vendor;
151
152 /// Human-readable identifier for log lines and the audit screen.
153 fn name(&self) -> &str;
154
155 /// Devices this executor owns. A given physical device belongs
156 /// to exactly one executor at any time.
157 fn devices(&self) -> &[BoundDevice];
158
159 /// Dtypes this executor advertises support for. The planner
160 /// will not ask for a dtype not in this list.
161 fn supported_dtypes(&self) -> &[Dtype];
162
163 /// Allocate an activation tensor of the given shape and dtype
164 /// on this executor's device(s).
165 ///
166 /// # Errors
167 ///
168 /// `UnsupportedDtype` if the dtype is not in `supported_dtypes`;
169 /// `OutOfMemory` if allocation fails.
170 fn alloc_activation(
171 &self,
172 shape: &TensorShape,
173 dtype: Dtype,
174 ) -> Result<ActivationHandle, ExecutorError>;
175
176 /// Release an activation previously returned by `alloc_activation`
177 /// or `execute`. Idempotent: re-releasing a handle is not an error.
178 fn release_activation(&self, handle: ActivationHandle);
179
180 /// Execute one planner-assigned sub-graph. The inputs are
181 /// activations local to this executor; the outputs are too.
182 ///
183 /// # Errors
184 ///
185 /// `VendorMismatch` if `plan.vendor` is not this executor's
186 /// vendor; `UnknownHandle` if any input was not issued by this
187 /// executor; `Backend` for vendor-SDK failures.
188 fn execute(
189 &self,
190 plan: &SubGraphPlan,
191 inputs: &[ActivationHandle],
192 ) -> Result<Vec<ActivationHandle>, ExecutorError>;
193
194 /// Export an activation as a portable buffer.
195 ///
196 /// Forward-compat for Thread A4 (cross-vendor): the supervisor
197 /// calls this on the producing executor, then calls
198 /// `import_activation` on the consuming executor of a different
199 /// vendor. Same-vendor pipelines should never hit this path —
200 /// the supervisor short-circuits to a direct handle hand-off
201 /// when source and destination executors are the same instance.
202 ///
203 /// Implementations are required from day one even when there is
204 /// no second vendor to hand off to. The minimum contract is a
205 /// device-to-host staging copy.
206 ///
207 /// # Errors
208 ///
209 /// `UnknownHandle` if the handle was not issued by this
210 /// executor; `Backend` for SDK failures during the staging copy.
211 fn export_activation(
212 &self,
213 handle: ActivationHandle,
214 ) -> Result<PortableActivation, ExecutorError>;
215
216 /// Import an activation produced by another executor.
217 ///
218 /// The byte layout contract is the one in `PortableActivation`'s
219 /// documentation: row-major, little-endian, dequantized to the
220 /// declared dtype. Vendor-specific re-quantization on import is
221 /// the executor's choice, not the caller's concern.
222 ///
223 /// # Errors
224 ///
225 /// `UnsupportedDtype` if the portable activation's dtype is not
226 /// in `supported_dtypes`; `OutOfMemory` if allocation fails;
227 /// `Backend` for SDK failures during the host-to-device copy.
228 fn import_activation(
229 &self,
230 portable: &PortableActivation,
231 ) -> Result<ActivationHandle, ExecutorError>;
232 }
233
234 #[cfg(test)]
235 mod tests {
236 use super::*;
237
238 #[test]
239 fn shape_element_count() {
240 let s = TensorShape {
241 dims: vec![2, 3, 4],
242 };
243 assert_eq!(s.element_count(), 24);
244 }
245
246 #[test]
247 fn error_display_renders() {
248 let e = ExecutorError::VendorMismatch {
249 plan: Vendor::Cuda,
250 executor: Vendor::Rocm,
251 };
252 let rendered = e.to_string();
253 assert!(rendered.contains("Cuda"));
254 assert!(rendered.contains("Rocm"));
255 }
256 }
257