ailloli_ui_bench/log.rs
1//! Tolerant benchmark artifact reader and metric extraction.
2//!
3//! The reader preserves unknown records but marks such runs unsuitable for
4//! regression gates. It validates protocol order separately from JSON syntax so
5//! callers can still inspect incomplete or legacy artifacts.
6
7use std::collections::{BTreeMap, BTreeSet};
8use std::fs::File;
9use std::io::{BufRead, BufReader};
10use std::path::{Path, PathBuf};
11
12use serde_json::Value;
13use thiserror::Error;
14
15use crate::model::{
16 BenchEventRecord, MetadataUpdateRecord, MetricRole, RunEndRecord, RunMetadata, RunStartRecord,
17 SamplePhase, SCHEMA_VERSION,
18};
19
20/// One finite numeric sample together with its explicit regression role.
21///
22/// # Examples
23///
24/// ```
25/// use ailloli_ui_bench::{MetricRole, MetricSample};
26/// let sample = MetricSample { value: 2.5, role: MetricRole::Diagnostic };
27/// assert_eq!(sample.value, 2.5);
28/// ```
29#[derive(Debug, Clone, Copy, PartialEq)]
30pub struct MetricSample {
31 /// Finite numeric measurement in the metric's declared unit.
32 pub value: f64,
33 /// Comparison and minimum-sample policy for this measurement.
34 pub role: MetricRole,
35}
36
37/// One parsed line from a benchmark artifact.
38///
39/// # Examples
40///
41/// ```
42/// use ailloli_ui_bench::LogRecord;
43/// let record = LogRecord::Unknown(serde_json::json!({"record_type": "future"}));
44/// assert!(matches!(record, LogRecord::Unknown(_)));
45/// ```
46#[derive(Debug, Clone)]
47#[non_exhaustive]
48pub enum LogRecord {
49 /// Initial run identity and metadata.
50 RunStart(RunStartRecord),
51 /// Sparse late metadata overlay.
52 MetadataUpdate(MetadataUpdateRecord),
53 /// Correlated benchmark event.
54 Event(BenchEventRecord),
55 /// Terminal validity and writer counts.
56 RunEnd(RunEndRecord),
57 /// A future record type retained verbatim by the tolerant reader.
58 Unknown(Value),
59}
60
61/// Parsed representation of one run.
62///
63/// Missing start/end records and protocol-order violations are retained for
64/// diagnostics but make [`Self::is_gate_valid`] return `false`.
65///
66/// # Examples
67///
68/// ```
69/// use ailloli_ui_bench::ParsedRun;
70/// let run = ParsedRun::default();
71/// assert!(!run.is_gate_valid());
72/// assert!(run.events.is_empty());
73/// ```
74#[derive(Debug, Clone, Default)]
75pub struct ParsedRun {
76 /// Artifact path supplied to [`read_run`].
77 pub path: PathBuf,
78 /// First and only parsed `run_start`, or `None` when absent.
79 pub start: Option<RunStartRecord>,
80 /// Metadata overlays in wire order.
81 pub metadata_updates: Vec<MetadataUpdateRecord>,
82 /// Event records in wire order.
83 pub events: Vec<BenchEventRecord>,
84 /// First and only parsed `run_end`, or `None` when absent.
85 pub end: Option<RunEndRecord>,
86 /// Unrecognized record objects retained verbatim.
87 pub unknown_records: Vec<Value>,
88 /// Whether start/event/end ordering, IDs, causes, and elapsed times agree.
89 protocol_order_valid: bool,
90 /// Count of nonblank wire lines, including the terminal record.
91 wire_record_count: u64,
92}
93
94impl ParsedRun {
95 /// Returns the last complete metadata view. A metadata update is an overlay
96 /// so providers can publish only the values learned after GPU creation.
97 ///
98 /// With no `run_start`, the result begins from [`RunMetadata::default`].
99 /// Sparse updates never erase existing optional fields.
100 ///
101 /// # Examples
102 ///
103 /// ```
104 /// use ailloli_ui_bench::ParsedRun;
105 /// assert_eq!(ParsedRun::default().final_metadata().scenario, None);
106 /// ```
107 pub fn final_metadata(&self) -> RunMetadata {
108 let mut metadata = self
109 .start
110 .as_ref()
111 .map(|start| start.metadata.clone())
112 .unwrap_or_default();
113 for update in &self.metadata_updates {
114 metadata.apply_update(&update.metadata);
115 }
116 metadata
117 }
118
119 /// Whether the artifact is complete and suitable for a regression gate.
120 ///
121 /// Gate-valid files require exactly one compatible start and end, matching
122 /// nonempty run IDs, ordered records/events, prior-only causes, no unknown
123 /// records, a valid end flag, zero drops, and an exact record count.
124 ///
125 /// # Examples
126 ///
127 /// ```
128 /// use ailloli_ui_bench::ParsedRun;
129 /// assert!(!ParsedRun::default().is_gate_valid());
130 /// ```
131 pub fn is_gate_valid(&self) -> bool {
132 let (Some(start), Some(end)) = (&self.start, &self.end) else {
133 return false;
134 };
135 self.protocol_order_valid
136 && !start.run_id.as_str().is_empty()
137 && start.schema_version <= SCHEMA_VERSION
138 && end.schema_version <= SCHEMA_VERSION
139 && end.run_id == start.run_id
140 && end.valid
141 && end.dropped_records == 0
142 && end.records_written == self.wire_record_count
143 && self.metadata_updates.iter().all(|record| {
144 record.schema_version <= SCHEMA_VERSION && record.run_id == start.run_id
145 })
146 && self.events.iter().all(|record| {
147 record.schema_version <= SCHEMA_VERSION && record.run_id == start.run_id
148 })
149 && self.unknown_records.is_empty()
150 }
151
152 /// Extracts finite numeric samples and their regression roles.
153 ///
154 /// Explicit `metric` events carry their serialized role. Metrics derived
155 /// from legacy provider events are diagnostics so an incidental renderer
156 /// field can never silently become a release gate. A missing or unknown
157 /// role in an older artifact also falls back to `Diagnostic`.
158 ///
159 /// Non-finite values and warmup events are omitted. A synthesized
160 /// `correctness.get_current_texture_err` sample is always present and counts
161 /// acquisition-error events with saturating `u64` arithmetic.
162 ///
163 /// # Examples
164 ///
165 /// ```
166 /// use ailloli_ui_bench::{MetricRole, ParsedRun};
167 /// let series = ParsedRun::default().metric_samples();
168 /// let sample = series["correctness.get_current_texture_err"][0];
169 /// assert_eq!(sample.value, 0.0);
170 /// assert_eq!(sample.role, MetricRole::Correctness);
171 /// ```
172 pub fn metric_samples(&self) -> BTreeMap<String, Vec<MetricSample>> {
173 let mut series = BTreeMap::<String, Vec<MetricSample>>::new();
174 let mut texture_errors = 0_u64;
175 for event in &self.events {
176 if event.context.sample_phase == SamplePhase::Warmup {
177 continue;
178 }
179 let Some(object) = event.event.as_object() else {
180 continue;
181 };
182 let Some(kind) = object.get("kind").and_then(Value::as_str) else {
183 continue;
184 };
185
186 if kind == "metric" {
187 if let (Some(name), Some(value)) = (
188 object.get("name").and_then(Value::as_str),
189 object.get("value").and_then(Value::as_f64),
190 ) {
191 if value.is_finite() {
192 let role = object
193 .get("role")
194 .cloned()
195 .and_then(|role| serde_json::from_value(role).ok())
196 .unwrap_or(MetricRole::Diagnostic);
197 series
198 .entry(name.to_string())
199 .or_default()
200 .push(MetricSample { value, role });
201 }
202 }
203 continue;
204 }
205
206 if kind == "get_current_texture_err" {
207 texture_errors = texture_errors.saturating_add(1);
208 }
209
210 for (field, value) in object {
211 if matches!(field.as_str(), "kind" | "ts_ms") {
212 continue;
213 }
214 let Some(value) = value.as_f64().filter(|value| value.is_finite()) else {
215 continue;
216 };
217 series
218 .entry(format!("{kind}.{field}"))
219 .or_default()
220 .push(MetricSample {
221 value,
222 role: MetricRole::Diagnostic,
223 });
224 }
225 }
226 series
227 .entry("correctness.get_current_texture_err".to_string())
228 .or_default()
229 .push(MetricSample {
230 value: texture_errors as f64,
231 role: MetricRole::Correctness,
232 });
233 series
234 }
235
236 /// Extracts only numeric values for callers that do not need gate roles.
237 ///
238 /// # Examples
239 ///
240 /// ```
241 /// use ailloli_ui_bench::ParsedRun;
242 /// let series = ParsedRun::default().metric_series();
243 /// assert_eq!(series["correctness.get_current_texture_err"], vec![0.0]);
244 /// ```
245 pub fn metric_series(&self) -> BTreeMap<String, Vec<f64>> {
246 self.metric_samples()
247 .into_iter()
248 .map(|(name, samples)| {
249 (
250 name,
251 samples.into_iter().map(|sample| sample.value).collect(),
252 )
253 })
254 .collect()
255 }
256}
257
258/// Failure while reading a benchmark artifact.
259///
260/// # Examples
261///
262/// ```
263/// use ailloli_ui_bench::BenchReadError;
264/// let error = BenchReadError::InvalidInput("missing".into());
265/// assert!(error.to_string().contains("neither a JSONL file nor a directory"));
266/// ```
267#[derive(Debug, Error)]
268#[non_exhaustive]
269pub enum BenchReadError {
270 /// Opening a JSONL artifact failed.
271 #[error("failed to open benchmark artifact {path}")]
272 Open {
273 /// Artifact path that could not be opened.
274 path: PathBuf,
275 /// Underlying filesystem error.
276 #[source]
277 source: std::io::Error,
278 },
279 /// Reading an already opened artifact failed.
280 #[error("failed to read benchmark artifact {path}")]
281 Read {
282 /// File or directory path being read.
283 path: PathBuf,
284 /// Underlying filesystem error.
285 #[source]
286 source: std::io::Error,
287 },
288 /// One physical line was not valid JSON.
289 #[error("invalid JSON on line {line} of benchmark artifact {path}")]
290 Json {
291 /// Artifact containing invalid JSON.
292 path: PathBuf,
293 /// One-based line number of the invalid JSON.
294 line: usize,
295 /// Underlying JSON parser error.
296 #[source]
297 source: serde_json::Error,
298 },
299 /// A recognized record type had an invalid typed payload.
300 #[error("malformed {record_type} record on line {line} of benchmark artifact {path}")]
301 MalformedRecord {
302 /// Artifact containing the malformed typed record.
303 path: PathBuf,
304 /// One-based line number of the malformed record.
305 line: usize,
306 /// `record_type` discriminator found on that line.
307 record_type: String,
308 /// Typed deserialization error.
309 #[source]
310 source: serde_json::Error,
311 },
312 #[error("duplicate run_start record in benchmark artifact {0}")]
313 /// More than one `run_start` line appeared in the artifact.
314 DuplicateRunStart(PathBuf),
315 #[error("duplicate run_end record in benchmark artifact {0}")]
316 /// More than one `run_end` line appeared in the artifact.
317 DuplicateRunEnd(PathBuf),
318 #[error("benchmark input path is neither a JSONL file nor a directory: {0}")]
319 /// Collection input was neither a regular file nor a directory.
320 InvalidInput(PathBuf),
321}
322
323/// Reads a version-tolerant benchmark JSONL artifact.
324///
325/// Blank lines are ignored. Unknown record types are retained. Structural
326/// protocol errors (records before start/after end, nonsequential IDs, future
327/// causes, decreasing elapsed time) do not fail parsing; they make the returned
328/// run ineligible for gates.
329///
330/// # Errors
331///
332/// Returns an error for open/read/JSON/typed-record failures or duplicate start
333/// and end records.
334///
335/// # Examples
336///
337/// ```no_run
338/// use ailloli_ui_bench::{read_run, BenchReadError, ParsedRun};
339///
340/// fn load(path: &std::path::Path) -> Result<ParsedRun, BenchReadError> {
341/// read_run(path)
342/// }
343/// ```
344pub fn read_run(path: impl AsRef<Path>) -> Result<ParsedRun, BenchReadError> {
345 let path = path.as_ref().to_path_buf();
346 let file = File::open(&path).map_err(|source| BenchReadError::Open {
347 path: path.clone(),
348 source,
349 })?;
350 let reader = BufReader::new(file);
351 let mut run = ParsedRun {
352 path: path.clone(),
353 protocol_order_valid: true,
354 ..ParsedRun::default()
355 };
356 let mut saw_start = false;
357 let mut saw_end = false;
358 let mut next_event_id = 1_u64;
359 let mut seen_event_ids = BTreeSet::new();
360 let mut last_elapsed_us = None;
361
362 for (line_index, line) in reader.lines().enumerate() {
363 let line_number = line_index + 1;
364 let line = line.map_err(|source| BenchReadError::Read {
365 path: path.clone(),
366 source,
367 })?;
368 if line.trim().is_empty() {
369 continue;
370 }
371 run.wire_record_count = run.wire_record_count.saturating_add(1);
372 if saw_end {
373 run.protocol_order_valid = false;
374 }
375 let mut value: Value =
376 serde_json::from_str(&line).map_err(|source| BenchReadError::Json {
377 path: path.clone(),
378 line: line_number,
379 source,
380 })?;
381 let raw_value = value.clone();
382 let record_type = value
383 .get("record_type")
384 .and_then(Value::as_str)
385 .unwrap_or("unknown")
386 .to_string();
387 if let Some(object) = value.as_object_mut() {
388 object.remove("record_type");
389 }
390
391 let malformed = |source| BenchReadError::MalformedRecord {
392 path: path.clone(),
393 line: line_number,
394 record_type: record_type.clone(),
395 source,
396 };
397 match record_type.as_str() {
398 "run_start" => {
399 if run.start.is_some() {
400 return Err(BenchReadError::DuplicateRunStart(path));
401 }
402 if run.wire_record_count != 1 {
403 run.protocol_order_valid = false;
404 }
405 let record = serde_json::from_value(value).map_err(malformed)?;
406 saw_start = true;
407 run.start = Some(record);
408 }
409 "metadata_update" => {
410 if !saw_start {
411 run.protocol_order_valid = false;
412 }
413 let record: MetadataUpdateRecord =
414 serde_json::from_value(value).map_err(malformed)?;
415 if last_elapsed_us.is_some_and(|last| record.elapsed_us < last) {
416 run.protocol_order_valid = false;
417 }
418 last_elapsed_us = Some(record.elapsed_us);
419 run.metadata_updates.push(record);
420 }
421 "event" => {
422 if !saw_start {
423 run.protocol_order_valid = false;
424 }
425 let record: BenchEventRecord = serde_json::from_value(value).map_err(malformed)?;
426 if record.event_id.get() != next_event_id {
427 run.protocol_order_valid = false;
428 }
429 next_event_id = next_event_id.saturating_add(1);
430 if record
431 .context
432 .cause_event_ids
433 .iter()
434 .any(|cause| !seen_event_ids.contains(&cause.get()))
435 {
436 run.protocol_order_valid = false;
437 }
438 seen_event_ids.insert(record.event_id.get());
439 if last_elapsed_us.is_some_and(|last| record.elapsed_us < last) {
440 run.protocol_order_valid = false;
441 }
442 last_elapsed_us = Some(record.elapsed_us);
443 run.events.push(record);
444 }
445 "run_end" => {
446 if run.end.is_some() {
447 return Err(BenchReadError::DuplicateRunEnd(path));
448 }
449 if !saw_start {
450 run.protocol_order_valid = false;
451 }
452 let record: RunEndRecord = serde_json::from_value(value).map_err(malformed)?;
453 if last_elapsed_us.is_some_and(|last| record.elapsed_us < last) {
454 run.protocol_order_valid = false;
455 }
456 saw_end = true;
457 run.end = Some(record);
458 }
459 _ => run.unknown_records.push(raw_value),
460 }
461 }
462
463 Ok(run)
464}
465
466/// Recursively finds `.jsonl` artifacts without following directory symlinks.
467///
468/// A regular file is returned even when it lacks a `.jsonl` extension; the
469/// extension filter applies only during directory traversal. Results are sorted
470/// lexicographically and directory symlinks are ignored.
471///
472/// # Errors
473///
474/// Returns [`BenchReadError::InvalidInput`] for a missing/special top-level path
475/// and [`BenchReadError::Read`] for directory iteration or metadata failures.
476///
477/// # Examples
478///
479/// ```no_run
480/// use ailloli_ui_bench::{collect_run_files, BenchReadError};
481///
482/// fn discover(path: &std::path::Path) -> Result<Vec<std::path::PathBuf>, BenchReadError> {
483/// collect_run_files(path)
484/// }
485/// ```
486pub fn collect_run_files(path: impl AsRef<Path>) -> Result<Vec<PathBuf>, BenchReadError> {
487 let path = path.as_ref();
488 if path.is_file() {
489 return Ok(vec![path.to_path_buf()]);
490 }
491 if !path.is_dir() {
492 return Err(BenchReadError::InvalidInput(path.to_path_buf()));
493 }
494
495 let mut pending = vec![path.to_path_buf()];
496 let mut files = Vec::new();
497 while let Some(directory) = pending.pop() {
498 let entries = std::fs::read_dir(&directory).map_err(|source| BenchReadError::Read {
499 path: directory.clone(),
500 source,
501 })?;
502 for entry in entries {
503 let entry = entry.map_err(|source| BenchReadError::Read {
504 path: directory.clone(),
505 source,
506 })?;
507 let file_type = entry.file_type().map_err(|source| BenchReadError::Read {
508 path: entry.path(),
509 source,
510 })?;
511 if file_type.is_dir() {
512 pending.push(entry.path());
513 } else if file_type.is_file()
514 && entry.path().extension().is_some_and(|ext| ext == "jsonl")
515 {
516 files.push(entry.path());
517 }
518 }
519 }
520 files.sort();
521 Ok(files)
522}