ailloli_ui_bench/
log.rs

1//! Tolerant benchmark artifact reader and metric extraction.
2//!
3//! The reader preserves unknown records but marks such runs unsuitable for
4//! regression gates. It validates protocol order separately from JSON syntax so
5//! callers can still inspect incomplete or legacy artifacts.
6
7use std::collections::{BTreeMap, BTreeSet};
8use std::fs::File;
9use std::io::{BufRead, BufReader};
10use std::path::{Path, PathBuf};
11
12use serde_json::Value;
13use thiserror::Error;
14
15use crate::model::{
16    BenchEventRecord, MetadataUpdateRecord, MetricRole, RunEndRecord, RunMetadata, RunStartRecord,
17    SamplePhase, SCHEMA_VERSION,
18};
19
20/// One finite numeric sample together with its explicit regression role.
21///
22/// # Examples
23///
24/// ```
25/// use ailloli_ui_bench::{MetricRole, MetricSample};
26/// let sample = MetricSample { value: 2.5, role: MetricRole::Diagnostic };
27/// assert_eq!(sample.value, 2.5);
28/// ```
29#[derive(Debug, Clone, Copy, PartialEq)]
30pub struct MetricSample {
31    /// Finite numeric measurement in the metric's declared unit.
32    pub value: f64,
33    /// Comparison and minimum-sample policy for this measurement.
34    pub role: MetricRole,
35}
36
37/// One parsed line from a benchmark artifact.
38///
39/// # Examples
40///
41/// ```
42/// use ailloli_ui_bench::LogRecord;
43/// let record = LogRecord::Unknown(serde_json::json!({"record_type": "future"}));
44/// assert!(matches!(record, LogRecord::Unknown(_)));
45/// ```
46#[derive(Debug, Clone)]
47#[non_exhaustive]
48pub enum LogRecord {
49    /// Initial run identity and metadata.
50    RunStart(RunStartRecord),
51    /// Sparse late metadata overlay.
52    MetadataUpdate(MetadataUpdateRecord),
53    /// Correlated benchmark event.
54    Event(BenchEventRecord),
55    /// Terminal validity and writer counts.
56    RunEnd(RunEndRecord),
57    /// A future record type retained verbatim by the tolerant reader.
58    Unknown(Value),
59}
60
61/// Parsed representation of one run.
62///
63/// Missing start/end records and protocol-order violations are retained for
64/// diagnostics but make [`Self::is_gate_valid`] return `false`.
65///
66/// # Examples
67///
68/// ```
69/// use ailloli_ui_bench::ParsedRun;
70/// let run = ParsedRun::default();
71/// assert!(!run.is_gate_valid());
72/// assert!(run.events.is_empty());
73/// ```
74#[derive(Debug, Clone, Default)]
75pub struct ParsedRun {
76    /// Artifact path supplied to [`read_run`].
77    pub path: PathBuf,
78    /// First and only parsed `run_start`, or `None` when absent.
79    pub start: Option<RunStartRecord>,
80    /// Metadata overlays in wire order.
81    pub metadata_updates: Vec<MetadataUpdateRecord>,
82    /// Event records in wire order.
83    pub events: Vec<BenchEventRecord>,
84    /// First and only parsed `run_end`, or `None` when absent.
85    pub end: Option<RunEndRecord>,
86    /// Unrecognized record objects retained verbatim.
87    pub unknown_records: Vec<Value>,
88    /// Whether start/event/end ordering, IDs, causes, and elapsed times agree.
89    protocol_order_valid: bool,
90    /// Count of nonblank wire lines, including the terminal record.
91    wire_record_count: u64,
92}
93
94impl ParsedRun {
95    /// Returns the last complete metadata view. A metadata update is an overlay
96    /// so providers can publish only the values learned after GPU creation.
97    ///
98    /// With no `run_start`, the result begins from [`RunMetadata::default`].
99    /// Sparse updates never erase existing optional fields.
100    ///
101    /// # Examples
102    ///
103    /// ```
104    /// use ailloli_ui_bench::ParsedRun;
105    /// assert_eq!(ParsedRun::default().final_metadata().scenario, None);
106    /// ```
107    pub fn final_metadata(&self) -> RunMetadata {
108        let mut metadata = self
109            .start
110            .as_ref()
111            .map(|start| start.metadata.clone())
112            .unwrap_or_default();
113        for update in &self.metadata_updates {
114            metadata.apply_update(&update.metadata);
115        }
116        metadata
117    }
118
119    /// Whether the artifact is complete and suitable for a regression gate.
120    ///
121    /// Gate-valid files require exactly one compatible start and end, matching
122    /// nonempty run IDs, ordered records/events, prior-only causes, no unknown
123    /// records, a valid end flag, zero drops, and an exact record count.
124    ///
125    /// # Examples
126    ///
127    /// ```
128    /// use ailloli_ui_bench::ParsedRun;
129    /// assert!(!ParsedRun::default().is_gate_valid());
130    /// ```
131    pub fn is_gate_valid(&self) -> bool {
132        let (Some(start), Some(end)) = (&self.start, &self.end) else {
133            return false;
134        };
135        self.protocol_order_valid
136            && !start.run_id.as_str().is_empty()
137            && start.schema_version <= SCHEMA_VERSION
138            && end.schema_version <= SCHEMA_VERSION
139            && end.run_id == start.run_id
140            && end.valid
141            && end.dropped_records == 0
142            && end.records_written == self.wire_record_count
143            && self.metadata_updates.iter().all(|record| {
144                record.schema_version <= SCHEMA_VERSION && record.run_id == start.run_id
145            })
146            && self.events.iter().all(|record| {
147                record.schema_version <= SCHEMA_VERSION && record.run_id == start.run_id
148            })
149            && self.unknown_records.is_empty()
150    }
151
152    /// Extracts finite numeric samples and their regression roles.
153    ///
154    /// Explicit `metric` events carry their serialized role. Metrics derived
155    /// from legacy provider events are diagnostics so an incidental renderer
156    /// field can never silently become a release gate. A missing or unknown
157    /// role in an older artifact also falls back to `Diagnostic`.
158    ///
159    /// Non-finite values and warmup events are omitted. A synthesized
160    /// `correctness.get_current_texture_err` sample is always present and counts
161    /// acquisition-error events with saturating `u64` arithmetic.
162    ///
163    /// # Examples
164    ///
165    /// ```
166    /// use ailloli_ui_bench::{MetricRole, ParsedRun};
167    /// let series = ParsedRun::default().metric_samples();
168    /// let sample = series["correctness.get_current_texture_err"][0];
169    /// assert_eq!(sample.value, 0.0);
170    /// assert_eq!(sample.role, MetricRole::Correctness);
171    /// ```
172    pub fn metric_samples(&self) -> BTreeMap<String, Vec<MetricSample>> {
173        let mut series = BTreeMap::<String, Vec<MetricSample>>::new();
174        let mut texture_errors = 0_u64;
175        for event in &self.events {
176            if event.context.sample_phase == SamplePhase::Warmup {
177                continue;
178            }
179            let Some(object) = event.event.as_object() else {
180                continue;
181            };
182            let Some(kind) = object.get("kind").and_then(Value::as_str) else {
183                continue;
184            };
185
186            if kind == "metric" {
187                if let (Some(name), Some(value)) = (
188                    object.get("name").and_then(Value::as_str),
189                    object.get("value").and_then(Value::as_f64),
190                ) {
191                    if value.is_finite() {
192                        let role = object
193                            .get("role")
194                            .cloned()
195                            .and_then(|role| serde_json::from_value(role).ok())
196                            .unwrap_or(MetricRole::Diagnostic);
197                        series
198                            .entry(name.to_string())
199                            .or_default()
200                            .push(MetricSample { value, role });
201                    }
202                }
203                continue;
204            }
205
206            if kind == "get_current_texture_err" {
207                texture_errors = texture_errors.saturating_add(1);
208            }
209
210            for (field, value) in object {
211                if matches!(field.as_str(), "kind" | "ts_ms") {
212                    continue;
213                }
214                let Some(value) = value.as_f64().filter(|value| value.is_finite()) else {
215                    continue;
216                };
217                series
218                    .entry(format!("{kind}.{field}"))
219                    .or_default()
220                    .push(MetricSample {
221                        value,
222                        role: MetricRole::Diagnostic,
223                    });
224            }
225        }
226        series
227            .entry("correctness.get_current_texture_err".to_string())
228            .or_default()
229            .push(MetricSample {
230                value: texture_errors as f64,
231                role: MetricRole::Correctness,
232            });
233        series
234    }
235
236    /// Extracts only numeric values for callers that do not need gate roles.
237    ///
238    /// # Examples
239    ///
240    /// ```
241    /// use ailloli_ui_bench::ParsedRun;
242    /// let series = ParsedRun::default().metric_series();
243    /// assert_eq!(series["correctness.get_current_texture_err"], vec![0.0]);
244    /// ```
245    pub fn metric_series(&self) -> BTreeMap<String, Vec<f64>> {
246        self.metric_samples()
247            .into_iter()
248            .map(|(name, samples)| {
249                (
250                    name,
251                    samples.into_iter().map(|sample| sample.value).collect(),
252                )
253            })
254            .collect()
255    }
256}
257
258/// Failure while reading a benchmark artifact.
259///
260/// # Examples
261///
262/// ```
263/// use ailloli_ui_bench::BenchReadError;
264/// let error = BenchReadError::InvalidInput("missing".into());
265/// assert!(error.to_string().contains("neither a JSONL file nor a directory"));
266/// ```
267#[derive(Debug, Error)]
268#[non_exhaustive]
269pub enum BenchReadError {
270    /// Opening a JSONL artifact failed.
271    #[error("failed to open benchmark artifact {path}")]
272    Open {
273        /// Artifact path that could not be opened.
274        path: PathBuf,
275        /// Underlying filesystem error.
276        #[source]
277        source: std::io::Error,
278    },
279    /// Reading an already opened artifact failed.
280    #[error("failed to read benchmark artifact {path}")]
281    Read {
282        /// File or directory path being read.
283        path: PathBuf,
284        /// Underlying filesystem error.
285        #[source]
286        source: std::io::Error,
287    },
288    /// One physical line was not valid JSON.
289    #[error("invalid JSON on line {line} of benchmark artifact {path}")]
290    Json {
291        /// Artifact containing invalid JSON.
292        path: PathBuf,
293        /// One-based line number of the invalid JSON.
294        line: usize,
295        /// Underlying JSON parser error.
296        #[source]
297        source: serde_json::Error,
298    },
299    /// A recognized record type had an invalid typed payload.
300    #[error("malformed {record_type} record on line {line} of benchmark artifact {path}")]
301    MalformedRecord {
302        /// Artifact containing the malformed typed record.
303        path: PathBuf,
304        /// One-based line number of the malformed record.
305        line: usize,
306        /// `record_type` discriminator found on that line.
307        record_type: String,
308        /// Typed deserialization error.
309        #[source]
310        source: serde_json::Error,
311    },
312    #[error("duplicate run_start record in benchmark artifact {0}")]
313    /// More than one `run_start` line appeared in the artifact.
314    DuplicateRunStart(PathBuf),
315    #[error("duplicate run_end record in benchmark artifact {0}")]
316    /// More than one `run_end` line appeared in the artifact.
317    DuplicateRunEnd(PathBuf),
318    #[error("benchmark input path is neither a JSONL file nor a directory: {0}")]
319    /// Collection input was neither a regular file nor a directory.
320    InvalidInput(PathBuf),
321}
322
323/// Reads a version-tolerant benchmark JSONL artifact.
324///
325/// Blank lines are ignored. Unknown record types are retained. Structural
326/// protocol errors (records before start/after end, nonsequential IDs, future
327/// causes, decreasing elapsed time) do not fail parsing; they make the returned
328/// run ineligible for gates.
329///
330/// # Errors
331///
332/// Returns an error for open/read/JSON/typed-record failures or duplicate start
333/// and end records.
334///
335/// # Examples
336///
337/// ```no_run
338/// use ailloli_ui_bench::{read_run, BenchReadError, ParsedRun};
339///
340/// fn load(path: &std::path::Path) -> Result<ParsedRun, BenchReadError> {
341///     read_run(path)
342/// }
343/// ```
344pub fn read_run(path: impl AsRef<Path>) -> Result<ParsedRun, BenchReadError> {
345    let path = path.as_ref().to_path_buf();
346    let file = File::open(&path).map_err(|source| BenchReadError::Open {
347        path: path.clone(),
348        source,
349    })?;
350    let reader = BufReader::new(file);
351    let mut run = ParsedRun {
352        path: path.clone(),
353        protocol_order_valid: true,
354        ..ParsedRun::default()
355    };
356    let mut saw_start = false;
357    let mut saw_end = false;
358    let mut next_event_id = 1_u64;
359    let mut seen_event_ids = BTreeSet::new();
360    let mut last_elapsed_us = None;
361
362    for (line_index, line) in reader.lines().enumerate() {
363        let line_number = line_index + 1;
364        let line = line.map_err(|source| BenchReadError::Read {
365            path: path.clone(),
366            source,
367        })?;
368        if line.trim().is_empty() {
369            continue;
370        }
371        run.wire_record_count = run.wire_record_count.saturating_add(1);
372        if saw_end {
373            run.protocol_order_valid = false;
374        }
375        let mut value: Value =
376            serde_json::from_str(&line).map_err(|source| BenchReadError::Json {
377                path: path.clone(),
378                line: line_number,
379                source,
380            })?;
381        let raw_value = value.clone();
382        let record_type = value
383            .get("record_type")
384            .and_then(Value::as_str)
385            .unwrap_or("unknown")
386            .to_string();
387        if let Some(object) = value.as_object_mut() {
388            object.remove("record_type");
389        }
390
391        let malformed = |source| BenchReadError::MalformedRecord {
392            path: path.clone(),
393            line: line_number,
394            record_type: record_type.clone(),
395            source,
396        };
397        match record_type.as_str() {
398            "run_start" => {
399                if run.start.is_some() {
400                    return Err(BenchReadError::DuplicateRunStart(path));
401                }
402                if run.wire_record_count != 1 {
403                    run.protocol_order_valid = false;
404                }
405                let record = serde_json::from_value(value).map_err(malformed)?;
406                saw_start = true;
407                run.start = Some(record);
408            }
409            "metadata_update" => {
410                if !saw_start {
411                    run.protocol_order_valid = false;
412                }
413                let record: MetadataUpdateRecord =
414                    serde_json::from_value(value).map_err(malformed)?;
415                if last_elapsed_us.is_some_and(|last| record.elapsed_us < last) {
416                    run.protocol_order_valid = false;
417                }
418                last_elapsed_us = Some(record.elapsed_us);
419                run.metadata_updates.push(record);
420            }
421            "event" => {
422                if !saw_start {
423                    run.protocol_order_valid = false;
424                }
425                let record: BenchEventRecord = serde_json::from_value(value).map_err(malformed)?;
426                if record.event_id.get() != next_event_id {
427                    run.protocol_order_valid = false;
428                }
429                next_event_id = next_event_id.saturating_add(1);
430                if record
431                    .context
432                    .cause_event_ids
433                    .iter()
434                    .any(|cause| !seen_event_ids.contains(&cause.get()))
435                {
436                    run.protocol_order_valid = false;
437                }
438                seen_event_ids.insert(record.event_id.get());
439                if last_elapsed_us.is_some_and(|last| record.elapsed_us < last) {
440                    run.protocol_order_valid = false;
441                }
442                last_elapsed_us = Some(record.elapsed_us);
443                run.events.push(record);
444            }
445            "run_end" => {
446                if run.end.is_some() {
447                    return Err(BenchReadError::DuplicateRunEnd(path));
448                }
449                if !saw_start {
450                    run.protocol_order_valid = false;
451                }
452                let record: RunEndRecord = serde_json::from_value(value).map_err(malformed)?;
453                if last_elapsed_us.is_some_and(|last| record.elapsed_us < last) {
454                    run.protocol_order_valid = false;
455                }
456                saw_end = true;
457                run.end = Some(record);
458            }
459            _ => run.unknown_records.push(raw_value),
460        }
461    }
462
463    Ok(run)
464}
465
466/// Recursively finds `.jsonl` artifacts without following directory symlinks.
467///
468/// A regular file is returned even when it lacks a `.jsonl` extension; the
469/// extension filter applies only during directory traversal. Results are sorted
470/// lexicographically and directory symlinks are ignored.
471///
472/// # Errors
473///
474/// Returns [`BenchReadError::InvalidInput`] for a missing/special top-level path
475/// and [`BenchReadError::Read`] for directory iteration or metadata failures.
476///
477/// # Examples
478///
479/// ```no_run
480/// use ailloli_ui_bench::{collect_run_files, BenchReadError};
481///
482/// fn discover(path: &std::path::Path) -> Result<Vec<std::path::PathBuf>, BenchReadError> {
483///     collect_run_files(path)
484/// }
485/// ```
486pub fn collect_run_files(path: impl AsRef<Path>) -> Result<Vec<PathBuf>, BenchReadError> {
487    let path = path.as_ref();
488    if path.is_file() {
489        return Ok(vec![path.to_path_buf()]);
490    }
491    if !path.is_dir() {
492        return Err(BenchReadError::InvalidInput(path.to_path_buf()));
493    }
494
495    let mut pending = vec![path.to_path_buf()];
496    let mut files = Vec::new();
497    while let Some(directory) = pending.pop() {
498        let entries = std::fs::read_dir(&directory).map_err(|source| BenchReadError::Read {
499            path: directory.clone(),
500            source,
501        })?;
502        for entry in entries {
503            let entry = entry.map_err(|source| BenchReadError::Read {
504                path: directory.clone(),
505                source,
506            })?;
507            let file_type = entry.file_type().map_err(|source| BenchReadError::Read {
508                path: entry.path(),
509                source,
510            })?;
511            if file_type.is_dir() {
512                pending.push(entry.path());
513            } else if file_type.is_file()
514                && entry.path().extension().is_some_and(|ext| ext == "jsonl")
515            {
516                files.push(entry.path());
517            }
518        }
519    }
520    files.sort();
521    Ok(files)
522}