micromegas_telemetry/
serde_byte_buf.rs

1//! Serde helper for encoding a `Vec<u8>` field as a CBOR byte string while
2//! remaining tolerant of the legacy array-of-integers encoding produced by
3//! serde's blanket `impl Serialize for Vec<T>`.
4//!
5//! # Why both paths exist
6//!
7//! `Vec<u8>`'s derived `Serialize` takes the generic sequence path, so
8//! ciborium previously emitted one CBOR item per byte (major type 4, array)
9//! instead of a single byte string (major type 2). For bytes >= 24 this costs
10//! two bytes per byte of payload instead of one, inflating stored block
11//! payloads by up to ~2x. `serialize` here switches to `serialize_bytes` to
12//! fix that going forward.
13//!
14//! Blocks written before this change are permanent in object storage (they
15//! are never rewritten), so `deserialize` must keep accepting the legacy
16//! array form indefinitely. This is **not a temporary compatibility shim** —
17//! do not delete the `visit_seq` path in a future cleanup; array-form blobs
18//! remain in the lake forever (`analytics/src/replication.rs` also copies
19//! payload blobs verbatim between lakes, so array-form objects can keep
20//! arriving after the cutover indefinitely).
21//!
22//! # Why `deserialize_byte_buf`, not `deserialize_bytes`
23//!
24//! ciborium's `deserialize_byte_buf` routes both CBOR hints to the visitor: a
25//! byte-string header goes to `visit_byte_buf`, and an array header goes to
26//! `visit_seq`. `deserialize_bytes` also routes array headers to `visit_seq`,
27//! but only accepts a byte-string header when its length fits in the
28//! decoder's fixed scratch buffer (4096 bytes) and otherwise errors out —
29//! rejecting both definite-length byte strings longer than that and
30//! indefinite-length byte strings entirely. Real block payloads run into the
31//! megabytes, so this module calls `deserialize_byte_buf`, never
32//! `deserialize_bytes`.
33//!
34//! This dual-hint routing works only because CBOR is self-describing (the
35//! byte-string vs. array distinction is carried in the header). Reusing this
36//! module with a non-self-describing format (bincode, postcard) would need a
37//! different approach.
38
39use serde::de::{Deserializer, SeqAccess, Visitor};
40use serde::ser::Serializer;
41use std::fmt;
42
43/// Serializes `v` as a CBOR byte string.
44pub fn serialize<S: Serializer>(v: &[u8], s: S) -> Result<S::Ok, S::Error> {
45    s.serialize_bytes(v)
46}
47
48/// Deserializes from either a CBOR byte string (current form) or a CBOR
49/// array of integers (legacy form, permanent in object storage).
50pub fn deserialize<'de, D: Deserializer<'de>>(d: D) -> Result<Vec<u8>, D::Error> {
51    d.deserialize_byte_buf(ByteBufVisitor)
52}
53
54struct ByteBufVisitor;
55
56impl<'de> Visitor<'de> for ByteBufVisitor {
57    type Value = Vec<u8>;
58
59    fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
60        formatter.write_str("a byte string or an array of integers")
61    }
62
63    fn visit_bytes<E: serde::de::Error>(self, v: &[u8]) -> Result<Self::Value, E> {
64        Ok(v.to_vec())
65    }
66
67    fn visit_byte_buf<E: serde::de::Error>(self, v: Vec<u8>) -> Result<Self::Value, E> {
68        Ok(v)
69    }
70
71    fn visit_seq<A: SeqAccess<'de>>(self, mut seq: A) -> Result<Self::Value, A::Error> {
72        // Legacy form: one CBOR item per byte. `seq.size_hint()` comes
73        // straight from an attacker-controlled CBOR array-length header on
74        // the public ingestion endpoint, so pre-allocating it directly would
75        // let a tiny request declare a multi-gigabyte capacity. Clamp it —
76        // deliberately stricter than serde's own 1 MiB `Vec<T>` guard — so
77        // this is no weaker than what it replaces.
78        let mut buf = Vec::with_capacity(seq.size_hint().unwrap_or(0).min(4096));
79        while let Some(byte) = seq.next_element::<u8>()? {
80            buf.push(byte);
81        }
82        Ok(buf)
83    }
84}
micromegas_telemetry/serde_byte_buf.rs

micromegas_telemetry/
serde_byte_buf.rs
