diff --git a/ergotree-ir/src/types/stype_param.rs b/ergotree-ir/src/types/stype_param.rs index 2b7473be2..0f6aa72eb 100644 --- a/ergotree-ir/src/types/stype_param.rs +++ b/ergotree-ir/src/types/stype_param.rs @@ -1,4 +1,3 @@ -use core::convert::TryInto; use core::fmt::Formatter; use core::hash::Hash; @@ -6,7 +5,6 @@ use alloc::string::String; use alloc::string::ToString; use alloc::vec; use alloc::vec::Vec; -use bounded_vec::BoundedVec; use crate::mir::expr::InvalidArgumentError; use crate::serialization::sigma_byte_reader::SigmaByteRead; @@ -19,7 +17,7 @@ use crate::serialization::SigmaSerializeResult; #[derive(PartialEq, Eq, Clone, Hash)] pub struct STypeVar { /// Type variable name (e.g. "T") - name_bytes: BoundedVec, + name_bytes: Vec, } impl core::fmt::Debug for STypeVar { @@ -29,33 +27,29 @@ impl core::fmt::Debug for STypeVar { } impl STypeVar { - /// Creates type variable from UTF8 text string of 1..255 length or returns an error + /// Creates a type variable from a UTF8 text string (name length is a `u8`, so 0..=255 bytes) pub fn new_from_str(name: &'static str) -> Result { Ok(Self { - name_bytes: name.to_string().into_bytes().try_into()?, + name_bytes: name.to_string().into_bytes(), }) } - /// Creates type variable from bytes of UTF8 text string of 1..255 length or returns an error + /// Creates a type variable from bytes of a UTF8 text string (name length 0..=255). + /// + /// Mirrors the JVM `TypeSerializer` (`new String(bytes, UTF_8)`): a non-UTF8 name is + /// lossily decoded -- malformed bytes become U+FFFD, matching the JVM's substitution + /// byte-for-byte (see [`jvm_lossy_utf8`]) -- and stored canonicalized, rather than + /// rejected. (`Result` is kept for API stability; this no longer errors.) pub fn new_from_bytes(bytes: Vec) -> Result { - // test if its UTF8 - Ok(match String::from_utf8(bytes.clone()) { - Ok(_) => Self { - name_bytes: bytes.try_into()?, - }, - Err(_) => { - return Err(InvalidArgumentError(format!( - "STypeVar: cannot decode {:?} from UTF8", - bytes - ))) - } + Ok(Self { + name_bytes: jvm_lossy_utf8(&bytes), }) } /// Returns text representation (e.g "T", etc.) pub fn as_string(&self) -> String { #[allow(clippy::unwrap_used)] - String::from_utf8(self.name_bytes.as_vec().clone()).unwrap() + String::from_utf8(self.name_bytes.clone()).unwrap() } /// "T" type variable @@ -76,6 +70,56 @@ impl STypeVar { } } +/// Lossily decodes `bytes` as UTF-8 the way the JVM `new String(bytes, UTF_8)` does, +/// returning the canonical UTF-8 bytes (valid input is returned unchanged). +/// +/// Deliberately NOT `String::from_utf8_lossy`: Rust replaces per the Unicode +/// "substitution of maximal subparts" rule, whereas the JVM replaces an entire +/// *attempted* multibyte sequence -- a valid lead byte plus the following continuation +/// bytes, up to the sequence's expected length -- with a single U+FFFD. They diverge on, +/// e.g., `ed a0 80` (a UTF-16 surrogate encoded in UTF-8): the JVM yields one U+FFFD, +/// `from_utf8_lossy` yields three. Matching the JVM keeps the re-serialized name -- and +/// thus the tree bytes and script hash -- identical across implementations. +fn jvm_lossy_utf8(bytes: &[u8]) -> Vec { + const REPLACEMENT: [u8; 3] = [0xEF, 0xBF, 0xBD]; // U+FFFD in UTF-8 + let mut out = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + let b = bytes[i]; + if b < 0x80 { + out.push(b); // ASCII + i += 1; + continue; + } + // Expected total length for a valid lead byte. 0xC0/0xC1 and 0xF5..=0xFF are + // never valid leads; a stray continuation (0x80..=0xBF) is not a lead either. + let expected = match b { + 0xC2..=0xDF => 2, + 0xE0..=0xEF => 3, + 0xF0..=0xF4 => 4, + _ => 0, + }; + if expected == 0 { + out.extend_from_slice(&REPLACEMENT); // one byte, one replacement + i += 1; + continue; + } + // Take the lead + following continuation bytes, up to `expected` total. + let mut j = i + 1; + while j < bytes.len() && j - i < expected && (0x80..=0xBF).contains(&bytes[j]) { + j += 1; + } + let seq = &bytes[i..j]; + if seq.len() == expected && core::str::from_utf8(seq).is_ok() { + out.extend_from_slice(seq); // a complete, valid char -- bytes unchanged + } else { + out.extend_from_slice(&REPLACEMENT); // one replacement for the attempt + } + i = j; + } + out +} + impl SigmaSerializable for STypeVar { fn sigma_serialize(&self, w: &mut W) -> SigmaSerializeResult { w.put_u8(self.name_bytes.len() as u8)?; @@ -96,3 +140,71 @@ impl SigmaSerializable for STypeVar { pub struct STypeParam { pub(crate) ident: STypeVar, } + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + // The JVM `TypeSerializer.deserialize` reads the type-var name length via + // `getUByte()` (0..=255, no bound) then `new String(getBytes(len))`, so an + // empty name (len 0) and a 255-byte name both deserialize. The previous + // `BoundedVec` over-rejected both -- a divergence from sigma-state. + // See sigma-state core/.../serialization/TypeSerializer.scala:202-206. + + #[test] + fn parse_name_length_0() { + // u8 length 0x00, zero name bytes -> STypeVar("") + let tv = STypeVar::sigma_parse_bytes(&[0x00u8]).unwrap(); + assert_eq!(tv.as_string(), ""); + assert_eq!(tv.sigma_serialize_bytes().unwrap(), vec![0x00u8]); + } + + #[test] + fn parse_name_length_255() { + // u8 length 0xff (255), then 255 'a' bytes -> STypeVar("a" * 255) + let mut bytes = vec![0xffu8]; + bytes.extend_from_slice(&[b'a'; 255]); + let tv = STypeVar::sigma_parse_bytes(&bytes).unwrap(); + assert_eq!(tv.as_string(), "a".repeat(255)); + assert_eq!(tv.sigma_serialize_bytes().unwrap(), bytes); + } + + #[test] + fn name_jvm_lossy_substitution() { + // A non-UTF8 name must canonicalize exactly as the JVM `new String(bytes, UTF_8)` + // does, so the re-serialized name (-> tree bytes -> script hash) matches. Oracle: + // sigma-state 6.0.3 / Java 17 (SANTA byte-exactness table). `from_utf8_lossy` would + // differ on the surrogate `ed a0 80` (3 U+FFFD vs the JVM's 1). + // (input bytes, JVM canonical name bytes) + let cases: &[(&[u8], &[u8])] = &[ + (&[0xff], &[0xef, 0xbf, 0xbd]), + (&[0xe2, 0x82], &[0xef, 0xbf, 0xbd]), + (&[0xc0, 0x80], &[0xef, 0xbf, 0xbd, 0xef, 0xbf, 0xbd]), + (&[0xed, 0xa0, 0x80], &[0xef, 0xbf, 0xbd]), + (&[0x61, 0xff, 0x62], &[0x61, 0xef, 0xbf, 0xbd, 0x62]), + ]; + for (input, expected_name) in cases { + let tv = STypeVar::new_from_bytes(input.to_vec()).unwrap(); + let mut expected_wire = vec![expected_name.len() as u8]; + expected_wire.extend_from_slice(expected_name); + assert_eq!( + tv.sigma_serialize_bytes().unwrap(), + expected_wire, + "input {:02x?}", + input + ); + } + } + + #[test] + fn name_valid_utf8_unchanged() { + // a valid multi-byte name round-trips byte-for-byte (euro sign U+20AC = e2 82 ac) + let tv = STypeVar::new_from_bytes(vec![0xe2, 0x82, 0xac]).unwrap(); + assert_eq!(tv.as_string(), "\u{20ac}"); + assert_eq!( + tv.sigma_serialize_bytes().unwrap(), + vec![0x03, 0xe2, 0x82, 0xac] + ); + } +}