diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e5aa9f..11c3d5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,22 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [1.0.1] - 2026-06-01 + +### Fixed + +- **Syslog: the PRI-less file format now parses.** rsyslog/syslog-ng write + `/var/log/syslog` without the `` wire header (an ISO-8601 or BSD timestamp, + then host and tag), but the parser's sniff required a `` — so a real + `/var/log/syslog` was misdetected as `ini` and collapsed to a single garbage + row. It is now recognized (timestamp + host + app) and parses one row per line; + `facility`/`severity` are present only when a `` is. Found by dogfooding + the host's real syslog (50k lines → `ini`/1 row, now → `syslog`/50k rows). +- **Column roles: `procid` is recognized as an identifier.** The syslog `procid` + (process id) column was classed a `measurement`, so PIDs were flagged as point + outliers (~18.5k noise findings on a 50k-line syslog). `procid` joins the + identifier name set, so it is skipped like other ids (→ 1 finding). + ## [1.0.0] - 2026-06-01 First stable release. No code changes from `0.9.0` — this commits the contract. @@ -327,7 +343,8 @@ Initial release — a contract-first anomaly-detection CLI over arbitrary corpor gates on every push. - Dual-licensed under MIT OR Apache-2.0. -[Unreleased]: https://github.com/copyleftdev/anomalyx/compare/v1.0.0...HEAD +[Unreleased]: https://github.com/copyleftdev/anomalyx/compare/v1.0.1...HEAD +[1.0.1]: https://github.com/copyleftdev/anomalyx/compare/v1.0.0...v1.0.1 [1.0.0]: https://github.com/copyleftdev/anomalyx/compare/v0.9.0...v1.0.0 [0.9.0]: https://github.com/copyleftdev/anomalyx/compare/v0.8.0...v0.9.0 [0.8.0]: https://github.com/copyleftdev/anomalyx/compare/v0.7.0...v0.8.0 diff --git a/Cargo.lock b/Cargo.lock index f5e94eb..9c18dac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -69,7 +69,7 @@ dependencies = [ [[package]] name = "anomalyx" -version = "1.0.0" +version = "1.0.1" dependencies = [ "anomalyx-core", "anomalyx-detect", @@ -80,7 +80,7 @@ dependencies = [ [[package]] name = "anomalyx-core" -version = "1.0.0" +version = "1.0.1" dependencies = [ "proptest", "serde", @@ -90,7 +90,7 @@ dependencies = [ [[package]] name = "anomalyx-detect" -version = "1.0.0" +version = "1.0.1" dependencies = [ "anomalyx-core", "proptest", @@ -101,7 +101,7 @@ dependencies = [ [[package]] name = "anomalyx-normalize" -version = "1.0.0" +version = "1.0.1" dependencies = [ "anomalyx-core", "apache-avro", @@ -129,7 +129,7 @@ dependencies = [ [[package]] name = "anomalyx-validate" -version = "1.0.0" +version = "1.0.1" dependencies = [ "anomalyx-core", "anomalyx-detect", diff --git a/Cargo.toml b/Cargo.toml index b8a5a9e..0e5be20 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ members = [ ] [workspace.package] -version = "1.0.0" +version = "1.0.1" edition = "2021" rust-version = "1.90" license = "MIT OR Apache-2.0" @@ -22,9 +22,9 @@ authors = ["copyleftdev"] # `ax-*` names were taken), but the import/extern name stays `ax_core` etc. via # the dependency key + `package` rename — so no source code changes are needed. # Keep versions in sync with workspace.package.version above. -ax-core = { path = "crates/ax-core", version = "1.0.0", package = "anomalyx-core" } -ax-normalize = { path = "crates/ax-normalize", version = "1.0.0", package = "anomalyx-normalize" } -ax-detect = { path = "crates/ax-detect", version = "1.0.0", package = "anomalyx-detect" } +ax-core = { path = "crates/ax-core", version = "1.0.1", package = "anomalyx-core" } +ax-normalize = { path = "crates/ax-normalize", version = "1.0.1", package = "anomalyx-normalize" } +ax-detect = { path = "crates/ax-detect", version = "1.0.1", package = "anomalyx-detect" } serde = { version = "1", features = ["derive"] } serde_json = "1" diff --git a/crates/ax-core/src/roles.rs b/crates/ax-core/src/roles.rs index 3398087..69be754 100644 --- a/crates/ax-core/src/roles.rs +++ b/crates/ax-core/src/roles.rs @@ -33,6 +33,7 @@ const ID_TOKENS: &[&str] = &[ "guid", "gid", "pid", + "procid", "ppid", "tid", "sid", @@ -238,6 +239,7 @@ mod tests { "SYSLOG_PID", "user_id", "uuid", + "procid", ] { assert!( name_is_identifier(id), diff --git a/crates/ax-normalize/src/parsers/syslog.rs b/crates/ax-normalize/src/parsers/syslog.rs index 4582f88..3e15689 100644 --- a/crates/ax-normalize/src/parsers/syslog.rs +++ b/crates/ax-normalize/src/parsers/syslog.rs @@ -14,8 +14,11 @@ //! year and UTC, so the same bytes always normalize identically (the real //! month/day/time are preserved; only the absent RFC 3164 year is a sentinel). //! -//! Detected by the `` header; claims `.syslog` (a plain `.log` is too -//! generic). A line without a valid `` is a clean parse error. +//! Detected by the `` header **or** the PRI-less file format that +//! rsyslog/syslog-ng actually write (ISO-8601 or BSD timestamp, then host and +//! tag) — recognized by `syslog_loose` extracting a timestamp + host + app. +//! `facility`/`severity` exist only when a `` is present. Claims `.syslog` +//! (a plain `.log` is too generic). A line that is neither is a clean parse error. use crate::parser::{Confidence, FormatParser, STRONG}; use crate::table::TableBuilder; @@ -31,6 +34,19 @@ pub struct SyslogParser; /// deterministic (the month/day/time carry the real information). const SENTINEL_YEAR: i32 = 1970; +/// Whether a line is syslog: either a `` wire header, or the PRI-less file +/// format (what rsyslog/syslog-ng actually write to `/var/log/syslog`) — which +/// we recognize by `syslog_loose` extracting a timestamp **and** a hostname +/// **and** an appname. Requiring all three keeps a timestamp-leading CSV row +/// (no space-delimited host/app after the time) from being mistaken for syslog. +fn looks_like_syslog(line: &str) -> bool { + if parse_pri(line).is_some() { + return true; + } + let m = parse_message_with_year_tz(line, |_| SENTINEL_YEAR, Some(Utc), Variant::Either); + m.timestamp.is_some() && m.hostname.is_some() && m.appname.is_some() +} + /// Parses the leading `` header into `(facility, severity)`. `PRI = facility /// * 8 + severity` and is 0–191; anything else is not a syslog priority. fn parse_pri(line: &str) -> Option<(i64, i64)> { @@ -59,7 +75,7 @@ impl FormatParser for SyslogParser { fn sniff(&self, bytes: &[u8]) -> Option { let text = std::str::from_utf8(bytes).ok()?; let line = text.lines().find(|l| !l.trim().is_empty())?; - parse_pri(line).map(|_| STRONG) + looks_like_syslog(line).then_some(STRONG) } fn parse(&self, _source: &str, bytes: &[u8]) -> Result, AxError> { let text = std::str::from_utf8(bytes).map_err(|e| self.err(e))?; @@ -68,14 +84,26 @@ impl FormatParser for SyslogParser { if line.trim().is_empty() { continue; } - let (facility, severity) = parse_pri(line) - .ok_or_else(|| self.err("not a syslog line: missing or invalid header"))?; + let pri = parse_pri(line); let msg = parse_message_with_year_tz(line, |_| SENTINEL_YEAR, Some(Utc), Variant::Either); + // Accept a line with a `` header (wire format) OR a recognizable + // timestamp (the PRI-less file format rsyslog/syslog-ng write). + // `syslog_loose` only yields a timestamp once it has also parsed the + // host/tag that follow it, so the timestamp alone is a sufficient gate. + if pri.is_none() && msg.timestamp.is_none() { + return Err( + self.err("not a syslog line: no header and no recognizable timestamp") + ); + } let mut row: BTreeMap = BTreeMap::new(); - row.insert("facility".into(), Value::Int(facility)); - row.insert("severity".into(), Value::Int(severity)); + // facility/severity come only from the `` header; a file-format + // line has none, so those columns are simply absent for it. + if let Some((facility, severity)) = pri { + row.insert("facility".into(), Value::Int(facility)); + row.insert("severity".into(), Value::Int(severity)); + } row.insert( "protocol".into(), Value::Str( @@ -242,6 +270,11 @@ mod tests { )); } + /// The PRI-less file formats that rsyslog/syslog-ng actually write to disk. + const ISO_FILE: &[u8] = + b"2026-06-01T09:14:57.403686-07:00 4ubox NetworkManager[3524]: dhcp4 beginning\n"; + const BSD_FILE: &[u8] = b"Jun 1 09:14:57 4ubox NetworkManager[3524]: dhcp4 beginning\n"; + #[test] fn sniff_keys_on_pri_header() { assert_eq!(SyslogParser.sniff(SYSLOG.as_bytes()), Some(STRONG)); @@ -256,6 +289,37 @@ mod tests { assert_eq!(SyslogParser.sniff(b"a,b,c\n1,2,3"), None); } + #[test] + fn sniff_recognizes_pri_less_file_format() { + // The real /var/log/syslog format (no ): ISO-8601 and BSD timestamps. + assert_eq!(SyslogParser.sniff(ISO_FILE), Some(STRONG)); + assert_eq!(SyslogParser.sniff(BSD_FILE), Some(STRONG)); + // But a timestamp-leading CSV (no space-delimited host/app) is NOT syslog. + assert_eq!(SyslogParser.sniff(b"2026-06-01T09:14:57,42,foo\n"), None); + // And it wins over the greedy `ini` sniff through the registry. + let reg = crate::parser::ParserRegistry::default(); + assert_eq!(reg.resolve("-", ISO_FILE).unwrap().id(), "syslog"); + } + + #[test] + fn pri_less_file_line_parses_without_facility_severity() { + let cols = SyslogParser.parse("-", ISO_FILE).unwrap(); + // The fields syslog_loose recovers are present... + assert_eq!(col(&cols, "hostname").cells[0], Value::Str("4ubox".into())); + assert_eq!( + col(&cols, "appname").cells[0], + Value::Str("NetworkManager".into()) + ); + assert_eq!(col(&cols, "procid").cells[0], Value::Int(3524)); + assert!( + matches!(&col(&cols, "timestamp").cells[0], Value::Str(s) if s.starts_with("2026-06-01")) + ); + // ...but facility/severity columns don't exist (no to derive them). + assert!(cols + .iter() + .all(|c| c.name != "facility" && c.name != "severity")); + } + #[test] fn claims_syslog_extension() { assert_eq!(SyslogParser.extensions(), &["syslog"]);