From 8007683600b5dbf9cf5fe840d2e43ec2ab084fbc Mon Sep 17 00:00:00 2001 From: Joshua Scarsbrook Date: Wed, 7 Oct 2026 11:14:05 +1000 Subject: [PATCH 1/2] Add native macOS media, HFS construction and Lion desktop hardware --- archive/bom/bom.go | 88 +++- archive/bom/bom_test.go | 63 ++- archive/bom/star.go | 42 ++ archive/gzip/materialize.go | 42 ++ archive/gzip/materialize_test.go | 64 +++ archive/gzip/reader.go | 10 +- archive/im4p/im4p.go | 166 ++++++ archive/im4p/im4p_test.go | 47 ++ archive/im4p/star.go | 57 ++ archive/lzfse/NOTICE | 23 + archive/lzfse/history_test.go | 43 ++ archive/lzfse/lzfse.go | 450 ++++++++++++++++ archive/lzfse/lzfse_test.go | 147 ++++++ archive/lzfse/lzvn.go | 88 ++++ archive/lzfse/star.go | 38 ++ archive/pbzx/pbzx.go | 145 ++++++ archive/pbzx/pbzx_test.go | 90 ++++ archive/pbzx/star.go | 38 ++ archive/udif/star.go | 62 +++ archive/udif/udif.go | 366 +++++++++++++ archive/udif/udif_test.go | 164 ++++++ archive/xar/star.go | 83 +++ archive/xar/verify_test.go | 31 ++ archive/xar/xar.go | 482 +++++++++++++++++ archive/xar/xar_test.go | 131 +++++ auto/imports/imports.go | 5 + binary/appledouble/appledouble.go | 83 +++ binary/appledouble/appledouble_test.go | 45 ++ binary/macho/macho.go | 247 +++++++++ binary/macho/macho_test.go | 118 +++++ binary/macho/names_test.go | 42 ++ binary/plist/plist.go | 601 ++++++++++++++++++++++ binary/plist/plist_test.go | 192 +++++++ binary/star/appledouble.go | 30 ++ binary/star/binary.go | 36 +- binary/star/macho.go | 47 ++ binary/star/plist.go | 196 +++++++ binary/star/plist_test.go | 68 +++ boot/darwin/handoff.go | 145 ++++++ boot/darwin/handoff_test.go | 49 ++ boot/darwin/kernel.go | 267 ++++++++++ boot/darwin/kernel_test.go | 152 ++++++ docs/darwin-direct-boot.md | 204 ++++++++ docs/macos-installers.md | 137 +++++ docs/starlark/namespaces/reference.md | 86 +++- filesystem/hfs/attributes.go | 168 ++++++ filesystem/hfs/attributes_test.go | 147 ++++++ filesystem/hfs/auto.go | 24 +- filesystem/hfs/build.go | 573 +++++++++++++++++++++ filesystem/hfs/build_links_test.go | 46 ++ filesystem/hfs/build_names_test.go | 31 ++ filesystem/hfs/build_star.go | 109 ++++ filesystem/hfs/build_test.go | 141 +++++ filesystem/hfs/build_tree_scale_test.go | 53 ++ filesystem/hfs/compressed.go | 199 +++++++ filesystem/hfs/compressed_test.go | 110 ++++ filesystem/hfs/hfs.go | 22 +- filesystem/hfs/plus.go | 19 +- filesystem/hfs/starlark.go | 26 +- firmware/acpi/pci.go | 20 +- firmware/acpi/pci_memory_test.go | 41 ++ firmware/acpi/pci_test.go | 7 + frontend/starlark/runtime.go | 13 + frontend/starlark/starlark_docs.go | 18 +- frontend/starlark/starlark_docs_native.go | 20 +- scripts/inspect/macos_layers.star | 196 +++++++ vmm/cc/acpi.go | 18 +- vmm/cc/backend.go | 71 ++- vmm/cc/bios.go | 13 + vmm/cc/darwin.go | 172 +++++++ vmm/cc/darwin_cpu.go | 57 ++ vmm/cc/darwin_cpu_test.go | 46 ++ vmm/cc/darwin_inspect.go | 105 ++++ vmm/cc/darwin_test.go | 104 ++++ vmm/cc/disassemble.go | 56 ++ vmm/cc/disassemble_test.go | 29 ++ vmm/cc/hpet.go | 55 +- vmm/cc/hpet_legacy_test.go | 63 +++ vmm/cc/ich7.go | 145 ++++++ vmm/cc/ich7_test.go | 121 +++++ vmm/cc/ide.go | 8 + vmm/cc/ide_dma.go | 2 +- vmm/cc/ide_identify_test.go | 48 ++ vmm/cc/inspect.go | 24 +- vmm/cc/keyboard.go | 14 +- vmm/cc/pc_io.go | 24 +- vmm/cc/pci.go | 76 ++- vmm/cc/pci_display.go | 42 ++ vmm/cc/pci_display_test.go | 63 +++ vmm/cc/serial.go | 50 ++ vmm/cc/smc.go | 200 +++++++ vmm/cc/smc_lion_test.go | 20 + vmm/cc/smc_test.go | 113 ++++ vmm/cc/star.go | 5 +- vmm/cc/uhci.go | 340 ++++++++++++ vmm/cc/uhci_tail_test.go | 31 ++ vmm/cc/uhci_test.go | 184 +++++++ vmm/cc/usb_hid.go | 326 ++++++++++++ vmm/cc/usb_input.go | 8 + vmm/darwin.go | 30 ++ vmm/qemu/config.go | 3 + vmm/star/backend.go | 11 + vmm/star/boot_test.go | 35 ++ vmm/star/darwin.go | 45 ++ vmm/star/value.go | 40 +- vmm/types.go | 1 + 106 files changed, 10378 insertions(+), 113 deletions(-) create mode 100644 archive/bom/star.go create mode 100644 archive/gzip/materialize.go create mode 100644 archive/gzip/materialize_test.go create mode 100644 archive/im4p/im4p.go create mode 100644 archive/im4p/im4p_test.go create mode 100644 archive/im4p/star.go create mode 100644 archive/lzfse/NOTICE create mode 100644 archive/lzfse/history_test.go create mode 100644 archive/lzfse/lzfse.go create mode 100644 archive/lzfse/lzfse_test.go create mode 100644 archive/lzfse/lzvn.go create mode 100644 archive/lzfse/star.go create mode 100644 archive/pbzx/pbzx.go create mode 100644 archive/pbzx/pbzx_test.go create mode 100644 archive/pbzx/star.go create mode 100644 archive/udif/star.go create mode 100644 archive/udif/udif.go create mode 100644 archive/udif/udif_test.go create mode 100644 archive/xar/star.go create mode 100644 archive/xar/verify_test.go create mode 100644 archive/xar/xar.go create mode 100644 archive/xar/xar_test.go create mode 100644 binary/appledouble/appledouble.go create mode 100644 binary/appledouble/appledouble_test.go create mode 100644 binary/macho/macho.go create mode 100644 binary/macho/macho_test.go create mode 100644 binary/macho/names_test.go create mode 100644 binary/plist/plist.go create mode 100644 binary/plist/plist_test.go create mode 100644 binary/star/appledouble.go create mode 100644 binary/star/macho.go create mode 100644 binary/star/plist.go create mode 100644 binary/star/plist_test.go create mode 100644 boot/darwin/handoff.go create mode 100644 boot/darwin/handoff_test.go create mode 100644 boot/darwin/kernel.go create mode 100644 boot/darwin/kernel_test.go create mode 100644 docs/darwin-direct-boot.md create mode 100644 docs/macos-installers.md create mode 100644 filesystem/hfs/attributes.go create mode 100644 filesystem/hfs/attributes_test.go create mode 100644 filesystem/hfs/build.go create mode 100644 filesystem/hfs/build_links_test.go create mode 100644 filesystem/hfs/build_names_test.go create mode 100644 filesystem/hfs/build_star.go create mode 100644 filesystem/hfs/build_test.go create mode 100644 filesystem/hfs/build_tree_scale_test.go create mode 100644 filesystem/hfs/compressed.go create mode 100644 filesystem/hfs/compressed_test.go create mode 100644 firmware/acpi/pci_memory_test.go create mode 100644 scripts/inspect/macos_layers.star create mode 100644 vmm/cc/darwin.go create mode 100644 vmm/cc/darwin_cpu.go create mode 100644 vmm/cc/darwin_cpu_test.go create mode 100644 vmm/cc/darwin_inspect.go create mode 100644 vmm/cc/darwin_test.go create mode 100644 vmm/cc/disassemble.go create mode 100644 vmm/cc/disassemble_test.go create mode 100644 vmm/cc/hpet_legacy_test.go create mode 100644 vmm/cc/ich7.go create mode 100644 vmm/cc/ich7_test.go create mode 100644 vmm/cc/ide_identify_test.go create mode 100644 vmm/cc/pci_display.go create mode 100644 vmm/cc/pci_display_test.go create mode 100644 vmm/cc/serial.go create mode 100644 vmm/cc/smc.go create mode 100644 vmm/cc/smc_lion_test.go create mode 100644 vmm/cc/smc_test.go create mode 100644 vmm/cc/uhci.go create mode 100644 vmm/cc/uhci_tail_test.go create mode 100644 vmm/cc/uhci_test.go create mode 100644 vmm/cc/usb_hid.go create mode 100644 vmm/cc/usb_input.go create mode 100644 vmm/darwin.go create mode 100644 vmm/star/darwin.go diff --git a/archive/bom/bom.go b/archive/bom/bom.go index 6d729ad..6ccacdd 100644 --- a/archive/bom/bom.go +++ b/archive/bom/bom.go @@ -30,7 +30,31 @@ func Open(prefix []byte, source storage.Reader, o auto.Options) (auto.View, erro if !bytes.HasPrefix(prefix, []byte("BOMStore")) { return nil, auto.ErrNoMatch } - limit := o.MaxEntries + entries, err := Read(source, o.MaxEntries) + if err != nil { + return nil, err + } + seen := map[string]bool{} + var treeEntries []auto.Entry + for i := range entries { + e := &entries[i] + if e.Name == "." { + continue + } + if seen[e.Name] { + e.Attributes["bom_path"] = e.Name + e.Name = fmt.Sprintf("%%00bom-variants/%v", e.Attributes["bom_id"]) + } else { + seen[e.Name] = true + } + treeEntries = append(treeEntries, *e) + } + return auto.Tree(treeEntries, o) +} + +// Read returns every flat inventory record, including architecture variants +// sharing a path. A BOM is metadata, never a substitute for payload contents. +func Read(source storage.Reader, limit int) ([]auto.Entry, error) { if limit <= 0 { limit = 100000 } @@ -46,7 +70,7 @@ func Open(prefix []byte, source storage.Reader, o auto.Options) (auto.View, erro if err != nil { return nil, err } - if be.Uint32(h[8:]) != 1 { + if string(h[:8]) != "BOMStore" || be.Uint32(h[8:]) != 1 { return nil, fmt.Errorf("bom: unsupported version") } index, err := read(uint64(be.Uint32(h[16:])), uint64(be.Uint32(h[20:]))) @@ -121,7 +145,23 @@ func Open(prefix []byte, source storage.Reader, o auto.Options) (auto.View, erro records := map[uint32]*record{} var order []uint32 var keyCount uint64 - for len(pending) > 0 { + leafLinks := map[uint32][2]uint32{} + followingLinks := false + for { + if len(pending) == 0 { + // Apple's branch index can omit the final partial leaf. The leaf + // chain is authoritative; retain branch validation, then follow + // its missing tails rather than dropping inventory records. + followingLinks = true + for _, link := range leafLinks { + if link[0] != 0 && !visited[link[0]] { + pending = append(pending, link[0]) + } + } + if len(pending) == 0 { + break + } + } id := pending[len(pending)-1] pending = pending[:len(pending)-1] if visited[id] { @@ -139,6 +179,12 @@ func Open(prefix []byte, source storage.Reader, o auto.Options) (auto.View, erro if leaf > 1 || n > (len(b)-12)/8 { return nil, fmt.Errorf("bom: invalid tree node") } + if followingLinks && leaf != 1 { + return nil, fmt.Errorf("bom: leaf chain points to branch") + } + if leaf == 1 { + leafLinks[id] = [2]uint32{be.Uint32(b[4:]), be.Uint32(b[8:])} + } keyCount += uint64(n) for i := 0; i < n; i++ { pair := b[12+i*8:] @@ -204,6 +250,34 @@ func Open(prefix []byte, source storage.Reader, o auto.Options) (auto.View, erro order = append(order, r.id) } } + // Validate the complete doubly-linked leaf chain, including tails missing + // from the branch index. Multiple heads and cycles are malformed. + var head uint32 + for id, link := range leafLinks { + if link[1] == 0 { + if head != 0 { + return nil, fmt.Errorf("bom: multiple leaf chain heads") + } + head = id + } else if back, ok := leafLinks[link[1]]; !ok || back[0] != id { + return nil, fmt.Errorf("bom: inconsistent leaf backlink") + } + if link[0] != 0 { + if next, ok := leafLinks[link[0]]; !ok || next[1] != id { + return nil, fmt.Errorf("bom: inconsistent leaf forward link") + } + } + } + seenLeaves := map[uint32]bool{} + for id := head; id != 0; id = leafLinks[id][0] { + if seenLeaves[id] { + return nil, fmt.Errorf("bom: cyclic leaf chain") + } + seenLeaves[id] = true + } + if len(seenLeaves) != len(leafLinks) { + return nil, fmt.Errorf("bom: disconnected leaf chain") + } // Apple also counts internal separator keys in some BOM generations. if uint32(len(records)) != expected && keyCount != uint64(expected) { return nil, fmt.Errorf("bom: path count mismatch: %d != %d", len(records), expected) @@ -240,7 +314,6 @@ func Open(prefix []byte, source storage.Reader, o auto.Options) (auto.View, erro return name, nil } var entries []auto.Entry - seenPaths := map[string]bool{} for _, id := range order { r := records[id] name, err := resolve(id, map[uint32]bool{}) @@ -251,14 +324,9 @@ func Open(prefix []byte, source storage.Reader, o auto.Options) (auto.View, erro if r.entry.Kind != "directory" { return nil, fmt.Errorf("bom: invalid root") } - continue - } - if seenPaths[name] { - return nil, fmt.Errorf("bom: duplicate inventory path") } - seenPaths[name] = true r.entry.Name = name entries = append(entries, r.entry) } - return auto.Tree(entries, o) + return entries, nil } diff --git a/archive/bom/bom_test.go b/archive/bom/bom_test.go index e369fa6..19d3b24 100644 --- a/archive/bom/bom_test.go +++ b/archive/bom/bom_test.go @@ -6,7 +6,8 @@ import ( "testing" ) -func fixture(branch bool) []byte { +func fixture(branch bool) []byte { return variantFixture(branch, false, false) } +func variantFixture(branch, variants, tail bool) []byte { b := make([]byte, 32) pointers := []pointer{{}} add := func(data []byte) uint32 { @@ -16,7 +17,11 @@ func fixture(branch bool) []byte { return id } var pairs [][2]uint32 - for i, name := range []string{".", "hello", "link"} { + names := []string{".", "hello", "link"} + if variants { + names = append(names, "hello") + } + for i, name := range names { meta := make([]byte, 27) meta[0] = 1 if i == 0 { @@ -29,6 +34,7 @@ func fixture(branch bool) []byte { be.PutUint32(meta[27:], 6) copy(meta[31:], "hello") } + be.PutUint16(meta[2:], uint16(i)) be.PutUint16(meta[4:], 0644) be.PutUint32(meta[6:], 501) be.PutUint32(meta[18:], 17) @@ -46,6 +52,10 @@ func fixture(branch bool) []byte { kid := add(key) pairs = append(pairs, [2]uint32{iid, kid}) } + allPairs := pairs + if tail { + pairs = pairs[:len(pairs)-1] + } leaf := make([]byte, 12+8*len(pairs)) be.PutUint16(leaf, 1) be.PutUint16(leaf[2:], uint16(len(pairs))) @@ -54,7 +64,18 @@ func fixture(branch bool) []byte { be.PutUint32(leaf[16+i*8:], p[1]) } child := add(leaf) - count := uint32(3) + count := uint32(len(allPairs)) + if tail { + last := allPairs[len(allPairs)-1] + node := make([]byte, 20) + be.PutUint16(node, 1) + be.PutUint16(node[2:], 1) + be.PutUint32(node[8:], child) + be.PutUint32(node[12:], last[0]) + be.PutUint32(node[16:], last[1]) + next := add(node) + be.PutUint32(b[pointers[child].off+4:], next) + } if branch { node := make([]byte, 20) be.PutUint16(node[2:], 1) @@ -132,3 +153,39 @@ func TestMalformedBOM(t *testing.T) { } } } + +func TestTrailingLeafAndArchitectureVariants(t *testing.T) { + for _, tail := range []bool{false, true} { + b := variantFixture(true, true, tail) + records, err := Read(&starfile.Bytes{Data: b}, 100) + if err != nil { + t.Fatal(err) + } + if len(records) != 4 { + t.Fatalf("dropped records: %d", len(records)) + } + var variants int + for _, e := range records { + if e.Name == "hello" { + variants++ + } + } + if variants != 2 { + t.Fatal("architecture variants lost") + } + root := auto.Open(&starfile.Bytes{Data: b}, "Bom", auto.Options{}) + n, err := root.Resolve("%00bom-variants/4") + if err != nil || n.Summary().Attributes["bom_path"] != "hello" { + t.Fatalf("variant tree %v %v", n, err) + } + if tail { + // The partial final leaf must point back to its predecessor. + index := be.Uint32(b[16:]) + last := be.Uint32(b[index+4+14*8:]) // leaf after four sets of metadata blocks + be.PutUint32(b[last+8:], 0) + if _, err := Read(&starfile.Bytes{Data: b}, 100); err == nil { + t.Fatal("accepted disconnected leaf chain") + } + } + } +} diff --git a/archive/bom/star.go b/archive/bom/star.go new file mode 100644 index 0000000..a6b5b91 --- /dev/null +++ b/archive/bom/star.go @@ -0,0 +1,42 @@ +package bom + +import ( + "fmt" + "github.com/tinyrange/trex/storage" + starfile "github.com/tinyrange/trex/storage/star" + "go.starlark.net/starlark" +) + +func Builtin(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var value starlark.Value + maximum := 1000000 + if err := starlark.UnpackArgs("bom", args, kwargs, "file", &value, "maximum_entries?", &maximum); err != nil { + return nil, err + } + source, ok := value.(storage.Reader) + if !ok || maximum <= 0 { + return nil, fmt.Errorf("bom: expected file and positive maximum_entries") + } + entries, err := Read(source, maximum) + if err != nil { + return nil, err + } + result := make([]starlark.Value, len(entries)) + for i, e := range entries { + record := starlark.StringDict{"path": starlark.String(e.Name), "kind": starlark.String(e.Kind)} + for k, v := range e.Attributes { + switch v := v.(type) { + case uint32: + record[k] = starlark.MakeUint64(uint64(v)) + case uint16: + record[k] = starlark.MakeUint64(uint64(v)) + case string: + record[k] = starlark.String(v) + case bool: + record[k] = starlark.Bool(v) + } + } + result[i] = starfile.NewRecord(record) + } + return starlark.NewList(result), nil +} diff --git a/archive/gzip/materialize.go b/archive/gzip/materialize.go new file mode 100644 index 0000000..7996db1 --- /dev/null +++ b/archive/gzip/materialize.go @@ -0,0 +1,42 @@ +package gzip + +import ( + "fmt" + "io" + + "github.com/tinyrange/trex/storage" +) + +// Materialize decodes and validates the entire stream into bounded volatile +// pages. Unlike Open's replaying window, the returned immutable reader supports +// arbitrary access without consulting the compressed source again. No result +// is returned after an integrity failure or limit overflow. +func Materialize(source storage.Reader, maximum int64) (storage.Reader, error) { + if maximum <= 0 { + return nil, fmt.Errorf("gzip: materialize requires positive maximum_bytes") + } + f, err := Open(source, maximum) + if err != nil { + return nil, err + } + defer f.decoder.Close() + store := storage.NewMemoryStore(maximum) + defer store.Close() + var buf [64 << 10]byte + var off int64 + for { + n, err := f.read(buf[:]) + if err != nil && err != io.EOF { + return nil, err + } + if n != 0 { + if _, e := store.WriteAt(buf[:n], off); e != nil { + return nil, e + } + off += int64(n) + } + if err == io.EOF { + return store.Snapshot() + } + } +} diff --git a/archive/gzip/materialize_test.go b/archive/gzip/materialize_test.go new file mode 100644 index 0000000..fd4e14a --- /dev/null +++ b/archive/gzip/materialize_test.go @@ -0,0 +1,64 @@ +package gzip + +import ( + "bytes" + "errors" + "io" + "testing" + + "github.com/tinyrange/trex/auto" + starfile "github.com/tinyrange/trex/storage/star" +) + +type guardedSource struct { + data []byte + forbidden bool + reads int +} + +func (s *guardedSource) Size() int64 { return int64(len(s.data)) } +func (s *guardedSource) ReadAt(p []byte, off int64) (int, error) { + s.reads++ + if s.forbidden { + return 0, errors.New("compressed source consulted after materialization") + } + return bytes.NewReader(s.data).ReadAt(p, off) +} +func TestMaterializeIndependentRandomAccessAndIntegrity(t *testing.T) { + data := bytes.Repeat([]byte("borrowed random-access package bytes"), 100000) + source := &guardedSource{data: encode(t, data)} + source.data = append(source.data, encode(t, []byte("tail"))...) + source.data = append(source.data, 0, 0, 0) + r, err := Materialize(source, int64(len(data)+4)) + if err != nil { + t.Fatal(err) + } + source.forbidden = true + reads := source.reads + want := append(data, []byte("tail")...) + for _, off := range []int64{int64(len(data)) - 10, 0, 1000000, 13} { + var got [14]byte + if _, err := r.ReadAt(got[:], off); err != nil || !bytes.Equal(got[:], want[off:off+14]) { + t.Fatal(off, err) + } + } + if source.reads != reads || r.Size() != int64(len(want)) { + t.Fatal("replayed source or incorrect size") + } + if n, err := r.ReadAt(make([]byte, 8), r.Size()-4); n != 4 || err != io.EOF { + t.Fatal(n, err) + } + bad := bytes.Clone(encode(t, data)) + bad[len(bad)-8] ^= 1 + for _, encoded := range [][]byte{bad, bad[:len(bad)-3]} { + if r, err := Materialize(&starfile.Bytes{Data: encoded}, int64(len(data))); r != nil || err == nil { + t.Fatal("invalid stream returned", r, err) + } + } + if r, err := Materialize(&starfile.Bytes{Data: encode(t, data)}, int64(len(data)-1)); r != nil || !errors.Is(err, auto.ErrLimit) { + t.Fatal(r, err) + } + if r, err := Materialize(source, 0); r != nil || err == nil { + t.Fatal("unbounded cache", r, err) + } +} diff --git a/archive/gzip/reader.go b/archive/gzip/reader.go index c499c73..a9d1a49 100644 --- a/archive/gzip/reader.go +++ b/archive/gzip/reader.go @@ -273,7 +273,8 @@ func (*File) AttrNames() []string { return starfile.AttrN func Builtin(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var source, maximum starlark.Value - if err := starlark.UnpackArgs("gzip", args, kwargs, "file", &source, "maximum_bytes?", &maximum); err != nil { + var cache bool + if err := starlark.UnpackArgs("gzip", args, kwargs, "file", &source, "maximum_bytes?", &maximum, "cache?", &cache); err != nil { return nil, err } reader, ok := source.(storage.Reader) @@ -289,5 +290,12 @@ func Builtin(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwarg return nil, fmt.Errorf("gzip: maximum_bytes must be positive") } } + if cache { + r, err := Materialize(reader, limit) + if err != nil { + return nil, err + } + return starfile.NewReader("gzip.cached", r), nil + } return Open(reader, limit) } diff --git a/archive/im4p/im4p.go b/archive/im4p/im4p.go new file mode 100644 index 0000000..5015203 --- /dev/null +++ b/archive/im4p/im4p.go @@ -0,0 +1,166 @@ +// Package im4p reads the ASN.1 IM4P firmware payload envelope. Payload bytes +// and optional DER metadata remain borrowed, without decryption/authentication. +package im4p + +import ( + "fmt" + "github.com/tinyrange/trex/storage" + "io" +) + +type Node struct { + Tag byte + Data storage.Reader + Raw storage.Reader + Children []Node +} +type Image struct { + Type, Description string + Payload storage.Reader + Extras []Node + ExpectedSize int64 +} + +func header(r storage.Reader, off, end int64) (byte, int64, int64, error) { + var b [10]byte + if off < 0 || off > end-2 { + return 0, 0, 0, fmt.Errorf("im4p: truncated DER header") + } + if _, err := r.ReadAt(b[:2], off); err != nil { + return 0, 0, 0, err + } + if b[0]&31 == 31 { + return 0, 0, 0, fmt.Errorf("im4p: high DER tags unsupported") + } + n := int64(b[1]) + h := int64(2) + if b[1]&128 != 0 { + width := int64(b[1] & 127) + if width == 0 || width > 8 || width > end-off-2 { + return 0, 0, 0, fmt.Errorf("im4p: invalid DER length") + } + if _, err := r.ReadAt(b[2:2+width], off+2); err != nil { + return 0, 0, 0, err + } + if b[2] == 0 { + return 0, 0, 0, fmt.Errorf("im4p: noncanonical DER length") + } + n = 0 + for _, v := range b[2 : 2+width] { + if n > (1<<63-1-int64(v))/256 { + return 0, 0, 0, fmt.Errorf("im4p: length overflow") + } + n = n*256 + int64(v) + } + if n < 128 { + return 0, 0, 0, fmt.Errorf("im4p: noncanonical DER length") + } + h += width + } + if n > end-off-h { + return 0, 0, 0, fmt.Errorf("im4p: DER extent outside source") + } + return b[0], h, n, nil +} +func node(r storage.Reader, off, end int64, depth int, count *int) (Node, int64, error) { + *count++ + if depth > 32 || *count > 1024 { + return Node{}, 0, fmt.Errorf("im4p: DER nesting/node limit") + } + tag, h, n, err := header(r, off, end) + if err != nil { + return Node{}, 0, err + } + result := Node{Tag: tag, Data: io.NewSectionReader(r, off+h, n), Raw: io.NewSectionReader(r, off, h+n)} + if tag&32 != 0 { + for pos := off + h; pos < off+h+n; { + child, next, err := node(r, pos, off+h+n, depth+1, count) + if err != nil { + return Node{}, 0, err + } + result.Children = append(result.Children, child) + pos = next + } + } + return result, off + h + n, nil +} +func text(n Node) (string, error) { + if n.Tag != 0x16 || n.Data.Size() > 4096 { + return "", fmt.Errorf("im4p: expected bounded IA5 string") + } + b := make([]byte, n.Data.Size()) + if _, err := io.ReadFull(io.NewSectionReader(n.Data, 0, n.Data.Size()), b); err != nil { + return "", err + } + for _, c := range b { + if c > 127 || c == 0 { + return "", fmt.Errorf("im4p: invalid IA5 string") + } + } + return string(b), nil +} +func integer(n Node) (int64, error) { + size := n.Data.Size() + if n.Tag != 2 || size <= 0 || size > 8 { + return 0, fmt.Errorf("im4p: invalid integer") + } + var b [8]byte + if _, err := n.Data.ReadAt(b[:size], 0); err != nil { + return 0, err + } + if b[0]&128 != 0 || size > 1 && b[0] == 0 && b[1]&128 == 0 { + return 0, fmt.Errorf("im4p: invalid positive integer") + } + v := int64(0) + for _, c := range b[:size] { + v = v*256 + int64(c) + } + return v, nil +} +func Open(r storage.Reader) (*Image, error) { + if r == nil { + return nil, fmt.Errorf("im4p: missing source") + } + count := 0 + root, end, err := node(r, 0, r.Size(), 0, &count) + if err != nil { + return nil, err + } + if end != r.Size() || root.Tag != 0x30 || len(root.Children) < 4 { + return nil, fmt.Errorf("im4p: invalid envelope/trailing data") + } + children := root.Children + magic, err := text(children[0]) + if err != nil || magic != "IM4P" { + return nil, fmt.Errorf("im4p: missing IM4P signature") + } + kind, err := text(children[1]) + if err != nil || len(kind) != 4 { + return nil, fmt.Errorf("im4p: invalid payload type") + } + desc, err := text(children[2]) + if err != nil { + return nil, err + } + if children[3].Tag != 4 { + return nil, fmt.Errorf("im4p: payload is not OCTET STRING") + } + image := &Image{Type: kind, Description: desc, Payload: children[3].Data, Extras: children[4:]} + for _, extra := range image.Extras { + if extra.Tag == 0x30 && len(extra.Children) == 2 && extra.Children[0].Tag == 2 && extra.Children[1].Tag == 2 { + method, err := integer(extra.Children[0]) + if err != nil { + return nil, err + } + size, err := integer(extra.Children[1]) + if err != nil { + return nil, err + } + if method != 1 || size <= 0 || image.ExpectedSize != 0 { + return nil, fmt.Errorf("im4p: invalid compression metadata") + } + image.ExpectedSize = size + } + } + return image, nil +} diff --git a/archive/im4p/im4p_test.go b/archive/im4p/im4p_test.go new file mode 100644 index 0000000..d488c15 --- /dev/null +++ b/archive/im4p/im4p_test.go @@ -0,0 +1,47 @@ +package im4p + +import ( + "bytes" + "github.com/tinyrange/trex/auto" + "io" + "testing" +) + +func fixture() []byte { + children := []byte{0x16, 4, 'I', 'M', '4', 'P', 0x16, 4, 'r', 'd', 's', 'k', 0x16, 1, '0', 4, 4, 'd', 'a', 't', 'a', 0x30, 6, 2, 1, 1, 2, 1, 4} + return append([]byte{0x30, byte(len(children))}, children...) +} +func TestBorrowedPayloadMetadataAndAuto(t *testing.T) { + b := fixture() + f, err := Open(bytes.NewReader(b)) + if err != nil { + t.Fatal(err) + } + got, err := io.ReadAll(io.NewSectionReader(f.Payload, 0, f.Payload.Size())) + if err != nil || string(got) != "data" || f.Type != "rdsk" || f.ExpectedSize != 4 || len(f.Extras) != 1 { + t.Fatalf("%+v %q %v", f, got, err) + } + b[19] = 'D' + f.Payload.ReadAt(got, 0) + if string(got) != "Data" { + t.Fatal("payload copied rather than borrowed") + } + id, err := auto.Identify(bytes.NewReader(b), auto.Options{}) + if err != nil || id.Format != "im4p" { + t.Fatalf("auto %v %v", id, err) + } +} +func TestRejectDERBoundsAndNesting(t *testing.T) { + b := fixture() + for _, bad := range [][]byte{b[:len(b)-1], append(append([]byte(nil), b...), 0), {0x30, 0x80, 0, 0}, {0x30, 0x81, 1, 0}} { + if _, err := Open(bytes.NewReader(bad)); err == nil { + t.Fatal("accepted malformed envelope") + } + } + for i := 0; i < 40; i++ { + b = append([]byte{0x30, byte(len(b))}, b...) + } + if _, err := Open(bytes.NewReader(b)); err == nil { + t.Fatal("unbounded nesting") + } +} diff --git a/archive/im4p/star.go b/archive/im4p/star.go new file mode 100644 index 0000000..c78e7b7 --- /dev/null +++ b/archive/im4p/star.go @@ -0,0 +1,57 @@ +package im4p + +import ( + "fmt" + "github.com/tinyrange/trex/auto" + "github.com/tinyrange/trex/storage" + starfile "github.com/tinyrange/trex/storage/star" + "go.starlark.net/starlark" +) + +func init() { + auto.Register("im4p", 15, func(p []byte, r storage.Reader, o auto.Options) (auto.View, error) { + if len(p) < 2 || p[0] != 0x30 { + return nil, auto.ErrNoMatch + } + // Inspect only the bounded first DER string; ordinary ASN.1 is not IM4P. + _, h, n, err := header(r, 0, r.Size()) + if err != nil { + return nil, auto.ErrNoMatch + } + tag, inner, size, err := header(r, h, h+n) + if err != nil || tag != 0x16 || size != 4 { + return nil, auto.ErrNoMatch + } + var sig [4]byte + if _, err := r.ReadAt(sig[:], h+inner); err != nil { + return nil, err + } + if string(sig[:]) != "IM4P" { + return nil, auto.ErrNoMatch + } + f, err := Open(r) + if err != nil { + return nil, err + } + return &auto.DecodedView{Reader: f.Payload, Format: "im4p", Attributes: map[string]any{"type": f.Type, "description": f.Description, "expected_size": f.ExpectedSize}}, nil + }) +} +func Builtin(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var value starlark.Value + if err := starlark.UnpackArgs("im4p", args, kwargs, "file", &value); err != nil { + return nil, err + } + r, ok := value.(storage.Reader) + if !ok { + return nil, fmt.Errorf("im4p: expected file") + } + f, err := Open(r) + if err != nil { + return nil, err + } + extras := make([]starlark.Value, len(f.Extras)) + for i, n := range f.Extras { + extras[i] = starfile.NewReader("metadata.der", n.Raw) + } + return starfile.NewRecord(starlark.StringDict{"payload_type": starlark.String(f.Type), "description": starlark.String(f.Description), "payload": starfile.NewReader("payload", f.Payload), "extras": starlark.NewList(extras), "expected_size": starlark.MakeInt64(f.ExpectedSize)}), nil +} diff --git a/archive/lzfse/NOTICE b/archive/lzfse/NOTICE new file mode 100644 index 0000000..c7b2223 --- /dev/null +++ b/archive/lzfse/NOTICE @@ -0,0 +1,23 @@ +Go decoder derived from Apple LZFSE format definitions and reference decoding algorithms. +Source: lzfse/lzfse, src/lzfse_decode_base.c, lzfse_fse.c, lzfse_internal.h, lzvn_decode_base.c. + +/* +Copyright (c) 2015-2016, Apple Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ diff --git a/archive/lzfse/history_test.go b/archive/lzfse/history_test.go new file mode 100644 index 0000000..ff59d55 --- /dev/null +++ b/archive/lzfse/history_test.go @@ -0,0 +1,43 @@ +package lzfse + +import ( + "bytes" + "testing" +) + +func TestLZVNCrossBlockHistory(t *testing.T) { + // The first explicit match references the preceding stored block, then + // reuses its distance with overlap. A cold random read must replay history. + stream := append([]byte("bvx-"), 3, 0, 0, 0, 'a', 'b', 'c') + tokens := []byte{0, 3, 0xf3, 6, 0, 0, 0, 0, 0, 0, 0} + h := make([]byte, 12) + copy(h, "bvxn") + le.PutUint32(h[4:], 6) + le.PutUint32(h[8:], uint32(len(tokens))) + stream = append(stream, h...) + stream = append(stream, tokens...) + stream = append(stream, []byte("bvx$")...) + f, err := Open(bytes.NewReader(stream), 0) + if err != nil { + t.Fatal(err) + } + var got [5]byte + if _, err := f.ReadAt(got[:], 4); err != nil || string(got[:]) != "bcabc" { + t.Fatalf("cross-block read %q: %v", got, err) + } + delete(f.cache, 1) + f.cached -= 6 + if _, err := f.ReadAt(got[:], 4); err != nil || string(got[:]) != "bcabc" { + t.Fatalf("uncached replay %q: %v", got, err) + } +} + +func TestV1CountsCannotOverflowHostInt(t *testing.T) { + for _, off := range []int{12, 16} { + stream := append(entropyFixture(false, 1, 3, 1), []byte("bvx$")...) + le.PutUint32(stream[off:], 0xfffffffc) + if _, err := Open(bytes.NewReader(stream), 0); err == nil { + t.Fatalf("accepted oversized v1 count at %d", off) + } + } +} diff --git a/archive/lzfse/lzfse.go b/archive/lzfse/lzfse.go new file mode 100644 index 0000000..edc4da8 --- /dev/null +++ b/archive/lzfse/lzfse.go @@ -0,0 +1,450 @@ +// Package lzfse decodes Apple's LZFSE/LZVN block streams as portable file views. +// Derived from Apple's BSD-licensed reference algorithm; see NOTICE. +package lzfse + +import ( + "encoding/binary" + "fmt" + "github.com/tinyrange/trex/storage" + "io" + "math" + "math/bits" + "sort" + "sync" +) + +const MaxBlock = 64 << 20 +const MaxBlocks = 100000 +const historySize = 262144 + +var le = binary.LittleEndian + +type block struct { + kind string + off, packed, start, size int64 + literals, matches, literalBytes, matchBytes, literalBits, matchBits int + states [7]int + freq [360]uint16 +} +type cached struct { + data []byte + tick uint64 +} +type File struct { + source storage.Reader + size int64 + blocks []block + mu sync.Mutex + next int + history []byte + cache map[int]cached + cached int + tick uint64 +} + +func (f *File) Size() int64 { return f.size } +func read(r storage.Reader, off, n int64) ([]byte, error) { + if off < 0 || n < 0 || n > MaxBlock || off > r.Size() || n > r.Size()-off { + return nil, fmt.Errorf("lzfse: invalid source extent") + } + b := make([]byte, int(n)) + _, err := io.ReadFull(io.NewSectionReader(r, off, n), b) + return b, err +} +func Open(source storage.Reader, maximum int64) (*File, error) { + if source == nil || maximum < 0 { + return nil, fmt.Errorf("lzfse: invalid source/maximum") + } + if maximum == 0 { + maximum = math.MaxInt64 + } + f := &File{source: source, cache: map[int]cached{}} + off := int64(0) + for { + sig, err := read(source, off, 4) + if err != nil { + return nil, err + } + if string(sig) == "bvx$" { + if off+4 != source.Size() { + return nil, fmt.Errorf("lzfse: trailing data") + } + return f, nil + } + if len(f.blocks) >= MaxBlocks { + return nil, fmt.Errorf("lzfse: block count exceeded") + } + h, err := read(source, off, 8) + if err != nil { + return nil, err + } + b := block{kind: string(sig), size: int64(le.Uint32(h[4:])), start: f.size} + header := int64(8) + switch b.kind { + case "bvx-": + b.packed = b.size + case "bvxn": + h, err = read(source, off, 12) + header = 12 + if err != nil { + return nil, err + } + b.packed = int64(le.Uint32(h[8:])) + case "bvx1": + h, err = read(source, off, 772) + header = 772 + if err != nil { + return nil, err + } + b.packed = int64(le.Uint32(h[8:])) + b.literals = int(le.Uint32(h[12:])) + b.matches = int(le.Uint32(h[16:])) + b.literalBytes = int(le.Uint32(h[20:])) + b.matchBytes = int(le.Uint32(h[24:])) + b.literalBits = int(int32(le.Uint32(h[28:]))) + b.matchBits = int(int32(le.Uint32(h[40:]))) + for i := 0; i < 4; i++ { + b.states[i] = int(le.Uint16(h[32+2*i:])) + } + for i := 0; i < 3; i++ { + b.states[4+i] = int(le.Uint16(h[44+2*i:])) + } + for i := range b.freq { + b.freq[i] = le.Uint16(h[50+2*i:]) + } + case "bvx2": + h, err = read(source, off, 32) + if err != nil { + return nil, err + } + a, c, d := le.Uint64(h[8:]), le.Uint64(h[16:]), le.Uint64(h[24:]) + header = int64(uint32(d)) + if header < 32 || header > 752 { + return nil, fmt.Errorf("lzfse: invalid v2 header size") + } + b.literals = int(a & 0xfffff) + b.literalBytes = int((a >> 20) & 0xfffff) + b.matches = int((a >> 40) & 0xfffff) + b.literalBits = int((a>>60)&7) - 7 + b.matchBytes = int((c >> 40) & 0xfffff) + b.matchBits = int((c>>60)&7) - 7 + b.packed = int64(b.literalBytes + b.matchBytes) + for i := 0; i < 4; i++ { + b.states[i] = int((c >> uint(i*10)) & 1023) + } + for i := 0; i < 3; i++ { + b.states[4+i] = int((d >> uint(32+10*i)) & 1023) + } + h, err = read(source, off+32, header-32) + if err != nil { + return nil, err + } + if err = frequencies(h, b.freq[:]); err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("lzfse: unsupported block %q", b.kind) + } + if b.size <= 0 || b.size > MaxBlock || b.packed <= 0 || b.packed > MaxBlock || b.size > maximum-f.size { + return nil, fmt.Errorf("lzfse: invalid block bounds") + } + if b.kind == "bvx1" || b.kind == "bvx2" { + if b.literals < 0 || b.literals > 40000 || b.literals%4 != 0 || b.matches < 0 || b.matches > 10000 || b.literalBytes < 0 || b.matchBytes < 0 || int64(b.literalBytes+b.matchBytes) != b.packed || b.literalBits < -7 || b.literalBits > 0 || b.matchBits < -7 || b.matchBits > 0 { + return nil, fmt.Errorf("lzfse: invalid entropy header") + } + } + b.off = off + header + if b.off > source.Size() || b.packed > source.Size()-b.off { + return nil, fmt.Errorf("lzfse: truncated block") + } + f.blocks = append(f.blocks, b) + f.size += b.size + off = b.off + b.packed + } +} +func frequencies(src []byte, dst []uint16) error { + if len(src) == 0 { + return fmt.Errorf("lzfse: missing frequency table") + } + widths := [32]int{2, 3, 2, 5, 2, 3, 2, 8, 2, 3, 2, 5, 2, 3, 2, 14, 2, 3, 2, 5, 2, 3, 2, 8, 2, 3, 2, 5, 2, 3, 2, 14} + values := [32]int{0, 2, 1, 4, 0, 3, 1, -1, 0, 2, 1, 5, 0, 3, 1, -1, 0, 2, 1, 6, 0, 3, 1, -1, 0, 2, 1, 7, 0, 3, 1, -1} + var acc uint32 + pos, n := 0, 0 + for i := range dst { + for pos < len(src) && n <= 24 { + acc |= uint32(src[pos]) << uint(n) + n += 8 + pos++ + } + width := widths[acc&31] + if width > n { + return fmt.Errorf("lzfse: truncated frequency table") + } + value := values[acc&31] + if width == 8 { + value = 8 + int((acc>>4)&15) + } + if width == 14 { + value = 24 + int((acc>>4)&1023) + } + dst[i] = uint16(value) + acc >>= uint(width) + n -= width + } + if pos != len(src) || n >= 8 || acc != 0 { + return fmt.Errorf("lzfse: trailing frequency bits") + } + return nil +} + +type reverseBits struct { + data []byte + end int + err error +} + +func reverse(data []byte, padding int) *reverseBits { + r := &reverseBits{data: data, end: len(data)*8 + padding} + if padding < -7 || padding > 0 || r.end < 0 || len(data) > 0 && padding != 0 && data[len(data)-1]>>uint(8+padding) != 0 { + r.err = fmt.Errorf("lzfse: invalid entropy padding") + } + return r +} +func (r *reverseBits) take(n int) int { + if r.err != nil { + return 0 + } + if n < 0 || n > 32 || n > r.end { + r.err = fmt.Errorf("lzfse: truncated entropy stream") + return 0 + } + r.end -= n + var v uint64 + start := r.end / 8 + count := (r.end%8 + n + 7) / 8 + for i := 0; i < count; i++ { + v |= uint64(r.data[start+i]) << uint(8*i) + } + return int((v >> uint(r.end%8)) & ((uint64(1) << uint(n)) - 1)) +} + +type stateEntry struct { + width, delta, value, extra int + valid bool +} + +func table(freq []uint16, states int, extra, base []int) ([]stateEntry, error) { + t := make([]stateEntry, states) + pos := 0 + for symbol, count := range freq { + f := int(count) + if f == 0 { + continue + } + if f > states-pos { + return nil, fmt.Errorf("lzfse: invalid frequencies") + } + k := bits.Len(uint(states)) - bits.Len(uint(f)) + j0 := ((2 * states) >> uint(k)) - f + for j := 0; j < f; j++ { + e := stateEntry{value: symbol, valid: true} + if j < j0 { + e.width = k + e.delta = ((f + j) << uint(k)) - states + } else { + e.width = k - 1 + e.delta = (j - j0) << uint(k-1) + } + if extra != nil { + e.extra = extra[symbol] + e.value = base[symbol] + } + t[pos] = e + pos++ + } + } + return t, nil +} +func symbol(t []stateEntry, state *int, r *reverseBits) int { + if *state < 0 || *state >= len(t) || !t[*state].valid { + r.err = fmt.Errorf("lzfse: invalid entropy state") + return 0 + } + e := t[*state] + v := r.take(e.width + e.extra) + *state = e.delta + (v >> uint(e.extra)) + return e.value + (v & ((1 << uint(e.extra)) - 1)) +} + +var lExtra = []int{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 5, 8} +var lBase = []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 20, 28, 60} +var mExtra = []int{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 8, 11} +var mBase = []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 24, 56, 312} + +func distances() ([]int, []int) { + extra, base := make([]int, 64), make([]int, 64) + for i := 0; i < 4; i++ { + base[i] = i + } + current := 4 + for i := 4; i < 64; i++ { + extra[i] = i / 4 + base[i] = current + current += 1 << uint(extra[i]) + } + return extra, base +} +func decode(b block, src, history []byte) ([]byte, error) { + if b.kind == "bvx-" { + return src, nil + } + if b.kind == "bvxn" { + return decodeLZVN(src, int(b.size), history) + } + lt, err := table(b.freq[104:], 1024, nil, nil) + if err != nil { + return nil, err + } + l, err := table(b.freq[:20], 64, lExtra, lBase) + if err != nil { + return nil, err + } + m, err := table(b.freq[20:40], 64, mExtra, mBase) + if err != nil { + return nil, err + } + de, db := distances() + d, err := table(b.freq[40:104], 256, de, db) + if err != nil { + return nil, err + } + literals := make([]byte, b.literals) + r := reverse(src[:b.literalBytes], b.literalBits) + states := b.states + for i := range literals { + literals[i] = byte(symbol(lt, &states[i%4], r)) + } + if r.err != nil { + return nil, r.err + } + r = reverse(src[b.literalBytes:], b.matchBits) + out := make([]byte, 0, int(b.size)) + literal := 0 + distance := -1 + for i := 0; i < b.matches; i++ { + length := symbol(l, &states[4], r) + match := symbol(m, &states[5], r) + newDistance := symbol(d, &states[6], r) + if r.err != nil { + return nil, r.err + } + if newDistance != 0 { + distance = newDistance + } + if length > len(literals)-literal || length+match > int(b.size)-len(out) { + return nil, fmt.Errorf("lzfse: literal/output overflow") + } + out = append(out, literals[literal:literal+length]...) + literal += length + if match > 0 && (distance <= 0 || distance > len(history)+len(out)) { + return nil, fmt.Errorf("lzfse: invalid match distance") + } + for j := 0; j < match; j++ { + index := len(out) - distance + if index < 0 { + out = append(out, history[len(history)+index]) + } else { + out = append(out, out[index]) + } + } + } + if int64(len(out)) != b.size { + return nil, fmt.Errorf("lzfse: decoded length mismatch") + } + return out, nil +} +func (f *File) retain(i int, data []byte) { + for f.cached+len(data) > MaxBlock { + oldest := -1 + for key, c := range f.cache { + if oldest < 0 || c.tick < f.cache[oldest].tick { + oldest = key + } + } + if oldest < 0 { + break + } + f.cached -= len(f.cache[oldest].data) + delete(f.cache, oldest) + } + if previous, ok := f.cache[i]; ok { + f.cached -= len(previous.data) + } + f.tick++ + f.cache[i] = cached{data, f.tick} + f.cached += len(data) +} +func (f *File) chunk(i int) ([]byte, error) { + if c, ok := f.cache[i]; ok { + f.tick++ + c.tick = f.tick + f.cache[i] = c + return c.data, nil + } + if i < f.next { + f.next = 0 + f.history = nil + } + for f.next <= i { + b := f.blocks[f.next] + src, err := read(f.source, b.off, b.packed) + if err != nil { + return nil, err + } + data, err := decode(b, src, f.history) + if err != nil { + return nil, fmt.Errorf("lzfse: block %d: %w", f.next, err) + } + if len(data) >= historySize { + f.history = append(f.history[:0], data[len(data)-historySize:]...) + } else { + keep := min(len(f.history), historySize-len(data)) + f.history = append(append([]byte(nil), f.history[len(f.history)-keep:]...), data...) + } + f.retain(f.next, data) + f.next++ + } + return f.cache[i].data, nil +} +func (f *File) ReadAt(p []byte, off int64) (int, error) { + f.mu.Lock() + defer f.mu.Unlock() + if off < 0 { + return 0, fmt.Errorf("lzfse: negative offset") + } + if len(p) == 0 { + return 0, nil + } + if off >= f.size { + return 0, io.EOF + } + wanted := len(p) + done := 0 + i := sort.Search(len(f.blocks), func(i int) bool { return f.blocks[i].start+f.blocks[i].size > off }) + for len(p) > 0 && i < len(f.blocks) { + b := f.blocks[i] + data, err := f.chunk(i) + if err != nil { + return done, err + } + n := copy(p, data[off-b.start:]) + p = p[n:] + off += int64(n) + done += n + i++ + } + if done != wanted { + return done, io.EOF + } + return done, nil +} diff --git a/archive/lzfse/lzfse_test.go b/archive/lzfse/lzfse_test.go new file mode 100644 index 0000000..d34a40a --- /dev/null +++ b/archive/lzfse/lzfse_test.go @@ -0,0 +1,147 @@ +package lzfse + +import ( + "bytes" + "io" + "testing" +) + +// Synthetic entropy fixture, independently specified from the wire layout: +// four A literals; each L/M/D token is (1,3,1), producing sixteen As. +func entropyFixture(v2 bool, length, match, distance int) []byte { + b := make([]byte, 772) + copy(b, "bvx1") + le.PutUint32(b[4:], 16) + le.PutUint32(b[8:], 14) + le.PutUint32(b[12:], 4) + le.PutUint32(b[16:], 4) + le.PutUint32(b[20:], 7) + le.PutUint32(b[24:], 7) + freq := make([]uint16, 360) + freq[length] = 64 + freq[20+match] = 64 + freq[40+distance] = 256 + freq[104+int('A')] = 1024 + for i, f := range freq { + le.PutUint16(b[50+2*i:], f) + } + if v2 { + var codes []byte + bit := 0 + put := func(value, n int) { + for i := 0; i < n; i++ { + if bit/8 >= len(codes) { + codes = append(codes, 0) + } + codes[bit/8] |= byte((value>>uint(i))&1) << uint(bit%8) + bit++ + } + } + for _, f := range freq { + if f == 0 { + put(0, 2) + } else { + put(15+(int(f)-24)*16, 14) + } + } + b = make([]byte, 32) + copy(b, "bvx2") + le.PutUint32(b[4:], 16) + le.PutUint64(b[8:], 4|7<<20|4<<40|7<<60) + le.PutUint64(b[16:], 7<<40|7<<60) + le.PutUint64(b[24:], uint64(32+len(codes))) + b = append(b, codes...) + } + return append(b, make([]byte, 14)...) +} +func TestEntropyV1V2AndHistory(t *testing.T) { + for _, v2 := range []bool{false, true} { + stream := append(entropyFixture(v2, 1, 3, 1), []byte("bvx$")...) + f, err := Open(bytes.NewReader(stream), 0) + if err != nil { + t.Fatal(err) + } + got, err := io.ReadAll(io.NewSectionReader(f, 0, f.Size())) + if err != nil || string(got) != "AAAAAAAAAAAAAAAA" { + t.Fatalf("v2=%v %q %v", v2, got, err) + } + // A raw B followed by match-only entropy requires cross-block history. + raw := append([]byte("bvx-"), 1, 0, 0, 0, 'B') + raw = append(raw, entropyFixture(v2, 0, 4, 1)...) + raw = append(raw, []byte("bvx$")...) + f, err = Open(bytes.NewReader(raw), 0) + if err != nil { + t.Fatal(err) + } + var p [3]byte + if _, err := f.ReadAt(p[:], 13); err != nil || string(p[:]) != "BBB" { + t.Fatalf("history %q %v", p, err) + } + if _, err := f.ReadAt(p[:], 0); err != nil || string(p[:]) != "BBB" { + t.Fatalf("backward %q %v", p, err) + } + } +} +func TestLZVNAndMalformedStreams(t *testing.T) { + // Literal abc + distance3/match3 + reused match3 + eight-byte EOS. + tokens := []byte{0xe3, 'a', 'b', 'c', 0, 3, 0xf3, 6, 0, 0, 0, 0, 0, 0, 0} + b := make([]byte, 12) + copy(b, "bvxn") + le.PutUint32(b[4:], 9) + le.PutUint32(b[8:], uint32(len(tokens))) + b = append(b, tokens...) + b = append(b, []byte("bvx$")...) + f, err := Open(bytes.NewReader(b), 0) + if err != nil { + t.Fatal(err) + } + got, err := io.ReadAll(io.NewSectionReader(f, 0, f.Size())) + if err != nil || string(got) != "abcabcabc" { + t.Fatalf("LZVN %q %v", got, err) + } + for _, bad := range [][]byte{b[:len(b)-1], append(append([]byte(nil), b...), 0), []byte("bvx2"), []byte("xxxx")} { + if _, err := Open(bytes.NewReader(bad), 0); err == nil { + t.Fatal("accepted malformed framing") + } + } + if _, err := Open(bytes.NewReader(b), 8); err == nil { + t.Fatal("maximum not enforced") + } + stream := append(entropyFixture(false, 0, 4, 1), []byte("bvx$")...) + f, err = Open(bytes.NewReader(stream), 0) + if err != nil { + t.Fatal(err) + } + if _, err = f.ReadAt(make([]byte, 1), 0); err == nil { + t.Fatal("invalid history distance accepted") + } + for _, tokens := range [][]byte{{0xf3, 6, 0, 0, 0, 0, 0, 0, 0}, {0x1e}, {0xe3, 'a'}} { + if _, err := decodeLZVN(tokens, 3, nil); err == nil { + t.Fatal("invalid LZVN accepted") + } + } +} +func TestBoundedCacheReplays(t *testing.T) { + var stream []byte + for i := 0; i < 3; i++ { + b := make([]byte, 8) + copy(b, "bvx-") + le.PutUint32(b[4:], 24<<20) + stream = append(stream, b...) + stream = append(stream, bytes.Repeat([]byte{byte(i)}, 24<<20)...) + } + stream = append(stream, []byte("bvx$")...) + f, err := Open(bytes.NewReader(stream), 0) + if err != nil { + t.Fatal(err) + } + var p [1]byte + for _, off := range []int64{70 << 20, 1, 50 << 20} { + if _, err := f.ReadAt(p[:], off); err != nil || p[0] != byte(off/(24<<20)) { + t.Fatalf("replay %d %v", off, err) + } + if f.cached > MaxBlock || len(f.history) > historySize { + t.Fatal("unbounded cache/history") + } + } +} diff --git a/archive/lzfse/lzvn.go b/archive/lzfse/lzvn.go new file mode 100644 index 0000000..0f406b8 --- /dev/null +++ b/archive/lzfse/lzvn.go @@ -0,0 +1,88 @@ +package lzfse + +import "fmt" + +// LZVN tokens encode literal/match lengths and an explicit or reused distance. +// End tokens occupy eight bytes; undefined opcodes never become empty output. +func decodeLZVN(src []byte, size int, history []byte) ([]byte, error) { + out := make([]byte, 0, size) + distance := 0 + pos := 0 + for pos < len(src) { + op := int(src[pos]) + n, l, m := 1, 0, 0 + switch { + case op == 6: + if pos+8 != len(src) || len(out) != size { + return nil, fmt.Errorf("lzfse: LZVN end/size mismatch") + } + return out, nil + case op == 14 || op == 22: + pos++ + continue + case op >= 0xe0 && op <= 0xef: + l = op & 15 + if op == 0xe0 { + n = 2 + if pos+2 > len(src) { + return nil, fmt.Errorf("lzfse: truncated LZVN token") + } + l = int(src[pos+1]) + 16 + } + case op >= 0xf0: + m = op & 15 + if op == 0xf0 { + n = 2 + if pos+2 > len(src) { + return nil, fmt.Errorf("lzfse: truncated LZVN token") + } + m = int(src[pos+1]) + 16 + } + case op >= 0xa0 && op <= 0xbf: + n = 3 + l = (op >> 3) & 3 + if pos+3 > len(src) { + return nil, fmt.Errorf("lzfse: truncated LZVN token") + } + v := int(le.Uint16(src[pos+1:])) + m = ((op&7)<<2 | v&3) + 3 + distance = v >> 2 + case op >= 0x70 && op <= 0x7f || op >= 0xd0 && op <= 0xdf || op&7 == 6 && op < 0x40: + return nil, fmt.Errorf("lzfse: undefined LZVN opcode") + default: + l = op >> 6 + m = ((op >> 3) & 7) + 3 + if op&7 == 7 { + n = 3 + if pos+3 > len(src) { + return nil, fmt.Errorf("lzfse: truncated LZVN token") + } + distance = int(le.Uint16(src[pos+1:])) + } else if op&7 != 6 { + n = 2 + if pos+2 > len(src) { + return nil, fmt.Errorf("lzfse: truncated LZVN token") + } + distance = (op&7)<<8 | int(src[pos+1]) + } + } + pos += n + if l > len(src)-pos || l+m > size-len(out) { + return nil, fmt.Errorf("lzfse: LZVN literal/output overflow") + } + out = append(out, src[pos:pos+l]...) + pos += l + if m > 0 && (distance <= 0 || distance > len(history)+len(out)) { + return nil, fmt.Errorf("lzfse: invalid LZVN distance") + } + for j := 0; j < m; j++ { + index := len(out) - distance + if index < 0 { + out = append(out, history[len(history)+index]) + } else { + out = append(out, out[index]) + } + } + } + return nil, fmt.Errorf("lzfse: missing LZVN end") +} diff --git a/archive/lzfse/star.go b/archive/lzfse/star.go new file mode 100644 index 0000000..7b96845 --- /dev/null +++ b/archive/lzfse/star.go @@ -0,0 +1,38 @@ +package lzfse + +import ( + "fmt" + "github.com/tinyrange/trex/auto" + "github.com/tinyrange/trex/storage" + starfile "github.com/tinyrange/trex/storage/star" + "go.starlark.net/starlark" +) + +func init() { + auto.Register("lzfse", 10, func(p []byte, r storage.Reader, o auto.Options) (auto.View, error) { + if len(p) < 4 || (string(p[:4]) != "bvx1" && string(p[:4]) != "bvx2" && string(p[:4]) != "bvx-" && string(p[:4]) != "bvxn") { + return nil, auto.ErrNoMatch + } + f, err := Open(r, o.StreamingMaximum()) + if err != nil { + return nil, err + } + return &auto.DecodedView{Reader: f, Format: "lzfse"}, nil + }) +} +func Builtin(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var value starlark.Value + var maximum int64 + if err := starlark.UnpackArgs("lzfse", args, kwargs, "file", &value, "maximum_bytes?", &maximum); err != nil { + return nil, err + } + r, ok := value.(storage.Reader) + if !ok { + return nil, fmt.Errorf("lzfse: expected file") + } + f, err := Open(r, maximum) + if err != nil { + return nil, err + } + return starfile.NewReader("decoded", f), nil +} diff --git a/archive/pbzx/pbzx.go b/archive/pbzx/pbzx.go new file mode 100644 index 0000000..e5d0e22 --- /dev/null +++ b/archive/pbzx/pbzx.go @@ -0,0 +1,145 @@ +// Package pbzx reads Apple's chunked installer payload streams without a host +// extractor or materialized intermediate. Chunk headers give decoded and stored +// sizes; unequal sizes wrap an independently checked XZ stream. +package pbzx + +import ( + "bytes" + "encoding/binary" + "fmt" + "github.com/tinyrange/trex/archive/xz" + "github.com/tinyrange/trex/storage" + bytecache "github.com/tinyrange/trex/storage/cache" + "io" + "math" + "sort" +) + +const MaxChunk = 64 << 20 +const MaxChunks = 100000 + +// Chunk retains physical and logical extents without decoding its bytes. +type Chunk struct { + Offset, Length, Start, Size int64 + Compressed bool + reader storage.Reader +} +type File struct { + source storage.Reader + size int64 + Chunks []Chunk + cache *bytecache.Cache +} + +func (f *File) Size() int64 { return f.size } +func Open(source storage.Reader, maximum int64) (*File, error) { + if source == nil || source.Size() < 12 || maximum < 0 { + return nil, fmt.Errorf("pbzx: invalid source/maximum") + } + if maximum == 0 { + maximum = math.MaxInt64 + } + var h [16]byte + if _, err := source.ReadAt(h[:12], 0); err != nil { + return nil, err + } + block := binary.BigEndian.Uint64(h[4:12]) + if string(h[:4]) != "pbzx" || block == 0 || block > MaxChunk { + return nil, fmt.Errorf("pbzx: invalid header/chunk bound") + } + f := &File{source: source, cache: bytecache.New(MaxChunk)} + off := int64(12) + for off < source.Size() { + if len(f.Chunks) >= MaxChunks || source.Size()-off < 16 { + return nil, fmt.Errorf("pbzx: truncated/excessive chunk headers") + } + if _, err := source.ReadAt(h[:], off); err != nil { + return nil, err + } + expanded, packed := binary.BigEndian.Uint64(h[:8]), binary.BigEndian.Uint64(h[8:]) + off += 16 + if expanded == 0 || expanded > block || packed == 0 || packed > MaxChunk || packed > uint64(source.Size()-off) || expanded > uint64(maximum-f.size) { + return nil, fmt.Errorf("pbzx: invalid chunk bounds") + } + r := io.NewSectionReader(source, off, int64(packed)) + c := Chunk{Offset: off, Length: int64(packed), Start: f.size, Size: int64(expanded), Compressed: packed != expanded, reader: r} + if c.Compressed { + var magic [6]byte + if _, err := r.ReadAt(magic[:], 0); err != nil { + return nil, err + } + if !bytes.Equal(magic[:], []byte{0xfd, '7', 'z', 'X', 'Z', 0}) { + return nil, fmt.Errorf("pbzx: compressed chunk is not XZ") + } + decoded, err := xz.Open(r, MaxChunk) + if err != nil { + return nil, fmt.Errorf("pbzx: chunk %d: %w", len(f.Chunks), err) + } + if decoded.Size() != c.Size { + return nil, fmt.Errorf("pbzx: chunk declared/XZ length mismatch") + } + } + f.Chunks = append(f.Chunks, c) + f.size += c.Size + off += c.Length + if c.Size < int64(block) && off != source.Size() { + return nil, fmt.Errorf("pbzx: data after short final chunk") + } + } + if len(f.Chunks) == 0 { + return nil, fmt.Errorf("pbzx: no chunks") + } + return f, nil +} +func (f *File) ReadAt(p []byte, off int64) (int, error) { + if off < 0 { + return 0, fmt.Errorf("pbzx: negative offset") + } + if len(p) == 0 { + return 0, nil + } + if off >= f.size { + return 0, io.EOF + } + wanted := len(p) + p = p[:min(int64(len(p)), f.size-off)] + done := 0 + i := sort.Search(len(f.Chunks), func(i int) bool { return f.Chunks[i].Start+f.Chunks[i].Size > off }) + for len(p) > 0 { + c := f.Chunks[i] + skip := off - c.Start + n := min(int64(len(p)), c.Size-skip) + if !c.Compressed { + if _, err := io.ReadFull(io.NewSectionReader(c.reader, skip, n), p[:n]); err != nil { + return done, err + } + } else { + data, err := f.cache.Get(bytecache.Key{Index: i}, func() ([]byte, error) { + // Keep only the borrowed stored extent in Chunk. Retaining one + // xz.File per chunk also retains each decoder's LZMA dictionary + // after EOF, growing memory with the entire decoded payload. + decoded, err := xz.Open(c.reader, MaxChunk) + if err != nil { + return nil, err + } + b := make([]byte, c.Size) + if _, err := io.ReadFull(io.NewSectionReader(decoded, 0, c.Size), b); err != nil { + return nil, err + } + return b, nil + }) + if err != nil { + return done, fmt.Errorf("pbzx: chunk %d: %w", i, err) + } + copy(p[:n], data[skip:skip+n]) + } + done += int(n) + off += n + p = p[n:] + i++ + } + if done != wanted { + return done, io.EOF + } + return done, nil +} diff --git a/archive/pbzx/pbzx_test.go b/archive/pbzx/pbzx_test.go new file mode 100644 index 0000000..88a17f4 --- /dev/null +++ b/archive/pbzx/pbzx_test.go @@ -0,0 +1,90 @@ +package pbzx + +import ( + "bytes" + "encoding/binary" + "github.com/tinyrange/trex/auto" + encoder "github.com/ulikunitz/xz" + "io" + "testing" +) + +func fixture(t *testing.T) []byte { + t.Helper() + var packed bytes.Buffer + w, err := (encoder.WriterConfig{DictCap: 1 << 20}).NewWriter(&packed) + if err != nil { + t.Fatal(err) + } + w.Write(bytes.Repeat([]byte{'X'}, 4096)) + if err := w.Close(); err != nil { + t.Fatal(err) + } + b := make([]byte, 12) + copy(b, "pbzx") + binary.BigEndian.PutUint64(b[4:], 4096) + for _, pair := range []struct { + size uint64 + data []byte + }{{4096, packed.Bytes()}, {17, bytes.Repeat([]byte{'R'}, 17)}} { + var h [16]byte + binary.BigEndian.PutUint64(h[:], pair.size) + binary.BigEndian.PutUint64(h[8:], uint64(len(pair.data))) + b = append(b, h[:]...) + b = append(b, pair.data...) + } + return b +} +func TestMixedChunksRandomAccess(t *testing.T) { + f, err := Open(bytes.NewReader(fixture(t)), 0) + if err != nil { + t.Fatal(err) + } + b := make([]byte, 22) + n, err := f.ReadAt(b, 4088) + if n != 22 || err != nil || string(b) != string(bytes.Repeat([]byte{'X'}, 8))+string(bytes.Repeat([]byte{'R'}, 14)) { + t.Fatalf("cross-chunk %q %d %v", b, n, err) + } + if _, err := f.ReadAt(b, 4100); err != io.EOF { + t.Fatal("EOF", err) + } + if _, err := f.ReadAt(b, 0); err != nil { + t.Fatal("backward", err) + } + if f.cache.Stats().Hits == 0 { + t.Fatal("missing cache reuse") + } + result, err := auto.Identify(bytes.NewReader(fixture(t)), auto.Options{}) + if err != nil || result.Format != "pbzx" { + t.Fatalf("auto %v %v", result, err) + } + if _, err := Open(bytes.NewReader(fixture(t)), 4096); err == nil { + t.Fatal("limit not honored") + } +} +func TestRejectBadFraming(t *testing.T) { + for name, mutate := range map[string]func([]byte) []byte{ + "truncated": func(b []byte) []byte { return b[:len(b)-1] }, + "size": func(b []byte) []byte { binary.BigEndian.PutUint64(b[12:], 4095); return b }, + "overflow": func(b []byte) []byte { binary.BigEndian.PutUint64(b[20:], 1<<63); return b }, + "trailing": func(b []byte) []byte { return append(b, 0) }, + "zero": func(b []byte) []byte { binary.BigEndian.PutUint64(b[12:], 0); return b }, + } { + t.Run(name, func(t *testing.T) { + if _, err := Open(bytes.NewReader(mutate(fixture(t))), 0); err == nil { + t.Fatal("accepted malformed PBZX") + } + }) + } +} +func TestRejectCorruptXZData(t *testing.T) { + b := fixture(t) + b[12+16+40] ^= 1 + f, err := Open(bytes.NewReader(b), 0) + if err == nil { + var p [1]byte + if _, err = f.ReadAt(p[:], 0); err == nil { + t.Fatal("accepted corrupt XZ") + } + } +} diff --git a/archive/pbzx/star.go b/archive/pbzx/star.go new file mode 100644 index 0000000..135585e --- /dev/null +++ b/archive/pbzx/star.go @@ -0,0 +1,38 @@ +package pbzx + +import ( + "fmt" + "github.com/tinyrange/trex/auto" + "github.com/tinyrange/trex/storage" + starfile "github.com/tinyrange/trex/storage/star" + "go.starlark.net/starlark" +) + +func init() { + auto.Register("pbzx", 10, func(p []byte, r storage.Reader, o auto.Options) (auto.View, error) { + if len(p) < 4 || string(p[:4]) != "pbzx" { + return nil, auto.ErrNoMatch + } + f, err := Open(r, o.StreamingMaximum()) + if err != nil { + return nil, err + } + return &auto.DecodedView{Reader: f, Format: "pbzx", Attributes: map[string]any{"chunks": len(f.Chunks), "decoded_size": f.Size()}}, nil + }) +} +func Builtin(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var value starlark.Value + var maximum int64 + if err := starlark.UnpackArgs("pbzx", args, kwargs, "file", &value, "maximum_bytes?", &maximum); err != nil { + return nil, err + } + r, ok := value.(storage.Reader) + if !ok { + return nil, fmt.Errorf("pbzx: expected file") + } + f, err := Open(r, maximum) + if err != nil { + return nil, err + } + return starfile.NewReader("payload.cpio", f), nil +} diff --git a/archive/udif/star.go b/archive/udif/star.go new file mode 100644 index 0000000..74df28b --- /dev/null +++ b/archive/udif/star.go @@ -0,0 +1,62 @@ +package udif + +import ( + "fmt" + "github.com/tinyrange/trex/auto" + "github.com/tinyrange/trex/storage" + starfile "github.com/tinyrange/trex/storage/star" + "go.starlark.net/starlark" +) + +func init() { + auto.Register("udif", 90, func(_ []byte, r storage.Reader, o auto.Options) (auto.View, error) { + if stream, ok := r.(interface{ KnownSize() (int64, bool) }); ok { + if _, known := stream.KnownSize(); !known { + return nil, auto.ErrNoMatch + } + } + if r.Size() < 512 { + return nil, auto.ErrNoMatch + } + var sig [4]byte + if _, err := r.ReadAt(sig[:], r.Size()-512); err != nil { + return nil, err + } + if string(sig[:]) != "koly" { + return nil, auto.ErrNoMatch + } + d, err := Open(r) + if err != nil { + return nil, err + } + return &auto.DecodedView{Reader: d, Name: "disk", Format: "udif", Attributes: map[string]any{"logical_size": d.Size(), "block_maps": len(d.Tables)}}, nil + }) +} +func Builtin(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var value starlark.Value + if err := starlark.UnpackArgs("udif", args, kwargs, "file", &value); err != nil { + return nil, err + } + r, ok := value.(storage.Reader) + if !ok { + return nil, fmt.Errorf("udif: expected file") + } + d, err := Open(r) + if err != nil { + return nil, err + } + var tables []starlark.Value + for _, t := range d.Tables { + tables = append(tables, starfile.NewRecord(starlark.StringDict{"name": starlark.String(t.Name), "offset": starlark.MakeInt64(t.Offset), "size": starlark.MakeInt64(t.Size), "raw": starlark.Bytes(t.Raw)})) + } + verify := starlark.NewBuiltin("udif.verify", func(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := starlark.UnpackArgs("verify", args, kwargs); err != nil { + return nil, err + } + if err := d.Verify(); err != nil { + return nil, err + } + return starlark.None, nil + }) + return starfile.NewRecord(starlark.StringDict{"disk": starfile.NewReader("disk", d), "xml": starfile.NewReader("plist.xml", d.XML), "trailer": starfile.NewReader("koly", d.Trailer), "tables": starlark.NewList(tables), "verify": verify}), nil +} diff --git a/archive/udif/udif.go b/archive/udif/udif.go new file mode 100644 index 0000000..564c0f3 --- /dev/null +++ b/archive/udif/udif.go @@ -0,0 +1,366 @@ +// Package udif reads XML-backed, single-segment Apple disk images lazily. +// Layout facts follow the published UDIF/BLKX declarations in libdmg-hfsplus; +// this reader and its fixtures are independent implementations. +package udif + +import ( + "bytes" + "compress/bzip2" + "compress/zlib" + "encoding/base64" + "encoding/binary" + "encoding/xml" + "fmt" + "hash/crc32" + "io" + "math" + "sort" + "strings" + + "github.com/tinyrange/trex/storage" + bytecache "github.com/tinyrange/trex/storage/cache" +) + +const MaxMetadata = 16 << 20 +const MaxChunk = 64 << 20 + +// Table describes a logical disk range; Raw retains the complete BLKX record. +type Table struct { + Name string + Offset, Size int64 + Raw []byte + ChecksumType, ChecksumBits, Checksum uint32 +} +type run struct { + kind uint32 + start, size, offset, length int64 +} + +// Image exposes the complete logical disk, plus original container metadata. +// Unsupported chunk codecs, overlapping/gapped maps and segmented images fail. +type Image struct { + source storage.Reader + size, dataOffset, dataLength int64 + runs []run + Tables []Table + XML, Trailer storage.Reader + cache *bytecache.Cache + checksumType, checksumBits, checksum uint32 +} + +func (d *Image) Size() int64 { return d.size } +func read(r storage.Reader, off, n int64) ([]byte, error) { + if off < 0 || n < 0 || off > r.Size() || n > r.Size()-off { + return nil, io.ErrUnexpectedEOF + } + b := make([]byte, n) + _, err := io.ReadFull(io.NewSectionReader(r, off, n), b) + return b, err +} +func rangeOK(off, n uint64, size int64) bool { + return size >= 0 && off <= uint64(size) && n <= uint64(size)-off +} +func Open(source storage.Reader) (*Image, error) { + if source == nil || source.Size() < 512 { + return nil, fmt.Errorf("udif: missing trailer") + } + h, err := read(source, source.Size()-512, 512) + if err != nil { + return nil, err + } + be := binary.BigEndian + if string(h[:4]) != "koly" || be.Uint32(h[4:]) != 4 || be.Uint32(h[8:]) != 512 { + return nil, fmt.Errorf("udif: invalid v4 trailer") + } + if be.Uint64(h[16:]) != 0 || be.Uint32(h[56:]) > 1 || be.Uint32(h[60:]) > 1 { + return nil, fmt.Errorf("udif: segmented image unsupported") + } + dataOff, dataLen := be.Uint64(h[24:]), be.Uint64(h[32:]) + xmlOff, xmlLen := be.Uint64(h[216:]), be.Uint64(h[224:]) + sectors := be.Uint64(h[492:]) + if !rangeOK(dataOff, dataLen, source.Size()-512) || !rangeOK(xmlOff, xmlLen, source.Size()-512) || xmlLen == 0 || xmlLen > MaxMetadata || sectors == 0 || sectors > math.MaxInt64/512 { + return nil, fmt.Errorf("udif: invalid data/XML/logical size bounds") + } + if dataOff+dataLen > xmlOff { + return nil, fmt.Errorf("udif: data fork overlaps XML") + } + d := &Image{source: source, size: int64(sectors) * 512, dataOffset: int64(dataOff), dataLength: int64(dataLen), + XML: io.NewSectionReader(source, int64(xmlOff), int64(xmlLen)), Trailer: io.NewSectionReader(source, source.Size()-512, 512), cache: bytecache.New(MaxChunk), + checksumType: be.Uint32(h[80:]), checksumBits: be.Uint32(h[84:]), checksum: be.Uint32(h[88:])} + metadata, err := read(d.XML, 0, int64(xmlLen)) + if err != nil { + return nil, err + } + root, err := parseXML(metadata) + if err != nil { + return nil, fmt.Errorf("udif: plist: %w", err) + } + if root.name != "plist" || len(root.children) != 1 { + return nil, fmt.Errorf("udif: invalid plist root") + } + maps := root.children[0].value("resource-fork").value("blkx") + if maps.name != "array" || len(maps.children) == 0 { + return nil, fmt.Errorf("udif: missing blkx array") + } + for _, item := range maps.children { + blob := item.value("Data") + if blob.name != "data" { + return nil, fmt.Errorf("udif: missing blkx data") + } + raw, err := base64.StdEncoding.DecodeString(strings.Join(strings.Fields(blob.text), "")) + if err != nil { + return nil, fmt.Errorf("udif: blkx base64: %w", err) + } + if len(raw) < 204 || string(raw[:4]) != "mish" || be.Uint32(raw[4:]) != 1 { + return nil, fmt.Errorf("udif: invalid blkx header") + } + count := be.Uint32(raw[200:]) + if count == 0 || uint64(count)*40 != uint64(len(raw)-204) { + return nil, fmt.Errorf("udif: invalid blkx run count") + } + first, length, dataStart := be.Uint64(raw[8:]), be.Uint64(raw[16:]), be.Uint64(raw[24:]) + if first > sectors || length > sectors-first || length == 0 || dataStart > dataLen { + return nil, fmt.Errorf("udif: blkx bounds") + } + table := Table{Name: item.value("Name").text, Offset: int64(first) * 512, Size: int64(length) * 512, Raw: raw, + ChecksumType: be.Uint32(raw[64:]), ChecksumBits: be.Uint32(raw[68:]), Checksum: be.Uint32(raw[72:])} + d.Tables = append(d.Tables, table) + var covered uint64 + for i := uint32(0); i < count; i++ { + p := raw[204+int(i)*40:] + kind := be.Uint32(p) + start, size, off, packed := be.Uint64(p[8:]), be.Uint64(p[16:]), be.Uint64(p[24:]), be.Uint64(p[32:]) + if kind == 0xffffffff { + if i != count-1 || size != 0 { + return nil, fmt.Errorf("udif: invalid terminator") + } + continue + } + if i == count-1 { + return nil, fmt.Errorf("udif: missing terminator") + } + if kind == 0x7ffffffe { + continue + } + if start != covered || size == 0 || size > length-covered { + return nil, fmt.Errorf("udif: noncontiguous blkx runs") + } + covered += size + switch kind { + case 0, 2: + if packed != 0 { + return nil, fmt.Errorf("udif: zero run has stored bytes") + } + case 1: + if packed != size*512 { + return nil, fmt.Errorf("udif: raw run length mismatch") + } + case 0x80000005, 0x80000006: + if size*512 > MaxChunk || packed == 0 || packed > MaxChunk { + return nil, fmt.Errorf("udif: chunk bounds exceeded") + } + default: + return nil, fmt.Errorf("udif: unsupported chunk codec %#x", kind) + } + if kind != 0 && kind != 2 && !rangeOK(off, packed, int64(dataLen-dataStart)) { + return nil, fmt.Errorf("udif: stored run outside data fork") + } + d.runs = append(d.runs, run{kind, int64(first+start) * 512, int64(size) * 512, int64(dataOff + dataStart + off), int64(packed)}) + } + if covered != length { + return nil, fmt.Errorf("udif: incomplete blkx range") + } + } + sort.Slice(d.runs, func(i, j int) bool { return d.runs[i].start < d.runs[j].start }) + var end int64 + for _, r := range d.runs { + if r.start != end { + return nil, fmt.Errorf("udif: overlapping or gapped block maps") + } + end += r.size + } + if end != d.size { + return nil, fmt.Errorf("udif: incomplete disk map") + } + return d, nil +} +func (d *Image) ReadAt(p []byte, off int64) (int, error) { + if off < 0 { + return 0, fmt.Errorf("udif: negative offset") + } + if len(p) == 0 { + return 0, nil + } + if off >= d.size { + return 0, io.EOF + } + wanted := len(p) + p = p[:min(int64(len(p)), d.size-off)] + done := 0 + i := sort.Search(len(d.runs), func(i int) bool { return d.runs[i].start+d.runs[i].size > off }) + for len(p) > 0 { + r := d.runs[i] + skip := off - r.start + n := min(int64(len(p)), r.size-skip) + switch r.kind { + case 0, 2: + clear(p[:n]) + case 1: + if _, err := io.ReadFull(io.NewSectionReader(d.source, r.offset+skip, n), p[:n]); err != nil { + return done, err + } + default: + data, err := d.cache.Get(bytecache.Key{Index: i}, func() ([]byte, error) { + packed, err := read(d.source, r.offset, r.length) + if err != nil { + return nil, err + } + input := bytes.NewReader(packed) + var decoded io.Reader + var close func() error + if r.kind == 0x80000005 { + z, err := zlib.NewReader(input) + if err != nil { + return nil, err + } + decoded = z + close = z.Close + } else { + decoded = bzip2.NewReader(input) + } + data, err := io.ReadAll(io.LimitReader(decoded, r.size+1)) + if close != nil { + close() + } + if err != nil { + return nil, err + } + if int64(len(data)) != r.size || input.Len() != 0 { + return nil, fmt.Errorf("udif: decoded chunk length/trailing data mismatch") + } + return data, nil + }) + if err != nil { + return done, fmt.Errorf("udif: chunk %d: %w", i, err) + } + copy(p[:n], data[skip:skip+n]) + } + done += int(n) + off += n + p = p[n:] + i++ + } + if done != wanted { + return done, io.EOF + } + return done, nil +} + +// Verify reads all stored bytes and logical table ranges and checks CRC32s when +// declared. It does not claim to authenticate an Apple code-signing signature. +func (d *Image) Verify() error { + check := func(r io.Reader, kind, bits, want uint32) error { + if kind == 0 { + _, err := io.Copy(io.Discard, r) + return err + } + if kind != 2 || bits != 32 { + return fmt.Errorf("udif: unsupported checksum %d/%d", kind, bits) + } + h := crc32.NewIEEE() + if _, err := io.Copy(h, r); err != nil { + return err + } + if h.Sum32() != want { + return fmt.Errorf("udif: CRC32 mismatch") + } + return nil + } + if err := check(io.NewSectionReader(d.source, d.dataOffset, d.dataLength), d.checksumType, d.checksumBits, d.checksum); err != nil { + return err + } + var sums []byte + for _, t := range d.Tables { + // BLOCK_IGNORE (2) reserves sectors but contributes no bytes to the + // BLKX checksum. It is distinct from explicit zero-fill (0). + var readers []io.Reader + first := sort.Search(len(d.runs), func(i int) bool { return d.runs[i].start >= t.Offset }) + for i := first; i < len(d.runs) && d.runs[i].start < t.Offset+t.Size; i++ { + r := d.runs[i] + if r.kind != 2 { + readers = append(readers, io.NewSectionReader(d, r.start, r.size)) + } + } + if err := check(io.MultiReader(readers...), t.ChecksumType, t.ChecksumBits, t.Checksum); err != nil { + return fmt.Errorf("udif: %s: %w", t.Name, err) + } + if t.ChecksumType == 2 { + sums = binary.BigEndian.AppendUint32(sums, t.Checksum) + } + } + trailer, err := read(d.Trailer, 352, 12) + if err != nil { + return err + } + return check(bytes.NewReader(sums), binary.BigEndian.Uint32(trailer), binary.BigEndian.Uint32(trailer[4:]), binary.BigEndian.Uint32(trailer[8:])) +} + +// XML is bounded in bytes, nodes and depth. No external entities are resolved. +type xmlNode struct { + name, text string + children []*xmlNode +} + +func (n *xmlNode) value(key string) *xmlNode { + if n != nil && n.name == "dict" { + for i := 0; i+1 < len(n.children); i += 2 { + if n.children[i].name == "key" && n.children[i].text == key { + return n.children[i+1] + } + } + } + return &xmlNode{} +} +func parseXML(data []byte) (*xmlNode, error) { + dec := xml.NewDecoder(bytes.NewReader(data)) + var root *xmlNode + var stack []*xmlNode + count := 0 + for { + tok, err := dec.Token() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + switch t := tok.(type) { + case xml.StartElement: + count++ + if len(stack) >= 64 || count > 1000000 { + return nil, fmt.Errorf("XML limits exceeded") + } + n := &xmlNode{name: t.Name.Local} + if len(stack) == 0 { + if root != nil { + return nil, fmt.Errorf("multiple XML roots") + } + root = n + } else { + parent := stack[len(stack)-1] + parent.children = append(parent.children, n) + } + stack = append(stack, n) + case xml.CharData: + if len(stack) > 0 { + stack[len(stack)-1].text += string(t) + } + case xml.EndElement: + stack = stack[:len(stack)-1] + } + } + if root == nil || len(stack) != 0 { + return nil, fmt.Errorf("incomplete XML") + } + return root, nil +} diff --git a/archive/udif/udif_test.go b/archive/udif/udif_test.go new file mode 100644 index 0000000..3b40a2d --- /dev/null +++ b/archive/udif/udif_test.go @@ -0,0 +1,164 @@ +package udif + +import ( + "bytes" + "compress/zlib" + "encoding/base64" + "encoding/binary" + "fmt" + "github.com/tinyrange/trex/auto" + "hash/crc32" + "io" + "strings" + "testing" +) + +func fixture(t *testing.T, mutate func([]byte), corrupt bool) []byte { + t.Helper() + plain := bytes.Repeat([]byte{0x57}, 512) + var packed bytes.Buffer + z := zlib.NewWriter(&packed) + z.Write(plain) + z.Close() + data := append(bytes.Repeat([]byte{0x41}, 512), packed.Bytes()...) + if corrupt { + data[len(data)-1] ^= 1 + } + be := binary.BigEndian + raw := make([]byte, 204+4*40) + copy(raw, "mish") + be.PutUint32(raw[4:], 1) + be.PutUint64(raw[16:], 3) + be.PutUint32(raw[200:], 4) + kinds := []uint32{1, 0x80000005, 2, 0xffffffff} + for i, k := range kinds { + p := raw[204+i*40:] + be.PutUint32(p, k) + be.PutUint64(p[8:], uint64(i)) + if i < 3 { + be.PutUint64(p[16:], 1) + } + } + be.PutUint64(raw[204+32:], 512) + be.PutUint64(raw[244+24:], 512) + be.PutUint64(raw[244+32:], uint64(packed.Len())) + logical := append(bytes.Repeat([]byte{0x41}, 512), plain...) + be.PutUint32(raw[64:], 2) + be.PutUint32(raw[68:], 32) + be.PutUint32(raw[72:], crc32.ChecksumIEEE(logical)) + if mutate != nil { + mutate(raw) + } + metadata := []byte(fmt.Sprintf(`resource-forkblkxNamefixtureData%s`, base64.StdEncoding.EncodeToString(raw))) + h := make([]byte, 512) + copy(h, "koly") + be.PutUint32(h[4:], 4) + be.PutUint32(h[8:], 512) + be.PutUint32(h[56:], 1) + be.PutUint32(h[60:], 1) + be.PutUint64(h[32:], uint64(len(data))) + be.PutUint64(h[216:], uint64(len(data))) + be.PutUint64(h[224:], uint64(len(metadata))) + be.PutUint64(h[492:], 3) + be.PutUint32(h[80:], 2) + be.PutUint32(h[84:], 32) + be.PutUint32(h[88:], crc32.ChecksumIEEE(data)) + return append(append(data, metadata...), h...) +} +func TestDiskRangesAndChecksums(t *testing.T) { + d, err := Open(bytes.NewReader(fixture(t, nil, false))) + if err != nil { + t.Fatal(err) + } + b := make([]byte, 40) + n, err := d.ReadAt(b, 500) + if err != nil || n != 40 || !bytes.Equal(b[:12], bytes.Repeat([]byte{0x41}, 12)) || !bytes.Equal(b[12:], bytes.Repeat([]byte{0x57}, 28)) { + t.Fatalf("cross-run read %x %d %v", b, n, err) + } + d.ReadAt(b, 1500) + if !bytes.Equal(b[:36], make([]byte, 36)) { + t.Fatal("zero-filled range") + } + if _, err = d.ReadAt(b, 1530); err != io.EOF { + t.Fatal("EOF", err) + } + if _, err = d.ReadAt(b, -1); err == nil { + t.Fatal("negative offset") + } + if err = d.Verify(); err != nil { + t.Fatal(err) + } + if d.cache.Stats().Hits == 0 { + t.Fatal("chunk not reused") + } +} +func TestRejectMalformedMaps(t *testing.T) { + for name, fn := range map[string]func([]byte){ + "overlap": func(b []byte) { binary.BigEndian.PutUint64(b[244+8:], 0) }, + "stored-bounds": func(b []byte) { binary.BigEndian.PutUint64(b[204+24:], 1<<63) }, + "run-count": func(b []byte) { binary.BigEndian.PutUint32(b[200:], 100) }, + "terminator": func(b []byte) { binary.BigEndian.PutUint32(b[324:], 2) }, + "codec": func(b []byte) { binary.BigEndian.PutUint32(b[244:], 0x80000007) }, + } { + t.Run(name, func(t *testing.T) { + if _, err := Open(bytes.NewReader(fixture(t, fn, false))); err == nil { + t.Fatal("accepted malformed map") + } + }) + } +} +func TestCorruptCompressedChunk(t *testing.T) { + d, err := Open(bytes.NewReader(fixture(t, nil, true))) + if err != nil { + t.Fatal(err) + } + b := make([]byte, 1) + if _, err := d.ReadAt(b, 512); err == nil { + t.Fatal("accepted corrupt zlib checksum") + } +} +func TestXMLDepthBound(t *testing.T) { + if _, err := parseXML([]byte(strings.Repeat("", 100) + strings.Repeat("", 100))); err == nil { + t.Fatal("accepted excessive XML depth") + } +} +func TestAutoDecodedDisk(t *testing.T) { + b := fixture(t, nil, false) + result, err := auto.Identify(bytes.NewReader(b), auto.Options{}) + if err != nil || result.Format != "udif" { + t.Fatalf("identify: %v %v", result, err) + } + if result.View.(*auto.DecodedView).Reader.Size() != 1536 { + t.Fatal("logical size") + } +} + +func TestExplicitZeroVersusIgnoreAndMasterChecksum(t *testing.T) { + var tableCRC uint32 + b := fixture(t, func(raw []byte) { + binary.BigEndian.PutUint32(raw[284:], 0) // Explicit zero-fill, not ignore. + logical := append(bytes.Repeat([]byte{0x41}, 512), bytes.Repeat([]byte{0x57}, 512)...) + logical = append(logical, make([]byte, 512)...) + tableCRC = crc32.ChecksumIEEE(logical) + binary.BigEndian.PutUint32(raw[72:], tableCRC) + }, false) + trailer := b[len(b)-512:] + binary.BigEndian.PutUint32(trailer[352:], 2) + binary.BigEndian.PutUint32(trailer[356:], 32) + binary.BigEndian.PutUint32(trailer[360:], crc32.ChecksumIEEE(binary.BigEndian.AppendUint32(nil, tableCRC))) + d, err := Open(bytes.NewReader(b)) + if err != nil { + t.Fatal(err) + } + if err := d.Verify(); err != nil { + t.Fatal(err) + } + trailer[360] ^= 1 + d, err = Open(bytes.NewReader(b)) + if err != nil { + t.Fatal(err) + } + if err := d.Verify(); err == nil { + t.Fatal("corrupt master checksum accepted") + } +} diff --git a/archive/xar/star.go b/archive/xar/star.go new file mode 100644 index 0000000..9b402bc --- /dev/null +++ b/archive/xar/star.go @@ -0,0 +1,83 @@ +package xar + +import ( + "fmt" + "github.com/tinyrange/trex/auto" + "github.com/tinyrange/trex/storage" + starfile "github.com/tinyrange/trex/storage/star" + "go.starlark.net/starlark" +) + +func init() { + auto.Register("xar", 20, func(p []byte, r storage.Reader, o auto.Options) (auto.View, error) { + if len(p) < 4 || string(p[:4]) != "xar!" { + return nil, auto.ErrNoMatch + } + a, err := Open(r, o.MaxEntries) + if err != nil { + return nil, err + } + var entries []auto.Entry + for _, e := range a.Entries { + kind := e.Kind + if kind == "hardlink" { + kind = "file" + } + entries = append(entries, auto.Entry{Name: e.Path, Kind: kind, Reader: e.Data, Attributes: map[string]any{"mode": e.Mode, "uid": e.UID, "gid": e.GID, "target": e.Target, "encoding": e.Encoding}}) + } + return auto.Tree(entries, o) + }) +} +func Builtin(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var value starlark.Value + maximum := 100000 + if err := starlark.UnpackArgs("xar", args, kwargs, "file", &value, "maximum_entries?", &maximum); err != nil { + return nil, err + } + r, ok := value.(storage.Reader) + if !ok { + return nil, fmt.Errorf("xar: expected file") + } + a, err := Open(r, maximum) + if err != nil { + return nil, err + } + var entries, paths []starlark.Value + index := map[string]starlark.Value{} + for i := range a.Entries { + e := &a.Entries[i] + var raw, data starlark.Value = starlark.None, starlark.None + var size int64 + if e.Raw != nil { + raw = starfile.NewReader(e.Path+".stored", e.Raw) + } + if e.Data != nil { + data = starfile.NewReader(e.Path, e.Data) + size = e.Data.Size() + } + verify := starlark.NewBuiltin("xar.entry.verify", func(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + if err := starlark.UnpackArgs("verify", args, kwargs); err != nil { + return nil, err + } + if err := e.Verify(); err != nil { + return nil, err + } + return starlark.None, nil + }) + v := starfile.NewRecord(starlark.StringDict{"path": starlark.String(e.Path), "id": starlark.String(e.ID), "kind": starlark.String(e.Kind), "mode": starlark.MakeUint64(e.Mode), "uid": starlark.MakeUint64(e.UID), "gid": starlark.MakeUint64(e.GID), "target": starlark.String(e.Target), "encoding": starlark.String(e.Encoding), "raw": raw, "data": data, "size": starlark.MakeInt64(size), "verify": verify}) + entries = append(entries, v) + paths = append(paths, starlark.String(e.Path)) + index[e.Path] = v + } + find := starlark.NewBuiltin("xar.find", func(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var name string + if err := starlark.UnpackArgs("find", args, kwargs, "path", &name); err != nil { + return nil, err + } + if v := index[name]; v != nil { + return v, nil + } + return starlark.None, nil + }) + return starfile.NewRecord(starlark.StringDict{"entries": starlark.NewList(entries), "files": starlark.NewList(paths), "toc": starlark.Bytes(a.TOC), "heap": starfile.NewReader("heap", a.Heap), "find": find}), nil +} diff --git a/archive/xar/verify_test.go b/archive/xar/verify_test.go new file mode 100644 index 0000000..45f8fcf --- /dev/null +++ b/archive/xar/verify_test.go @@ -0,0 +1,31 @@ +package xar + +import ( + "bytes" + "compress/zlib" + "io" + "testing" +) + +func TestEmptyZlibVerification(t *testing.T) { + for _, bad := range []bool{false, true} { + var b bytes.Buffer + z := zlib.NewWriter(&b) + z.Close() + if bad { + b.Bytes()[b.Len()-1] ^= 1 + } + e := Entry{Data: &zlibFile{source: bytes.NewReader(b.Bytes())}} + err := e.Verify() + if (err != nil) != bad { + t.Fatalf("empty zlib bad=%v: %v", bad, err) + } + } +} +func TestNoChecksumStillReadsAllBytes(t *testing.T) { + e := Entry{Data: io.NewSectionReader(bytes.NewReader([]byte("x")), 0, 3)} + // SectionReader can end early: full declared-size validation must catch it. + if err := e.Verify(); err == nil { + t.Fatal("accepted truncated checksum-free member") + } +} diff --git a/archive/xar/xar.go b/archive/xar/xar.go new file mode 100644 index 0000000..a23f756 --- /dev/null +++ b/archive/xar/xar.go @@ -0,0 +1,482 @@ +// Package xar indexes XAR v1 installer archives and exposes borrowed members. +// Format facts follow the Apple open-source xar header and XML TOC definitions. +package xar + +import ( + "bufio" + "bytes" + "compress/zlib" + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding/binary" + "encoding/hex" + "encoding/xml" + "fmt" + "hash" + "io" + "path" + "strconv" + "strings" + "sync" + + "github.com/tinyrange/trex/archive/bzip2" + "github.com/tinyrange/trex/storage" +) + +const MaxTOC = 16 << 20 +const MaxMember = 64 << 30 + +type checksum struct { + Style string `xml:"style,attr"` + Value string `xml:",chardata"` + Offset int64 `xml:"offset"` + Size int64 `xml:"size"` +} +type dataRecord struct { + Offset int64 `xml:"offset"` + Length int64 `xml:"length"` + Size int64 `xml:"size"` + Encoding struct { + Style string `xml:"style,attr"` + } `xml:"encoding"` + Archived checksum `xml:"archived-checksum"` + Extracted checksum `xml:"extracted-checksum"` +} +type fileRecord struct { + ID string `xml:"id,attr"` + Name string `xml:"name"` + Type struct { + Value string `xml:",chardata"` + Link string `xml:"link,attr"` + } `xml:"type"` + Link string `xml:"link"` + Mode string `xml:"mode"` + UID uint64 `xml:"uid"` + GID uint64 `xml:"gid"` + Data *dataRecord `xml:"data"` + Children []fileRecord `xml:"file"` +} + +// Entry retains Unix metadata; symlinks are never followed. Raw denotes the +// archived member bytes, Data the decoded byte source. Checksums are verified +// explicitly by Verify, without requiring whole-member reads during listing. +type Entry struct { + Path, ID, Kind, Target, Encoding string + Mode, UID, GID uint64 + Raw, Data storage.Reader + archived, extracted checksum +} +type Archive struct { + Entries []Entry + TOC []byte + Heap storage.Reader +} + +func sum(style string) (hash.Hash, error) { + switch strings.ToLower(style) { + case "sha1": + return sha1.New(), nil + case "md5": + return md5.New(), nil + case "sha256": + return sha256.New(), nil + case "sha512": + return sha512.New(), nil + default: + return nil, fmt.Errorf("xar: unsupported checksum %q", style) + } +} +func check(r storage.Reader, c checksum) error { + if c.Style == "" || c.Style == "none" { + n, err := io.Copy(io.Discard, io.NewSectionReader(r, 0, r.Size())) + if err == nil && n != r.Size() { + err = io.ErrUnexpectedEOF + } + return err + } + h, err := sum(c.Style) + if err != nil { + return err + } + expected, err := hex.DecodeString(strings.TrimSpace(c.Value)) + if err != nil || len(expected) != h.Size() { + return fmt.Errorf("xar: invalid checksum value") + } + n, err := io.Copy(h, io.NewSectionReader(r, 0, r.Size())) + if err != nil { + return err + } + if n != r.Size() { + return io.ErrUnexpectedEOF + } + if !bytes.Equal(expected, h.Sum(nil)) { + return fmt.Errorf("xar: %s checksum mismatch", c.Style) + } + return nil +} +func (e *Entry) Verify() error { + if e.Raw != nil { + if err := check(e.Raw, e.archived); err != nil { + return fmt.Errorf("xar: %s archived: %w", e.Path, err) + } + } + if e.Data != nil { + if stream, ok := e.Data.(interface{ validateEmpty() error }); ok && e.Data.Size() == 0 { + if err := stream.validateEmpty(); err != nil { + return err + } + } + if err := check(e.Data, e.extracted); err != nil { + return fmt.Errorf("xar: %s extracted: %w", e.Path, err) + } + } + return nil +} +func Open(source storage.Reader, maximumEntries int) (*Archive, error) { + if source == nil || source.Size() < 28 || maximumEntries <= 0 { + return nil, fmt.Errorf("xar: invalid source/entry limit") + } + var h [28]byte + if _, err := io.ReadFull(io.NewSectionReader(source, 0, 28), h[:]); err != nil { + return nil, err + } + be := binary.BigEndian + header := int64(be.Uint16(h[4:])) + packed := be.Uint64(h[8:]) + expanded := be.Uint64(h[16:]) + algorithm := be.Uint32(h[24:]) + if string(h[:4]) != "xar!" || header < 28 || be.Uint16(h[6:]) != 1 || packed == 0 || packed > MaxTOC || expanded == 0 || expanded > MaxTOC || header > source.Size() || packed > uint64(source.Size()-header) { + return nil, fmt.Errorf("xar: invalid v1 header/TOC bounds") + } + compressed := make([]byte, packed) + if _, err := io.ReadFull(io.NewSectionReader(source, header, int64(packed)), compressed); err != nil { + return nil, err + } + input := bytes.NewReader(compressed) + z, err := zlib.NewReader(input) + if err != nil { + return nil, err + } + toc, err := io.ReadAll(io.LimitReader(z, int64(expanded)+1)) + z.Close() + if err != nil { + return nil, err + } + if uint64(len(toc)) != expanded || input.Len() != 0 { + return nil, fmt.Errorf("xar: TOC length/trailing bytes mismatch") + } + // Bound recursive XML decoding before constructing records. Entity expansion + // and external resources are not enabled by encoding/xml. + dec := xml.NewDecoder(bytes.NewReader(toc)) + depth, nodes := 0, 0 + for { + tok, err := dec.Token() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + switch tok.(type) { + case xml.StartElement: + depth++ + nodes++ + if depth > 128 || nodes > 1000000 { + return nil, fmt.Errorf("xar: XML limits exceeded") + } + case xml.EndElement: + depth-- + } + } + var doc struct { + XMLName xml.Name `xml:"xar"` + TOC struct { + Checksum checksum `xml:"checksum"` + Files []fileRecord `xml:"file"` + } `xml:"toc"` + } + if err := xml.Unmarshal(toc, &doc); err != nil { + return nil, fmt.Errorf("xar: XML: %w", err) + } + heapOff := header + int64(packed) + heap := io.NewSectionReader(source, heapOff, source.Size()-heapOff) + c := doc.TOC.Checksum + if algorithm != 0 { + style := "" + switch algorithm { + case 1: + style = "sha1" + case 2: + style = "md5" + default: + return nil, fmt.Errorf("xar: unsupported TOC checksum algorithm %d", algorithm) + } + h, err := sum(style) + if err != nil { + return nil, err + } + if c.Style != style || c.Offset < 0 || c.Size != int64(h.Size()) || c.Offset > heap.Size() || c.Size > heap.Size()-c.Offset { + return nil, fmt.Errorf("xar: TOC checksum bounds/type") + } + expected := make([]byte, h.Size()) + if _, err = heap.ReadAt(expected, c.Offset); err != nil { + return nil, err + } + h.Write(compressed) + if !bytes.Equal(h.Sum(nil), expected) { + return nil, fmt.Errorf("xar: TOC checksum mismatch") + } + } else if c.Style != "" && c.Style != "none" { + return nil, fmt.Errorf("xar: unexpected TOC checksum") + } + archive := &Archive{TOC: toc, Heap: heap} + ids := map[string]int{} + names := map[string]bool{} + var walk func([]fileRecord, string) error + walk = func(files []fileRecord, base string) error { + for _, f := range files { + if len(archive.Entries) >= maximumEntries { + return fmt.Errorf("xar: entry limit exceeded") + } + if f.Name == "" || f.Name == "." || f.Name == ".." || strings.ContainsAny(f.Name, "/\x00") { + return fmt.Errorf("xar: unsafe component %q", f.Name) + } + name := path.Join(base, f.Name) + if names[name] || f.ID == "" { + return fmt.Errorf("xar: duplicate path/missing ID") + } + names[name] = true + if _, exists := ids[f.ID]; exists { + return fmt.Errorf("xar: duplicate ID") + } + e := Entry{Path: name, ID: f.ID, Kind: f.Type.Value, Target: f.Link, UID: f.UID, GID: f.GID} + if f.Mode != "" { + v, err := strconv.ParseUint(f.Mode, 8, 32) + if err != nil { + return fmt.Errorf("xar: invalid mode") + } + e.Mode = v + } + if f.Data != nil { + d := f.Data + if d.Offset < 0 || d.Length < 0 || d.Size < 0 || d.Size > MaxMember || d.Offset > heap.Size() || d.Length > heap.Size()-d.Offset { + return fmt.Errorf("xar: %s data bounds", name) + } + e.Raw = io.NewSectionReader(heap, d.Offset, d.Length) + e.Encoding = d.Encoding.Style + e.archived = d.Archived + e.extracted = d.Extracted + switch e.Encoding { + case "", "application/octet-stream": + if d.Length != d.Size { + return fmt.Errorf("xar: stored length mismatch") + } + e.Data = e.Raw + case "application/x-gzip": + e.Data = &zlibFile{source: e.Raw, size: d.Size, offset: -1} + case "application/x-bzip2": + e.Data = &sizedReader{Reader: bzip2.NewReader(e.Raw, d.Size), size: d.Size} + default: + return fmt.Errorf("xar: unsupported encoding %q", e.Encoding) + } + } + switch e.Kind { + case "file": + if e.Data == nil { + return fmt.Errorf("xar: missing file data") + } + case "directory": + if e.Data != nil { + return fmt.Errorf("xar: directory has data") + } + case "symlink": + if e.Data != nil { + return fmt.Errorf("xar: symlink has data") + } + case "hardlink": + e.Target = f.Type.Link + default: + return fmt.Errorf("xar: unsupported entry type %q", e.Kind) + } + ids[e.ID] = len(archive.Entries) + archive.Entries = append(archive.Entries, e) + if len(f.Children) > 0 { + if e.Kind != "directory" { + return fmt.Errorf("xar: children under non-directory") + } + if err := walk(f.Children, name); err != nil { + return err + } + } + } + return nil + } + if err := walk(doc.TOC.Files, ""); err != nil { + return nil, err + } + for i := range archive.Entries { + e := &archive.Entries[i] + if e.Kind == "hardlink" { + index, ok := ids[e.Target] + if !ok || archive.Entries[index].Kind != "file" { + return nil, fmt.Errorf("xar: unresolved hardlink") + } + target := archive.Entries[index] + e.Data = target.Data + e.Raw = target.Raw + e.Target = target.Path + e.archived = target.archived + e.extracted = target.extracted + } + } + return archive, nil +} + +type sizedReader struct { + storage.Reader + size int64 +} + +func (s *sizedReader) Size() int64 { return s.size } +func (s *sizedReader) validateEmpty() error { + var extra [1]byte + n, err := s.Reader.ReadAt(extra[:], s.size) + if n != 0 || err != io.EOF || s.Reader.Size() != s.size { + return fmt.Errorf("xar: bzip2 declared size mismatch: %v", err) + } + return nil +} +func (s *sizedReader) ReadAt(p []byte, off int64) (int, error) { + if off < 0 { + return 0, fmt.Errorf("xar: negative decoded offset") + } + if len(p) == 0 { + return 0, nil + } + if off >= s.size { + return 0, io.EOF + } + wanted := len(p) + p = p[:min(int64(len(p)), s.size-off)] + n, err := s.Reader.ReadAt(p, off) + if err != nil && err != io.EOF { + return n, err + } + if n != len(p) { + return n, io.ErrUnexpectedEOF + } + if off+int64(n) == s.size { + if err := s.validateEmpty(); err != nil { + return n, err + } + } + if n < wanted { + return n, io.EOF + } + return n, nil +} + +// zlibFile uses declared size with a bounded look-behind window. Listing is +// metadata-only, forward scans reuse decoder state, backwards reads replay. +type zlibFile struct { + source storage.Reader + size int64 + mu sync.Mutex + input *bufio.Reader + decoder io.ReadCloser + offset int64 + cache []byte + cacheOffset int64 +} + +func (f *zlibFile) Size() int64 { return f.size } +func (f *zlibFile) validateEmpty() error { + f.mu.Lock() + defer f.mu.Unlock() + if err := f.reset(); err != nil { + return err + } + return f.finish() +} +func (f *zlibFile) finish() error { + var extra [1]byte + n, err := f.decoder.Read(extra[:]) + if n != 0 || err != io.EOF { + return fmt.Errorf("xar: zlib size/checksum mismatch: %v", err) + } + if _, err := f.input.ReadByte(); err != io.EOF { + return fmt.Errorf("xar: trailing compressed data") + } + return nil +} +func (f *zlibFile) reset() error { + if f.decoder != nil { + f.decoder.Close() + } + f.input = bufio.NewReader(io.NewSectionReader(f.source, 0, f.source.Size())) + z, err := zlib.NewReader(f.input) + if err != nil { + return err + } + f.decoder = z + f.offset = 0 + f.cache = f.cache[:0] + return nil +} +func (f *zlibFile) ReadAt(p []byte, off int64) (int, error) { + f.mu.Lock() + defer f.mu.Unlock() + if off < 0 { + return 0, fmt.Errorf("xar: negative decoded offset") + } + if len(p) == 0 { + return 0, nil + } + if off >= f.size { + return 0, io.EOF + } + wanted := len(p) + p = p[:min(int64(len(p)), f.size-off)] + if off >= f.cacheOffset && off+int64(len(p)) <= f.cacheOffset+int64(len(f.cache)) { + copy(p, f.cache[off-f.cacheOffset:]) + if len(p) < wanted { + return len(p), io.EOF + } + return len(p), nil + } + if f.decoder == nil || off < f.offset { + if err := f.reset(); err != nil { + return 0, err + } + } + if off > f.offset { + n, err := io.CopyN(io.Discard, f.decoder, off-f.offset) + f.offset += n + if err != nil { + return 0, err + } + } + n, err := io.ReadFull(f.decoder, p) + f.offset += int64(n) + if err != nil { + return n, fmt.Errorf("xar: truncated zlib member: %w", err) + } + if f.offset == f.size { + if err := f.finish(); err != nil { + return n, err + } + } + if len(p) <= 1<<20 { + f.cache = append(f.cache[:0], p...) + f.cacheOffset = off + } else { + f.cache = append(f.cache[:0], p[len(p)-(1<<20):]...) + f.cacheOffset = f.offset - int64(len(f.cache)) + } + if n < wanted { + return n, io.EOF + } + return n, nil +} diff --git a/archive/xar/xar_test.go b/archive/xar/xar_test.go new file mode 100644 index 0000000..46e864f --- /dev/null +++ b/archive/xar/xar_test.go @@ -0,0 +1,131 @@ +package xar + +import ( + "bytes" + "compress/zlib" + "crypto/sha1" + "encoding/binary" + "encoding/hex" + "fmt" + "github.com/tinyrange/trex/auto" + "io" + "strings" + "testing" +) + +func fixture(t *testing.T, memberSize int, modify func(string) string) ([]byte, []byte) { + t.Helper() + plain := bytes.Repeat([]byte("member-data\n"), memberSize) + var data bytes.Buffer + z := zlib.NewWriter(&data) + z.Write(plain) + z.Close() + stored := sha1.Sum(data.Bytes()) + expanded := sha1.Sum(plain) + toc := fmt.Sprintf(`020dirdirectory0755memberfile06445012020%d%d%s%saliashardlinklinksymlinkmember`, data.Len(), len(plain), hex.EncodeToString(stored[:]), hex.EncodeToString(expanded[:])) + if modify != nil { + toc = modify(toc) + } + var packed bytes.Buffer + z = zlib.NewWriter(&packed) + z.Write([]byte(toc)) + z.Close() + digest := sha1.Sum(packed.Bytes()) + h := make([]byte, 28) + copy(h, "xar!") + binary.BigEndian.PutUint16(h[4:], 28) + binary.BigEndian.PutUint16(h[6:], 1) + binary.BigEndian.PutUint64(h[8:], uint64(packed.Len())) + binary.BigEndian.PutUint64(h[16:], uint64(len(toc))) + binary.BigEndian.PutUint32(h[24:], 1) + b := append(h, packed.Bytes()...) + b = append(b, digest[:]...) + b = append(b, data.Bytes()...) + return b, plain +} +func TestMembersLinksAndChecksums(t *testing.T) { + b, plain := fixture(t, 150000, nil) + a, err := Open(bytes.NewReader(b), 100) + if err != nil { + t.Fatal(err) + } + e := a.Entries[1] + if e.Path != "dir/member" || e.UID != 501 || e.Mode != 0644 { + t.Fatalf("metadata %+v", e) + } + for _, off := range []int64{int64(len(plain) - 8), 100, 110, 0} { + var p [8]byte + if _, err := e.Data.ReadAt(p[:], off); err != nil || !bytes.Equal(p[:], plain[off:off+8]) { + t.Fatalf("read %d %x %v", off, p, err) + } + } + if a.Entries[2].Data != e.Data || a.Entries[2].Target != e.Path || a.Entries[3].Target != "member" { + t.Fatal("link metadata") + } + if err := e.Verify(); err != nil { + t.Fatal(err) + } + result, err := auto.Identify(bytes.NewReader(b), auto.Options{}) + if err != nil || result.Format != "xar" { + t.Fatalf("auto %v %v", result, err) + } +} +func TestRejectUnsafeMetadataAndLimits(t *testing.T) { + for _, change := range []func(string) string{ + func(s string) string { return strings.Replace(s, "member", "../escape", 1) }, + func(s string) string { + return strings.Replace(s, "20", "999999999999", 1) + }, + func(s string) string { return strings.Replace(s, `id="3"`, `id="2"`, 1) }, + func(s string) string { return strings.Replace(s, `link="2"`, `link="3"`, 1) }, + } { + b, _ := fixture(t, 10, change) + if _, err := Open(bytes.NewReader(b), 100); err == nil { + t.Fatal("accepted malformed TOC") + } + } + b, _ := fixture(t, 10, nil) + if _, err := Open(bytes.NewReader(b), 1); err == nil { + t.Fatal("entry cap") + } + b[len(b)-1] ^= 1 + a, err := Open(bytes.NewReader(b), 100) + if err != nil { + t.Fatal(err) + } + if err := a.Entries[1].Verify(); err == nil { + t.Fatal("accepted corrupt member") + } +} +func TestRejectTOCChecksum(t *testing.T) { + b, _ := fixture(t, 1, nil) + offset := 28 + int(binary.BigEndian.Uint64(b[8:])) + b[offset] ^= 1 + if _, err := Open(bytes.NewReader(b), 100); err == nil { + t.Fatal("accepted corrupt TOC checksum") + } +} +func TestZlibSizeAndTrailingValidation(t *testing.T) { + for _, kind := range []string{"short", "long", "trailing"} { + t.Run(kind, func(t *testing.T) { + var b bytes.Buffer + z := zlib.NewWriter(&b) + z.Write([]byte("abcd")) + z.Close() + size := int64(4) + if kind == "short" { + size = 3 + } + if kind == "long" { + size = 5 + } + if kind == "trailing" { + b.WriteByte(7) + } + f := &zlibFile{source: bytes.NewReader(b.Bytes()), size: size} + if _, err := io.ReadAll(io.NewSectionReader(f, 0, size)); err == nil { + t.Fatal("accepted zlib size/trailing mismatch") + } + }) + } +} diff --git a/auto/imports/imports.go b/auto/imports/imports.go index bc48df2..43ef9af 100644 --- a/auto/imports/imports.go +++ b/auto/imports/imports.go @@ -17,10 +17,13 @@ import ( _ "github.com/tinyrange/trex/archive/gzip" _ "github.com/tinyrange/trex/archive/hunk" _ "github.com/tinyrange/trex/archive/ibmisave" + _ "github.com/tinyrange/trex/archive/im4p" _ "github.com/tinyrange/trex/archive/irix" _ "github.com/tinyrange/trex/archive/kwaj" _ "github.com/tinyrange/trex/archive/lha" + _ "github.com/tinyrange/trex/archive/lzfse" _ "github.com/tinyrange/trex/archive/macresource" + _ "github.com/tinyrange/trex/archive/pbzx" _ "github.com/tinyrange/trex/archive/rar" _ "github.com/tinyrange/trex/archive/sevenzip" _ "github.com/tinyrange/trex/archive/sfp" @@ -28,8 +31,10 @@ import ( _ "github.com/tinyrange/trex/archive/szdd" _ "github.com/tinyrange/trex/archive/tar" _ "github.com/tinyrange/trex/archive/tome" + _ "github.com/tinyrange/trex/archive/udif" _ "github.com/tinyrange/trex/archive/vmsbackup" _ "github.com/tinyrange/trex/archive/wim" + _ "github.com/tinyrange/trex/archive/xar" _ "github.com/tinyrange/trex/archive/xz" _ "github.com/tinyrange/trex/archive/zip" _ "github.com/tinyrange/trex/archive/zstd" diff --git a/binary/appledouble/appledouble.go b/binary/appledouble/appledouble.go new file mode 100644 index 0000000..ccede29 --- /dev/null +++ b/binary/appledouble/appledouble.go @@ -0,0 +1,83 @@ +// Package appledouble reads RFC1740 version2 AppleDouble fork containers. +// Entry payloads remain borrowed views. Unknown IDs are retained, not discarded. +package appledouble + +import ( + "encoding/binary" + "fmt" + "github.com/tinyrange/trex/storage" + "io" +) + +type Entry struct { + ID uint32 + Data storage.Reader +} +type part struct { + storage.Reader + off, size int64 +} + +func (p part) Size() int64 { return p.size } +func (p part) ReadAt(b []byte, off int64) (int, error) { + if off < 0 { + return 0, fmt.Errorf("appledouble: negative offset") + } + if len(b) == 0 { + return 0, nil + } + if off >= p.size { + return 0, io.EOF + } + count := len(b) + if int64(count) > p.size-off { + b = b[:p.size-off] + } + n, err := p.Reader.ReadAt(b, p.off+off) + if n < count && err == nil { + err = io.EOF + } + return n, err +} +func Open(r storage.Reader) ([]Entry, error) { + if r == nil || r.Size() < 26 { + return nil, fmt.Errorf("appledouble: truncated header") + } + h := make([]byte, 26) + if _, err := io.ReadFull(io.NewSectionReader(r, 0, 26), h); err != nil { + return nil, err + } + be := binary.BigEndian + if be.Uint32(h) != 0x00051607 || be.Uint32(h[4:]) != 0x00020000 { + return nil, fmt.Errorf("appledouble: expected version2 AppleDouble") + } + count := int(be.Uint16(h[24:])) + if count > 256 || int64(26+count*12) > r.Size() { + return nil, fmt.Errorf("appledouble: entry table bounds") + } + table := make([]byte, count*12) + if _, err := io.ReadFull(io.NewSectionReader(r, 26, int64(len(table))), table); err != nil { + return nil, err + } + type span struct{ off, end uint64 } + var spans []span + ids := map[uint32]bool{} + var entries []Entry + for i := 0; i < count; i++ { + e := table[i*12:] + id := be.Uint32(e) + off, size := uint64(be.Uint32(e[4:])), uint64(be.Uint32(e[8:])) + if ids[id] || off < uint64(26+len(table)) || off > uint64(r.Size()) || size > uint64(r.Size())-off { + return nil, fmt.Errorf("appledouble: invalid/duplicate extent") + } + for _, s := range spans { + if size > 0 && off < s.end && s.off < off+size { + return nil, fmt.Errorf("appledouble: overlapping entries") + } + } + ids[id] = true + spans = append(spans, span{off, off + size}) + entries = append(entries, Entry{id, part{r, int64(off), int64(size)}}) + } + return entries, nil +} diff --git a/binary/appledouble/appledouble_test.go b/binary/appledouble/appledouble_test.go new file mode 100644 index 0000000..c685a62 --- /dev/null +++ b/binary/appledouble/appledouble_test.go @@ -0,0 +1,45 @@ +package appledouble + +import ( + "bytes" + "encoding/binary" + "testing" +) + +type source []byte + +func (s source) Size() int64 { return int64(len(s)) } +func (s source) ReadAt(b []byte, off int64) (int, error) { return bytes.NewReader(s).ReadAt(b, off) } +func TestForksRemainSeparateAndBorrowed(t *testing.T) { + b := make(source, 86) + be := binary.BigEndian + be.PutUint32(b, 0x51607) + be.PutUint32(b[4:], 0x20000) + be.PutUint16(b[24:], 2) + be.PutUint32(b[26:], 9) + be.PutUint32(b[30:], 50) + be.PutUint32(b[34:], 32) + be.PutUint32(b[38:], 2) + be.PutUint32(b[42:], 82) + be.PutUint32(b[46:], 4) + copy(b[82:], "fork") + entries, err := Open(b) + if err != nil || len(entries) != 2 { + t.Fatalf("%v %v", entries, err) + } + b[82] = 'F' + got := make([]byte, 4) + entries[1].Data.ReadAt(got, 0) + if string(got) != "Fork" { + t.Fatal("copied resource fork") + } + be.PutUint32(b[42:], 81) + if _, err := Open(b); err == nil { + t.Fatal("accepted overlap") + } + be.PutUint32(b[42:], 82) + be.PutUint32(b[38:], 9) + if _, err := Open(b); err == nil { + t.Fatal("accepted duplicate ID") + } +} diff --git a/binary/macho/macho.go b/binary/macho/macho.go new file mode 100644 index 0000000..2c2febb --- /dev/null +++ b/binary/macho/macho.go @@ -0,0 +1,247 @@ +// Package macho provides bounded, read-only inspection of 64-bit Mach-O images. +// Segment contents are borrowed storage views; this is not a linker or loader. +package macho + +import ( + "bytes" + "encoding/binary" + "fmt" + "github.com/tinyrange/trex/storage" + "io" +) + +const AMD64 uint32 = 0x01000007 +const maxCommands = 1 << 20 +const maxSymbols = 1 << 20 +const maxStrings = 64 << 20 + +type Segment struct { + Name string + Address, Size, Offset, FileSize uint64 + Flags uint32 + Data storage.Reader +} +type Symbol struct { + Name string + Address uint64 + Type, Section byte + Description uint16 +} +type Image struct { + CPU, Type uint32 + Source storage.Reader + Segments []Segment + Symbols []Symbol +} +type view struct { + storage.Reader + offset, size int64 +} + +func (v view) Size() int64 { return v.size } +func (v view) ReadAt(b []byte, off int64) (int, error) { + if off < 0 { + return 0, fmt.Errorf("macho: negative read offset") + } + if off >= v.size { + return 0, io.EOF + } + size := len(b) + if int64(size) > v.size-off { + b = b[:v.size-off] + } + n, err := v.Reader.ReadAt(b, v.offset+off) + if n < size && err == nil { + err = io.EOF + } + return n, err +} +func read(r storage.Reader, off, size uint64) ([]byte, error) { + if off > uint64(r.Size()) || size > uint64(r.Size())-off { + return nil, fmt.Errorf("macho: range outside image") + } + b := make([]byte, size) + _, err := io.ReadFull(io.NewSectionReader(r, int64(off), int64(size)), b) + return b, err +} +func span(r storage.Reader, off, size uint64) (storage.Reader, error) { + if off > uint64(r.Size()) || size > uint64(r.Size())-off { + return nil, fmt.Errorf("macho: range outside image") + } + return view{r, int64(off), int64(size)}, nil +} + +// Open selects the requested CPU from a universal image, or validates a thin +// image of that CPU. It accepts little-endian 64-bit images and fat32/fat64 +// big-endian containers. Load commands and symbol allocations are bounded. +func Open(r storage.Reader, cpu uint32) (*Image, error) { + if r == nil || r.Size() < 32 { + return nil, fmt.Errorf("macho: truncated header") + } + h, err := read(r, 0, 32) + if err != nil { + return nil, err + } + magic := binary.BigEndian.Uint32(h) + if magic == 0xcafebabe || magic == 0xcafebabf { + count := binary.BigEndian.Uint32(h[4:]) + width := uint64(20) + if magic == 0xcafebabf { + width = 32 + } + if count == 0 || count > 64 { + return nil, fmt.Errorf("macho: invalid architecture count") + } + table, err := read(r, 8, uint64(count)*width) + if err != nil { + return nil, err + } + var selected storage.Reader + for i := uint32(0); i < count; i++ { + a := table[uint64(i)*width:] + off, size := uint64(binary.BigEndian.Uint32(a[8:])), uint64(binary.BigEndian.Uint32(a[12:])) + align := binary.BigEndian.Uint32(a[16:]) + if width == 32 { + off, size = binary.BigEndian.Uint64(a[8:]), binary.BigEndian.Uint64(a[16:]) + align = binary.BigEndian.Uint32(a[24:]) + } + if align > 63 || off < 8+uint64(count)*width || off&((uint64(1)< 4096 || size > maxCommands || uint64(count)*8 > uint64(size) { + return nil, fmt.Errorf("macho: invalid load command bounds") + } + commands, err := read(r, 32, uint64(size)) + if err != nil { + return nil, err + } + image := &Image{CPU: cpu, Type: u32(h[12:]), Source: r} + var symtab []byte + for i := uint32(0); i < count; i++ { + if len(commands) < 8 { + return nil, fmt.Errorf("macho: truncated command") + } + kind, length := u32(commands), u32(commands[4:]) + if length < 8 || length%8 != 0 || uint64(length) > uint64(len(commands)) { + return nil, fmt.Errorf("macho: invalid command length") + } + c := commands[:length] + commands = commands[length:] + switch kind { + case 0x19: + if length < 72 || uint64(u32(c[64:]))*80+72 != uint64(length) { + return nil, fmt.Errorf("macho: invalid segment command") + } + s := Segment{Name: string(bytes.TrimRight(c[8:24], "\x00")), Address: u64(c[24:]), Size: u64(c[32:]), Offset: u64(c[40:]), FileSize: u64(c[48:]), Flags: u32(c[68:])} + if s.Size > ^uint64(0)-s.Address { + return nil, fmt.Errorf("macho: wrapping virtual segment") + } + // Zero-VM-size metadata segments (e.g. __CTF) may still have file data. + s.Data, err = span(r, s.Offset, s.FileSize) + if err != nil { + return nil, err + } + image.Segments = append(image.Segments, s) + case 2: + if length != 24 || symtab != nil { + return nil, fmt.Errorf("macho: invalid symbol command") + } + symtab = c + } + } + if len(commands) != 0 { + return nil, fmt.Errorf("macho: unconsumed load commands") + } + if symtab != nil { + count, stringsSize := u32(symtab[12:]), u32(symtab[20:]) + if count > maxSymbols || stringsSize > maxStrings { + return nil, fmt.Errorf("macho: symbol resource limit") + } + symbols, err := read(r, uint64(u32(symtab[8:])), uint64(count)*16) + if err != nil { + return nil, err + } + strings, err := read(r, uint64(u32(symtab[16:])), uint64(stringsSize)) + if err != nil { + return nil, err + } + // Duplicate string indices share a single name allocation. Distinct + // overlapping suffixes must not amplify a bounded table into unbounded + // copied names (or repeatedly scan a huge duplicate name). + names := map[uint32]string{} + nameBytes := 0 + for i := uint32(0); i < count; i++ { + s := symbols[i*16 : (i+1)*16] + index := u32(s) + if index >= stringsSize { + return nil, fmt.Errorf("macho: symbol name outside string table") + } + name, exists := names[index] + if !exists { + end := bytes.IndexByte(strings[index:], 0) + if end < 0 { + return nil, fmt.Errorf("macho: unterminated symbol name") + } + if end > maxStrings-nameBytes { + return nil, fmt.Errorf("macho: aggregate symbol name resource limit") + } + nameBytes += end + name = string(strings[index : uint64(index)+uint64(end)]) + names[index] = name + } + image.Symbols = append(image.Symbols, Symbol{name, u64(s[8:]), s[4], s[5], binary.LittleEndian.Uint16(s[6:])}) + } + } + return image, nil +} + +// At returns a borrowed view of file-backed virtual bytes. It does not fabricate +// zero-fill contents, perform relocations, or resolve overlapping segments. +func (i *Image) At(address, size uint64) (storage.Reader, error) { + if size > ^uint64(0)-address { + return nil, fmt.Errorf("macho: wrapping address range") + } + var found storage.Reader + for _, s := range i.Segments { + if address < s.Address || address-s.Address > s.FileSize || size > s.FileSize-(address-s.Address) { + continue + } + if found != nil { + return nil, fmt.Errorf("macho: ambiguous virtual range") + } + var err error + found, err = span(s.Data, address-s.Address, size) + if err != nil { + return nil, err + } + } + if found == nil { + return nil, fmt.Errorf("macho: range not file-backed") + } + return found, nil +} diff --git a/binary/macho/macho_test.go b/binary/macho/macho_test.go new file mode 100644 index 0000000..13af6ea --- /dev/null +++ b/binary/macho/macho_test.go @@ -0,0 +1,118 @@ +package macho + +import ( + "bytes" + "debug/macho" + "encoding/binary" + "io" + "testing" +) + +type source []byte + +func (s source) Size() int64 { return int64(len(s)) } +func (s source) ReadAt(b []byte, off int64) (int, error) { return bytes.NewReader(s).ReadAt(b, off) } +func fixture() source { + b := make(source, 160) + u32, u64 := binary.LittleEndian.PutUint32, binary.LittleEndian.PutUint64 + u32(b, 0xfeedfacf) + u32(b[4:], AMD64) + u32(b[12:], 11) + u32(b[16:], 2) + u32(b[20:], 96) + c := b[32:] + u32(c, 0x19) + u32(c[4:], 72) + copy(c[8:], "__TEXT") + u64(c[24:], 0x1000) + u64(c[32:], 160) + u64(c[48:], 160) + u32(c[68:], 8) + c = b[104:] + u32(c, 2) + u32(c[4:], 24) + u32(c[8:], 128) + u32(c[12:], 1) + u32(c[16:], 144) + u32(c[20:], 16) + c = b[128:] + u32(c, 1) + c[4] = 0xf + c[5] = 1 + u64(c[8:], 0x1008) + copy(b[145:], "_probe") + return b +} +func TestThinSymbolsAndBorrowedVirtualBytes(t *testing.T) { + b := fixture() + image, err := Open(b, AMD64) + if err != nil { + t.Fatal(err) + } + native, err := macho.NewFile(bytes.NewReader(b)) + if err != nil { + t.Fatal(err) + } + if len(image.Symbols) != 1 || image.Symbols[0].Name != native.Symtab.Syms[0].Name || image.Symbols[0].Address != native.Symtab.Syms[0].Value { + t.Fatalf("symbols %+v", image.Symbols) + } + if image.Segments[0].Flags != native.Segment("__TEXT").Flag { + t.Fatal("protection flags differ from independent reader") + } + r, err := image.At(0x1091, 6) + if err != nil { + t.Fatal(err) + } + b[145] = 'X' + got, err := io.ReadAll(io.NewSectionReader(r, 0, r.Size())) + if err != nil || string(got) != "Xprobe" { + t.Fatalf("borrowed bytes %q %v", got, err) + } + for _, v := range [][2]uint64{{0xfff, 1}, {0x109f, 2}, {^uint64(0), 2}} { + if _, err := image.At(v[0], v[1]); err == nil { + t.Fatalf("accepted virtual range %x", v) + } + } +} +func TestFatSelectionAndInvalidMetadata(t *testing.T) { + thin := fixture() + fat := make(source, 4096+len(thin)) + binary.BigEndian.PutUint32(fat, 0xcafebabe) + binary.BigEndian.PutUint32(fat[4:], 1) + a := fat[8:] + binary.BigEndian.PutUint32(a, AMD64) + binary.BigEndian.PutUint32(a[8:], 4096) + binary.BigEndian.PutUint32(a[12:], uint32(len(thin))) + binary.BigEndian.PutUint32(a[16:], 12) + copy(fat[4096:], thin) + if _, err := Open(fat, AMD64); err != nil { + t.Fatal(err) + } + if _, err := Open(fat, 0x0100000c); err == nil { + t.Fatal("selected missing architecture") + } + for name, edit := range map[string]func(source){ + "commands": func(b source) { binary.LittleEndian.PutUint32(b[20:], maxCommands+1) }, + "symbols": func(b source) { binary.LittleEndian.PutUint32(b[116:], maxSymbols+1) }, + "string-index": func(b source) { binary.LittleEndian.PutUint32(b[128:], 16) }, + "unterminated": func(b source) { + for i := 145; i < len(b); i++ { + b[i] = 'x' + } + }, + "extent": func(b source) { binary.LittleEndian.PutUint64(b[80:], 161) }, + "wrapping": func(b source) { binary.LittleEndian.PutUint64(b[56:], ^uint64(0)) }, + } { + t.Run(name, func(t *testing.T) { + b := fixture() + edit(b) + if _, err := Open(b, AMD64); err == nil { + t.Fatal("accepted malformed image") + } + }) + } + binary.BigEndian.PutUint32(fat[20:], uint32(len(fat))) + if _, err := Open(fat, AMD64); err == nil { + t.Fatal("accepted fat extent past EOF") + } +} diff --git a/binary/macho/names_test.go b/binary/macho/names_test.go new file mode 100644 index 0000000..2451142 --- /dev/null +++ b/binary/macho/names_test.go @@ -0,0 +1,42 @@ +package macho + +import ( + "encoding/binary" + "strings" + "testing" +) + +func manyNames(count int, distinct bool) source { + const stringSize = 256 << 10 + off := 56 + count*16 + b := make(source, off+stringSize) + p := binary.LittleEndian.PutUint32 + p(b, 0xfeedfacf) + p(b[4:], AMD64) + p(b[16:], 1) + p(b[20:], 24) + p(b[32:], 2) + p(b[36:], 24) + p(b[40:], 56) + p(b[44:], uint32(count)) + p(b[48:], uint32(off)) + p(b[52:], stringSize) + for i := 0; i < count; i++ { + if distinct { + p(b[56+i*16:], uint32(i)) + } + } + for i := off; i < len(b)-1; i++ { + b[i] = 'a' + } + return b +} +func TestSymbolNamesBoundAmplification(t *testing.T) { + if _, err := Open(manyNames(300, true), AMD64); err == nil || !strings.Contains(err.Error(), "aggregate") { + t.Fatalf("overlapping name allocation not bounded: %v", err) + } + image, err := Open(manyNames(4096, false), AMD64) + if err != nil || len(image.Symbols) != 4096 || len(image.Symbols[4095].Name) != (256<<10)-1 { + t.Fatalf("duplicate string indices must be shared: %v", err) + } +} diff --git a/binary/plist/plist.go b/binary/plist/plist.go new file mode 100644 index 0000000..d5fb001 --- /dev/null +++ b/binary/plist/plist.go @@ -0,0 +1,601 @@ +// Package plist reads bounded Apple XML 1.0 and binary bplist00 property lists. +// Binary markers/trailer semantics follow Apple's CF-635.21 CFBinaryPList.c; +// this is an independent implementation, not a copy of that source. +package plist + +import ( + "bytes" + "encoding/base64" + "encoding/binary" + "encoding/xml" + "fmt" + "io" + "math" + "sort" + "strconv" + "strings" + "time" + "unicode/utf16" + "unicode/utf8" + + "github.com/tinyrange/trex/storage" +) + +const MaxBytes = 64 << 20 +const maxNodes = 100000 +const maxDepth = 128 + +type Date struct{ time.Time } +type budget struct{ nodes, bytes int } + +func (b *budget) use(depth, size int) error { + b.nodes++ + b.bytes += size + if depth > maxDepth || b.nodes > maxNodes || b.bytes > MaxBytes { + return fmt.Errorf("plist: decoded value limit") + } + return nil +} + +// Open returns maps with string keys, []any arrays, []byte data, strings, +// int64, float64, bool, Date, or nil (binary null only). It never follows URLs. +func Open(r storage.Reader) (any, error) { + if r == nil || r.Size() < 1 || r.Size() > MaxBytes { + return nil, fmt.Errorf("plist: input size limit") + } + raw := make([]byte, int(r.Size())) + if _, err := io.ReadFull(io.NewSectionReader(r, 0, r.Size()), raw); err != nil { + return nil, err + } + return Decode(raw) +} +func Decode(raw []byte) (any, error) { + if len(raw) == 0 || len(raw) > MaxBytes { + return nil, fmt.Errorf("plist: input size limit") + } + if bytes.HasPrefix(raw, []byte("bplist00")) { + return decodeBinary(raw) + } + p := xmlParser{d: xml.NewDecoder(bytes.NewReader(raw))} + start, err := p.nextStart() + if err != nil || start.Name.Local != "plist" || start.Name.Space != "" { + return nil, fmt.Errorf("plist: expected XML plist root") + } + token, err := p.nextContent() + child, ok := token.(xml.StartElement) + if err != nil || !ok { + return nil, fmt.Errorf("plist: expected one root value") + } + value, err := p.value(child, 0) + if err != nil { + return nil, err + } + token, err = p.nextContent() + end, ok := token.(xml.EndElement) + if err != nil || !ok || end.Name.Local != "plist" { + return nil, fmt.Errorf("plist: multiple root values") + } + if _, err = p.nextContent(); err != io.EOF { + return nil, fmt.Errorf("plist: trailing XML content") + } + return value, nil +} + +type xmlParser struct { + d *xml.Decoder + budget budget +} + +func (p *xmlParser) nextContent() (xml.Token, error) { + for { + t, e := p.d.Token() + if e != nil { + return nil, e + } + switch v := t.(type) { + case xml.Comment, xml.ProcInst, xml.Directive: + continue + case xml.CharData: + if strings.TrimSpace(string(v)) != "" { + return nil, fmt.Errorf("plist: unexpected text") + } + continue + default: + return t, nil + } + } +} +func (p *xmlParser) nextStart() (xml.StartElement, error) { + t, e := p.nextContent() + s, ok := t.(xml.StartElement) + if e != nil { + return s, e + } + if !ok { + return s, fmt.Errorf("plist: expected element") + } + return s, nil +} +func (p *xmlParser) text(s xml.StartElement) (string, error) { + var buf strings.Builder + for { + t, e := p.d.Token() + if e != nil { + return "", e + } + switch v := t.(type) { + case xml.CharData: + if buf.Len()+len(v) > MaxBytes { + return "", fmt.Errorf("plist: text size limit") + } + buf.Write(v) + case xml.Comment: + continue + case xml.EndElement: + if v.Name != s.Name { + return "", fmt.Errorf("plist: mismatched element") + } + return buf.String(), nil + default: + return "", fmt.Errorf("plist: scalar contains element") + } + } +} +func (p *xmlParser) value(s xml.StartElement, depth int) (any, error) { + if e := p.budget.use(depth, 0); e != nil { + return nil, e + } + if s.Name.Space != "" { + return nil, fmt.Errorf("plist: namespaced element") + } + switch s.Name.Local { + case "array", "dict": + array := []any{} + dict := map[string]any{} + for { + t, e := p.nextContent() + if e != nil { + return nil, e + } + if end, ok := t.(xml.EndElement); ok { + if end.Name != s.Name { + return nil, fmt.Errorf("plist: mismatched container") + } + if s.Name.Local == "dict" { + return dict, nil + } + return array, nil + } + child, ok := t.(xml.StartElement) + if !ok { + return nil, fmt.Errorf("plist: expected value") + } + if s.Name.Local == "array" { + v, e := p.value(child, depth+1) + if e != nil { + return nil, e + } + array = append(array, v) + continue + } + if child.Name.Local != "key" || child.Name.Space != "" { + return nil, fmt.Errorf("plist: expected dictionary key") + } + key, e := p.text(child) + if e != nil { + return nil, e + } + if e = p.budget.use(depth+1, len(key)); e != nil { + return nil, e + } + if _, ok := dict[key]; ok { + return nil, fmt.Errorf("plist: duplicate key %q", key) + } + next, e := p.nextStart() + if e != nil { + return nil, e + } + v, e := p.value(next, depth+1) + if e != nil { + return nil, e + } + dict[key] = v + } + default: + text, e := p.text(s) + if e != nil { + return nil, e + } + if e = p.budget.use(depth, len(text)); e != nil { + return nil, e + } + switch s.Name.Local { + case "string": + return text, nil + case "integer": + return strconv.ParseInt(strings.TrimSpace(text), 10, 64) + case "real": + v, e := strconv.ParseFloat(strings.TrimSpace(text), 64) + if math.IsNaN(v) || math.IsInf(v, 0) { + return nil, fmt.Errorf("plist: nonfinite real") + } + return v, e + case "data": + return base64.StdEncoding.DecodeString(strings.Join(strings.Fields(text), "")) + case "date": + v, e := time.Parse(time.RFC3339Nano, strings.TrimSpace(text)) + return Date{v}, e + case "true", "false": + if strings.TrimSpace(text) != "" { + return nil, fmt.Errorf("plist: boolean text") + } + return s.Name.Local == "true", nil + default: + return nil, fmt.Errorf("plist: unsupported XML value %s", s.Name.Local) + } + } +} + +type binaryParser struct { + raw []byte + offsets []uint64 + end uint64 + refSize int + active map[uint64]bool + budget budget +} + +func sized(raw []byte) uint64 { + var v uint64 + for _, b := range raw { + v = v<<8 | uint64(b) + } + return v +} +func decodeBinary(raw []byte) (any, error) { + if len(raw) < 40 { + return nil, fmt.Errorf("plist: truncated binary trailer") + } + t := raw[len(raw)-32:] + width, refs := int(t[6]), int(t[7]) + n, top, table := sized(t[8:16]), sized(t[16:24]), sized(t[24:]) + if width < 1 || width > 8 || refs < 1 || refs > 8 || n == 0 || n > maxNodes || top >= n || table < 8 || table > uint64(len(raw)-32) || n > uint64(len(raw)-32)/uint64(width) || n*uint64(width) != uint64(len(raw)-32)-table { + return nil, fmt.Errorf("plist: binary table bounds") + } + p := binaryParser{raw: raw, end: table, refSize: refs, active: map[uint64]bool{}} + p.offsets = make([]uint64, int(n)) + for i := range p.offsets { + off := sized(raw[int(table)+i*width : int(table)+(i+1)*width]) + if off < 8 || off >= table { + return nil, fmt.Errorf("plist: object offset bounds") + } + p.offsets[i] = off + } + return p.value(top, 0) +} +func (p *binaryParser) span(off, n uint64) ([]byte, error) { + if off > p.end || n > p.end-off { + return nil, fmt.Errorf("plist: object extent bounds") + } + return p.raw[off : off+n], nil +} +func (p *binaryParser) count(off uint64, marker byte) (uint64, uint64, error) { + if marker&15 != 15 { + return uint64(marker & 15), off + 1, nil + } + b, e := p.span(off+1, 1) + if e != nil { + return 0, 0, e + } + if b[0]>>4 != 1 || b[0]&15 > 3 { + return 0, 0, fmt.Errorf("plist: invalid extended length") + } + width := uint64(1) << uint(b[0]&15) + data, e := p.span(off+2, width) + if e != nil { + return 0, 0, e + } + return sized(data), off + 2 + width, nil +} +func (p *binaryParser) ref(off uint64) (uint64, error) { + data, e := p.span(off, uint64(p.refSize)) + if e != nil { + return 0, e + } + id := sized(data) + if id >= uint64(len(p.offsets)) { + return 0, fmt.Errorf("plist: object reference bounds") + } + return id, nil +} +func (p *binaryParser) value(id uint64, depth int) (any, error) { + if e := p.budget.use(depth, 0); e != nil { + return nil, e + } + if p.active[id] { + return nil, fmt.Errorf("plist: cyclic object graph") + } + p.active[id] = true + defer delete(p.active, id) + off := p.offsets[id] + m := p.raw[off] + switch m >> 4 { + case 0: + switch m { + case 0: + return nil, nil + case 8: + return false, nil + case 9: + return true, nil + } + return nil, fmt.Errorf("plist: unsupported simple marker") + case 1: + if m&15 > 3 { + return nil, fmt.Errorf("plist: integer wider than int64") + } + data, e := p.span(off+1, uint64(1)<>4 == 3 && m != 0x33) || (m>>4 == 2 && width != 4 && width != 8) { + return nil, fmt.Errorf("plist: invalid real/date width") + } + data, e := p.span(off+1, width) + if e != nil { + return nil, e + } + v := math.Float64frombits(sized(data)) + if width == 4 { + v = float64(math.Float32frombits(uint32(sized(data)))) + } + if math.IsNaN(v) || math.IsInf(v, 0) { + return nil, fmt.Errorf("plist: nonfinite real/date") + } + if m>>4 == 3 { + if v < -63113904000 || v >= 252423993600 { + return nil, fmt.Errorf("plist: date range") + } + seconds, fraction := math.Modf(v) + return Date{time.Unix(978307200+int64(seconds), int64(fraction*1e9)).UTC()}, nil + } + return v, nil + case 4, 5, 6: + n, start, e := p.count(off, m) + if e != nil { + return nil, e + } + width := uint64(1) + if m>>4 == 6 { + width = 2 + } + if n > MaxBytes/width { + return nil, fmt.Errorf("plist: scalar length bounds") + } + data, e := p.span(start, n*width) + if e != nil { + return nil, e + } + if e = p.budget.use(depth, int(n*width)); e != nil { + return nil, e + } + if m>>4 == 4 { + return append([]byte(nil), data...), nil + } + if m>>4 == 5 { + for _, b := range data { + if b > 127 { + return nil, fmt.Errorf("plist: non-ASCII marker5 string") + } + } + return string(data), nil + } + units := make([]uint16, int(n)) + for i := range units { + units[i] = binary.BigEndian.Uint16(data[i*2:]) + } + decodedBytes := 0 + for i, u := range units { + if u >= 0xd800 && u <= 0xdbff { + decodedBytes += 4 + if i+1 >= len(units) || units[i+1] < 0xdc00 || units[i+1] > 0xdfff { + return nil, fmt.Errorf("plist: unpaired UTF16 surrogate") + } + } else if u >= 0xdc00 && u <= 0xdfff { + if i == 0 || units[i-1] < 0xd800 || units[i-1] > 0xdbff { + return nil, fmt.Errorf("plist: unpaired UTF16 surrogate") + } + } else if u < 0x80 { + decodedBytes++ + } else if u < 0x800 { + decodedBytes += 2 + } else { + decodedBytes += 3 + } + } + // Count UTF8 expansion before allocating the decoded string. + if extra := decodedBytes - len(data); extra > 0 { + if e := p.budget.use(depth, extra); e != nil { + return nil, e + } + } + return string(utf16.Decode(units)), nil + case 10, 13: + n, start, e := p.count(off, m) + if e != nil { + return nil, e + } + mult := uint64(1) + if m>>4 == 13 { + mult = 2 + } + if n > uint64(maxNodes-p.budget.nodes)/mult || start > p.end || n*mult*uint64(p.refSize) > p.end-start { + return nil, fmt.Errorf("plist: container length bounds") + } + if m>>4 == 10 { + out := make([]any, int(n)) + for i := range out { + ref, e := p.ref(start + uint64(i*p.refSize)) + if e != nil { + return nil, e + } + out[i], e = p.value(ref, depth+1) + if e != nil { + return nil, e + } + } + return out, nil + } + out := map[string]any{} + for i := uint64(0); i < n; i++ { + ref, e := p.ref(start + i*uint64(p.refSize)) + if e != nil { + return nil, e + } + key, e := p.value(ref, depth+1) + if e != nil { + return nil, e + } + s, ok := key.(string) + if !ok { + return nil, fmt.Errorf("plist: dictionary key not string") + } + if _, ok := out[s]; ok { + return nil, fmt.Errorf("plist: duplicate key %q", s) + } + ref, e = p.ref(start + (n+i)*uint64(p.refSize)) + if e != nil { + return nil, e + } + out[s], e = p.value(ref, depth+1) + if e != nil { + return nil, e + } + } + return out, nil + default: + return nil, fmt.Errorf("plist: unsupported binary marker %#x", m) + } +} + +// EncodeXML emits deterministic XML1.0, preserving data and scalar types. +func EncodeXML(value any) ([]byte, error) { + var out bytes.Buffer + out.WriteString("\n\n") + e := xml.NewEncoder(&limitWriter{out: &out, count: out.Len()}) + b := budget{} + if err := encode(e, value, &b, 0); err != nil { + return nil, err + } + if err := e.Flush(); err != nil { + return nil, err + } + out.WriteString("\n") + if out.Len() > MaxBytes { + return nil, fmt.Errorf("plist: encoded size limit") + } + return out.Bytes(), nil +} +func encode(e *xml.Encoder, v any, b *budget, depth int) error { + if err := b.use(depth, 0); err != nil { + return err + } + scalar := func(tag, text string) error { + if err := b.use(depth, len(text)); err != nil { + return err + } + if !utf8.ValidString(text) { + return fmt.Errorf("plist: invalid UTF8") + } + for _, r := range text { + if r != 9 && r != 10 && r != 13 && (r < 32 || r == 0xfffe || r == 0xffff) { + return fmt.Errorf("plist: invalid XML character") + } + } + return e.EncodeElement(text, xml.StartElement{Name: xml.Name{Local: tag}}) + } + container := func(tag string, emit func() error) error { + s := xml.StartElement{Name: xml.Name{Local: tag}} + if err := e.EncodeToken(s); err != nil { + return err + } + if err := emit(); err != nil { + return err + } + return e.EncodeToken(s.End()) + } + switch x := v.(type) { + case string: + return scalar("string", x) + case []byte: + if len(x) > MaxBytes/4*3 || base64.StdEncoding.EncodedLen(len(x)) > MaxBytes-b.bytes { + return fmt.Errorf("plist: data size limit") + } + return scalar("data", base64.StdEncoding.EncodeToString(x)) + case bool: + if x { + return scalar("true", "") + } + return scalar("false", "") + case int64: + return scalar("integer", strconv.FormatInt(x, 10)) + case float64: + if math.IsNaN(x) || math.IsInf(x, 0) { + return fmt.Errorf("plist: nonfinite real") + } + return scalar("real", strconv.FormatFloat(x, 'g', -1, 64)) + case Date: + return scalar("date", x.UTC().Format(time.RFC3339Nano)) + case []any: + if len(x) > maxNodes-b.nodes { + return fmt.Errorf("plist: encoded node limit") + } + return container("array", func() error { + for _, item := range x { + if err := encode(e, item, b, depth+1); err != nil { + return err + } + } + return nil + }) + case map[string]any: + if len(x) > (maxNodes-b.nodes)/2 { + return fmt.Errorf("plist: encoded node limit") + } + return container("dict", func() error { + keys := make([]string, 0, len(x)) + for k := range x { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + if err := scalar("key", k); err != nil { + return err + } + if err := encode(e, x[k], b, depth+1); err != nil { + return err + } + } + return nil + }) + default: + return fmt.Errorf("plist: cannot encode %T as XML1.0", v) + } +} + +type limitWriter struct { + out *bytes.Buffer + count int +} + +func (w *limitWriter) Write(p []byte) (int, error) { + if len(p) > MaxBytes-w.count { + return 0, fmt.Errorf("plist: encoded size limit") + } + n, e := w.out.Write(p) + w.count += n + return n, e +} diff --git a/binary/plist/plist_test.go b/binary/plist/plist_test.go new file mode 100644 index 0000000..ea7a81b --- /dev/null +++ b/binary/plist/plist_test.go @@ -0,0 +1,192 @@ +package plist + +import ( + "encoding/binary" + "encoding/hex" + "math" + "reflect" + "strings" + "testing" + "time" +) + +// Explicit object bytes exercise the external format rather than our encoder. +func fixture(objects ...string) []byte { + raw := []byte("bplist00") + offsets := []byte{} + for _, object := range objects { + offsets = append(offsets, byte(len(raw))) + data, e := hex.DecodeString(object) + if e != nil { + panic(e) + } + raw = append(raw, data...) + } + table := len(raw) + raw = append(raw, offsets...) + trailer := make([]byte, 32) + trailer[6] = 1 + trailer[7] = 1 + binary.BigEndian.PutUint64(trailer[8:], uint64(len(objects))) + binary.BigEndian.PutUint64(trailer[24:], uint64(table)) + return append(raw, trailer...) +} +func TestBinaryTypedValues(t *testing.T) { + // {user:[501,true,"😀",<0001ff>,-1,1.5,2001-01-01T00:00:00Z]} + raw := fixture("d10102", "5475736572", "a703040506070809", "1101f5", "09", "62d83dde00", "430001ff", "13ffffffffffffffff", "233ff8000000000000", "330000000000000000") + got, e := Decode(raw) + if e != nil { + t.Fatal(e) + } + want := map[string]any{"user": []any{int64(501), true, "😀", []byte{0, 1, 255}, int64(-1), 1.5, Date{time.Unix(978307200, 0).UTC()}}} + if !reflect.DeepEqual(got, want) { + t.Fatalf("got %#v", got) + } + xml, e := EncodeXML(got) + if e != nil { + t.Fatal(e) + } + again, e := Decode(xml) + if e != nil || !reflect.DeepEqual(again, want) { + t.Fatalf("roundtrip %#v %v", again, e) + } +} +func TestXMLSemantics(t *testing.T) { + raw := `a&b<x>08 AA H/ ` + got, e := Decode([]byte(raw)) + if e != nil { + t.Fatal(e) + } + want := map[string]any{"a&b": []any{"", int64(8), []byte{0, 1, 255}, false}} + if !reflect.DeepEqual(got, want) { + t.Fatalf("%#v", got) + } + encoded, e := EncodeXML(want) + if e != nil { + t.Fatal(e) + } + got, e = Decode(encoded) + if e != nil || !reflect.DeepEqual(got, want) { + t.Fatalf("%#v %v", got, e) + } +} +func TestRejectMalformedGraphAndXML(t *testing.T) { + cases := [][]byte{ + fixture("a100"), // self-cycle + fixture("d201010202", "516b", "09"), // duplicate keys + fixture("d10102", "1001", "09"), // non-string key + fixture("a1ff"), // invalid ref + fixture("4f13ffffffffffffffff"), // overflowing extended length + fixture("51ff"), // marker5 must be ASCII + fixture("61d800"), // unpaired surrogate + fixture("80ff"), // UID is not a plist1.0 scalar + []byte(`aa`), + []byte(`a`), + []byte(``), + []byte(`x`), + []byte(`NaN`), + []byte(`yes`), + []byte(`!`), + []byte(`` + strings.Repeat(``, 129) + strings.Repeat(``, 129) + ``), + } + for i, raw := range cases { + if _, e := Decode(raw); e == nil { + t.Errorf("case %d accepted", i) + } + } + for _, v := range []any{math.NaN(), math.Inf(1), "\x00", nil} { + if _, e := EncodeXML(v); e == nil { + t.Errorf("accepted %v", v) + } + } + cyclic := map[string]any{} + cyclic["self"] = cyclic + if _, e := EncodeXML(cyclic); e == nil { + t.Fatal("encoded cycle") + } +} +func TestRejectBadOffsetsAndExpansion(t *testing.T) { + raw := fixture("09") + binary.BigEndian.PutUint64(raw[len(raw)-24:], math.MaxUint64) + if _, e := Decode(raw); e == nil { + t.Fatal("accepted object count") + } + raw = fixture("09") + raw[9] = 1 + if _, e := Decode(raw); e == nil { + t.Fatal("accepted header object offset") + } + raw = fixture("a101", "09") + raw[9] = 2 + if _, e := Decode(raw); e == nil { + t.Fatal("accepted invalid reference") + } + // An acyclic doubling DAG must not expand exponentially without a bound. + objects := []string{} + for i := 0; i < 19; i++ { + objects = append(objects, "a2"+hex.EncodeToString([]byte{byte(i + 1), byte(i + 1)})) + } + objects = append(objects, "09") + if _, e := Decode(fixture(objects...)); e == nil { + t.Fatal("accepted exponential DAG") + } +} +func FuzzDecode(f *testing.F) { + f.Add(fixture("09")) + f.Add([]byte(``)) + f.Fuzz(func(t *testing.T, raw []byte) { + if len(raw) > 1<<20 { + return + } + v, e := Decode(raw) + if e == nil { + encoded, e := EncodeXML(v) + if e == nil { + if _, e = Decode(encoded); e != nil { + t.Fatal(e) + } + } + } + }) +} + +func TestAllocationBudgetsAndDateEpoch(t *testing.T) { + // UTF16 U+0800 expands two encoded bytes into three decoded UTF8 bytes. + raw := []byte{0x61, 0x08, 0x00} + p := binaryParser{raw: raw, offsets: []uint64{0}, end: 3, active: map[uint64]bool{}, budget: budget{bytes: MaxBytes - 2}} + if _, err := p.value(0, 0); err == nil { + t.Fatal("UTF16 expansion bypassed decoded byte budget") + } + p = binaryParser{raw: []byte{0xa2, 0, 0}, offsets: []uint64{0}, end: 3, refSize: 1, active: map[uint64]bool{}, budget: budget{nodes: maxNodes - 1}} + if _, err := p.value(0, 0); err == nil { + t.Fatal("container bypassed remaining node budget") + } + for _, seconds := range []float64{-63113904000, 252423993599} { + encoded := make([]byte, 9) + encoded[0] = 0x33 + binary.BigEndian.PutUint64(encoded[1:], math.Float64bits(seconds)) + got, err := Decode(fixture(hex.EncodeToString(encoded))) + if err != nil { + t.Fatal(err) + } + year := got.(Date).Year() + if year != 1 && year != 9999 { + t.Fatalf("wrong CF epoch year: %d", year) + } + xml, err := EncodeXML(got) + if err != nil { + t.Fatal(err) + } + if _, err := Decode(xml); err != nil { + t.Fatal(err) + } + } + for _, seconds := range []float64{-63113904001, 252423993600} { + encoded := make([]byte, 9) + encoded[0] = 0x33 + binary.BigEndian.PutUint64(encoded[1:], math.Float64bits(seconds)) + if _, err := Decode(fixture(hex.EncodeToString(encoded))); err == nil { + t.Fatal("accepted date outside XML representable range") + } + } +} diff --git a/binary/star/appledouble.go b/binary/star/appledouble.go new file mode 100644 index 0000000..57052c9 --- /dev/null +++ b/binary/star/appledouble.go @@ -0,0 +1,30 @@ +package star + +import ( + "fmt" + "github.com/tinyrange/trex/binary/appledouble" + starfile "github.com/tinyrange/trex/storage/star" + "go.starlark.net/starlark" +) + +func appleDoubleBuiltin(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var value starlark.Value + if err := starlark.UnpackArgs("appledouble", args, kwargs, "file", &value); err != nil { + return nil, err + } + file, ok := value.(starfile.File) + if !ok { + return nil, fmt.Errorf("appledouble: expected file") + } + entries, err := appledouble.Open(file) + if err != nil { + return nil, err + } + out := starlark.NewDict(len(entries)) + for _, e := range entries { + if err := out.SetKey(starlark.MakeUint(uint(e.ID)), starfile.NewReader("AppleDouble entry", e.Data)); err != nil { + return nil, err + } + } + return out, nil +} diff --git a/binary/star/binary.go b/binary/star/binary.go index c7c4895..7e85a4b 100644 --- a/binary/star/binary.go +++ b/binary/star/binary.go @@ -63,22 +63,26 @@ func binaryScalarCodecNamed(name string) (binaryScalarCodec, bool) { func Builtins() starlark.StringDict { attrs := starlark.StringDict{ - "annotate": starlark.NewBuiltin("annotate", binaryAnnotateBuiltin), - "base64": starlark.NewBuiltin("base64", binaryBase64Builtin), - "bits": starlark.NewBuiltin("bits", binaryBitsBuiltin), - "builder": starlark.NewBuiltin("builder", binaryBuilderBuiltin), - "concat": starlark.NewBuiltin("concat", binaryConcatBuiltin), - "cursor": starlark.NewBuiltin("cursor", binaryCursorBuiltin), - "decode": starlark.NewBuiltin("decode", binaryDecodeBuiltin), - "encode": starlark.NewBuiltin("encode", binaryEncodeBuiltin), - "extents": starlark.NewBuiltin("extents", binaryExtentsBuiltin), - "hex": starlark.NewBuiltin("hex", binaryHexBuiltin), - "layout": starlark.NewBuiltin("layout", binaryLayoutBuiltin), - "replace": starlark.NewBuiltin("replace", binaryReplaceBuiltin), - "strings": starlark.NewBuiltin("strings", binaryStringsBuiltin), - "text": starlark.NewBuiltin("text", binaryTextBuiltin), - "view": starlark.NewBuiltin("view", binaryViewBuiltin), - "xml": starlark.NewBuiltin("xml", binaryXMLBuiltin), + "annotate": starlark.NewBuiltin("annotate", binaryAnnotateBuiltin), + "appledouble": starlark.NewBuiltin("appledouble", appleDoubleBuiltin), + "base64": starlark.NewBuiltin("base64", binaryBase64Builtin), + "bits": starlark.NewBuiltin("bits", binaryBitsBuiltin), + "builder": starlark.NewBuiltin("builder", binaryBuilderBuiltin), + "concat": starlark.NewBuiltin("concat", binaryConcatBuiltin), + "cursor": starlark.NewBuiltin("cursor", binaryCursorBuiltin), + "decode": starlark.NewBuiltin("decode", binaryDecodeBuiltin), + "encode": starlark.NewBuiltin("encode", binaryEncodeBuiltin), + "extents": starlark.NewBuiltin("extents", binaryExtentsBuiltin), + "hex": starlark.NewBuiltin("hex", binaryHexBuiltin), + "layout": starlark.NewBuiltin("layout", binaryLayoutBuiltin), + "macho": starlark.NewBuiltin("macho", machoBuiltin), + "plist": starlark.NewBuiltin("plist", plistBuiltin), + "plist_encode": starlark.NewBuiltin("plist_encode", plistEncodeBuiltin), + "replace": starlark.NewBuiltin("replace", binaryReplaceBuiltin), + "strings": starlark.NewBuiltin("strings", binaryStringsBuiltin), + "text": starlark.NewBuiltin("text", binaryTextBuiltin), + "view": starlark.NewBuiltin("view", binaryViewBuiltin), + "xml": starlark.NewBuiltin("xml", binaryXMLBuiltin), } for _, codec := range binaryScalarCodecs { codec := codec diff --git a/binary/star/macho.go b/binary/star/macho.go new file mode 100644 index 0000000..5e5792f --- /dev/null +++ b/binary/star/macho.go @@ -0,0 +1,47 @@ +package star + +import ( + "fmt" + "github.com/tinyrange/trex/binary/macho" + starfile "github.com/tinyrange/trex/storage/star" + "go.starlark.net/starlark" +) + +func machoBuiltin(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var input starlark.Value + architecture := "x86_64" + if err := starlark.UnpackArgs("macho", args, kwargs, "file", &input, "architecture?", &architecture); err != nil { + return nil, err + } + file, ok := input.(starfile.File) + if !ok { + return nil, fmt.Errorf("macho: expected file") + } + if architecture != "x86_64" { + return nil, fmt.Errorf("macho: only x86_64 inspection supported") + } + image, err := macho.Open(file, macho.AMD64) + if err != nil { + return nil, err + } + symbols := make([]starlark.Value, 0, len(image.Symbols)) + for _, s := range image.Symbols { + symbols = append(symbols, starfile.NewRecord(starlark.StringDict{"name": starlark.String(s.Name), "address": starlark.MakeUint64(s.Address), "type": starlark.MakeInt(int(s.Type)), "section": starlark.MakeInt(int(s.Section)), "description": starlark.MakeInt(int(s.Description))})) + } + segments := make([]starlark.Value, 0, len(image.Segments)) + for _, s := range image.Segments { + segments = append(segments, starfile.NewRecord(starlark.StringDict{"name": starlark.String(s.Name), "address": starlark.MakeUint64(s.Address), "size": starlark.MakeUint64(s.Size), "file_offset": starlark.MakeUint64(s.Offset), "file_size": starlark.MakeUint64(s.FileSize), "flags": starlark.MakeUint(uint(s.Flags)), "data": starfile.NewReader(s.Name, s.Data)})) + } + at := starlark.NewBuiltin("macho.at", func(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var address, size uint64 + if err := starlark.UnpackArgs("at", args, kwargs, "address", &address, "size", &size); err != nil { + return nil, err + } + r, err := image.At(address, size) + if err != nil { + return nil, err + } + return starfile.NewReader("macho virtual bytes", r), nil + }) + return starfile.NewRecord(starlark.StringDict{"architecture": starlark.String(architecture), "type": starlark.MakeUint(uint(image.Type)), "file": starfile.NewReader("macho selected image", image.Source), "segments": starlark.NewList(segments), "symbols": starlark.NewList(symbols), "at": at}), nil +} diff --git a/binary/star/plist.go b/binary/star/plist.go new file mode 100644 index 0000000..2c9e0d3 --- /dev/null +++ b/binary/star/plist.go @@ -0,0 +1,196 @@ +package star + +import ( + "bytes" + "fmt" + "sort" + + "github.com/tinyrange/trex/binary/plist" + starfile "github.com/tinyrange/trex/storage/star" + "go.starlark.net/starlark" +) + +type plistDate struct{ value plist.Date } + +func (d *plistDate) String() string { + return fmt.Sprintf("plist_date(%q)", d.value.Format("2006-01-02T15:04:05.999999999Z07:00")) +} +func (d *plistDate) Type() string { return "plist_date" } +func (d *plistDate) Freeze() {} +func (d *plistDate) Truth() starlark.Bool { return true } +func (d *plistDate) Hash() (uint32, error) { return starlark.String(d.String()).Hash() } + +func plistBuiltin(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var value starlark.Value + if err := starlark.UnpackArgs("plist", args, kwargs, "source", &value); err != nil { + return nil, err + } + view, err := newBinaryByteView(value) + if err != nil { + return nil, err + } + if view.size > plist.MaxBytes { + return nil, fmt.Errorf("plist: input size limit") + } + raw := make([]byte, int(view.size)) + if _, err = view.ReadAt(raw, 0); err != nil { + return nil, err + } + result, err := plist.Decode(raw) + if err != nil { + return nil, err + } + return plistValue(result) +} +func plistValue(value any) (starlark.Value, error) { + switch x := value.(type) { + case nil: + return starlark.None, nil + case bool: + return starlark.Bool(x), nil + case string: + return starlark.String(x), nil + case []byte: + return starlark.Bytes(x), nil + case int64: + return starlark.MakeInt64(x), nil + case float64: + return starlark.Float(x), nil + case plist.Date: + return &plistDate{value: x}, nil + case []any: + items := make([]starlark.Value, len(x)) + for i, v := range x { + item, err := plistValue(v) + if err != nil { + return nil, err + } + items[i] = item + } + return starlark.NewList(items), nil + case map[string]any: + d := starlark.NewDict(len(x)) + keys := make([]string, 0, len(x)) + for k := range x { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + v, err := plistValue(x[k]) + if err != nil { + return nil, err + } + if err = d.SetKey(starlark.String(k), v); err != nil { + return nil, err + } + } + return d, nil + default: + return nil, fmt.Errorf("plist: unsupported value %T", value) + } +} + +type plistBudget struct{ nodes, bytes int } + +func (b *plistBudget) use(size int) error { + if size > plist.MaxBytes-b.bytes { + return fmt.Errorf("plist_encode: byte limit") + } + b.bytes += size + return nil +} +func plistNative(value starlark.Value, depth int, budget *plistBudget) (any, error) { + budget.nodes++ + if depth > 128 || budget.nodes > 100000 { + return nil, fmt.Errorf("plist_encode: value limit") + } + switch x := value.(type) { + case starlark.Bool: + return bool(x), nil + case starlark.String: + if err := budget.use(len(x)); err != nil { + return nil, err + } + return string(x), nil + case starlark.Bytes: + if err := budget.use(len(x)); err != nil { + return nil, err + } + return []byte(x), nil + case starlark.Int: + n, ok := x.Int64() + if !ok { + return nil, fmt.Errorf("plist_encode: integer exceeds int64") + } + return n, nil + case starlark.Float: + return float64(x), nil + case *plistDate: + return x.value, nil + case *starlark.List: + if x.Len() > 100000-budget.nodes { + return nil, fmt.Errorf("plist_encode: value limit") + } + items := make([]any, x.Len()) + for i := range items { + v, err := plistNative(x.Index(i), depth+1, budget) + if err != nil { + return nil, err + } + items[i] = v + } + return items, nil + case starlark.Tuple: + if len(x) > 100000-budget.nodes { + return nil, fmt.Errorf("plist_encode: value limit") + } + items := make([]any, len(x)) + for i, v := range x { + converted, err := plistNative(v, depth+1, budget) + if err != nil { + return nil, err + } + items[i] = converted + } + return items, nil + case *starlark.Dict: + if x.Len() > (100000-budget.nodes)/2 { + return nil, fmt.Errorf("plist_encode: value limit") + } + d := make(map[string]any, x.Len()) + for _, item := range x.Items() { + k, ok := starlark.AsString(item[0]) + if !ok { + return nil, fmt.Errorf("plist_encode: key must be string") + } + budget.nodes++ + if err := budget.use(len(k)); err != nil { + return nil, err + } + v, err := plistNative(item[1], depth+1, budget) + if err != nil { + return nil, err + } + d[k] = v + } + return d, nil + default: + return nil, fmt.Errorf("plist_encode: unsupported %s", value.Type()) + } +} +func plistEncodeBuiltin(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var value starlark.Value + if err := starlark.UnpackArgs("plist_encode", args, kwargs, "value", &value); err != nil { + return nil, err + } + budget := plistBudget{} + native, err := plistNative(value, 0, &budget) + if err != nil { + return nil, err + } + raw, err := plist.EncodeXML(native) + if err != nil { + return nil, err + } + return starfile.NewReader("plist.xml", bytes.NewReader(raw)), nil +} diff --git a/binary/star/plist_test.go b/binary/star/plist_test.go new file mode 100644 index 0000000..dd42781 --- /dev/null +++ b/binary/star/plist_test.go @@ -0,0 +1,68 @@ +package star + +import ( + "github.com/tinyrange/trex/binary/plist" + "go.starlark.net/starlark" + "strings" + "testing" +) + +func TestPlistAdapterBudgetsAndCycles(t *testing.T) { + for _, value := range []starlark.Value{starlark.String("abcd"), starlark.Bytes("abcd")} { + budget := plistBudget{bytes: plist.MaxBytes - 3} + if _, err := plistNative(value, 0, &budget); err == nil { + t.Fatal("accepted oversized leaf before copying") + } + } + budget := plistBudget{nodes: 99999} + if _, err := plistNative(starlark.NewList([]starlark.Value{starlark.True}), 0, &budget); err == nil { + t.Fatal("accepted container beyond remaining node budget") + } + dictionary := starlark.NewDict(1) + if err := dictionary.SetKey(starlark.String("long-key"), starlark.True); err != nil { + t.Fatal(err) + } + budget = plistBudget{bytes: plist.MaxBytes - 7} + if _, err := plistNative(dictionary, 0, &budget); err == nil { + t.Fatal("dictionary key did not count towards byte budget") + } + cycle := starlark.NewList(nil) + if err := cycle.Append(cycle); err != nil { + t.Fatal(err) + } + budget = plistBudget{} + if _, err := plistNative(cycle, 0, &budget); err == nil { + t.Fatal("accepted recursive list") + } + shared := starlark.Bytes(strings.Repeat("x", 65536)) + items := make([]starlark.Value, 1025) + for i := range items { + items[i] = shared + } + budget = plistBudget{} + if _, err := plistNative(starlark.NewList(items), 0, &budget); err == nil { + t.Fatal("shared leaves bypassed aggregate byte limit") + } +} + +func TestPlistAdapterTypedRoundTrip(t *testing.T) { + original := starlark.NewDict(2) + if err := original.SetKey(starlark.String("data"), starlark.Bytes("\x00\xff")); err != nil { + t.Fatal(err) + } + if err := original.SetKey(starlark.String("list"), starlark.NewList([]starlark.Value{starlark.True, starlark.MakeInt(-1), starlark.String("é")})); err != nil { + t.Fatal(err) + } + encoded, err := plistEncodeBuiltin(nil, nil, starlark.Tuple{original}, nil) + if err != nil { + t.Fatal(err) + } + decoded, err := plistBuiltin(nil, nil, starlark.Tuple{encoded}, nil) + if err != nil { + t.Fatal(err) + } + equal, err := starlark.Equal(original, decoded) + if err != nil || !equal { + t.Fatalf("typed round trip: %v %v", decoded, err) + } +} diff --git a/boot/darwin/handoff.go b/boot/darwin/handoff.go new file mode 100644 index 0000000..dd359bd --- /dev/null +++ b/boot/darwin/handoff.go @@ -0,0 +1,145 @@ +package darwin + +import ( + "encoding/binary" + "fmt" + "sort" + "strings" +) + +// Node is Apple's flattened device tree (not the Linux FDT format). +// Property names are NUL-terminated in 32 bytes; values are padded to 4 bytes. +type Node struct { + Properties map[string][]byte + Children []Node +} + +func (n Node) Encode() ([]byte, error) { + var out []byte + var visit func(Node, int) error + visit = func(n Node, depth int) error { + if depth > 32 || len(n.Properties) > 1024 || len(n.Children) > 1024 { + return fmt.Errorf("darwin: device tree limit") + } + if len(out) > (1<<20)-8 { + return fmt.Errorf("darwin: device tree too large") + } + out = binary.LittleEndian.AppendUint32(out, uint32(len(n.Properties))) + out = binary.LittleEndian.AppendUint32(out, uint32(len(n.Children))) + keys := make([]string, 0, len(n.Properties)) + for k := range n.Properties { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + v := n.Properties[k] + if len(k) == 0 || len(k) > 31 || strings.ContainsRune(k, 0) || len(v) > 1<<20 { + return fmt.Errorf("darwin: invalid device tree property") + } + if 36+(len(v)+3)&^3 > (1<<20)-len(out) { + return fmt.Errorf("darwin: device tree too large") + } + var name [32]byte + copy(name[:], k) + out = append(out, name[:]...) + out = binary.LittleEndian.AppendUint32(out, uint32(len(v))) + out = append(out, v...) + for len(out)%4 != 0 { + out = append(out, 0) + } + } + for _, c := range n.Children { + if err := visit(c, depth+1); err != nil { + return err + } + } + if len(out) > 1<<20 { + return fmt.Errorf("darwin: device tree too large") + } + return nil + } + if err := visit(n, 0); err != nil { + return nil, err + } + return out, nil +} +func CString(s string) []byte { return append([]byte(s), 0) } +func U32(v uint32) []byte { return binary.LittleEndian.AppendUint32(nil, v) } +func U64(v uint64) []byte { return binary.LittleEndian.AppendUint64(nil, v) } +func Range(address, size uint32) []byte { return append(U32(address), U32(size)...) } + +// MemoryRange is a 40-byte EFI_MEMORY_DESCRIPTOR v1. +type MemoryRange struct { + Type uint32 + Physical, Virtual, Pages, Attributes uint64 +} + +func MemoryMap(ranges []MemoryRange) ([]byte, error) { + var b []byte + var end uint64 + for _, r := range ranges { + if r.Physical%4096 != 0 || r.Virtual%4096 != 0 || r.Pages > (^uint64(0)-r.Virtual)/4096 || r.Pages == 0 || r.Physical < end || r.Pages > (^uint64(0)-r.Physical)/4096 { + return nil, fmt.Errorf("darwin: invalid/overlapping EFI memory range") + } + end = r.Physical + r.Pages*4096 + b = binary.LittleEndian.AppendUint32(b, r.Type) + b = binary.LittleEndian.AppendUint32(b, 0) + for _, v := range []uint64{r.Physical, r.Virtual, r.Pages, r.Attributes} { + b = binary.LittleEndian.AppendUint64(b, v) + } + } + return b, nil +} + +// Arguments encodes the 4096-byte XNU boot_args version 2, revision 0 ABI. +// All pointers except EFI runtime virtual addresses are physical and below 4GiB. +type Arguments struct { + CommandLine string + MemoryMap, MemoryMapSize, DeviceTree, DeviceTreeSize uint32 + KernelAddress, KernelSize, EFISystemTable uint32 + RuntimePageStart, RuntimePageCount uint32 + RuntimeVirtualPage uint64 + Video [6]uint32 // base, display, stride, width, height, depth + BootMemoryStart, BootMemorySize, PhysicalMemorySize, FSBFrequency uint64 +} + +func (a Arguments) Encode() ([]byte, error) { + if len(a.CommandLine) > 1023 || strings.ContainsRune(a.CommandLine, 0) { + return nil, fmt.Errorf("darwin: command line must be NUL-free and shorter than 1024 bytes") + } + if a.MemoryMap == 0 || a.MemoryMapSize == 0 || a.MemoryMapSize%40 != 0 || a.DeviceTree == 0 || a.DeviceTreeSize == 0 || a.KernelAddress == 0 || a.KernelSize == 0 { + return nil, fmt.Errorf("darwin: missing boot metadata") + } + for _, r := range [][2]uint32{{a.MemoryMap, a.MemoryMapSize}, {a.DeviceTree, a.DeviceTreeSize}, {a.KernelAddress, a.KernelSize}} { + if uint64(r[0])+uint64(r[1]) > 1<<32 { + return nil, fmt.Errorf("darwin: metadata range wraps 32-bit address space") + } + } + if uint64(a.RuntimePageStart)+uint64(a.RuntimePageCount) > 1<<20 || a.RuntimeVirtualPage > (^uint64(0)>>12)-uint64(a.RuntimePageCount) { + return nil, fmt.Errorf("darwin: invalid runtime page range") + } + b := make([]byte, 4096) + binary.LittleEndian.PutUint16(b[2:], 2) + b[4] = 64 + copy(b[8:1032], a.CommandLine) + put := func(off int, v uint32) { binary.LittleEndian.PutUint32(b[off:], v) } + put(1032, a.MemoryMap) + put(1036, a.MemoryMapSize) + put(1040, 40) + put(1044, 1) + for i, v := range a.Video { + put(1048+i*4, v) + } + put(1072, a.DeviceTree) + put(1076, a.DeviceTreeSize) + put(1080, a.KernelAddress) + put(1084, a.KernelSize) + put(1088, a.RuntimePageStart) + put(1092, a.RuntimePageCount) + binary.LittleEndian.PutUint64(b[1096:], a.RuntimeVirtualPage) + put(1104, a.EFISystemTable) + for i, v := range []uint64{a.BootMemoryStart, a.BootMemorySize, a.PhysicalMemorySize, a.FSBFrequency} { + binary.LittleEndian.PutUint64(b[1128+i*8:], v) + } + return b, nil +} diff --git a/boot/darwin/handoff_test.go b/boot/darwin/handoff_test.go new file mode 100644 index 0000000..2b20feb --- /dev/null +++ b/boot/darwin/handoff_test.go @@ -0,0 +1,49 @@ +package darwin + +import ( + "encoding/binary" + "testing" +) + +func TestBootArgsABI(t *testing.T) { + b, err := (Arguments{CommandLine: "-v rd=md0", MemoryMap: 0x4000, MemoryMapSize: 80, DeviceTree: 0x5000, DeviceTreeSize: 96, KernelAddress: 0x100000, KernelSize: 0x800000, EFISystemTable: 0x6000, PhysicalMemorySize: 2 << 30, FSBFrequency: 100000000, Video: [6]uint32{0xe0001000, 1, 5120, 1280, 720, 32}}).Encode() + if err != nil { + t.Fatal(err) + } + if len(b) != 4096 || binary.LittleEndian.Uint16(b[2:]) != 2 || b[4] != 64 || string(b[8:17]) != "-v rd=md0" || b[17] != 0 || binary.LittleEndian.Uint32(b[1072:]) != 0x5000 || binary.LittleEndian.Uint32(b[1104:]) != 0x6000 || binary.LittleEndian.Uint64(b[1144:]) != 2<<30 { + t.Fatal("ABI offsets wrong") + } +} +func TestDeviceTreeWirePadding(t *testing.T) { + tree := Node{Properties: map[string][]byte{"name": CString("/")}, Children: []Node{{Properties: map[string][]byte{"name": CString("chosen"), "RAMDisk": Range(0x200000, 0x300000)}}}} + b, err := tree.Encode() + if err != nil { + t.Fatal(err) + } + u := binary.LittleEndian.Uint32 + if u(b) != 1 || u(b[4:]) != 1 || string(b[8:12]) != "name" || u(b[40:]) != 2 || b[44] != '/' || b[45] != 0 || u(b[48:]) != 2 || u(b[52:]) != 0 || string(b[56:63]) != "RAMDisk" || u(b[88:]) != 8 || u(b[92:]) != 0x200000 { + t.Fatalf("wrong flattened tree: %x", b) + } + if _, err = (Node{Properties: map[string][]byte{string(make([]byte, 32)): nil}}).Encode(); err == nil { + t.Fatal("accepted invalid name") + } + if _, err = MemoryMap([]MemoryRange{{Type: 7, Physical: 4096, Pages: 2}, {Type: 7, Physical: 8192, Pages: 1}}); err == nil { + t.Fatal("accepted overlap") + } +} + +func TestDeviceTreeAggregateBoundAndRuntimeAlignment(t *testing.T) { + value := make([]byte, 1<<19) + if _, err := (Node{Properties: map[string][]byte{"a": value, "b": value}}).Encode(); err == nil { + t.Fatal("accepted excessive aggregate tree") + } + for _, r := range []MemoryRange{{Physical: 4096, Virtual: 1, Pages: 1}, {Physical: 4096, Virtual: 0xfffffffffffff000, Pages: 2}, {Physical: 0xfffffffffffff000, Pages: 2}} { + if _, err := MemoryMap([]MemoryRange{r}); err == nil { + t.Fatal("accepted wrapping/unaligned descriptor") + } + } + a := Arguments{MemoryMap: 0xfffffff0, MemoryMapSize: 80, DeviceTree: 4096, DeviceTreeSize: 8, KernelAddress: 0x100000, KernelSize: 4096} + if _, err := a.Encode(); err == nil { + t.Fatal("accepted wrapping boot metadata") + } +} diff --git a/boot/darwin/kernel.go b/boot/darwin/kernel.go new file mode 100644 index 0000000..447652b --- /dev/null +++ b/boot/darwin/kernel.go @@ -0,0 +1,267 @@ +// Package darwin implements portable Intel XNU direct-boot wire formats. +// No bootloader is executed and no kernel instructions are patched. +package darwin + +import ( + "bytes" + "encoding/binary" + "fmt" + "github.com/tinyrange/trex/storage" + "hash/adler32" + "io" + "strings" +) + +const MaxKernelSize = 256 << 20 +const kernelBase uint64 = 0xffffff8000000000 + +type Segment struct { + Name string + Address, Size, FileOffset, FileSize uint64 +} +type Image struct { + Data []byte + Entry uint64 // physical 32-bit _pstart + Segments []Segment + Symbols map[string]uint64 // original virtual addresses for debugging + Base, End uint64 +} + +func Open(r storage.Reader) (*Image, error) { + if r == nil || r.Size() < 4 || r.Size() > MaxKernelSize { + return nil, fmt.Errorf("darwin: invalid kernel size") + } + b := make([]byte, r.Size()) + if _, err := io.ReadFull(io.NewSectionReader(r, 0, r.Size()), b); err != nil { + return nil, err + } + for depth := 0; depth < 3; depth++ { + if len(b) >= 8 && binary.BigEndian.Uint32(b) == 0xcafebabe { + n := binary.BigEndian.Uint32(b[4:]) + if n == 0 || n > 32 || uint64(n)*20+8 > uint64(len(b)) { + return nil, fmt.Errorf("darwin: invalid fat table") + } + var slice []byte + for i := uint32(0); i < n; i++ { + a := b[8+i*20 : 8+(i+1)*20] + off, size := uint64(binary.BigEndian.Uint32(a[8:])), uint64(binary.BigEndian.Uint32(a[12:])) + if off < 8+uint64(n)*20 || off > uint64(len(b)) || size > uint64(len(b))-off { + return nil, fmt.Errorf("darwin: fat slice outside file") + } + if binary.BigEndian.Uint32(a) == 0x01000007 { + if slice != nil { + return nil, fmt.Errorf("darwin: duplicate amd64 slice") + } + slice = b[off : off+size] + } + } + if slice == nil { + return nil, fmt.Errorf("darwin: no amd64 slice") + } + b = slice + continue + } + if len(b) >= 8 && string(b[:4]) == "comp" { + if string(b[4:8]) != "lzss" || len(b) < 384 { + return nil, fmt.Errorf("darwin: unsupported compressed cache") + } + size, packed := binary.BigEndian.Uint32(b[12:]), binary.BigEndian.Uint32(b[16:]) + if uint64(packed) != uint64(len(b)-384) { + return nil, fmt.Errorf("darwin: packed length mismatch") + } + out, err := decodeLZSS(b[384:], int(size)) + if err != nil { + return nil, err + } + if adler32.Checksum(out) != binary.BigEndian.Uint32(b[8:]) { + return nil, fmt.Errorf("darwin: kernelcache Adler32 mismatch") + } + b = out + continue + } + return parseMachO(b) + } + return nil, fmt.Errorf("darwin: kernel wrapper nesting limit") +} + +// Original decoder for the wire format: LSB-first flags and 12-bit ring indices. +func decodeLZSS(src []byte, size int) ([]byte, error) { + if size < 32 || size > MaxKernelSize { + return nil, fmt.Errorf("darwin: decoded size outside bounds") + } + out := make([]byte, 0, size) + var ring [4096]byte + for i := range ring { + ring[i] = ' ' + } + r, pos := 4096-18, 0 + put := func(v byte) { out = append(out, v); ring[r] = v; r = (r + 1) & 4095 } + for len(out) < size { + if pos >= len(src) { + return nil, io.ErrUnexpectedEOF + } + flags := src[pos] + pos++ + for bit := 0; bit < 8 && len(out) < size; bit++ { + if flags&(1<= len(src) { + return nil, io.ErrUnexpectedEOF + } + put(src[pos]) + pos++ + } else { + if len(src)-pos < 2 { + return nil, io.ErrUnexpectedEOF + } + a, c := int(src[pos]), int(src[pos+1]) + pos += 2 + index := a | ((c & 0xf0) << 4) + count := (c & 15) + 3 + if count > size-len(out) { + return nil, fmt.Errorf("darwin: LZSS match exceeds decoded size") + } + for j := 0; j < count; j++ { + put(ring[(index+j)&4095]) + } + } + } + } + if pos != len(src) { + return nil, fmt.Errorf("darwin: trailing compressed bytes") + } + return out, nil +} +func physical(v uint64) (uint64, error) { + if v < 1<<32 { + return v, nil + } + if v >= kernelBase && v < kernelBase+(1<<32) { + return v - kernelBase, nil + } + return 0, fmt.Errorf("darwin: address %#x outside unslid kernel window", v) +} +func parseMachO(b []byte) (*Image, error) { + u32, u64 := binary.LittleEndian.Uint32, binary.LittleEndian.Uint64 + if len(b) < 32 || u32(b) != 0xfeedfacf || u32(b[4:]) != 0x01000007 || u32(b[12:]) != 2 { + return nil, fmt.Errorf("darwin: expected amd64 Mach-O executable") + } + n, cmdsize := u32(b[16:]), u32(b[20:]) + if n == 0 || n > 4096 || uint64(cmdsize) > uint64(len(b)-32) { + return nil, fmt.Errorf("darwin: invalid load command bounds") + } + end, off := 32+int(cmdsize), 32 + img := &Image{Data: b, Base: 1 << 32, Symbols: map[string]uint64{}} + var symtab []byte + for i := uint32(0); i < n; i++ { + if end-off < 8 { + return nil, fmt.Errorf("darwin: truncated command") + } + c := b[off:end] + sz := int(u32(c[4:])) + if sz < 8 || sz%4 != 0 || sz > len(c) { + return nil, fmt.Errorf("darwin: invalid command size") + } + c = c[:sz] + switch u32(c) { + case 0x19: + if sz < 72 || uint64(u32(c[64:]))*80+72 != uint64(sz) { + return nil, fmt.Errorf("darwin: malformed segment") + } + name := strings.TrimRight(string(c[8:24]), "\x00") + va, vs, fo, fs := u64(c[24:]), u64(c[32:]), u64(c[40:]), u64(c[48:]) + if fo > uint64(len(b)) || fs > uint64(len(b))-fo || (vs != 0 && fs > vs) { + return nil, fmt.Errorf("darwin: segment %s invalid file range %#x+%#x (VM size %#x, file size %#x)", name, fo, fs, vs, len(b)) + } + if vs == 0 { + break + } + pa, err := physical(va) + if err != nil { + return nil, err + } + if vs > MaxKernelSize || pa < 0x100000 || pa >= 1<<32 || vs > (1<<32)-pa { + return nil, fmt.Errorf("darwin: invalid segment memory range") + } + for _, s := range img.Segments { + if pa < s.Address+s.Size && s.Address < pa+vs { + return nil, fmt.Errorf("darwin: overlapping segments") + } + } + img.Segments = append(img.Segments, Segment{name, pa, vs, fo, fs}) + img.Base = min(img.Base, pa) + img.End = max(img.End, pa+vs) + case 5: + if sz != 184 || u32(c[8:]) != 4 || u32(c[12:]) != 42 { + return nil, fmt.Errorf("darwin: unsupported UNIXTHREAD state") + } + if img.Entry != 0 { + return nil, fmt.Errorf("darwin: duplicate entry") + } + var err error + img.Entry, err = physical(u64(c[144:])) + if err != nil { + return nil, err + } + case 2: + if sz != 24 || symtab != nil { + return nil, fmt.Errorf("darwin: invalid symtab command") + } + symtab = c + } + off += sz + } + if off != end || len(img.Segments) == 0 || img.End-img.Base > MaxKernelSize { + return nil, fmt.Errorf("darwin: invalid kernel layout") + } + mapped := false + for _, s := range img.Segments { + if img.Entry >= s.Address && img.Entry < s.Address+s.FileSize { + mapped = true + } + } + if !mapped { + return nil, fmt.Errorf("darwin: entry outside file-backed segments") + } + if symtab != nil { + so, ns, st, ss := uint64(u32(symtab[8:])), uint64(u32(symtab[12:])), uint64(u32(symtab[16:])), uint64(u32(symtab[20:])) + if so > uint64(len(b)) || ns > (uint64(len(b))-so)/16 || st > uint64(len(b)) || ss > uint64(len(b))-st { + return nil, fmt.Errorf("darwin: symbol table out of bounds") + } + for i := uint64(0); i < ns; i++ { + s := b[so+i*16 : so+(i+1)*16] + index := uint64(u32(s)) + if index >= ss { + return nil, fmt.Errorf("darwin: invalid symbol string") + } + if s[4]&0xe0 != 0 || s[4]&0xe == 0 { + continue + } + str := b[st+index : st+ss] + zero := bytes.IndexByte(str, 0) + if zero < 0 { + return nil, fmt.Errorf("darwin: unterminated symbol") + } + img.Symbols[string(str[:zero])] = u64(s[8:]) + } + } + return img, nil +} + +// Load validates destinations before writing and clears each zero-fill tail. +func (i *Image) Load(ram []byte) error { + if i.End > uint64(len(ram)) { + return fmt.Errorf("darwin: kernel exceeds RAM") + } + // Image fields are exported for inspection; reject changed or fabricated + // ranges atomically instead of panicking or partially modifying RAM. + for _, s := range i.Segments { + if s.Address > uint64(len(ram)) || s.Size > uint64(len(ram))-s.Address || s.FileSize > s.Size || s.FileOffset > uint64(len(i.Data)) || s.FileSize > uint64(len(i.Data))-s.FileOffset { + return fmt.Errorf("darwin: invalid load range") + } + } + for _, s := range i.Segments { + clear(ram[s.Address : s.Address+s.Size]) + copy(ram[s.Address:s.Address+s.FileSize], i.Data[s.FileOffset:s.FileOffset+s.FileSize]) + } + return nil +} diff --git a/boot/darwin/kernel_test.go b/boot/darwin/kernel_test.go new file mode 100644 index 0000000..e86d39f --- /dev/null +++ b/boot/darwin/kernel_test.go @@ -0,0 +1,152 @@ +package darwin + +import ( + "bytes" + "encoding/binary" + "hash/adler32" + "strings" + "testing" +) + +type source struct{ *bytes.Reader } + +func (s source) Size() int64 { return s.Reader.Size() } +func fixture() []byte { + b := make([]byte, 4096) + p := binary.LittleEndian.PutUint32 + q := binary.LittleEndian.PutUint64 + p(b, 0xfeedfacf) + p(b[4:], 0x01000007) + p(b[12:], 2) + p(b[16:], 2) + p(b[20:], 72+184) + c := b[32:] + p(c, 0x19) + p(c[4:], 72) + copy(c[8:], "__TEXT") + q(c[24:], kernelBase+0x200000) + q(c[32:], 8192) + q(c[48:], 4096) + p(c[56:], 7) + p(c[60:], 5) + c = b[104:] + p(c, 5) + p(c[4:], 184) + p(c[8:], 4) + p(c[12:], 42) + q(c[144:], kernelBase+0x200200) + b[512] = 0xfa + return b +} +func packed(b []byte) []byte { + out := make([]byte, 384) + copy(out, "complzss") + binary.BigEndian.PutUint32(out[8:], adler32.Checksum(b)) + binary.BigEndian.PutUint32(out[12:], uint32(len(b))) + for off := 0; off < len(b); off += 8 { + n := min(8, len(b)-off) + out = append(out, byte((1<-desktop.png`. It then opens the original Calculator through Finder, +enters 2+2 and requires the exact result glyph4 (different from its initial0). +`.png` and `.json` record the application proof. The VM is +stopped, waited and closed. No terminal or injected application launcher is used. + +## Portable intent and native board + +```python +boot = vmm.darwin_boot(original_kernel_file, + command_line="-v keepsyms=1 rd=disk0s3 -s") +machine = vmm.machine(architecture="x86_64", memory=2048 << 20, + boot=boot, disks=[vmm.disk(original_disk_file, + bus="ide", unit=0, snapshot=True)], + display=vmm.display("capturable")) +vm = vmm.start(machine, cc.backend(acpi=True, hpet=True, + pci_ide=True, ide_model="ich7-pata")) +``` + +The root partition in a real recipe must be derived from the complete map; +it is not always slice2. Lion BaseSystem's 2048-byte APM includes map and driver +entries before HFS+ slice3. The example above is specific to verified media. +Linux and Darwin boot intents are mutually exclusive. Darwin requires the +`boot.darwin` backend capability. Native cc requires Linux/amd64 KVM, one CPU, +x86_64, and no UEFI-image execution. QEMU rejects Darwin intent explicitly. +The default synthetic PCI IDE and legacy Windows CPU policy remain unchanged. + +`ide_model="ich7-pata"` supplies Intel 8086:27df identification and relevant +register semantics over the existing native bounded ATA PIO/PRD-DMA engine. +Primary compatibility interrupts use IRQ14, while native INTA is routed through +ACPI _PRT to GSI16. These must not alias. Command/control and bus-master BARs +can be relocated. Decode, native interrupt-disable and timing-register masks +are honored; electrical cable timing is not simulated. The secondary channel +has no device (enabled reads return zero; disabled reads return 0xff). + +## Handoff contracts + +* `boot/darwin` selects the amd64 fat slice, verifies Apple's `comp/lzss` + length and Adler32, and validates unslid Mach-O64 executable segments and + LC_UNIXTHREAD. Kernel files and decoded caches are capped at 256MiB. + Zero-VM-size file-only __CTF data is not loaded. Destination bounds are + checked before copying segments or clearing their zero-fill tails. +* The native board supplies XNU boot_args v2/revision0, Apple's flattened + device tree (not Linux FDT), EFI64 system/runtime/configuration tables with + CRCs, and a sorted EFI memory map. Metadata fits the original low-1GiB + bootstrap mapping and leaves at least 128MiB free RAM. The first usable + range is conventional low memory, not the loaded kernel. +* Original pstart enters in flat 32-bit protected mode with IF and paging off, + EAX pointing to physical boot_args. It performs its own long-mode transition. + No external loader or injected transition code is run. +* EFI runtime pointers use XNU's high virtual mapping. Runtime functions + return EFI_UNSUPPORTED rather than simulating persistent firmware variables. + ACPI's device-tree configuration entry is named by its uppercase GUID, + `8868E871-E4F1-11D3-BC22-0080C73C8881`, with a physical RSDP address. +* The single virtual Penryn CPU retains only accelerator-supported instruction + features. The truthful CPUID hypervisor bit selects Apple's original virtual + power-management implementation. FSB frequency is derived from the stopped + VCPU's observed TSC frequency and PERF_STATUS multiplier, not a guessed + host clock. Missing observations fail closed. +* The original console requires framebuffer base bit0 set to indicate a + physical address after VM initialization. Exact `-v`/`-s` command-line tokens + select FB_TEXT_MODE; the native capture uses the same 1280x720 BGRX buffer. + +These wire formats and device semantics were implemented independently from +Apple's XNU 1699.32.7 ABI declarations/consumer behavior, AppleIntelPIIXATA +251.0.1 register use, and Intel ICH7 document 307013. Apple kernel/kext source +implementation bodies were not copied into the boot path. Tests use synthetic +Mach-O, EFI and disk fixtures, not distributed Apple payloads. + +## Bounded native observations + +`vm.extension("cc.v1")` is tied to the owning session and offers: + +* `symbol(name)` and `symbols(prefix)`: original Mach-O kernel symbols; prefix + selection is limited to 128 results. No prelinked-kext symbol table is implied. +* `symbolize(address)`: nearest preceding kernel symbol and unsigned offset, + or None if no predecessor exists. It is not a function-extent or kext resolver. +* `read_virtual(address, size)`: read-only, nonwrapping 0..65536-byte access + through the current original guest page tables, including cross-page reads. +* `disassemble(address, size=256, mode=64)`: read-only Intel-syntax instruction + rows; size is 0..4096, mode is 16/32/64. Invalid or partial final instructions + produce a terminal error row while preserving the decoded prefix. +* `breakpoint(address)`: native hardware execution breakpoint, with no guest + instruction modifications. Address0 disables it. +* `state()`: existing bounded device/CPU state plus Darwin handoff metadata, + full PCI IDE configuration and the last32 configuration transfers. Darwin + transfers retain at most six readable frame-pointer links from transaction + time; they are observations, not a guaranteed unwinder. + +The private recovery smoke uses a panic breakpoint, the bounded original msgbuf +ring and allproc list, and verifies its generation-specific process-name offset +against the supplied kernel's own proc_name instructions. It polls for actual +HFS-root/launchd/sh readiness within120 seconds, not generic changed pixels. +The final recovery PNG permits separate visual verification of the original +shell surface; the recovery smoke does not send keyboard commands. + +Run from the private repository (media paths must be supplied explicitly): + +```sh +go run ./trex/cmd/trex scripts/smoke/macos_lion_cc.star \ + /explicit/original/Lion-10.7.dmg /explicit/output/lion-rootboot +``` + +Only final `.png` and `.json` evidence is written. The VM is +stopped, waited and closed after successful observation. diff --git a/docs/macos-installers.md b/docs/macos-installers.md new file mode 100644 index 0000000..583aa62 --- /dev/null +++ b/docs/macos-installers.md @@ -0,0 +1,137 @@ +# Intel macOS installer layers + +The native readers cover the Apple installer media for Lion (10.7), Mountain +Lion (10.8), Yosemite (10.10), El Capitan (10.11) and Sierra (10.12). No host +mount, conversion or extraction program is needed. Each stage borrows portable +`storage.Reader` / Starlark `file` views; decoded host intermediates are not used. + +## Layer order + +1. `archive.udif(media)` reconstructs the logical disk from UDIF v4 `koly` + metadata and BLKX maps. `disk`, `xml`, `trailer` and original table records + remain accessible. `verify()` checks declared stored-data, BLKX and master + CRC32s. Ignored sectors contribute no bytes to BLKX CRCs; zero-fill does. +2. Parse **all** partitions using `filesystem.apm` (2048-byte map blocks on + these older images) or `filesystem.gpt`. Keep driver/map/free partitions; + do not mistake them for filesystems. EFI system partitions are FAT32 and + are accessible with `filesystem.fat(partition)["/"]`. +3. `filesystem.hfs(partition)` exposes HFS+/HFSX catalog entries, fragmented + data/resource forks, BSD metadata, links and extended attributes. Paths + percent-escape control characters, slash and percent reversibly. Hardlink + aliases retain `raw_data` / `raw_resource` and target inode paths; symbolic + links are never followed. `raw_forks=True` is an explicit forensic option, + not decoded-file evidence. decmpfs types 3/4 decode inline/resource zlib + files with stored-block escapes; unsupported types fail by default. +4. `archive.xar(package)` exposes the complete decoded XML TOC, original heap + and member records. Keep PackageInfo, Distribution, Bom, Scripts, Payload, + resources and signature metadata, rather than selecting Payload alone. + `member.verify()` checks archived/extracted hashes and decoded lengths. +5. In the outer packages, `InstallESD.dmg` is a **sibling member** of Payload, + not a file inside its CPIO application tree. Follow its DMG/partition/HFS+ + layers, plus BaseSystem.dmg and AppleDiagnostics.dmg where present. +6. Package Payload/Scripts are independently compressed CPIO streams: Lion + uses gzip or stored odc; Mountain Lion uses bzip2 or stored odc; later + releases use PBZX/XZ. `archive.gzip`, `archive.bzip2`, `archive.pbzx` return + decoded file views. Then `archive.cpio(..., maximum_entries=1000000)` keeps + Unix entry metadata and borrowed data. Read a stream sequentially with + `digest(decoded)` to validate all bytes/checksums; hashing each gzip member + in pathname order causes expensive backward replays. Nested packages in + CPIO contents must also be inspected. +7. `archive.bom` retains every inventory record, including architecture + variants with the same path and partial trailing leaves omitted from a + branch index. A BOM contains metadata, **not file contents**. The automatic + tree keeps variants under `%00bom-variants`, with original `bom_path`. +8. Sierra EmbeddedOSFirmware.pkg includes `.dmg` files that are actually + ASN.1 IM4P envelopes. `archive.im4p` preserves their payload and optional + DER metadata. One payload is LZFSE-compressed (`archive.lzfse`), the other + stored HFS+. Their HFS+ files include decmpfs type 3/4 compression. Firmware + contents are browsable/readable, but no firmware is executed. + +Header signatures take precedence over a trailer belonging to a nested member: +these outer XAR packages can end with the `koly` of their embedded DMG. + +## Explicit example + +```python +image = archive.udif(open("Lion-10.7.dmg")) +image.verify() +partitions = filesystem.apm(image.disk, block_size=2048) +hfs_part = [p.data for p in partitions.partitions + if p.partition_type == b"Apple_HFS"][0] +volume = filesystem.hfs(hfs_part) +package = archive.xar(volume.find("/InstallMacOSX.pkg").data) +for member in package.entries: + member.verify() +esd_member = [e for e in package.entries + if e.path.endswith("/InstallESD.dmg")][0] +esd = archive.udif(esd_member.data) +# Select explicit partition and package members to continue; do not materialize. +``` + +For a complete content traversal and standalone JSON validation report: + +```sh +go run ./cmd/trex scripts/inspect/macos_layers.star \ + /explicit/media/Sierra-10.12.dmg /explicit/output/Sierra-report.json +``` + +Run from the public checkout, or adjust script/media/report paths explicitly +when invoking the root workspace CLI. Original images are opened read-only. +Only the final report is written. `auto(file)` also recognizes UDIF, XAR, PBZX, +IM4P and LZFSE for interactive browsing; explicit APIs expose more metadata. + +## Verified original media + +Complete native traversals of the five original Apple DMGs read all discovered +installer containers, every XAR member, every decoded CPIO stream, and all HFS+ +data/resource/extended-attribute forks. Counts include outer installer packages, +nested packages inside payloads, and firmware filesystems (not just InstallESD). + +| Release | UDIF disks | HFS volumes | XAR packages | CPIO streams | +| --- | ---: | ---: | ---: | ---: | +| Lion 10.7 | 3 | 3 | 58 | 72 | +| Mountain Lion 10.8 | 3 | 3 | 17 | 21 | +| Yosemite 10.10 | 3 | 3 | 15 | 17 | +| El Capitan 10.11 | 4 | 4 | 8 | 10 | +| Sierra 10.12 | 4 | 6 | 7 | 11 | + +El Capitan and Sierra also expose two EFI FAT volumes each. Their payloads +contain nested USB-C adapter firmware packages; El Capitan additionally contains +NEProviderTargetTemplates.pkg. Lion/Mountain Lion contain a nested +RemoteDesktopClient.pkg. Sierra's two IM4P firmware payloads add two HFS volumes; +one contains 34 type-3 and 120 type-4 decmpfs catalog files, including aliases. +Reports retain partition inventories, decoded-stream SHA-256s, XAR TOC hashes, +flat BOM record counts and hashes of complete HFS fork inventories. These are +content-reading proofs, not installation or boot tests. + +## Bounds and verification scope + +UDIF plist and XAR TOCs are bounded to 16 MiB, with XML node/depth limits. +UDIF, PBZX and LZFSE chunks and decoded caches are bounded to 64 MiB; +PBZX/LZFSE have at most 100,000 chunks/blocks. LZFSE additionally retains +256 KiB of match history and replays dependent blocks on an uncached backward +read. decmpfs zlib chunks are at most 64 KiB, cached one at a time. Filesystem +and archive indexes use explicit entry limits. XAR members are capped at +64 GiB. Total decoded PBZX/LZFSE bytes may be bounded with `maximum_bytes`; +omitting it deliberately permits these multi-gigabyte installer streams. + +Checksum validation proves stored/decoded format consistency, **not Apple +signature authentication, successful installation or bootability**. Unknown +codecs and malformed/truncated inputs fail instead of silently yielding empty +files. These readers do not claim encrypted/segmented UDIF, APFS, newer UDIF +codecs or every decmpfs compression variant. IM4P exposes encrypted bytes if +present, but does not decrypt them; the witnessed firmware payloads are plain. + +## Format provenance + +UDIF facts: libdmg-hfsplus `dmg/dmg.h` and BLKX checksum conventions. +XAR facts: Apple's open-source xar header/XML TOC definitions. +HFS+ facts: Apple Technical Note TN1150 and HFSPlus format declarations. +PBZX: chunk length pairs and independent XZ streams, confirmed on the actual +Yosemite/El Capitan/Sierra installers. BOM layout: bomutils declarations, +with linked-leaf and architecture behavior confirmed on the original media. +IM4P: bounded ASN.1 envelope structure confirmed on Sierra firmware. +LZFSE/LZVN: Go implementation derived from Apple's BSD-licensed LZFSE +reference algorithms; copyright/license and file provenance are retained in +`archive/lzfse/NOTICE`. All other new fixtures are synthetic, not extracted +Apple payloads. Parsing/decompression remains native Go, without CGO. diff --git a/docs/starlark/namespaces/reference.md b/docs/starlark/namespaces/reference.md index eefe53f..3470b01 100644 --- a/docs/starlark/namespaces/reference.md +++ b/docs/starlark/namespaces/reference.md @@ -2098,6 +2098,12 @@ Reads an AWS tape image into logical records and tape marks, checking block fram Reads single-volume AIX by-name backup archives with extended name records (FS_NAME_X=11), checking every header checksum, record and security bounds, declared sizes and end framing. Stored payloads, raw headers and ACL/PCL bytes remain borrowed file views; packed Huffman bodies decode natively using the enclosing logical size and a cumulative decoded-byte limit. Entries preserve original names, paths, modes, inode/link identity, owners, device numbers and timestamps without applying them. The returned trailer retains final unused distribution-block bytes, which may be nonzero. This initial generation handles exact-end or 1KiB-rounded distribution archives; other record generations, continuation volumes and larger physical tails fail explicitly. It does not run scripts, plan installation, interpret XCOFF or validate uncompressed file contents with a nonexistent payload checksum. +### `archive.bom` + +`archive.bom(file, maximum_entries=1M) -> list[record]` + +Returns every flat Apple BOMStore path-inventory record, including architecture variants sharing a path and trailing leaf-chain nodes missing from branch indexes. Records preserve kind, BOM ID, mode, uid/gid, mtime, declared size, architecture, checksum or link target/device number. BOM describes metadata only and has no file contents. maximum_entries bounds indexing. The auto tree preserves duplicate variants under %00bom-variants with bom_path metadata. + ### `archive.bru` `archive.bru(file, maximum_entries=1M) -> record` @@ -2114,7 +2120,7 @@ Reads full, single-volume historical BSD dumps with 1024-byte records, magic6001 `archive.bzip2(file, maximum_bytes=) -> file` -Opens concatenated bzip2 streams, including legacy randomized blocks, as a lazy read-only file with a bounded decoded block cache. Block and stream checksums are checked during reads; size scans the complete stream. Omitted maximum_bytes has no decoded-length ceiling; an explicit positive maximum is enforced. Does not interpret an archive inside the stream. +Opens concatenated bzip2 streams, including legacy randomized blocks, as a lazy read-only file with a bounded decoded block cache. Block and stream checksums are checked during reads; size scans the complete stream. Omitted maximum_bytes has no decoded-length ceiling; an explicit positive maximum is enforced. cache=True requires explicit positive maximum_bytes, eagerly validates every member to EOF, and returns an immutable paged RAM snapshot supporting random access without replay or host files; integrity or overflow failures return no partial result. Does not interpret an archive inside the stream. ### `archive.cab` @@ -2160,9 +2166,9 @@ Constructs a deterministic Linux-compatible newc archive from Unix entry diction ### `archive.gzip` -`archive.gzip(file, maximum_bytes=) -> file` +`archive.gzip(file, maximum_bytes=, cache=False) -> file` -Opens concatenated gzip streams as a lazy read-only file with a 1 MiB decoded cache. Forward reads reuse the decoder; backward reads outside the cache replay it. Checksums are validated as data is consumed; size scans the complete stream without retaining it. Omitted maximum_bytes has no decoded-length ceiling; an explicit positive maximum is enforced. Does not interpret an archive inside the stream. +Opens concatenated gzip streams as a lazy read-only file with a 1 MiB decoded cache. Forward reads reuse the decoder; backward reads outside the cache replay it. Checksums are validated as data is consumed; size scans the complete stream without retaining it. Omitted maximum_bytes has no decoded-length ceiling; an explicit positive maximum is enforced. cache=True requires explicit positive maximum_bytes, eagerly validates every member to EOF, and returns an immutable paged RAM snapshot supporting random access without replay or host files; integrity or overflow failures return no partial result. Does not interpret an archive inside the stream. ### `archive.gzip_members` @@ -2188,6 +2194,12 @@ Reads concatenated classic Amiga and EHF HUNK_UNIT object libraries, preserving Reads IBM i 7.5 optical save streams: save groups, EBCDIC object names, descriptor types, raw headers and stored sections. Preserves logical sizes, opaque addresses and uninterpreted object trailers without inventing restored contents. Validates page framing and section bounds; standalone 528-byte SAVF transport and other release layouts are not supported. Auto browsing exposes named object directories and downloadable stored bytes; this is structural decoding, not object restoration. +### `archive.im4p` + +`archive.im4p(file) -> record` + +Parses an ASN.1 IM4P firmware envelope with bounded DER depth/node counts. Returns payload_type, description, borrowed payload, extras (original optional DER records) and expected_size (LZFSE metadata when present, otherwise zero). Does not decrypt, decompress, authenticate signatures or infer a filesystem; pass payload explicitly to archive.lzfse or the appropriate format reader. + ### `archive.installer` `archive.installer(file, maximum_scan=256MiB, cache=True, maximum_bytes=512MiB) -> installer` @@ -2254,6 +2266,12 @@ Returns the decoded KWAJ file together with its original name, method and compre Reads level-0 LHA headers and stored lh0 or Huffman/LZSS lh5 payloads. Checks header byte sums, file CRC-16, declared sizes, terminal marker, path components and duplicate paths. Returns entries, files and find(path). Entries preserve raw name/header bytes, the packed DOS timestamp, DOS attributes, method, CRC, decoded data and a borrowed stored view. Backslashes become path separators; parent traversal and empty components are rejected. maximum_entries bounds records; maximum_decoded_bytes bounds total decoded bytes and each stored payload. Other header levels, directory records, compression methods and self-extractors are explicit gaps. Nested containers are not automatically decoded. +### `archive.lzfse` + +`archive.lzfse(file, maximum_bytes=) -> file` + +Opens Apple LZFSE v1/v2, LZVN and stored-block streams as a read-only file. Bounds block size/count and retains a 64 MiB decoded cache plus 256 KiB match history. Backward reads outside cache replay preceding dependent blocks. Validates entropy states, source/output lengths, match distances and end marker; LZFSE has no content checksum. Explicit maximum_bytes bounds total decoded bytes. No host or C decoder is used. + ### `archive.mac_resource` `archive.mac_resource(file, maximum_entries=1M, maximum_decoded_bytes=256MiB) -> record` @@ -2278,6 +2296,12 @@ Lists File instructions from the NSIS 2 ANSI non-solid stored/NSIS-DEFLATE metad Decodes a UNIX pack (1f1e) Huffman stream, checking its symbol tree, end marker, padding and declared output size. Enforces maximum_bytes and returns a file; useful for compressed IRIX manual pages nested inside inst images. +### `archive.pbzx` + +`archive.pbzx(file, maximum_bytes=) -> file` + +Opens chunked Apple installer payloads as a lazy read-only concatenated file. Validates decoded/stored chunk lengths, XZ indexes and final short-chunk framing; stored chunks are borrowed. Uses a 64 MiB decoded cache and 64 MiB per-chunk bound. Omitted maximum_bytes has no total decoded ceiling; explicit positive limits are enforced. Read complete contents to force XZ checksums. Parse the resulting CPIO separately. + ### `archive.rar` `archive.rar(file, maximum_entries=100000) -> RAR archive` @@ -2326,6 +2350,12 @@ With a directory, serializes its entries into tar bytes, optionally compressed. Reads classic kc0001 Apple installer Tome catalogs and decodes both forks using chunked InstaCompOne in memory. Validates catalog counts, unique IDs, payload ranges, nonoverlap, exact fork sizes, both decoded-fork checksums and complete compressed-input consumption. Returns entries with ID, raw name bytes, reversible ID-qualified paths, file type/creator, Finder flags, version, Mac-epoch dates, decoded data/resource files and original stored views. maximum_decoded_bytes bounds total decoded output and each stored fork. Catalog checksum values are preserved as data_checksum and resource_checksum; checksums_verified is True after validation. Supports the observed 00010000 binary and 00000000 seven-bit-text chunk framing, plus stored chunks selected by a leading 01 byte, with retained 32KiB history and command-aligned 64KiB blocks; other encodings fail explicitly. Resource maps and nested archives require separate decoders. Installer code never runs. catalog_kind distinguishes file catalogs (1) from individual-resource catalogs (2). Kind2 exposes resource_type and signed resource_id, with the decoded resource payload in data and an empty resource fork; file-only metadata is omitted. Unknown catalog kinds are rejected. +### `archive.udif` + +`archive.udif(file) -> record` + +Opens XML-backed, single-segment UDIF v4 disk images. Returns disk (complete logical disk), original xml and trailer, BLKX tables (name, offset, size, raw), and verify(). Raw, zero-fill, ignored, zlib and bzip2 chunks use bounded caches. verify() reads stored data and checks declared data-fork, table and master CRC32s; ignored sectors are excluded from table checksums. Rejects encryption, segmented images, unsupported codecs and overlapping/gapped maps. Does not authenticate signatures or infer filesystems. + ### `archive.vmsbackup` `archive.vmsbackup(file, maximum_blocks=1M, maximum_records=1M, maximum_block_size=16MiB, maximum_files=1M, maximum_attributes=256) -> record` @@ -2344,6 +2374,12 @@ Reads a complete single-volume VMS BACKUP block stream, validating header CRC-16 Parses a Windows Imaging Format archive and exposes its image contents through file views. The archive reader handles its supported compression internally without mounting an image. +### `archive.xar` + +`archive.xar(file, maximum_entries=100000) -> record` + +Indexes XAR v1 without extraction. Returns entries, files, original decoded toc, raw heap and find(path). Entries preserve path, kind, ID, mode, uid/gid, link target, encoding, raw/data file views, size and verify(). Verifies declared compressed-TOC checksum on open; member verify() checks archived/extracted checksums and exact decoded lengths. Supports stored, zlib and bzip2 members. Safe paths, XML depth/size, heap extents and index sizes are bounded. Signature metadata remains in TOC/heap but signatures are not authenticated. + ### `archive.xz` `archive.xz(file, max_dictionary=64MiB) -> file` @@ -2368,6 +2404,12 @@ Opens a Zstandard stream as a portable file with a 64 MiB decoder window limit. Wraps a file with caller-supplied attributes while retaining its bytes. Use this to attach construction metadata without rebuilding the file's contents. +### `binary.appledouble` + +`binary.appledouble(file) -> dict[int,file]` + +Parses AppleDouble version 2 headers and returns an integer-entry-ID dictionary of borrowed files. Validates table bounds, duplicate IDs and non-overlapping extents. Preserves unknown entries without interpreting them. FinderInfo (9), resource fork (2) and any ATTR extension remain raw bytes; does not apply metadata or modify files. + ### `binary.base64` `binary.base64(value, url=False, padding=True, maximum=512MiB)` @@ -2494,6 +2536,24 @@ Encodes a 8-bit signed integer as 1 bytes in single-byte order. Returns new byte Compiles a fixed-size binary layout from a format string and optional field names. The returned value encodes records or decodes a source at a byte offset. +### `binary.macho` + +`binary.macho(file, architecture="x86_64") -> record` + +Inspects a bounded little-endian Mach-O64 x86_64 image, optionally selected from a big-endian fat32/fat64 universal container. Exposes the selected borrowed file, segments (name, address, size, file_offset, file_size, flags), and original symbols (name, address, type, section, description). at(address, size) returns borrowed file-backed virtual bytes; zero-fill, unmapped, ambiguous and overflowing ranges fail. Header, command, string and symbol counts and aggregate symbol-name allocations are bounded. Does not relocate, link or execute the image. + +### `binary.plist` + +`binary.plist(source) -> value` + +Decodes bounded XML 1.0 or bplist00 property lists into dict/list, bytes, string, bool, int64, float and opaque round-trippable plist_date values. Binary null becomes None; unsupported UID and 16-byte integers fail explicitly. Enforces 64MiB input/decoded data, 100000 visited values and depth128, validates reference/offset bounds, rejects cycles, duplicate keys and invalid Unicode. Does not follow XML external entities. + +### `binary.plist_encode` + +`binary.plist_encode(value) -> file` + +Encodes supported plist values as deterministic XML1.0 in an in-memory file. Preserves plist_date values from decoding. Enforces aggregate conversion/output byte, node and depth limits, including dictionary keys. Rejects None, non-finite floats, non-string keys, invalid XML characters and integers outside int64. + ### `binary.read_f32be` `binary.read_f32be(source, offset=0) -> float` @@ -2706,9 +2766,9 @@ Returns a read-only file view of a live block device. Later device changes remai ### `cc.backend` -`cc.backend(acpi=False, pci_ide=False, hpet=False, uefi=False, ide_dma=True, overlay_limit=268435456) -> vmm_backend` +`cc.backend(acpi=False, pci_ide=False, hpet=False, uefi=False, ide_model='synthetic', ide_dma=True, overlay_limit=268435456) -> vmm_backend` -Creates an in-process CrumbleCracker PC backend for Linux/amd64 KVM. The i386 and x86_64 platforms provide Go BIOS services, one ATA disk, VGA capture and PS/2 input. ACPI is automatic for x86_64 and optional for i386. uefi=True selects native Go UEFI for x86_64 GPT disks, enables PCI IDE, and provides a 1280x720 GOP framebuffer. RAM supports 16 MiB through 2 GiB. hpet=True adds a 100 MHz HPET with three comparators and enables ACPI. pci_ide enables ACPI and a generic PCI IDE controller with PIO and bus-master DMA; ide_dma=False selects a PIO-only disk. overlay_limit bounds dirty snapshot memory in bytes (default 256 MiB). vmm.start creates the guest from portable memory and disk intent. +Creates an in-process CrumbleCracker PC backend for Linux/amd64 KVM. The i386 and x86_64 platforms provide Go BIOS services, one ATA disk, VGA capture and PS/2 input. ACPI is automatic for x86_64 and optional for i386. uefi=True selects native Go UEFI for x86_64 GPT disks, enables PCI IDE, and provides a 1280x720 GOP framebuffer. RAM supports 16 MiB through 2 GiB. hpet=True adds a 100 MHz HPET with three comparators and enables ACPI. pci_ide enables ACPI and a generic PCI IDE controller with PIO and bus-master DMA; ide_dma=False selects a PIO-only disk. ide_model='synthetic' preserves the generic compatibility controller; 'ich7-pata' requires PCI IDE and selects the 8086:27df profile with one primary disk, compatibility IRQ14 and native INTA routed to GSI16. The secondary channel has no disk. Timing registers are recorded; electrical cable timing is not simulated. The cc.v1 extension provides bounded read_virtual, original kernel symbol lookup and read-only disassemble observations for direct Darwin boot. overlay_limit bounds dirty snapshot memory in bytes (default 256 MiB). vmm.start creates the guest from portable memory and disk intent. ### `channel.expose_tcp` @@ -3042,9 +3102,15 @@ Decodes standard and extended HD-Copy images in memory with strict RLE validatio ### `filesystem.hfs` -`filesystem.hfs(file, maximum_entries=1M) -> record` +`filesystem.hfs(file, maximum_entries=1M, raw_forks=False) -> record` + +Reads classic HFS, HFS+/HFSX and embedded HFS+ volumes without mounting. Returns raw volume name, entries, paths and exact find(path). Entries expose distinct data/resource forks, xattrs, reversible escaped paths, Finder/catalog metadata, Unix modes/uid/gid and link targets. Hardlink inode aliases share decoded inode contents while raw_data/raw_resource retain physical alias forks. Symlinks are not followed. Inline and fork-backed attributes have independent extent-overflow support. decmpfs types 3/4 (inline/resource zlib with stored-block escape) decode lazily with exact lengths; other compression types fail unless raw_forks=True explicitly requests physical forensic forks. compression_type identifies decoded files. Does not mount, repair or execute filesystem contents. + +### `filesystem.hfs_build` -Reads a classic HFS volume's catalog and extent-overflow records without mounting. Returns raw volume name, entries, paths and exact find(path) lookup. Every regular entry exposes separate data and resource fork views, their sizes, raw name bytes, catalog/parent IDs, Finder information, flags and Mac-epoch timestamps. Path components percent-escape non-ASCII/control bytes, slash and percent, keeping legacy names reversible without assuming a script encoding. HFS Plus and fragmented extents-file bootstrap are not yet supported; compressed resources require a separate resource decoder. Aliases are not followed. +`filesystem.hfs_build(entries, size, label="Untitled") -> file` + +Builds a bounded unjournaled case-sensitive HFSX version-5 volume natively in memory from UNIX entry dictionaries. size is the volume byte length; payloads are borrowed file/string/bytes values. Entries support path, mode, uid, gid, mtime, data, target, hardlink, major, minor, finder_info (32 bytes), resource, xattrs (name to file/bytes), owner_flags and admin_flags. Preserves regular-file hardlinks through native private inodes; directory hardlinks are unsupported. Paths use POSIX components (colon becomes catalog slash); names use HFS canonical decomposition. Allocation, catalog and attribute trees include free growth nodes with bounded metadata. Does not build partitions, journal, execute guest tools or write intermediate host files. ### `filesystem.host` @@ -3439,6 +3505,12 @@ Lists the VMM backends registered in this runtime and their advertised capabilit Describes a named VM byte channel of the requested kind. required determines whether lack of backend support is a validation failure. +### `vmm.darwin_boot` + +`vmm.darwin_boot(kernel, command_line='', smc_osk=b'') -> vmm_darwin_boot` + +Creates an unslid amd64 XNU v2 direct handoff from an immutable original kernelcache or Mach-O executable. Supports amd64 fat slices and Apple comp/lzss caches with checksum validation. Native cc requires Linux/amd64 KVM, x86_64 and no UEFI image execution; QEMU rejects this intent. No guest bootloader is executed or kernel instructions patched. RAM disks are not supported. Optional smc_osk must be empty or exactly 64 bytes; it is explicit caller-supplied SMC material, never bundled or logged. Native cc exposes SMC APP0001/IRQ6 and omits OSK0/OSK1 when no material is supplied. Verified scope is Lion HFS-root boot to launchd and a visible single-user shell, not keyboard input, a desktop or a complete installation. + ### `vmm.disk` `vmm.disk(source, name='disk0', bus='auto', media='disk', unit=-1, chs=None, read_only=None, snapshot=False, required=True)` diff --git a/filesystem/hfs/attributes.go b/filesystem/hfs/attributes.go new file mode 100644 index 0000000..e8eefe6 --- /dev/null +++ b/filesystem/hfs/attributes.go @@ -0,0 +1,168 @@ +package hfs + +import ( + "fmt" + starfile "github.com/tinyrange/trex/storage/star" + "strings" +) + +type attrKey struct { + id uint32 + name string +} +type attrRecord struct { + inline starfile.File + fork []byte + extents map[uint32][]plusExtent +} + +// HFS+ attributes are an independent B-tree. Attribute forks use their own +// overflow records, not the ordinary data/resource extents-overflow tree. +func (r *plusReader) attributes(desc []byte, v *Volume, ids map[uint32]int, maximum int) error { + tree, err := r.fork(8, 0, desc) + if err != nil { + return err + } + if tree.Size() == 0 { + return nil + } + records := map[attrKey]*attrRecord{} + err = leafRecords(tree, maximum, func(b []byte) error { + if len(b) < 18 { + return fmt.Errorf("hfs+: truncated attribute key") + } + keySize := int(be.Uint16(b)) + n := int(be.Uint16(b[12:])) + off := 2 + keySize + if keySize != 12+2*n || n == 0 || n > 127 || off+4 > len(b) { + return fmt.Errorf("hfs+: invalid attribute key") + } + name, err := unicodeName(b[14:off]) + if err != nil { + return err + } + key := attrKey{be.Uint32(b[4:]), string(name)} + start := be.Uint32(b[8:]) + data := b[off:] + record := records[key] + if record == nil { + record = &attrRecord{extents: map[uint32][]plusExtent{}} + records[key] = record + } + switch be.Uint32(data) { + case 0x10: + if start != 0 || len(data) < 16 || record.inline != nil || record.fork != nil { + return fmt.Errorf("hfs+: invalid/duplicate inline attribute") + } + size := be.Uint32(data[12:]) + if uint64(size) > uint64(len(data)-16) { + return fmt.Errorf("hfs+: truncated inline attribute") + } + record.inline = &starfile.Bytes{Data: append([]byte(nil), data[16:16+int(size)]...)} + case 0x20: + if start != 0 || len(data) != 88 || record.fork != nil || record.inline != nil { + return fmt.Errorf("hfs+: invalid/duplicate fork attribute") + } + record.fork = append([]byte(nil), data[8:]...) + case 0x30: + if start == 0 || len(data) != 72 || record.extents[start] != nil { + return fmt.Errorf("hfs+: invalid attribute overflow") + } + record.extents[start] = plusExtents(data[8:]) + default: + return fmt.Errorf("hfs+: unsupported attribute record") + } + return nil + }) + if err != nil { + return err + } + for key, record := range records { + index, ok := ids[key.id] + if !ok { + return fmt.Errorf("hfs+: attribute for missing catalog ID %d", key.id) + } + value := record.inline + if record.fork != nil { + forkReader := *r + forkReader.overflow = map[plusKey][]plusExtent{} + for start, extents := range record.extents { + forkReader.overflow[plusKey{key.id, 0, start}] = extents + } + value, err = forkReader.fork(key.id, 0, record.fork) + if err != nil { + return err + } + } else if len(record.extents) != 0 { + return fmt.Errorf("hfs+: orphan attribute extents") + } + if value == nil { + return fmt.Errorf("hfs+: missing attribute data") + } + entry := &v.Entries[index] + if entry.Xattrs == nil { + entry.Xattrs = map[string]starfile.File{} + } + entry.Xattrs[key.name] = value + } + return nil +} + +// Resolve only the documented file-inode alias, never symbolic links. Keep +// both the original alias fork and inode path available for forensic access. +func plusLinks(v *Volume, raw ...bool) error { + var private uint32 + for _, e := range v.Entries { + if e.Parent == 2 && string(e.Name) == "\x00\x00\x00\x00HFS+ Private Data" && e.Kind == "directory" { + private = e.ID + } + } + inodes := map[string]int{} + for i, e := range v.Entries { + if private != 0 && e.Parent == private { + inodes[string(e.Name)] = i + } + } + for i := range v.Entries { + e := &v.Entries[i] + if e.Kind != "file" { + continue + } + if len(e.FinderInfo) >= 8 && string(e.FinderInfo[:8]) == "hlnkhfs+" { + index, ok := inodes[fmt.Sprintf("iNode%d", e.Special)] + if !ok { + return fmt.Errorf("hfs+: unresolved hardlink inode %d", e.Special) + } + inode := &v.Entries[index] + if inode.Kind != "file" || len(inode.FinderInfo) >= 8 && string(inode.FinderInfo[:8]) == "hlnkhfs+" { + return fmt.Errorf("hfs+: invalid hardlink target") + } + e.RawData = e.Data + e.RawResource = e.Resource + e.Data = inode.Data + e.CompressionType = inode.CompressionType + e.Resource = inode.Resource + e.Xattrs = inode.Xattrs + e.Target = inode.Path + } + if e.Mode&0170000 == 0120000 { + if e.Data == nil || e.Data.Size() > 65536 { + return fmt.Errorf("hfs+: invalid symlink size") + } + data, err := starfile.ReadAll(e.Data) + if err != nil { + return err + } + if len(data) == 0 || strings.ContainsRune(string(data), 0) { + return fmt.Errorf("hfs+: invalid symlink target") + } + e.Kind = "symlink" + e.Target = string(data) + } + // Unsupported compressed files must never look like empty regular files. + if e.OwnerFlags&0x20 != 0 && e.CompressionType == 0 && (len(raw) == 0 || !raw[0]) { + return fmt.Errorf("hfs+: decmpfs-compressed file unsupported: %s", e.Path) + } + } + return nil +} diff --git a/filesystem/hfs/attributes_test.go b/filesystem/hfs/attributes_test.go new file mode 100644 index 0000000..eba1f82 --- /dev/null +++ b/filesystem/hfs/attributes_test.go @@ -0,0 +1,147 @@ +package hfs + +import ( + "bytes" + "github.com/tinyrange/trex/auto" + starfile "github.com/tinyrange/trex/storage/star" + "testing" + "unicode/utf16" +) + +func attributeKey(id, start uint32, name string, data []byte) []byte { + u := utf16.Encode([]rune(name)) + b := make([]byte, 14+len(u)*2+len(data)) + be.PutUint16(b, uint16(12+len(u)*2)) + be.PutUint32(b[4:], id) + be.PutUint32(b[8:], start) + be.PutUint16(b[12:], uint16(len(u))) + for i, c := range u { + be.PutUint16(b[14+2*i:], c) + } + copy(b[14+len(u)*2:], data) + return b +} +func inlineAttribute(id uint32, name string, value []byte) []byte { + b := make([]byte, 16+len(value)) + be.PutUint32(b, 0x10) + be.PutUint32(b[12:], uint32(len(value))) + copy(b[16:], value) + return attributeKey(id, 0, name, b) +} +func attributeFixture(records ...[]byte) []byte { + b := plusFixture() + desc := b[1024+352:] + be.PutUint64(desc, 1024) + be.PutUint32(desc[12:], 2) + be.PutUint32(desc[16:], 40) + be.PutUint32(desc[20:], 2) + copy(b[40*512:], tree(records...)) + return b +} +func TestPlusInlineAttributesAndAuto(t *testing.T) { + b := attributeFixture(inlineAttribute(16, "com.apple.test", []byte{0, 1, 255}), inlineAttribute(2, "directory", []byte("root"))) + v, err := Open(&starfile.Bytes{Data: b}, 100) + if err != nil { + t.Fatal(err) + } + for i, name := range []string{"directory", "com.apple.test"} { + got, err := starfile.ReadAll(v.Entries[i].Xattrs[name]) + want := []byte("root") + if i == 1 { + want = []byte{0, 1, 255} + } + if err != nil || !bytes.Equal(got, want) { + t.Fatalf("attribute %q %x %v", name, got, err) + } + } + node := auto.Open(&starfile.Bytes{Data: b}, "disk", auto.Options{}) + for path, want := range map[string][]byte{"é%2F😀/xattrs/com.apple.test": {0, 1, 255}, "é%2F😀/resource": []byte("res")} { + n, err := node.Resolve(path) + if err != nil { + t.Fatal(err) + } + got, err := starfile.ReadAll(n.Reader()) + if err != nil || !bytes.Equal(got, want) { + t.Fatalf("auto attribute %x %v", got, err) + } + } + // Root attributes must not replace the directory view and hide children. + if _, err := node.Resolve("é%2F😀/data"); err != nil { + t.Fatal(err) + } +} +func TestPlusForkAttributeOverflow(t *testing.T) { + data := make([]byte, 88) + be.PutUint32(data, 0x20) + fork := data[8:] + be.PutUint64(fork, 4097) + be.PutUint32(fork[12:], 9) + for i := 0; i < 8; i++ { + be.PutUint32(fork[16+i*8:], uint32(10+2*i)) + be.PutUint32(fork[20+i*8:], 1) + } + overflow := make([]byte, 72) + be.PutUint32(overflow, 0x30) + be.PutUint32(overflow[8:], 30) + be.PutUint32(overflow[12:], 1) + b := attributeFixture(attributeKey(16, 0, "large", data), attributeKey(16, 8, "large", overflow)) + v, err := Open(&starfile.Bytes{Data: b}, 100) + if err != nil { + t.Fatal(err) + } + got, err := starfile.ReadAll(v.Entries[1].Xattrs["large"]) + if err != nil || len(got) != 4097 || got[4096] != 9 || got[0] != 1 || got[512] != 2 { + t.Fatalf("fork attr %d %v", len(got), err) + } + // Attribute overflow must not accidentally use the ordinary data-fork tree. + b = attributeFixture(attributeKey(16, 0, "large", data)) + if _, err := Open(&starfile.Bytes{Data: b}, 100); err == nil { + t.Fatal("missing independent attribute overflow accepted") + } +} +func TestPlusRejectAttributeCorruption(t *testing.T) { + valid := inlineAttribute(16, "a", []byte("test")) + for name, records := range map[string][][]byte{ + "duplicate": {valid, valid}, + "missing ID": {inlineAttribute(999, "a", nil)}, + "truncated": {valid[:len(valid)-1]}, + "overflow without fork": {attributeKey(16, 1, "a", append([]byte{0, 0, 0, 0x30}, make([]byte, 68)...))}, + } { + t.Run(name, func(t *testing.T) { + if _, err := Open(&starfile.Bytes{Data: attributeFixture(records...)}, 100); err == nil { + t.Fatal("accepted malformed attributes") + } + }) + } +} +func TestPlusLinksPreserveRawForksAndRejectInvalidTargets(t *testing.T) { + raw := &starfile.Bytes{Data: []byte("alias")} + rawRes := &starfile.Bytes{Data: []byte("alias-resource")} + data := &starfile.Bytes{Data: []byte("inode data")} + res := &starfile.Bytes{Data: []byte("resource")} + v := &Volume{Entries: []Entry{ + {ID: 20, Parent: 2, Name: []byte("\x00\x00\x00\x00HFS+ Private Data"), Kind: "directory"}, + {ID: 21, Parent: 20, Name: []byte("iNode123"), Path: "/private/iNode123", Kind: "file", Data: data, Resource: res, Xattrs: map[string]starfile.File{"attr": data}}, + {ID: 22, Parent: 2, Path: "/alias", Kind: "file", FinderInfo: []byte("hlnkhfs+"), Special: 123, Data: raw, Resource: rawRes}, + {ID: 23, Parent: 2, Path: "/link", Kind: "file", Mode: 0120777, Data: &starfile.Bytes{Data: []byte("../target")}}, + }} + if err := plusLinks(v); err != nil { + t.Fatal(err) + } + e := v.Entries[2] + if e.Data != data || e.Resource != res || e.RawData != raw || e.RawResource != rawRes || e.Xattrs["attr"] != data || e.Target != "/private/iNode123" { + t.Fatal("inode alias lost forks/metadata") + } + if v.Entries[3].Kind != "symlink" || v.Entries[3].Target != "../target" { + t.Fatal("symlink not decoded") + } + for _, e := range []Entry{ + {Kind: "file", FinderInfo: []byte("hlnkhfs+"), Special: 456}, + {Kind: "file", Mode: 0120777, Data: &starfile.Bytes{Data: []byte("a\x00b")}}, + {Kind: "file", OwnerFlags: 0x20}, + } { + if err := plusLinks(&Volume{Entries: []Entry{e}}); err == nil { + t.Fatal("accepted unresolved/compressed/invalid link") + } + } +} diff --git a/filesystem/hfs/auto.go b/filesystem/hfs/auto.go index d29a9a5..05b8273 100644 --- a/filesystem/hfs/auto.go +++ b/filesystem/hfs/auto.go @@ -17,13 +17,29 @@ func init() { } var entries []auto.Entry for _, e := range volume.Entries { - item := auto.Entry{Name: e.Path, Kind: e.Kind, Reader: e.Data, Attributes: map[string]any{"name": e.Name, "id": e.ID, "created": e.Created, "modified": e.Modified, "finder_info": e.FinderInfo}} - if e.Resource != nil { - forks := []auto.Entry{{Name: "resource", Kind: "file", Reader: e.Resource}} + item := auto.Entry{Name: e.Path, Kind: e.Kind, Reader: e.Data, Attributes: map[string]any{"name": e.Name, "id": e.ID, "created": e.Created, "modified": e.Modified, "finder_info": e.FinderInfo, "mode": e.Mode, "uid": e.UID, "gid": e.GID, "owner_flags": e.OwnerFlags, "admin_flags": e.AdminFlags, "target": e.Target}} + if e.Kind != "directory" && (e.Resource != nil && e.Resource.Size() != 0 || len(e.Xattrs) != 0 || e.RawData != nil) { + var forks []auto.Entry + if e.Resource != nil { + forks = append(forks, auto.Entry{Name: "resource", Kind: "file", Reader: e.Resource}) + } + if e.RawData != nil { + forks = append(forks, auto.Entry{Name: "raw_data", Kind: "file", Reader: e.RawData}) + } + if e.RawResource != nil { + forks = append(forks, auto.Entry{Name: "raw_resource", Kind: "file", Reader: e.RawResource}) + } + for name, value := range e.Xattrs { + forks = append(forks, auto.Entry{Name: "xattrs/" + plusComponent([]byte(name)), Kind: "file", Reader: value}) + } if e.Data != nil { forks = append(forks, auto.Entry{Name: "data", Kind: "file", Reader: e.Data}) } - item.View = auto.ViewFunc(func() ([]auto.Entry, error) { return forks, nil }) + forkView, err := auto.Tree(forks, o) + if err != nil { + return nil, err + } + item.View = forkView } entries = append(entries, item) } diff --git a/filesystem/hfs/build.go b/filesystem/hfs/build.go new file mode 100644 index 0000000..b23a851 --- /dev/null +++ b/filesystem/hfs/build.go @@ -0,0 +1,573 @@ +package hfs + +import ( + "bytes" + "fmt" + "io" + "math" + "path" + "sort" + "strings" + "unicode/utf16" + "unicode/utf8" + + "github.com/tinyrange/trex/filesystem/unixfs" + "github.com/tinyrange/trex/storage" + "golang.org/x/text/unicode/norm" +) + +// BuildEntry describes native Unix metadata, separate data/resource forks and +// extended attributes. Payload readers are borrowed, not extracted or copied. +type BuildEntry struct { + unixfs.Entry + FinderInfo []byte + Resource storage.Reader + Xattrs map[string]storage.Reader + OwnerFlags, AdminFlags byte +} +type BuildOptions struct { + Size int64 + Label string +} + +const allocationBlock = 4096 +const treeNodeSize = 8192 +const hfsEpoch = 2082844800 + +type hfsFragment struct { + offset int64 + source storage.Reader +} +type hfsBuilder struct { + next, total uint32 + fragments []hfsFragment +} + +func (b *hfsBuilder) allocate(r storage.Reader) ([]byte, error) { + desc := make([]byte, 80) + if r == nil || r.Size() == 0 { + return desc, nil + } + if r.Size() < 0 || uint64(r.Size()) > uint64(b.total)*allocationBlock { + return nil, fmt.Errorf("hfsx: invalid fork size") + } + blocks := uint32((r.Size() + allocationBlock - 1) / allocationBlock) + if blocks > b.total-1-b.next { + return nil, fmt.Errorf("hfsx: volume full") + } + be.PutUint64(desc, uint64(r.Size())) + be.PutUint32(desc[12:], blocks) + be.PutUint32(desc[16:], b.next) + be.PutUint32(desc[20:], blocks) + b.fragments = append(b.fragments, hfsFragment{int64(b.next) * allocationBlock, r}) + b.next += blocks + return desc, nil +} + +// hfsName uses canonical decomposition with the TN1150 excluded ranges. HFSX +// binary catalogs compare UTF-16 code units, not UTF-8 bytes or locale ordering. +func hfsName(name string, maximum int) ([]uint16, error) { + if !utf8.ValidString(name) || strings.ContainsAny(name, "\x00/") { + return nil, fmt.Errorf("hfsx: invalid name %q", name) + } + var decomposed, run strings.Builder + flush := func() { decomposed.WriteString(norm.NFD.String(run.String())); run.Reset() } + for _, r := range name { + if r >= 0x2000 && r <= 0x2fff || r >= 0xf900 && r <= 0xfaff || r >= 0x2f800 && r <= 0x2faff { + flush() + decomposed.WriteRune(r) + } else { + run.WriteRune(r) + } + } + flush() + u := utf16.Encode([]rune(decomposed.String())) + if len(u) > maximum { + return nil, fmt.Errorf("hfsx: name exceeds %d UTF-16 units", maximum) + } + return u, nil +} +func catalogKey(id uint32, name []uint16) []byte { + b := make([]byte, 8+2*len(name)) + be.PutUint16(b, uint16(len(b)-2)) + be.PutUint32(b[2:], id) + be.PutUint16(b[6:], uint16(len(name))) + for i, v := range name { + be.PutUint16(b[8+i*2:], v) + } + return b +} + +type treeRecord struct{ key, data []byte } +type buildTreeNode struct { + records []treeRecord + kind byte + height byte + id uint32 +} + +// buildTree builds actual index levels and linked leaves, with free nodes for +// guest growth. All metadata allocations are bounded independently of disk size. +func buildTree(records []treeRecord, maxKey uint16, compare byte) ([]byte, error) { + nodes := []buildTreeNode{{kind: 1}} + pack := func(rows []treeRecord, kind, height byte) ([]buildTreeNode, error) { + var out []buildTreeNode + n := buildTreeNode{kind: kind, height: height} + used := 14 + for _, r := range rows { + need := len(r.key) + len(r.data) + if need&1 != 0 { + need++ + } + if need+14+4 > treeNodeSize { + return nil, fmt.Errorf("hfsx: B-tree record too large") + } + if used+need+2*(len(n.records)+2) > treeNodeSize { + out = append(out, n) + n = buildTreeNode{kind: kind, height: height} + used = 14 + } + n.records = append(n.records, r) + used += need + } + if len(n.records) > 0 { + out = append(out, n) + } + return out, nil + } + level, err := pack(records, 255, 1) + if err != nil { + return nil, err + } + first, last := uint32(0), uint32(0) + depth := byte(0) + root := uint32(0) + for len(level) > 0 { + for i := range level { + level[i].id = uint32(len(nodes)) + nodes = append(nodes, level[i]) + } + if depth == 0 { + first = level[0].id + last = level[len(level)-1].id + } + depth = level[0].height + if len(level) == 1 { + root = level[0].id + break + } + var indices []treeRecord + for _, n := range level { + child := make([]byte, 4) + be.PutUint32(child, n.id) + indices = append(indices, treeRecord{n.records[0].key, child}) + } + if depth >= 16 { + return nil, fmt.Errorf("hfsx: B-tree depth exceeded") + } + level, err = pack(indices, 0, depth+1) + if err != nil { + return nil, err + } + } + total := max(8, len(nodes)*2) + // A single header map describes this deliberately bounded tree (no map nodes). + mapSize := treeNodeSize - 14 - 106 - 128 - 8 + if total > mapSize*8 || total*treeNodeSize > 256<<20 { + return nil, fmt.Errorf("hfsx: B-tree metadata limit") + } + out := make([]byte, total*treeNodeSize) + write := func(id uint32, kind, height byte, rows [][]byte) { + n := out[int(id)*treeNodeSize : (int(id)+1)*treeNodeSize] + n[8], n[9] = kind, height + be.PutUint16(n[10:], uint16(len(rows))) + pos := 14 + for i, r := range rows { + be.PutUint16(n[treeNodeSize-2*(i+1):], uint16(pos)) + copy(n[pos:], r) + pos += (len(r) + 1) &^ 1 + } + be.PutUint16(n[treeNodeSize-2*(len(rows)+1):], uint16(pos)) + } + header := make([]byte, 106) + be.PutUint16(header, uint16(depth)) + be.PutUint32(header[2:], root) + be.PutUint32(header[6:], uint32(len(records))) + be.PutUint32(header[10:], first) + be.PutUint32(header[14:], last) + be.PutUint16(header[18:], treeNodeSize) + be.PutUint16(header[20:], maxKey) + be.PutUint32(header[22:], uint32(total)) + be.PutUint32(header[26:], uint32(total-len(nodes))) + be.PutUint32(header[32:], treeNodeSize*8) + header[37] = compare + be.PutUint32(header[38:], 6) // big keys, variable index keys. + bitmap := make([]byte, mapSize) + for i := range nodes { + bitmap[i/8] |= 0x80 >> uint(i%8) + } + write(0, 1, 0, [][]byte{header, make([]byte, 128), bitmap}) + for _, n := range nodes[1:] { + rows := make([][]byte, len(n.records)) + for i, r := range n.records { + rows[i] = append(append([]byte(nil), r.key...), r.data...) + } + write(n.id, n.kind, n.height, rows) + // Sibling links at every level; leaves are also traversable without indexes. + if n.id > 1 && nodes[n.id-1].height == n.height { + be.PutUint32(out[int(n.id)*treeNodeSize+4:], n.id-1) + } + if int(n.id)+1 < len(nodes) && nodes[n.id+1].height == n.height { + be.PutUint32(out[int(n.id)*treeNodeSize:], n.id+1) + } + } + return out, nil +} + +// Build creates an unjournaled, case-sensitive HFSX volume (HFS+ generation 5). +// It does not create partitions, execute mkfs or modify borrowed payloads. +// Regular-file hardlinks use the TN1150 private inode directory and hlnk/hfs+ +// catalog aliases; directory hardlinks and journaling are not constructed. +func Build(input []BuildEntry, options BuildOptions) (storage.Reader, error) { + if options.Size < 16<<20 || options.Size%allocationBlock != 0 || options.Size/allocationBlock > math.MaxUint32 || len(input) > 1000000 { + return nil, fmt.Errorf("hfsx: invalid volume geometry or entry limit") + } + label, err := hfsName(options.Label, 255) + if err != nil || len(label) == 0 { + return nil, fmt.Errorf("hfsx: invalid volume label") + } + originals := map[string]BuildEntry{} + plain := make([]unixfs.Entry, 0, len(input)) + for _, e := range input { + p, err := unixfs.Clean(e.Path) + if err != nil { + return nil, err + } + if _, ok := originals[p]; ok { + return nil, fmt.Errorf("hfsx: duplicate path %q", p) + } + if len(e.FinderInfo) != 0 && len(e.FinderInfo) != 32 { + return nil, fmt.Errorf("hfsx: Finder info must be 32 bytes") + } + e.Path = p + originals[p] = e + plain = append(plain, e.Entry) + } + entries, err := unixfs.Normalize(plain) + if err != nil { + return nil, err + } + // Normalize has resolved hardlink chains and validated shared Unix metadata. + byPath := map[string]unixfs.Entry{} + groups := map[string][]string{} + for _, e := range entries { + byPath[e.Path] = e + if e.Hardlink { + groups[e.Target] = append(groups[e.Target], e.Path) + } + } + aliasTarget := map[string]string{} + inodeLinks := map[string]uint32{} + const privatePath = ".hfs-private-inodes" + if len(groups) > 0 { + if _, exists := byPath[privatePath]; exists { + return nil, fmt.Errorf("hfsx: reserved private inode path") + } + dir := unixfs.Entry{Path: privatePath, Mode: unixfs.Directory | 0700, Mtime: entries[0].Mtime} + entries = append(entries, dir) + finder := make([]byte, 32) + be.PutUint16(finder[8:], 0x4000) + originals[privatePath] = BuildEntry{Entry: dir, FinderInfo: finder} + targets := make([]string, 0, len(groups)) + for target := range groups { + targets = append(targets, target) + } + sort.Strings(targets) + for index, target := range targets { + // Reference numbers are deterministic, nonzero and independent of + // catalog IDs. Both the original path and every alias share the inode. + ref := uint32(index + 16) + inodePath := fmt.Sprintf("%s/iNode%d", privatePath, ref) + inode := byPath[target] + inode.Path = inodePath + inode.Hardlink = false + inode.Target = "" + entries = append(entries, inode) + metadata := originals[target] + metadata.Entry = inode + originals[inodePath] = metadata + inodeLinks[inodePath] = uint32(len(groups[target]) + 1) + aliasTarget[target] = inodePath + for _, alias := range groups[target] { + aliasTarget[alias] = inodePath + } + } + sort.Slice(entries, func(i, j int) bool { return entries[i].Path < entries[j].Path }) + } + if uint64(options.Size/allocationBlock+7)/8 > 64<<20 { + return nil, fmt.Errorf("hfsx: allocation bitmap limit") + } + b := &hfsBuilder{next: 1, total: uint32(options.Size / allocationBlock)} + // Reserve the allocation bitmap early; its bits are populated after all forks. + bitmap := make([]byte, (uint64(b.total)+7)/8) + bitmapDesc, err := b.allocate(bytes.NewReader(bitmap)) + if err != nil { + return nil, err + } + overflow, err := buildTree(nil, 10, 0) + if err != nil { + return nil, err + } + overflowDesc, err := b.allocate(bytes.NewReader(overflow)) + if err != nil { + return nil, err + } + ids := map[string]uint32{".": 2} + names := map[string][]uint16{".": label} + valences := map[string]uint32{} + nextID := uint32(16) + for _, e := range entries { + if e.Path == "." { + continue + } + name, err := hfsName(path.Base(e.Path), 255) + // The HFS catalog slash is the POSIX colon (TN1150 name translation). + // Directory separators are already consumed by path.Base. + for i, c := range name { + if c == ':' { + name[i] = '/' + } + } + if e.Path == privatePath && len(groups) > 0 { + name = utf16.Encode([]rune("\x00\x00\x00\x00HFS+ Private Data")) + err = nil + } + if err != nil || len(name) == 0 { + return nil, fmt.Errorf("hfsx: invalid catalog name %q", e.Path) + } + names[e.Path] = name + ids[e.Path] = nextID + nextID++ + valences[path.Dir(e.Path)]++ + } + var catalog, attributes []treeRecord + var files, folders uint32 + for _, e := range entries { + metadata := originals[e.Path] + name := names[e.Path] + id := ids[e.Path] + parent := uint32(1) + if e.Path != "." { + parent = ids[path.Dir(e.Path)] + } + directory := e.Mode&0170000 == unixfs.Directory + kind := uint16(2) + length := 248 + if directory { + kind = 1 + length = 88 + if e.Path != "." { + folders++ + } + } else { + files++ + } + value := make([]byte, length) + be.PutUint16(value, kind) + be.PutUint16(value[2:], 2) // thread record exists. + if directory { + be.PutUint32(value[4:], valences[e.Path]) + } + be.PutUint32(value[8:], id) + if uint64(e.Mtime)+hfsEpoch > math.MaxUint32 { + return nil, fmt.Errorf("hfsx: date outside HFS epoch") + } + date := e.Mtime + hfsEpoch + for _, off := range []int{12, 16, 20, 24} { + be.PutUint32(value[off:], date) + } + be.PutUint32(value[32:], e.UID) + be.PutUint32(value[36:], e.GID) + value[40], value[41] = metadata.AdminFlags, metadata.OwnerFlags + be.PutUint16(value[42:], uint16(e.Mode)) + copy(value[48:80], metadata.FinderInfo) + if !directory { + var data storage.Reader + switch e.Mode & 0170000 { + case unixfs.Regular: + data = e.Data + be.PutUint32(value[44:], max(1, inodeLinks[e.Path])) + if inode, alias := aliasTarget[e.Path]; alias { + data = nil + metadata.Resource = nil + metadata.Xattrs = nil + clear(value[48:80]) + copy(value[48:], "hlnkhfs+") + be.PutUint16(value[56:], 0x100) + var ref uint32 + if _, err := fmt.Sscanf(path.Base(inode), "iNode%d", &ref); err != nil { + return nil, err + } + be.PutUint32(value[44:], ref) + // Alias creation date is the volume/private-directory date. + be.PutUint32(value[12:], entries[0].Mtime+hfsEpoch) + } + case unixfs.Symlink: + data = bytes.NewReader([]byte(e.Target)) + copy(value[48:], "slnkrhap") + be.PutUint32(value[44:], 1) + case unixfs.Character, unixfs.Block: + if e.Major > 255 || e.Minor > 0xffffff { + return nil, fmt.Errorf("hfsx: device number out of range") + } + be.PutUint32(value[44:], e.Major<<24|e.Minor) + case unixfs.FIFO, unixfs.Socket: + default: + return nil, fmt.Errorf("hfsx: unsupported file mode") + } + fork, err := b.allocate(data) + if err != nil { + return nil, fmt.Errorf("hfsx: %s: %w", e.Path, err) + } + copy(value[88:], fork) + resource, err := b.allocate(metadata.Resource) + if err != nil { + return nil, err + } + copy(value[168:], resource) + } else if metadata.Resource != nil && metadata.Resource.Size() != 0 { + return nil, fmt.Errorf("hfsx: directory resource fork") + } + for key, r := range metadata.Xattrs { + u, err := hfsName(key, 127) + if err != nil || len(u) == 0 || r == nil || r.Size() < 0 { + return nil, fmt.Errorf("hfsx: invalid extended attribute") + } + k := make([]byte, 14+len(u)*2) + be.PutUint16(k, uint16(len(k)-2)) + be.PutUint32(k[4:], id) + be.PutUint16(k[12:], uint16(len(u))) + for i, v := range u { + be.PutUint16(k[14+i*2:], v) + } + var v []byte + if r.Size() <= 2048 { + v = make([]byte, 16+r.Size()) + be.PutUint32(v, 0x10) + be.PutUint32(v[12:], uint32(r.Size())) + if _, err := io.ReadFull(io.NewSectionReader(r, 0, r.Size()), v[16:]); err != nil { + return nil, err + } + } else { + v = make([]byte, 88) + be.PutUint32(v, 0x20) + fork, err := b.allocate(r) + if err != nil { + return nil, err + } + copy(v[8:], fork) + } + attributes = append(attributes, treeRecord{k, v}) + } + if len(metadata.Xattrs) > 0 { + be.PutUint16(value[2:], be.Uint16(value[2:])|4) + } + catalog = append(catalog, treeRecord{catalogKey(parent, name), value}) + thread := make([]byte, 10+2*len(name)) + be.PutUint16(thread, kind+2) + be.PutUint32(thread[4:], parent) + be.PutUint16(thread[8:], uint16(len(name))) + for i, v := range name { + be.PutUint16(thread[10+i*2:], v) + } + catalog = append(catalog, treeRecord{catalogKey(id, nil), thread}) + } + // Length fields must not participate in key comparison: parentID, then name + // code units, ignoring the UTF-16 name length prefix. + sort.Slice(catalog, func(i, j int) bool { + a, c := catalog[i].key, catalog[j].key + if be.Uint32(a[2:]) != be.Uint32(c[2:]) { + return be.Uint32(a[2:]) < be.Uint32(c[2:]) + } + return bytes.Compare(a[8:], c[8:]) < 0 + }) + for i := 1; i < len(catalog); i++ { + if bytes.Equal(catalog[i].key, catalog[i-1].key) { + return nil, fmt.Errorf("hfsx: duplicate decomposed catalog key") + } + } + catData, err := buildTree(catalog, 516, 0xbc) + if err != nil { + return nil, err + } + catDesc, err := b.allocate(bytes.NewReader(catData)) + if err != nil { + return nil, err + } + var attrDesc []byte + if len(attributes) > 0 { + sort.Slice(attributes, func(i, j int) bool { + a, c := attributes[i].key, attributes[j].key + if be.Uint32(a[4:]) != be.Uint32(c[4:]) { + return be.Uint32(a[4:]) < be.Uint32(c[4:]) + } + return bytes.Compare(a[14:], c[14:]) < 0 + }) + attrData, err := buildTree(attributes, 266, 0) + if err != nil { + return nil, err + } + attrDesc, err = b.allocate(bytes.NewReader(attrData)) + if err != nil { + return nil, err + } + } + // First and last allocation blocks reserve boot/volume-header sectors. The + // bitmap uses MSB-first allocation bits and marks unused tail bits allocated. + for i := uint32(0); i < b.next; i++ { + bitmap[i/8] |= 0x80 >> uint(i%8) + } + for i := uint64(b.total) - 1; i < uint64(len(bitmap))*8; i++ { + bitmap[i/8] |= 0x80 >> uint(i%8) + } + header := make([]byte, 512) + be.PutUint16(header, 0x4858) + be.PutUint16(header[2:], 5) + be.PutUint32(header[4:], 1<<8) // cleanly unmounted, no journal. + copy(header[8:], "10.0") + be.PutUint32(header[32:], files) + be.PutUint32(header[36:], folders) + be.PutUint32(header[40:], allocationBlock) + be.PutUint32(header[44:], b.total) + be.PutUint32(header[48:], b.total-b.next-1) + be.PutUint32(header[52:], b.next) + be.PutUint32(header[56:], allocationBlock*16) + be.PutUint32(header[60:], allocationBlock*16) + be.PutUint32(header[64:], nextID) + be.PutUint32(header[68:], 1) + root := entries[0] + date := root.Mtime + hfsEpoch + for _, off := range []int{16, 20, 28} { + be.PutUint32(header[off:], date) + } + copy(header[112:], bitmapDesc) + copy(header[192:], overflowDesc) + copy(header[272:], catDesc) + copy(header[352:], attrDesc) + b.fragments = append(b.fragments, hfsFragment{1024, bytes.NewReader(header)}, hfsFragment{options.Size - 1024, bytes.NewReader(header)}) + sort.Slice(b.fragments, func(i, j int) bool { return b.fragments[i].offset < b.fragments[j].offset }) + var ranges []storage.Range + var position int64 + for _, f := range b.fragments { + if f.offset < position { + return nil, fmt.Errorf("hfsx: overlapping fragments") + } + if f.offset > position { + ranges = append(ranges, storage.Range{Source: unixfs.Zero(f.offset - position), Length: f.offset - position}) + } + ranges = append(ranges, storage.Range{Source: f.source, Length: f.source.Size()}) + position = f.offset + f.source.Size() + } + ranges = append(ranges, storage.Range{Source: unixfs.Zero(options.Size - position), Length: options.Size - position}) + return storage.Compose(ranges...) +} diff --git a/filesystem/hfs/build_links_test.go b/filesystem/hfs/build_links_test.go new file mode 100644 index 0000000..72ccd8f --- /dev/null +++ b/filesystem/hfs/build_links_test.go @@ -0,0 +1,46 @@ +package hfs + +import ( + "bytes" + "github.com/tinyrange/trex/filesystem/unixfs" + starfile "github.com/tinyrange/trex/storage/star" + "testing" +) + +func TestBuildHFSXHardlinkInodes(t *testing.T) { + entries := []BuildEntry{ + {Entry: unixfs.Entry{Path: "usr/bin/tool", Mode: unixfs.Regular | 0755, Data: bytes.NewReader([]byte("executable"))}}, + {Entry: unixfs.Entry{Path: "usr/bin/alias", Mode: unixfs.Regular | 0755, Target: "usr/bin/tool", Hardlink: true}}, + {Entry: unixfs.Entry{Path: "usr/bin/chain", Mode: unixfs.Regular | 0755, Target: "usr/bin/alias", Hardlink: true}}, + } + r, err := Build(entries, BuildOptions{Size: 16 << 20, Label: "Links"}) + if err != nil { + t.Fatal(err) + } + v, err := Open(starfile.NewReader("links", r), 100) + if err != nil { + t.Fatal(err) + } + var target string + count := 0 + for _, e := range v.Entries { + if e.Path == "/usr/bin/tool" || e.Path == "/usr/bin/alias" || e.Path == "/usr/bin/chain" { + count++ + if e.Data.Size() != 10 || string(e.FinderInfo[:8]) != "hlnkhfs+" { + t.Fatal("invalid native alias") + } + if target != "" && target != e.Target { + t.Fatal("aliases do not share inode") + } + target = e.Target + } + } + if count != 3 || target == "" { + t.Fatal("missing hardlink aliases") + } + for _, e := range v.Entries { + if e.Path == target && e.Special != 3 { + t.Fatal("inode link count") + } + } +} diff --git a/filesystem/hfs/build_names_test.go b/filesystem/hfs/build_names_test.go new file mode 100644 index 0000000..303cc0c --- /dev/null +++ b/filesystem/hfs/build_names_test.go @@ -0,0 +1,31 @@ +package hfs + +import ( + "bytes" + "github.com/tinyrange/trex/filesystem/unixfs" + starfile "github.com/tinyrange/trex/storage/star" + "testing" +) + +func TestBuildPOSIXColonCatalogSlash(t *testing.T) { + r, err := Build([]BuildEntry{{Entry: unixfs.Entry{Path: "folder/Farallon LAN:Modem", Mode: unixfs.Regular | 0644, Data: bytes.NewReader([]byte("driver"))}}}, BuildOptions{Size: 32 << 20, Label: "names"}) + if err != nil { + t.Fatal(err) + } + v, err := Open(starfile.NewReader("test", r), 100) + if err != nil { + t.Fatal(err) + } + found := false + for _, e := range v.Entries { + if e.Path == "/folder/Farallon LAN%2FModem" { + found = true + if string(e.Name) != "Farallon LAN/Modem" { + t.Fatal(string(e.Name)) + } + } + } + if !found { + t.Fatal("POSIX colon not represented by native catalog slash") + } +} diff --git a/filesystem/hfs/build_star.go b/filesystem/hfs/build_star.go new file mode 100644 index 0000000..bb6aee1 --- /dev/null +++ b/filesystem/hfs/build_star.go @@ -0,0 +1,109 @@ +package hfs + +import ( + "fmt" + "github.com/tinyrange/trex/filesystem/unixfs" + "github.com/tinyrange/trex/storage" + starfile "github.com/tinyrange/trex/storage/star" + "go.starlark.net/starlark" +) + +func BuildBuiltin(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var input starlark.Value + options := BuildOptions{Label: "Untitled"} + if err := starlark.UnpackArgs("hfs_build", args, kwargs, "entries", &input, "size", &options.Size, "label?", &options.Label); err != nil { + return nil, err + } + iter := starlark.Iterate(input) + if iter == nil { + return nil, fmt.Errorf("hfs_build: expected entries iterable") + } + defer iter.Done() + var entries []BuildEntry + var v starlark.Value + for iter.Next(&v) { + if len(entries) >= 1000000 { + return nil, fmt.Errorf("hfs_build: entry limit") + } + d, ok := v.(*starlark.Dict) + if !ok { + return nil, fmt.Errorf("hfs_build: entries must be dictionaries") + } + plain := starlark.NewDict(d.Len()) + var e BuildEntry + for _, pair := range d.Items() { + name, ok := starlark.AsString(pair[0]) + if !ok { + return nil, fmt.Errorf("hfs_build: invalid field") + } + switch name { + case "finder_info": + b, err := starfile.BytesForValue(pair[1], 32) + if err != nil { + return nil, err + } + e.FinderInfo = b + case "resource": + var err error + e.Resource, err = buildPayload(pair[1]) + if err != nil { + return nil, err + } + case "owner_flags", "admin_flags": + var n uint8 + if err := starlark.AsInt(pair[1], &n); err != nil { + return nil, err + } + if name == "owner_flags" { + e.OwnerFlags = n + } else { + e.AdminFlags = n + } + case "xattrs": + attrs, ok := pair[1].(*starlark.Dict) + if !ok { + return nil, fmt.Errorf("hfs_build: xattrs must be dict") + } + e.Xattrs = map[string]storage.Reader{} + for _, item := range attrs.Items() { + key, ok := starlark.AsString(item[0]) + if !ok { + return nil, fmt.Errorf("hfs_build: attribute name must be string") + } + r, err := buildPayload(item[1]) + if err != nil { + return nil, err + } + e.Xattrs[key] = r + } + default: + if err := plain.SetKey(pair[0], pair[1]); err != nil { + return nil, err + } + } + } + parsed, err := unixfs.ParseEntries(starlark.NewList([]starlark.Value{plain})) + if err != nil { + return nil, err + } + e.Entry = parsed[0] + entries = append(entries, e) + } + r, err := Build(entries, options) + if err != nil { + return nil, err + } + return starfile.NewReader("HFSX volume", r), nil +} +func buildPayload(v starlark.Value) (storage.Reader, error) { + switch v := v.(type) { + case storage.Reader: + return v, nil + case starlark.String: + return &starfile.Bytes{Data: []byte(v)}, nil + case starlark.Bytes: + return &starfile.Bytes{Data: []byte(v)}, nil + default: + return nil, fmt.Errorf("hfs_build: expected file, string or bytes") + } +} diff --git a/filesystem/hfs/build_test.go b/filesystem/hfs/build_test.go new file mode 100644 index 0000000..15ebef8 --- /dev/null +++ b/filesystem/hfs/build_test.go @@ -0,0 +1,141 @@ +package hfs + +import ( + "bytes" + "fmt" + "github.com/tinyrange/trex/filesystem/unixfs" + "github.com/tinyrange/trex/storage" + starfile "github.com/tinyrange/trex/storage/star" + "io" + "testing" +) + +func TestBuildHFSXNativeMetadataAndIndexLookup(t *testing.T) { + data := []byte("borrowed app payload") + entries := []BuildEntry{ + {Entry: unixfs.Entry{Path: ".", Mode: unixfs.Directory | 0755, Mtime: 1345765197}}, + {Entry: unixfs.Entry{Path: "Applications/App", Mode: unixfs.Regular | 0755, UID: 501, GID: 20, Data: bytes.NewReader(data), Mtime: 1345765197}, FinderInfo: make([]byte, 32), Resource: bytes.NewReader([]byte("resource")), Xattrs: map[string]storage.Reader{"com.example.inline": bytes.NewReader([]byte("attr")), "com.example.large": bytes.NewReader(bytes.Repeat([]byte{0x5a}, 4097))}}, + {Entry: unixfs.Entry{Path: "alias", Mode: unixfs.Symlink | 0777, Target: "Applications/App"}}, + {Entry: unixfs.Entry{Path: "dev/console", Mode: unixfs.Character | 0600, Major: 3, Minor: 0x123456}}, + } + for i := 0; i < 1000; i++ { + entries = append(entries, BuildEntry{Entry: unixfs.Entry{Path: fmt.Sprintf("many/file%04d", i), Mode: unixfs.Regular | 0644, Data: bytes.NewReader(nil)}}) + } + image, err := Build(entries, BuildOptions{Size: 32 << 20, Label: "Lion"}) + if err != nil { + t.Fatal(err) + } + file := starfile.NewReader("image", image) + v, err := Open(file, 2000) + if err != nil { + t.Fatal(err) + } + found := map[string]Entry{} + for _, e := range v.Entries { + found[e.Path] = e + } + app := found["/Applications/App"] + if app.Mode != 0100755 || app.UID != 501 || app.GID != 20 || app.Modified != 1345765197+hfsEpoch { + t.Fatalf("metadata %+v", app) + } + data[0] = 'B' + b, err := io.ReadAll(io.NewSectionReader(app.Data, 0, app.Data.Size())) + if err != nil || string(b) != "Borrowed app payload" { + t.Fatalf("data %q %v", b, err) + } + if app.Resource.Size() != 8 || app.Xattrs["com.example.large"].Size() != 4097 || found["/alias"].Target != "Applications/App" || found["/dev/console"].Special != 0x03123456 { + t.Fatal("native forks, attributes or special metadata lost") + } + // Independently walk catalog index keys as a guest does: the greatest key + // <= the target selects the child. Leaf enumeration alone cannot prove this. + header := make([]byte, 512) + image.ReadAt(header, 1024) + r := plusReader{file: file, block: 4096, blocks: 8192, overflow: map[plusKey][]plusExtent{}} + catalog, err := r.fork(4, 0, header[272:352]) + if err != nil { + t.Fatal(err) + } + first := make([]byte, treeNodeSize) + catalog.ReadAt(first, 0) + depth := be.Uint16(first[14:]) + root := be.Uint32(first[16:]) + if depth < 2 { + t.Fatal("catalog did not exercise index levels") + } + want := catalogKey(found["/many"].ID, []uint16{'f', 'i', 'l', 'e', '0', '7', '5', '0'}) + compare := func(a, b []byte) int { + if be.Uint32(a[2:]) < be.Uint32(b[2:]) { + return -1 + } + if be.Uint32(a[2:]) > be.Uint32(b[2:]) { + return 1 + } + return bytes.Compare(a[8:2+int(be.Uint16(a))], b[8:2+int(be.Uint16(b))]) + } + id := root + for level := depth; level > 0; level-- { + node := make([]byte, treeNodeSize) + if _, err := catalog.ReadAt(node, int64(id)*treeNodeSize); err != nil { + t.Fatal(err) + } + count := int(be.Uint16(node[10:])) + var child uint32 + matched := false + for i := 0; i < count; i++ { + off := int(be.Uint16(node[treeNodeSize-2*(i+1):])) + end := int(be.Uint16(node[treeNodeSize-2*(i+2):])) + rec := node[off:end] + keySize := 2 + int(be.Uint16(rec)) + cmp := compare(rec[:keySize], want) + if level == 1 && cmp == 0 { + matched = true + break + } + if level > 1 && cmp <= 0 { + child = be.Uint32(rec[keySize:]) + } + } + if level == 1 { + if !matched { + t.Fatal("guest-style catalog lookup failed") + } + } else { + if child == 0 { + t.Fatal("invalid index child") + } + id = child + } + } + alternate := make([]byte, 512) + image.ReadAt(alternate, image.Size()-1024) + if !bytes.Equal(header, alternate) { + t.Fatal("alternate volume header differs") + } + bitmap, err := r.fork(6, 0, header[112:192]) + if err != nil { + t.Fatal(err) + } + bits := make([]byte, bitmap.Size()) + bitmap.ReadAt(bits, 0) + var allocated uint32 + for i := uint32(0); i < r.blocks; i++ { + if bits[i/8]&(0x80>>uint(i%8)) != 0 { + allocated++ + } + } + if r.blocks-allocated != be.Uint32(header[48:]) { + t.Fatal("allocation bitmap accounting mismatch") + } +} +func TestBuildHFSXRejectsInvalidConstruction(t *testing.T) { + for _, entries := range [][]BuildEntry{ + {{Entry: unixfs.Entry{Path: "../escape", Mode: unixfs.Regular | 0644}}}, + {{Entry: unixfs.Entry{Path: "a", Mode: unixfs.Regular | 0644}}, {Entry: unixfs.Entry{Path: "a/b", Mode: unixfs.Regular | 0644}}}, + {{Entry: unixfs.Entry{Path: "é", Mode: unixfs.Regular | 0644}}, {Entry: unixfs.Entry{Path: "é", Mode: unixfs.Regular | 0644}}}, + {{Entry: unixfs.Entry{Path: "a", Mode: unixfs.Regular | 0644, Hardlink: true, Target: "b"}}}, + } { + if _, err := Build(entries, BuildOptions{Size: 16 << 20, Label: "Test"}); err == nil { + t.Fatal("accepted invalid construction") + } + } +} diff --git a/filesystem/hfs/build_tree_scale_test.go b/filesystem/hfs/build_tree_scale_test.go new file mode 100644 index 0000000..9607998 --- /dev/null +++ b/filesystem/hfs/build_tree_scale_test.go @@ -0,0 +1,53 @@ +package hfs + +import ( + "testing" +) + +func TestBuildTreeInstalledVolumeScale(t *testing.T) { + // A complete original desktop volume exceeds the former 64 MiB cap once + // thread records and free growth nodes are included. Check the on-disk + // accounting independently, not just the leaf-enumerating format reader. + rows := make([]treeRecord, 100000) + data := make([]byte, 400) + for i := range rows { + rows[i] = treeRecord{catalogKey(uint32(i+2), nil), data} + } + tree, err := buildTree(rows, 516, 0xbc) + if err != nil { + t.Fatal(err) + } + if len(tree) <= 64<<20 || len(tree) > 256<<20 { + t.Fatal("unexpected tree bound", len(tree)) + } + h := tree[14:120] + total, free := be.Uint32(h[22:]), be.Uint32(h[26:]) + mapOffset := int(be.Uint16(tree[treeNodeSize-6:])) + bitmap := tree[mapOffset : treeNodeSize-8] + var used uint32 + for id := uint32(0); id < total; id++ { + if bitmap[id/8]&(0x80>>uint(id%8)) != 0 { + used++ + } + } + if used+free != total || int(total)*treeNodeSize != len(tree) { + t.Fatal("node map accounting", used, free, total) + } + var records uint32 + id := be.Uint32(h[10:]) + previous := uint32(0) + for id != 0 { + if id >= total || id <= previous { + t.Fatal("leaf chain cycle or invalid node", id) + } + n := tree[int(id)*treeNodeSize:] + if n[8] != 255 || n[9] != 1 || be.Uint32(n[4:]) != previous { + t.Fatal("leaf descriptor") + } + records += uint32(be.Uint16(n[10:])) + previous, id = id, be.Uint32(n) + } + if records != uint32(len(rows)) || previous != be.Uint32(h[14:]) { + t.Fatal("incomplete leaf chain", records) + } +} diff --git a/filesystem/hfs/compressed.go b/filesystem/hfs/compressed.go new file mode 100644 index 0000000..e01edb7 --- /dev/null +++ b/filesystem/hfs/compressed.go @@ -0,0 +1,199 @@ +package hfs + +import ( + "bytes" + "compress/zlib" + "encoding/binary" + "fmt" + "github.com/tinyrange/trex/archive/macresource" + "github.com/tinyrange/trex/storage" + starfile "github.com/tinyrange/trex/storage/star" + "io" + "math" + "sync" +) + +const compressionBlock = 65536 +const compressionBlocks = 1000000 + +type compressionChunk struct { + source storage.Reader + offset, size int64 +} +type compressedFork struct { + size int64 + chunks []compressionChunk + mu sync.Mutex + cached int + data []byte +} + +func (f *compressedFork) Size() int64 { return f.size } +func inflate(source storage.Reader, off, size, wanted int64) ([]byte, error) { + if wanted < 0 || wanted > compressionBlock || size <= 0 || size > compressionBlock+1024 { + return nil, fmt.Errorf("hfs+: invalid decmpfs chunk bounds") + } + b := make([]byte, size) + if _, err := io.ReadFull(io.NewSectionReader(source, off, size), b); err != nil { + return nil, err + } + if b[0] == 0xff { + if int64(len(b)-1) != wanted { + return nil, fmt.Errorf("hfs+: stored decmpfs size mismatch") + } + return b[1:], nil + } + input := bytes.NewReader(b) + z, err := zlib.NewReader(input) + if err != nil { + return nil, err + } + decoded, err := io.ReadAll(io.LimitReader(z, wanted+1)) + closeErr := z.Close() + if err != nil { + return nil, err + } + if closeErr != nil { + return nil, closeErr + } + if int64(len(decoded)) != wanted || input.Len() != 0 { + return nil, fmt.Errorf("hfs+: decmpfs length/trailing mismatch") + } + return decoded, nil +} +func (f *compressedFork) ReadAt(p []byte, off int64) (int, error) { + f.mu.Lock() + defer f.mu.Unlock() + if off < 0 { + return 0, fmt.Errorf("hfs+: negative compressed fork offset") + } + if len(p) == 0 { + return 0, nil + } + if off >= f.size { + return 0, io.EOF + } + wanted := len(p) + done := 0 + for len(p) > 0 && off < f.size { + index := int(off / compressionBlock) + if f.cached != index || f.data == nil { + c := f.chunks[index] + data, err := inflate(c.source, c.offset, c.size, min(compressionBlock, f.size-int64(index)*compressionBlock)) + if err != nil { + return done, fmt.Errorf("hfs+: decmpfs chunk %d: %w", index, err) + } + f.cached = index + f.data = data + } + n := copy(p, f.data[off%compressionBlock:]) + done += n + off += int64(n) + p = p[n:] + } + if done != wanted { + return done, io.EOF + } + return done, nil +} +func compressedData(attr, resource starfile.File) (starfile.File, uint32, error) { + var header [16]byte + if attr == nil || attr.Size() < 16 { + return nil, 0, fmt.Errorf("hfs+: missing/truncated decmpfs attribute") + } + if _, err := attr.ReadAt(header[:], 0); err != nil { + return nil, 0, err + } + le := binary.LittleEndian + kind, size := le.Uint32(header[4:]), le.Uint64(header[8:]) + if string(header[:4]) != "fpmc" || size > math.MaxInt64 || size > compressionBlocks*compressionBlock { + return nil, 0, fmt.Errorf("hfs+: invalid decmpfs header/size") + } + f := &compressedFork{size: int64(size), cached: -1} + switch kind { + case 3: + if size > compressionBlock { + return nil, 0, fmt.Errorf("hfs+: inline decmpfs size exceeds one block") + } + f.chunks = []compressionChunk{{source: attr, offset: 16, size: attr.Size() - 16}} + case 4: + if attr.Size() != 16 || resource == nil { + return nil, 0, fmt.Errorf("hfs+: invalid resource decmpfs attribute") + } + fork, err := macresource.Open(resource, 1000) + if err != nil { + return nil, 0, err + } + var data starfile.File + for _, e := range fork.Entries { + if string(e.Type[:]) == "cmpf" { + if data != nil { + return nil, 0, fmt.Errorf("hfs+: duplicate cmpf resource") + } + data = e.Data + } + } + if data == nil || data.Size() < 4 { + return nil, 0, fmt.Errorf("hfs+: missing cmpf resource") + } + var countBytes [4]byte + if _, err := data.ReadAt(countBytes[:], 0); err != nil { + return nil, 0, err + } + count := int64(le.Uint32(countBytes[:])) + if count != (int64(size)+compressionBlock-1)/compressionBlock || count > compressionBlocks || count > (data.Size()-4)/8 { + return nil, 0, fmt.Errorf("hfs+: invalid decmpfs chunk count") + } + table := make([]byte, 8*count) + if _, err := data.ReadAt(table, 4); err != nil { + return nil, 0, err + } + previous := 4 + 8*count + for i := int64(0); i < count; i++ { + off, n := int64(le.Uint32(table[8*i:])), int64(le.Uint32(table[8*i+4:])) + if off != previous || n <= 0 || n > compressionBlock+1024 || off > data.Size() || n > data.Size()-off { + return nil, 0, fmt.Errorf("hfs+: invalid decmpfs chunk extent") + } + f.chunks = append(f.chunks, compressionChunk{data, off, n}) + previous = off + n + } + if previous != data.Size() { + return nil, 0, fmt.Errorf("hfs+: trailing decmpfs resource bytes") + } + default: + return nil, 0, fmt.Errorf("hfs+: unsupported decmpfs compression %d", kind) + } + if size == 0 { + if kind != 3 { + return nil, 0, fmt.Errorf("hfs+: empty resource decmpfs") + } + c := f.chunks[0] + if _, err := inflate(c.source, c.offset, c.size, 0); err != nil { + return nil, 0, err + } + } + return starfile.NewReader("decmpfs", f), kind, nil +} +func decodeCompressed(v *Volume, raw bool) error { + for i := range v.Entries { + e := &v.Entries[i] + if e.Kind != "file" || e.OwnerFlags&0x20 == 0 { + continue + } + if len(e.FinderInfo) >= 8 && string(e.FinderInfo[:8]) == "hlnkhfs+" { + continue + } + attr := e.Xattrs["com.apple.decmpfs"] + if raw { + continue + } + data, kind, err := compressedData(attr, e.Resource) + if err != nil { + return fmt.Errorf("hfs+: %s: %w", e.Path, err) + } + e.RawData = e.Data + e.Data = data + e.CompressionType = kind + } + return nil +} diff --git a/filesystem/hfs/compressed_test.go b/filesystem/hfs/compressed_test.go new file mode 100644 index 0000000..91de0bb --- /dev/null +++ b/filesystem/hfs/compressed_test.go @@ -0,0 +1,110 @@ +package hfs + +import ( + "bytes" + "compress/zlib" + "encoding/binary" + starfile "github.com/tinyrange/trex/storage/star" + "testing" +) + +func packedData(t *testing.T, data []byte) []byte { + t.Helper() + var b bytes.Buffer + z := zlib.NewWriter(&b) + if _, err := z.Write(data); err != nil { + t.Fatal(err) + } + if err := z.Close(); err != nil { + t.Fatal(err) + } + return b.Bytes() +} +func compressionAttr(kind uint32, size uint64, data []byte) *starfile.Bytes { + b := make([]byte, 16) + copy(b, "fpmc") + binary.LittleEndian.PutUint32(b[4:], kind) + binary.LittleEndian.PutUint64(b[8:], size) + return &starfile.Bytes{Data: append(b, data...)} +} +func cmpfResource(payload []byte) *starfile.Bytes { + mapOffset := 260 + len(payload) + b := make([]byte, mapOffset+50) + be.PutUint32(b, 256) + be.PutUint32(b[4:], uint32(mapOffset)) + be.PutUint32(b[8:], uint32(4+len(payload))) + be.PutUint32(b[12:], 50) + be.PutUint32(b[256:], uint32(len(payload))) + copy(b[260:], payload) + m := b[mapOffset:] + be.PutUint16(m[24:], 28) + be.PutUint16(m[26:], 50) + copy(m[30:], "cmpf") + be.PutUint16(m[36:], 10) + be.PutUint16(m[40:], 65535) + return &starfile.Bytes{Data: b} +} +func TestDecmpfsInlineChecksumsStoredAndRaw(t *testing.T) { + plain := []byte("decoded inode") + for _, payload := range [][]byte{packedData(t, plain), append([]byte{255}, plain...)} { + attr := compressionAttr(3, uint64(len(plain)), payload) + data, kind, err := compressedData(attr, nil) + if err != nil { + t.Fatal(err) + } + got, err := starfile.ReadAll(data) + if err != nil || !bytes.Equal(got, plain) || kind != 3 { + t.Fatalf("%q %v", got, err) + } + } + payload := packedData(t, plain) + payload[len(payload)-1] ^= 1 + data, _, err := compressedData(compressionAttr(3, uint64(len(plain)), payload), nil) + if err != nil { + t.Fatal(err) + } + if _, err := starfile.ReadAll(data); err == nil { + t.Fatal("corrupt zlib accepted") + } + raw := &starfile.Bytes{} + v := &Volume{Entries: []Entry{{Kind: "file", OwnerFlags: 0x20, Data: raw, Xattrs: map[string]starfile.File{"com.apple.decmpfs": compressionAttr(99, 2, nil)}}}} + if err := decodeCompressed(v, true); err != nil || v.Entries[0].Data != raw { + t.Fatal("explicit raw access failed") + } + if err := decodeCompressed(v, false); err == nil { + t.Fatal("unsupported compression silently empty") + } +} +func TestDecmpfsResourceCrossBlockAndBounds(t *testing.T) { + first := bytes.Repeat([]byte{'A'}, 65536) + second := []byte("end") + a := packedData(t, first) + b := append([]byte{255}, second...) + table := make([]byte, 20) + le := binary.LittleEndian + le.PutUint32(table, 2) + le.PutUint32(table[4:], 20) + le.PutUint32(table[8:], uint32(len(a))) + le.PutUint32(table[12:], uint32(20+len(a))) + le.PutUint32(table[16:], uint32(len(b))) + resource := cmpfResource(append(append(table, a...), b...)) + attr := compressionAttr(4, 65539, nil) + data, _, err := compressedData(attr, resource) + if err != nil { + t.Fatal(err) + } + var p [5]byte + if _, err := data.ReadAt(p[:], 65534); err != nil || string(p[:]) != "AAend" { + t.Fatalf("crossblock %q %v", p, err) + } + if _, err := data.ReadAt(p[:], 0); err != nil || string(p[:]) != "AAAAA" { + t.Fatalf("backward %q %v", p, err) + } + le.PutUint32(resource.Data[264:], 19) + if _, _, err := compressedData(attr, resource); err == nil { + t.Fatal("overlapping chunk table accepted") + } + if _, _, err := compressedData(compressionAttr(3, 65537, []byte{255}), nil); err == nil { + t.Fatal("inline output bound not enforced") + } +} diff --git a/filesystem/hfs/hfs.go b/filesystem/hfs/hfs.go index 71f0b5f..c51310d 100644 --- a/filesystem/hfs/hfs.go +++ b/filesystem/hfs/hfs.go @@ -24,6 +24,13 @@ type Entry struct { ID, Parent, Created, Modified, Backup uint32 Flags uint16 FinderInfo []byte + CompressionType uint32 + UID, GID, Special uint32 + Mode uint16 + OwnerFlags, AdminFlags byte + RawData, RawResource starfile.File + Target string + Xattrs map[string]starfile.File Data, Resource starfile.File } type Volume struct { @@ -161,7 +168,16 @@ func component(name []byte) string { } func Open(file starfile.File, maximumEntries int) (*Volume, error) { - if maximumEntries <= 0 || maximumEntries > int(^uint(0)>>1)/3 { + return open(file, maximumEntries, false) +} + +// OpenRaw preserves physical forks even for compressed files. It is an explicit +// forensic view, not a decoded empty-file substitute. +func OpenRaw(file starfile.File, maximumEntries int) (*Volume, error) { + return open(file, maximumEntries, true) +} +func open(file starfile.File, maximumEntries int, raw bool) (*Volume, error) { + if maximumEntries <= 0 || maximumEntries > int(^uint(0)>>1)/4 { return nil, fmt.Errorf("hfs: invalid entry limit") } var m [162]byte @@ -169,7 +185,7 @@ func Open(file starfile.File, maximumEntries int) (*Volume, error) { return nil, err } if be.Uint16(m[:]) == 0x482b || be.Uint16(m[:]) == 0x4858 { - return openPlus(file, maximumEntries) + return openPlus(file, maximumEntries, raw) } if be.Uint16(m[:]) != 0x4244 { return nil, fmt.Errorf("hfs: expected classic HFS signature") @@ -182,7 +198,7 @@ func Open(file starfile.File, maximumEntries int) (*Volume, error) { return nil, fmt.Errorf("hfs: invalid embedded HFS+ range") } embedded := filesystem.NewGeneratedImage("embedded HFS+", size, []filesystem.ExtentSpec{{Start: 0, Size: size, File: file, Offset: base}}) - return openPlus(embedded, maximumEntries) + return openPlus(embedded, maximumEntries, raw) } r := reader{file: file, base: int64(be.Uint16(m[28:])) * 512, block: int64(be.Uint32(m[20:])), blocks: uint32(be.Uint16(m[18:])), overflow: map[forkKey][]extent{}} if r.block < 512 || r.block%512 != 0 || r.blocks == 0 || r.base < 1536 || r.base > file.Size() || int64(r.blocks)*r.block > file.Size()-r.base || m[36] > 27 { diff --git a/filesystem/hfs/plus.go b/filesystem/hfs/plus.go index 7eed65b..9628b66 100644 --- a/filesystem/hfs/plus.go +++ b/filesystem/hfs/plus.go @@ -111,7 +111,7 @@ func plusComponent(b []byte) string { } return s } -func openPlus(file starfile.File, maximum int) (*Volume, error) { +func openPlus(file starfile.File, maximum int, raw bool) (*Volume, error) { var h [512]byte if _, err := starfile.ReadFullAt(file, h[:], 1024); err != nil { return nil, err @@ -188,6 +188,9 @@ func openPlus(file starfile.File, maximum int) (*Volume, error) { e.Modified = be.Uint32(data[16:]) e.Backup = be.Uint32(data[28:]) e.FinderInfo = append([]byte(nil), data[48:80]...) + e.UID, e.GID = be.Uint32(data[32:]), be.Uint32(data[36:]) + e.AdminFlags, e.OwnerFlags = data[40], data[41] + e.Mode, e.Special = be.Uint16(data[42:]), be.Uint32(data[44:]) if kind == 1 { e.Kind = "directory" } else { @@ -214,5 +217,17 @@ func openPlus(file starfile.File, maximum int) (*Volume, error) { if err != nil { return nil, err } - return finishPaths(v, ids, plusComponent) + if err := r.attributes(h[352:432], v, ids, maximum*4); err != nil { + return nil, err + } + if _, err := finishPaths(v, ids, plusComponent); err != nil { + return nil, err + } + if err := decodeCompressed(v, raw); err != nil { + return nil, err + } + if err := plusLinks(v, raw); err != nil { + return nil, err + } + return v, nil } diff --git a/filesystem/hfs/starlark.go b/filesystem/hfs/starlark.go index 29634ec..46fb543 100644 --- a/filesystem/hfs/starlark.go +++ b/filesystem/hfs/starlark.go @@ -9,14 +9,21 @@ import ( func Builtin(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var value starlark.Value maximum := 1000000 - if err := starlark.UnpackArgs("hfs", args, kwargs, "file", &value, "maximum_entries?", &maximum); err != nil { + raw := false + if err := starlark.UnpackArgs("hfs", args, kwargs, "file", &value, "maximum_entries?", &maximum, "raw_forks?", &raw); err != nil { return nil, err } file, ok := value.(starfile.File) if !ok { return nil, fmt.Errorf("hfs: expected file") } - volume, err := Open(file, maximum) + var volume *Volume + var err error + if raw { + volume, err = OpenRaw(file, maximum) + } else { + volume, err = Open(file, maximum) + } if err != nil { return nil, err } @@ -24,6 +31,13 @@ func Builtin(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwarg paths := make([]starlark.Value, len(entries)) index := map[string]starlark.Value{} for i, e := range volume.Entries { + var rawData, rawResource starlark.Value = starlark.None, starlark.None + if e.RawData != nil { + rawData = e.RawData + } + if e.RawResource != nil { + rawResource = e.RawResource + } var data, resource starlark.Value = starlark.None, starlark.None var size, resourceSize int64 if e.Data != nil { @@ -34,7 +48,13 @@ func Builtin(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwarg resource = e.Resource resourceSize = e.Resource.Size() } - record := starfile.NewRecord(starlark.StringDict{"path": starlark.String(e.Path), "name": starlark.Bytes(e.Name), "entry_type": starlark.String(e.Kind), "id": starlark.MakeUint(uint(e.ID)), "parent_id": starlark.MakeUint(uint(e.Parent)), "created": starlark.MakeUint(uint(e.Created)), "modified": starlark.MakeUint(uint(e.Modified)), "backup": starlark.MakeUint(uint(e.Backup)), "flags": starlark.MakeUint(uint(e.Flags)), "finder_info": starlark.Bytes(e.FinderInfo), "data": data, "resource": resource, "size": starlark.MakeInt64(size), "resource_size": starlark.MakeInt64(resourceSize)}) + xattrs := starlark.NewDict(len(e.Xattrs)) + for name, value := range e.Xattrs { + if err := xattrs.SetKey(starlark.String(name), value); err != nil { + return nil, err + } + } + record := starfile.NewRecord(starlark.StringDict{"path": starlark.String(e.Path), "name": starlark.Bytes(e.Name), "entry_type": starlark.String(e.Kind), "id": starlark.MakeUint(uint(e.ID)), "parent_id": starlark.MakeUint(uint(e.Parent)), "created": starlark.MakeUint(uint(e.Created)), "modified": starlark.MakeUint(uint(e.Modified)), "backup": starlark.MakeUint(uint(e.Backup)), "flags": starlark.MakeUint(uint(e.Flags)), "finder_info": starlark.Bytes(e.FinderInfo), "uid": starlark.MakeUint(uint(e.UID)), "gid": starlark.MakeUint(uint(e.GID)), "mode": starlark.MakeUint(uint(e.Mode)), "owner_flags": starlark.MakeUint(uint(e.OwnerFlags)), "admin_flags": starlark.MakeUint(uint(e.AdminFlags)), "special": starlark.MakeUint(uint(e.Special)), "compression_type": starlark.MakeUint(uint(e.CompressionType)), "xattrs": xattrs, "raw_data": rawData, "raw_resource": rawResource, "target": starlark.String(e.Target), "data": data, "resource": resource, "size": starlark.MakeInt64(size), "resource_size": starlark.MakeInt64(resourceSize)}) entries[i] = record paths[i] = starlark.String(e.Path) index[e.Path] = record diff --git a/firmware/acpi/pci.go b/firmware/acpi/pci.go index 38dcfc3..1cab6a4 100644 --- a/firmware/acpi/pci.go +++ b/firmware/acpi/pci.go @@ -74,9 +74,21 @@ func (p PCIRoot) AML() ([]byte, error) { for _, v := range []uint16{0, uint16(p.FirstBus), uint16(p.LastBus), 0, uint16(p.LastBus) - uint16(p.FirstBus) + 1} { bus = binary.LittleEndian.AppendUint16(bus, v) } - resources := append(bus, 0x8a, 43, 0, 0, 0x0c, 1) - for _, v := range []uint64{0, p.MemoryBase, p.MemoryBase + p.MemorySize - 1, 0, p.MemorySize} { - resources = binary.LittleEndian.AppendUint64(resources, v) + resources := bus + end := p.MemoryBase + p.MemorySize - 1 + // Prefer DWord descriptors for windows that fit below 4 GiB. Older + // ACPI PCI roots (including Lion's AppleACPIPlatform) ignore QWord + // memory windows while accepting the equivalent DWord producer range. + if end <= 0xffffffff && p.MemorySize <= 0xffffffff { + resources = append(resources, 0x87, 23, 0, 0, 0x0c, 1) + for _, v := range []uint32{0, uint32(p.MemoryBase), uint32(end), 0, uint32(p.MemorySize)} { + resources = binary.LittleEndian.AppendUint32(resources, v) + } + } else { + resources = append(resources, 0x8a, 43, 0, 0, 0x0c, 1) + for _, v := range []uint64{0, p.MemoryBase, end, 0, p.MemorySize} { + resources = binary.LittleEndian.AppendUint64(resources, v) + } } if p.Legacy { // DWord I/O window; 65536 bytes cannot fit in a Word descriptor. @@ -90,7 +102,7 @@ func (p PCIRoot) AML() ([]byte, error) { body = append(body, amlName("_CRS", amlPackage([]byte{0x11}, buffer))...) routes := []byte{byte(len(p.Interrupts))} for _, r := range p.Interrupts { - if r.Device > 31 || r.Pin > 3 || r.Interrupt < 32 { + if r.Device > 31 || r.Pin > 3 { return nil, fmt.Errorf("invalid PCI interrupt route") } row := []byte{4} diff --git a/firmware/acpi/pci_memory_test.go b/firmware/acpi/pci_memory_test.go new file mode 100644 index 0000000..d137ab2 --- /dev/null +++ b/firmware/acpi/pci_memory_test.go @@ -0,0 +1,41 @@ +package acpi + +import ( + "bytes" + "encoding/binary" + "testing" +) + +func TestPCIRootMemoryDescriptorWidth(t *testing.T) { + for _, tc := range []struct { + base, size uint64 + width int + tag byte + }{ + {0xc0000000, 0x3ec00000, 4, 0x87}, + {0x100000000, 0x20000000, 8, 0x8a}, + {0, 0x100000000, 8, 0x8a}, // A 4 GiB length cannot fit a DWord. + } { + aml, err := (PCIRoot{Legacy: true, MemoryBase: tc.base, MemorySize: tc.size}).AML() + if err != nil { + t.Fatal(err) + } + // Find the fixed memory producer descriptor, independently decode all + // address fields, and verify no truncation at the width boundary. + prefix := []byte{tc.tag, byte(3 + 5*tc.width), 0, 0, 0x0c, 1} + at := bytes.Index(aml, prefix) + if at < 0 { + t.Fatal("missing memory producer descriptor", tc) + } + fields := aml[at+6 : at+6+5*tc.width] + read := func(n int) uint64 { + if tc.width == 4 { + return uint64(binary.LittleEndian.Uint32(fields[n*4:])) + } + return binary.LittleEndian.Uint64(fields[n*8:]) + } + if read(0) != 0 || read(1) != tc.base || read(2) != tc.base+tc.size-1 || read(3) != 0 || read(4) != tc.size { + t.Fatal("resource address fields truncated", tc) + } + } +} diff --git a/firmware/acpi/pci_test.go b/firmware/acpi/pci_test.go index 5f277b3..b3a0931 100644 --- a/firmware/acpi/pci_test.go +++ b/firmware/acpi/pci_test.go @@ -49,3 +49,10 @@ func TestARM64PlatformTables(t *testing.T) { t.Fatal("accepted invalid INTx pin") } } + +func TestX86LowGSIINTARoute(t *testing.T) { + aml, err := (PCIRoot{MemoryBase: 0xe0000000, MemorySize: 0x10000000, Interrupts: []PCIInterrupt{{Device: 1, Pin: 0, Interrupt: 16}}}).AML() + if err != nil || !bytes.Contains(aml, []byte("_PRT")) { + t.Fatal("x86 IOAPIC GSI16 route rejected", err) + } +} diff --git a/frontend/starlark/runtime.go b/frontend/starlark/runtime.go index 203fc95..814e14c 100644 --- a/frontend/starlark/runtime.go +++ b/frontend/starlark/runtime.go @@ -18,6 +18,7 @@ import ( "github.com/tinyrange/trex/archive/arsenic" "github.com/tinyrange/trex/archive/aws" "github.com/tinyrange/trex/archive/bff" + "github.com/tinyrange/trex/archive/bom" "github.com/tinyrange/trex/archive/bru" "github.com/tinyrange/trex/archive/bsddump" "github.com/tinyrange/trex/archive/bzip2" @@ -29,10 +30,13 @@ import ( "github.com/tinyrange/trex/archive/gzip" "github.com/tinyrange/trex/archive/hunk" "github.com/tinyrange/trex/archive/ibmisave" + "github.com/tinyrange/trex/archive/im4p" "github.com/tinyrange/trex/archive/irix" "github.com/tinyrange/trex/archive/kwaj" "github.com/tinyrange/trex/archive/lha" + "github.com/tinyrange/trex/archive/lzfse" "github.com/tinyrange/trex/archive/macresource" + "github.com/tinyrange/trex/archive/pbzx" "github.com/tinyrange/trex/archive/rar" "github.com/tinyrange/trex/archive/rms" "github.com/tinyrange/trex/archive/sevenzip" @@ -40,8 +44,10 @@ import ( "github.com/tinyrange/trex/archive/stuffit" "github.com/tinyrange/trex/archive/szdd" "github.com/tinyrange/trex/archive/tome" + "github.com/tinyrange/trex/archive/udif" "github.com/tinyrange/trex/archive/vmsbackup" "github.com/tinyrange/trex/archive/wim" + "github.com/tinyrange/trex/archive/xar" "github.com/tinyrange/trex/archive/xz" ziparchive "github.com/tinyrange/trex/archive/zip" "github.com/tinyrange/trex/archive/zstd" @@ -147,6 +153,12 @@ func predeclared() starlark.StringDict { "rms_variable": starlark.NewBuiltin("rms_variable", rms.VariableBuiltin), "gzip": starlark.NewBuiltin("gzip", gzip.Builtin), "gzip_members": starlark.NewBuiltin("gzip_members", gzip.MembersBuiltin), + "pbzx": starlark.NewBuiltin("pbzx", pbzx.Builtin), + "im4p": starlark.NewBuiltin("im4p", im4p.Builtin), + "bom": starlark.NewBuiltin("bom", bom.Builtin), + "lzfse": starlark.NewBuiltin("lzfse", lzfse.Builtin), + "xar": starlark.NewBuiltin("xar", xar.Builtin), + "udif": starlark.NewBuiltin("udif", udif.Builtin), "cpio": starlark.NewBuiltin("cpio", cpio.Builtin), "cpio_build": starlark.NewBuiltin("cpio_build", cpio.BuildBuiltin), "bzip2": starlark.NewBuiltin("bzip2", bzip2.Builtin), @@ -194,6 +206,7 @@ func predeclared() starlark.StringDict { "ext4": starlark.NewBuiltin("ext4", filesystemext4.Builtin), "ext4_build": starlark.NewBuiltin("ext4_build", filesystemext4.BuildBuiltin), "hfs": starlark.NewBuiltin("hfs", filesystemhfs.Builtin), + "hfs_build": starlark.NewBuiltin("hfs_build", filesystemhfs.BuildBuiltin), "ufs": starlark.NewBuiltin("ufs", filesystemufs.Builtin), "ods2": starlark.NewBuiltin("ods2", filesystemods2.Builtin), "ultrix_label": starlark.NewBuiltin("ultrix_label", filesystemultrix.Builtin), diff --git a/frontend/starlark/starlark_docs.go b/frontend/starlark/starlark_docs.go index 4849161..40dbff6 100644 --- a/frontend/starlark/starlark_docs.go +++ b/frontend/starlark/starlark_docs.go @@ -214,7 +214,7 @@ var nativeStarlarkSignatures = map[string]string{ "filesystem.sgi": "filesystem.sgi(file) -> record", "filesystem.apm": "filesystem.apm(file, block_size=512, maximum_entries=1M) -> record", "filesystem.efs": "filesystem.efs(file, maximum_entries=1M) -> record", - "filesystem.hfs": "filesystem.hfs(file, maximum_entries=1M) -> record", + "filesystem.hfs": "filesystem.hfs(file, maximum_entries=1M, raw_forks=False) -> record", "filesystem.ufs": "filesystem.ufs(file, maximum_entries=1M, maximum_blocks=1M) -> record", "filesystem.ods2": "filesystem.ods2(file, maximum_entries=1M, maximum_depth=64) -> record", "filesystem.ultrix_label": "filesystem.ultrix_label(file) -> record", @@ -227,12 +227,23 @@ var nativeStarlarkSignatures = map[string]string{ "filesystem.ckd_attribute_page": "filesystem.ckd_attribute_page(data) -> record", "filesystem.openbsd_label": "filesystem.openbsd_label(file, label_offset=512) -> record", "filesystem.xfs": "filesystem.xfs(file, maximum_entries=1M) -> record", + "archive.udif": "archive.udif(file) -> record", + "archive.xar": "archive.xar(file, maximum_entries=100000) -> record", + "archive.pbzx": "archive.pbzx(file, maximum_bytes=) -> file", + "archive.lzfse": "archive.lzfse(file, maximum_bytes=) -> file", + "archive.im4p": "archive.im4p(file) -> record", + "archive.bom": "archive.bom(file, maximum_entries=1M) -> list[record]", "archive.cpio": "archive.cpio(file, maximum_entries=100000) -> list[dict]", "archive.cpio_build": "archive.cpio_build(entries) -> file", "archive.gzip_members": "archive.gzip_members(file, maximum_bytes=512MiB, maximum_members=1024) -> list[file]", "filesystem.ext4": "filesystem.ext4(file, maximum_entries=100000, maximum_depth=32) -> record", "filesystem.ext4_build": "filesystem.ext4_build(entries, size, label='', uuid='') -> file", - "archive.gzip": "archive.gzip(file, maximum_bytes=) -> file", + "binary.macho": "binary.macho(file, architecture=\"x86_64\") -> record", + "binary.plist": "binary.plist(source) -> value", + "binary.plist_encode": "binary.plist_encode(value) -> file", + "binary.appledouble": "binary.appledouble(file) -> dict[int,file]", + "filesystem.hfs_build": "filesystem.hfs_build(entries, size, label=\"Untitled\") -> file", + "archive.gzip": "archive.gzip(file, maximum_bytes=, cache=False) -> file", "archive.bzip2": "archive.bzip2(file, maximum_bytes=) -> file", "archive.sevenzip": "archive.sevenzip(file, maximum_entries=1M, maximum_metadata=64MiB, max_dictionary=256MiB) -> sevenzip", "archive.szdd": "archive.szdd(file, maximum=512MiB) -> file", @@ -284,7 +295,7 @@ var nativeStarlarkSignatures = map[string]string{ "windows.win9x_vxd_library": "windows.win9x_vxd_library(base, members, exclude=[]) -> file", "windows.win9x_vxd_library_members": "windows.win9x_vxd_library_members(file) -> list[string]", "qemu.acpi_table": "qemu.acpi_table(file)", - "cc.backend": "cc.backend(acpi=False, pci_ide=False, hpet=False, uefi=False, ide_dma=True, overlay_limit=268435456) -> vmm_backend", + "cc.backend": "cc.backend(acpi=False, pci_ide=False, hpet=False, uefi=False, ide_model='synthetic', ide_dma=True, overlay_limit=268435456) -> vmm_backend", "qemu.backend": "qemu.backend(binary='', machine='pc', machine_properties={}, accelerator='auto', firmware='bios', display_frontend='auto', display_zoom_to_fit=False, block_transport='auto', overlay_limit=256MiB, stderr_limit=1MiB, devices=[], netdevs=[], chardevs=[], options=[], acpi_tables=[])", "qemu.chardev": "qemu.chardev(name, **properties)", "qemu.device": "qemu.device(name, **properties)", @@ -295,6 +306,7 @@ var nativeStarlarkSignatures = map[string]string{ "vmm.channel": "vmm.channel(kind, name, required=True)", "vmm.disk": "vmm.disk(source, name='disk0', bus='auto', media='disk', unit=-1, chs=None, read_only=None, snapshot=False, required=True)", "vmm.display": "vmm.display(mode, required=True)", + "vmm.darwin_boot": "vmm.darwin_boot(kernel, command_line='', smc_osk=b'') -> vmm_darwin_boot", "vmm.linux_boot": "vmm.linux_boot(kernel, initramfs=None, command_line='')", "vmm.machine": "vmm.machine(architecture, memory, cpus=1, disks=[], networks=[], display=vmm.display('none'), channels=[], start_paused=False, required_capabilities=[], boot=None)", "vmm.network": "vmm.network(kind, name='net0', required=True, switch=None, mac='')", diff --git a/frontend/starlark/starlark_docs_native.go b/frontend/starlark/starlark_docs_native.go index 0c87138..9c1aa87 100644 --- a/frontend/starlark/starlark_docs_native.go +++ b/frontend/starlark/starlark_docs_native.go @@ -66,7 +66,7 @@ var nativeStarlarkDescriptions = map[string]string{ "archive.tome": "Reads classic kc0001 Apple installer Tome catalogs and decodes both forks using chunked InstaCompOne in memory. Validates catalog counts, unique IDs, payload ranges, nonoverlap, exact fork sizes, both decoded-fork checksums and complete compressed-input consumption. Returns entries with ID, raw name bytes, reversible ID-qualified paths, file type/creator, Finder flags, version, Mac-epoch dates, decoded data/resource files and original stored views. maximum_decoded_bytes bounds total decoded output and each stored fork. Catalog checksum values are preserved as data_checksum and resource_checksum; checksums_verified is True after validation. Supports the observed 00010000 binary and 00000000 seven-bit-text chunk framing, plus stored chunks selected by a leading 01 byte, with retained 32KiB history and command-aligned 64KiB blocks; other encodings fail explicitly. Resource maps and nested archives require separate decoders. Installer code never runs. catalog_kind distinguishes file catalogs (1) from individual-resource catalogs (2). Kind2 exposes resource_type and signed resource_id, with the decoded resource payload in data and an empty resource fork; file-only metadata is omitted. Unknown catalog kinds are rejected.", "archive.stuffit": "Reads the classic 22-byte/112-byte StuffIt archive generation, including self-extractors with an archive data fork. Validates declared archive size, header CRC-16/ARC, nested directory markers and each decoded fork CRC. Classic top-level counts are checked; STi2/ST46/ST50/ST60/ST65 installer counts are preserved separately as declared_count/top_level_count with root_count_verified=False because their selection semantics are not interpreted. STi2 repeated directory records remain in order. ST46 preserves repeated STcp/STde/STal/DIFF records with creator STin, exposing installer_record, one-based occurrence and raw declared fork sizes; copy/delete/alias metadata bytes are retained even when their logical data size is zero. ST60/ST65 additionally preserve STda/STmv metadata, including metadata naming existing directories without replacing them. No installation operations or patches are applied. ST60 alternative ordinary files are also retained as separate occurrences; other variants reject ordinary duplicate files, and ordinary file/directory conflicts still fail. Supports stored forks, method13 LZSS/Huffman with embedded or predefined tables, and method14 installer blocks with recursive Huffman tables and retained 256KiB history. Method14 validates block framing, decoded sizes and final fork CRCs; unavailable history is never invented. Returns version and entries preserving raw name bytes, reversible percent-escaped paths, file type/creator, Finder flags, Mac-epoch dates, resource/data methods and CRCs, separate data/resource files and original stored views. maximum_decoded_bytes bounds total decoded fork bytes and each stored fork. Also reads the StuffIt5 banner container with method15 (Arsenic) forks, validating the global, record and metadata CRC16 fields, parent/previous/file-next links, directory child counts and in-stream fork CRC32. StuffIt5 fork CRC16 attributes are None because integrity comes from the method15 stream. Other StuffIt5 fork methods, other compression methods, encryption and StuffItX remain explicit gaps; executable self-extractor code never runs. Nested containers and resource payloads require their own decoders.", "archive.compactpro": "Decodes a single-volume CompactPro archive, including self-extractors whose data fork contains the archive. Validates catalog CRC, directory descendant counts, payload ranges and the combined CRC of each file's decoded resource and data forks. Supports RLE and block-based LZH followed by RLE, retaining both forks and their stored views. Returns entries with reversible percent-escaped paths, raw name bytes, file type/creator, Finder flags, Mac-epoch dates, compression flags, CRC, data/resource and their sizes. maximum_decoded_bytes bounds total decoded fork bytes and each stored fork. Encrypted and multi-volume archives fail explicitly; no self-extractor code runs. Nested archives and resource payload formats remain separate layers.", - "filesystem.hfs": "Reads a classic HFS volume's catalog and extent-overflow records without mounting. Returns raw volume name, entries, paths and exact find(path) lookup. Every regular entry exposes separate data and resource fork views, their sizes, raw name bytes, catalog/parent IDs, Finder information, flags and Mac-epoch timestamps. Path components percent-escape non-ASCII/control bytes, slash and percent, keeping legacy names reversible without assuming a script encoding. HFS Plus and fragmented extents-file bootstrap are not yet supported; compressed resources require a separate resource decoder. Aliases are not followed.", + "filesystem.hfs": "Reads classic HFS, HFS+/HFSX and embedded HFS+ volumes without mounting. Returns raw volume name, entries, paths and exact find(path). Entries expose distinct data/resource forks, xattrs, reversible escaped paths, Finder/catalog metadata, Unix modes/uid/gid and link targets. Hardlink inode aliases share decoded inode contents while raw_data/raw_resource retain physical alias forks. Symlinks are not followed. Inline and fork-backed attributes have independent extent-overflow support. decmpfs types 3/4 (inline/resource zlib with stored-block escape) decode lazily with exact lengths; other compression types fail unless raw_forks=True explicitly requests physical forensic forks. compression_type identifies decoded files. Does not mount, repair or execute filesystem contents.", "archive.mac_resource": "Parses a classic Macintosh resource fork, checking map, reference, name and payload bounds and repeated type groups. Duplicate IDs retain every record in map order; duplicate_ids reports collisions, occurrence is one-based, and later occurrences have ordinal path suffixes. Returns fork attributes and entries with resource_type (four raw bytes), signed id, name (raw bytes or None), attributes, offset, size and data. Uncompressed data is a borrowed view; supported Apple dcmp 0/1/2/3 compressed resources are decoded and validated in memory. Compression requires both attribute bit0 and the a89f6572 payload tag, matching Resource Manager; raw attributes remain available when the bit is set on ordinary data. compressed reports actual decoding, while stored_data and stored_size preserve the original representation. maximum_decoded_bytes bounds total decoded compressed payload bytes and each stored compressed input. Unknown codecs and unsupported token variants fail explicitly. The synthetic path uses hexadecimal type and signed ID without interpreting names as host paths. Payload-specific formats remain separate layers.", "filesystem.xfs": "Reads a legacy version-4 XFS directory tree using borrowed file extents, without mounting or replaying the journal. Returns entries and paths; find(path) returns an entry or None. Entries expose data, entry_type, inode, mode, uid, gid, modified and size. Supports local forks and inode-resident extent lists, sparse and unwritten data, and version-1/version-2 directories (including version-1 directory btrees). Symlinks are not followed. Version-5 metadata, extent btrees, realtime data and file-type directory extensions are currently rejected explicitly.", "windows.memory_image": "Borrows a file as a read-only physical capture. Optional ranges map (physical_start, file_offset, size); omitted ranges cover the whole file, while an empty list captures nothing. Gaps remain unavailable, not zero-filled. Keep the source unchanged while views exist; this does not parse crash dumps or discover RAM ranges.", @@ -81,13 +81,24 @@ var nativeStarlarkDescriptions = map[string]string{ "archive.bru": "Reads classic uncompressed BRU backups, validating each 2 KiB record's signed-byte checksum, archive identity, sequence and member name. Returns ordered entries with type, mode, ownership, link target and data file views assembled across record headers. Checks the end record and trailing buffer padding, including padding-block checksums. Compressed/extended members and incomplete multivolume inputs fail explicitly; no restore commands are executed.", "filesystem.sgi": "Validates an SGI volume-header checksum and returns partition and boot-directory file views. Partitions expose index, partition_type, start_block, blocks, data and complete; addresses are 512-byte basic blocks. A whole-volume descriptor may exceed a CD image: its declared blocks are retained, data covers only available bytes, and complete is false. Filesystem partitions and boot files must fit in full. Does not infer an inner filesystem from a filename.", "filesystem.efs": "Reads an IRIX EFS volume into entries and case-sensitive paths, resolving direct and indirect extents without mounting. find(path) returns an entry or None; entries expose data, entry_type, size, inode, mode, uid, gid and modified. Accepts a trimmed free tail only when the allocation bitmap proves the omitted region is free and all referenced extents fit the original input. Symlink data remains the link target and is never followed; device entries have no data.", + "archive.udif": "Opens XML-backed, single-segment UDIF v4 disk images. Returns disk (complete logical disk), original xml and trailer, BLKX tables (name, offset, size, raw), and verify(). Raw, zero-fill, ignored, zlib and bzip2 chunks use bounded caches. verify() reads stored data and checks declared data-fork, table and master CRC32s; ignored sectors are excluded from table checksums. Rejects encryption, segmented images, unsupported codecs and overlapping/gapped maps. Does not authenticate signatures or infer filesystems.", + "archive.xar": "Indexes XAR v1 without extraction. Returns entries, files, original decoded toc, raw heap and find(path). Entries preserve path, kind, ID, mode, uid/gid, link target, encoding, raw/data file views, size and verify(). Verifies declared compressed-TOC checksum on open; member verify() checks archived/extracted checksums and exact decoded lengths. Supports stored, zlib and bzip2 members. Safe paths, XML depth/size, heap extents and index sizes are bounded. Signature metadata remains in TOC/heap but signatures are not authenticated.", + "archive.pbzx": "Opens chunked Apple installer payloads as a lazy read-only concatenated file. Validates decoded/stored chunk lengths, XZ indexes and final short-chunk framing; stored chunks are borrowed. Uses a 64 MiB decoded cache and 64 MiB per-chunk bound. Omitted maximum_bytes has no total decoded ceiling; explicit positive limits are enforced. Read complete contents to force XZ checksums. Parse the resulting CPIO separately.", + "archive.lzfse": "Opens Apple LZFSE v1/v2, LZVN and stored-block streams as a read-only file. Bounds block size/count and retains a 64 MiB decoded cache plus 256 KiB match history. Backward reads outside cache replay preceding dependent blocks. Validates entropy states, source/output lengths, match distances and end marker; LZFSE has no content checksum. Explicit maximum_bytes bounds total decoded bytes. No host or C decoder is used.", + "archive.im4p": "Parses an ASN.1 IM4P firmware envelope with bounded DER depth/node counts. Returns payload_type, description, borrowed payload, extras (original optional DER records) and expected_size (LZFSE metadata when present, otherwise zero). Does not decrypt, decompress, authenticate signatures or infer a filesystem; pass payload explicitly to archive.lzfse or the appropriate format reader.", + "archive.bom": "Returns every flat Apple BOMStore path-inventory record, including architecture variants sharing a path and trailing leaf-chain nodes missing from branch indexes. Records preserve kind, BOM ID, mode, uid/gid, mtime, declared size, architecture, checksum or link target/device number. BOM describes metadata only and has no file contents. maximum_entries bounds indexing. The auto tree preserves duplicate variants under %00bom-variants with bom_path metadata.", "archive.cpio": "Reads ASCII CPIO into flat metadata dictionaries without following links or instantiating special files. Supports newc, CRC newc, odc and concatenated archives; maximum_entries bounds the index. Regular data is exposed as lazy file views.", "archive.cpio_build": "Constructs a deterministic Linux-compatible newc archive from Unix entry dictionaries. Preserves modes, ownership, timestamps, symbolic links, hardlinks and device numbers. Synthesizes missing parent directories, rejects duplicate or unsafe paths, and borrows payload readers without extracting them.", "archive.gzip_members": "Validates individual gzip members and returns exact compressed-member file views. CRC and size checks cover every member; maximum_bytes bounds total expanded data and maximum_members bounds indexing. No expanded payload is retained. Trailing junk is rejected.", "filesystem.ext4": "Reads a clean ext2/3/4 image without mounting. Returns flat entry records with Unix metadata and lazy file data; symbolic links and device nodes are metadata, never followed. Handles multi-level extents, sparse/unwritten ranges, indirect blocks, indexed directories, 64-bit descriptors and metadata checksums. Rejects unsupported incompatible features and images requiring journal replay. Positive entry/depth limits bound traversal.", "filesystem.ext4_build": "Builds a sparse ext4 image from Unix entry dictionaries using native 4096-byte blocks, extents and linear directories. Size is explicit and block-aligned; metadata is bounded at 256 MiB. Preserves ownership, permissions, links and devices. Images have no journal; file payloads are borrowed immutable readers. UUID is 32 hexadecimal digits; label is at most 16 bytes.", - "archive.gzip": "Opens concatenated gzip streams as a lazy read-only file with a 1 MiB decoded cache. Forward reads reuse the decoder; backward reads outside the cache replay it. Checksums are validated as data is consumed; size scans the complete stream without retaining it. Omitted maximum_bytes has no decoded-length ceiling; an explicit positive maximum is enforced. Does not interpret an archive inside the stream.", - "archive.bzip2": "Opens concatenated bzip2 streams, including legacy randomized blocks, as a lazy read-only file with a bounded decoded block cache. Block and stream checksums are checked during reads; size scans the complete stream. Omitted maximum_bytes has no decoded-length ceiling; an explicit positive maximum is enforced. Does not interpret an archive inside the stream.", + "binary.macho": "Inspects a bounded little-endian Mach-O64 x86_64 image, optionally selected from a big-endian fat32/fat64 universal container. Exposes the selected borrowed file, segments (name, address, size, file_offset, file_size, flags), and original symbols (name, address, type, section, description). at(address, size) returns borrowed file-backed virtual bytes; zero-fill, unmapped, ambiguous and overflowing ranges fail. Header, command, string and symbol counts and aggregate symbol-name allocations are bounded. Does not relocate, link or execute the image.", + "binary.plist": "Decodes bounded XML 1.0 or bplist00 property lists into dict/list, bytes, string, bool, int64, float and opaque round-trippable plist_date values. Binary null becomes None; unsupported UID and 16-byte integers fail explicitly. Enforces 64MiB input/decoded data, 100000 visited values and depth128, validates reference/offset bounds, rejects cycles, duplicate keys and invalid Unicode. Does not follow XML external entities.", + "binary.plist_encode": "Encodes supported plist values as deterministic XML1.0 in an in-memory file. Preserves plist_date values from decoding. Enforces aggregate conversion/output byte, node and depth limits, including dictionary keys. Rejects None, non-finite floats, non-string keys, invalid XML characters and integers outside int64.", + "binary.appledouble": "Parses AppleDouble version 2 headers and returns an integer-entry-ID dictionary of borrowed files. Validates table bounds, duplicate IDs and non-overlapping extents. Preserves unknown entries without interpreting them. FinderInfo (9), resource fork (2) and any ATTR extension remain raw bytes; does not apply metadata or modify files.", + "filesystem.hfs_build": "Builds a bounded unjournaled case-sensitive HFSX version-5 volume natively in memory from UNIX entry dictionaries. size is the volume byte length; payloads are borrowed file/string/bytes values. Entries support path, mode, uid, gid, mtime, data, target, hardlink, major, minor, finder_info (32 bytes), resource, xattrs (name to file/bytes), owner_flags and admin_flags. Preserves regular-file hardlinks through native private inodes; directory hardlinks are unsupported. Paths use POSIX components (colon becomes catalog slash); names use HFS canonical decomposition. Allocation, catalog and attribute trees include free growth nodes with bounded metadata. Does not build partitions, journal, execute guest tools or write intermediate host files.", + "archive.gzip": "Opens concatenated gzip streams as a lazy read-only file with a 1 MiB decoded cache. Forward reads reuse the decoder; backward reads outside the cache replay it. Checksums are validated as data is consumed; size scans the complete stream without retaining it. Omitted maximum_bytes has no decoded-length ceiling; an explicit positive maximum is enforced. cache=True requires explicit positive maximum_bytes, eagerly validates every member to EOF, and returns an immutable paged RAM snapshot supporting random access without replay or host files; integrity or overflow failures return no partial result. Does not interpret an archive inside the stream.", + "archive.bzip2": "Opens concatenated bzip2 streams, including legacy randomized blocks, as a lazy read-only file with a bounded decoded block cache. Block and stream checksums are checked during reads; size scans the complete stream. Omitted maximum_bytes has no decoded-length ceiling; an explicit positive maximum is enforced. cache=True requires explicit positive maximum_bytes, eagerly validates every member to EOF, and returns an immutable paged RAM snapshot supporting random access without replay or host files; integrity or overflow failures return no partial result. Does not interpret an archive inside the stream.", "archive.cab": "Parses a Microsoft Cabinet file and exposes its members as file views. Folder decompression is shared when caching is enabled, so several entries in one compressed folder need not decode it repeatedly.", "archive.cab_set": "Opens related Cabinet files as one set, resolving files and compressed data that span cabinet boundaries. The caller supplies the cabinet files; the parser does not search host directories.", "archive.installer": "Recognizes a supported installer container, including supported embedded payloads, and returns an inspection object with files and a declarative installation plan. It does not run the installer or apply that plan.", @@ -215,7 +226,7 @@ var nativeStarlarkDescriptions = map[string]string{ "path.from_windows": "Converts a Windows-style path into the library's logical slash-separated path representation. It does not translate it into a host path or access a drive.", "path.join": "Joins logical path components and normalizes the result. The operation is independent of the host operating system's path rules.", "qemu.acpi_table": "Creates a QEMU backend descriptor for an ACPI table file. The table is supplied through the backend when the VM starts.", - "cc.backend": "Creates an in-process CrumbleCracker PC backend for Linux/amd64 KVM. The i386 and x86_64 platforms provide Go BIOS services, one ATA disk, VGA capture and PS/2 input. ACPI is automatic for x86_64 and optional for i386. uefi=True selects native Go UEFI for x86_64 GPT disks, enables PCI IDE, and provides a 1280x720 GOP framebuffer. RAM supports 16 MiB through 2 GiB. hpet=True adds a 100 MHz HPET with three comparators and enables ACPI. pci_ide enables ACPI and a generic PCI IDE controller with PIO and bus-master DMA; ide_dma=False selects a PIO-only disk. overlay_limit bounds dirty snapshot memory in bytes (default 256 MiB). vmm.start creates the guest from portable memory and disk intent.", + "cc.backend": "Creates an in-process CrumbleCracker PC backend for Linux/amd64 KVM. The i386 and x86_64 platforms provide Go BIOS services, one ATA disk, VGA capture and PS/2 input. ACPI is automatic for x86_64 and optional for i386. uefi=True selects native Go UEFI for x86_64 GPT disks, enables PCI IDE, and provides a 1280x720 GOP framebuffer. RAM supports 16 MiB through 2 GiB. hpet=True adds a 100 MHz HPET with three comparators and enables ACPI. pci_ide enables ACPI and a generic PCI IDE controller with PIO and bus-master DMA; ide_dma=False selects a PIO-only disk. ide_model='synthetic' preserves the generic compatibility controller; 'ich7-pata' requires PCI IDE and selects the 8086:27df profile with one primary disk, compatibility IRQ14 and native INTA routed to GSI16. The secondary channel has no disk. Timing registers are recorded; electrical cable timing is not simulated. The cc.v1 extension provides bounded read_virtual, original kernel symbol lookup and read-only disassemble observations for direct Darwin boot. overlay_limit bounds dirty snapshot memory in bytes (default 256 MiB). vmm.start creates the guest from portable memory and disk intent.", "qemu.audiodev": "Creates a QEMU audio-backend descriptor from its name and properties. It is configuration data; creating it does not start audio or launch QEMU.", "qemu.backend": "Creates a QEMU implementation of the portable VMM backend with selected machine, acceleration, devices and transport policies. A separate vmm.start call launches the guest.", "qemu.chardev": "Creates a QEMU character-device backend descriptor from a backend name and properties. Use it in qemu.backend configuration.", @@ -238,6 +249,7 @@ var nativeStarlarkDescriptions = map[string]string{ "vmm.channel": "Describes a named VM byte channel of the requested kind. required determines whether lack of backend support is a validation failure.", "vmm.disk": "Describes a guest disk or optical medium backed by a file or block device, including bus, unit and write/snapshot policy. It does not boot or materialize the source.", "vmm.display": "Describes the requested guest display mode and whether support is mandatory. This is part of a portable machine specification.", + "vmm.darwin_boot": "Creates an unslid amd64 XNU v2 direct handoff from an immutable original kernelcache or Mach-O executable. Supports amd64 fat slices and Apple comp/lzss caches with checksum validation. Native cc requires Linux/amd64 KVM, x86_64 and no UEFI image execution; QEMU rejects this intent. No guest bootloader is executed or kernel instructions patched. RAM disks are not supported. Optional smc_osk must be empty or exactly 64 bytes; it is explicit caller-supplied SMC material, never bundled or logged. Native cc exposes SMC APP0001/IRQ6 and omits OSK0/OSK1 when no material is supplied. Verified scope is Lion HFS-root boot to launchd and a visible single-user shell, not keyboard input, a desktop or a complete installation.", "vmm.linux_boot": "Creates portable direct Linux boot intent from borrowed immutable kernel and optional initramfs files plus a command line. Requires backend boot.linux capability; QEMU supports Linux hosts and x86 guests using anonymous inherited descriptors.", "vmm.machine": "Creates a portable machine specification from architecture, memory, CPUs, storage, networking and channels. It does not start a VM.", "vmm.network": "Describes a named guest network connection and whether its requested kind is required. The selected backend implements the transport.", diff --git a/scripts/inspect/macos_layers.star b/scripts/inspect/macos_layers.star new file mode 100644 index 0000000..0b3dfb9 --- /dev/null +++ b/scripts/inspect/macos_layers.star @@ -0,0 +1,196 @@ +"""Inspect Intel macOS installer layers without mounting or extracted intermediates. + +Usage: trex scripts/inspect/macos_layers.star MEDIA.dmg REPORT.json +All byte sources remain native borrowed file views. Reports are final outputs. +This validates content and format checksums, not signatures or boot/install behavior. +""" + +def fingerprint(file): + return hex(digest(file)) + +def decode_payload(file): + header = file.bytes(0, min(6, file.size)) + if header[:4] == b"pbzx": + return "pbzx/xz", archive.pbzx(file) + if header[:3] == b"BZh": + return "bzip2", archive.bzip2(file) + if header[:2] == b"\x1f\x8b": + return "gzip", archive.gzip(file) + if header in [b"070707", b"070701", b"070702"]: + return "stored", file + fail("unknown installer payload header: " + hex(header)) + +def add_container(queue, path, file, depth): + if depth > 16: + fail("nested installer depth exceeded: " + path) + if file.size >= 4 and file.bytes(0, 4) == b"xar!": + queue.append((path, "xar", file, depth)) + elif file.size >= 512 and file.bytes(file.size - 512, 4) == b"koly": + queue.append((path, "udif", file, depth)) + elif b"IM4P" in file.bytes(0,min(32,file.size)): + queue.append((path, "im4p", file, depth)) + else: + fail("unrecognized installer container: " + path) + +def inspect_partitions(disk): + if disk.bytes(512, 8) == b"EFI PART": + table = filesystem.gpt(disk) + return "gpt", [{"name": p["name"], "offset": p["offset"], "size": p["file"].size, + "type": p["type_guid"], "data": p["file"]} for p in table.partitions] + for block in [512, 1024, 2048]: + if disk.bytes(block, 2) == b"PM": + table = filesystem.apm(disk, block_size=block) + return "apm", [{"name": str(p.name), "offset": p.start_block * p.block_size, + "size": p.data.size if p.data != None else 0, + "type": str(p.partition_type), "data": p.data} for p in table.partitions] + # Some embedded firmware DMGs contain a filesystem with no partition map. + if disk.bytes(1024, 2) in [b"H+", b"HX", b"BD"]: + return "unpartitioned", [{"name": "filesystem", "offset": 0, "size": disk.size, "type": "hfs", "data": disk}] + fail("unknown disk partition layout") + +def inspect_volume(path, file, queue, depth): + print("HFS", path) + volume = filesystem.hfs(file) + kinds = {} + hashes = [] + byte_count = 0 + attributes = 0 + resources = 0 + aliases = 0 + compressed = {} + for e in volume.entries: + kinds[e.entry_type] = kinds.get(e.entry_type, 0) + 1 + if e.compression_type: + compressed[str(e.compression_type)] = compressed.get(str(e.compression_type),0) + 1 + if e.finder_info[:8] == b"hlnkhfs+": + aliases += 1 + forks = [] + for label, source in [("data", e.data), ("resource", e.resource), + ("raw_data", e.raw_data), ("raw_resource", e.raw_resource)]: + if source != None: + forks.append((label, source.size, fingerprint(source))) + byte_count += source.size + if label == "resource" and source.size: + resources += 1 + for label, source in sorted(e.xattrs.items()): + forks.append(("xattr:" + label, source.size, fingerprint(source))) + byte_count += source.size + attributes += 1 + hashes.append((e.path, e.mode, e.uid, e.gid, e.target, forks)) + if e.entry_type == "file" and e.data != None and (e.path.endswith(".dmg") or e.path.endswith(".pkg")): + add_container(queue, path + ":" + e.path, e.data, depth + 1) + return {"layer": "hfs", "path": path, "name": str(volume.name), "entries": len(volume.entries), + "kinds": kinds, "fork_bytes_read": byte_count, "extended_attributes": attributes, + "nonempty_resource_forks": resources, "hardlink_aliases": aliases, "compressed_files": compressed, + "inventory_sha256": fingerprint(json.encode(hashes)), "all_forks_read": True} + +def inspect_cpio(path, file, queue, depth): + print("PAYLOAD", path) + encoding, decoded = decode_payload(file) + size = decoded.size + sha256 = fingerprint(decoded) # Sequential scan forces all codec checksums. + entries = archive.cpio(decoded, maximum_entries=1000000) + kinds = {} + for e in entries: + kind = e["kind"] + kinds[kind] = kinds.get(kind, 0) + 1 + source = e.get("data") + if kind == "file" and source != None and (e["path"].endswith(".dmg") or e["path"].endswith(".pkg")): + add_container(queue, path + ":" + e["path"], source, depth + 1) + return {"layer": "cpio", "path": path, "encoding": encoding, "decoded_bytes": size, + "sha256": sha256, "entries": len(entries), "kinds": kinds, "complete_stream_read": True} + +def inspect_xar(path, file, queue, depth): + print("XAR", path) + package = archive.xar(file) + payloads = [] + inventories = [] + for e in package.entries: + e.verify() + if e.data == None: + continue + if e.path.endswith("Payload") or e.path.endswith("Scripts"): + payloads.append(inspect_cpio(path + ":" + e.path, e.data, queue, depth + 1)) + elif e.path.endswith(".dmg") or e.path.endswith(".pkg"): + add_container(queue, path + ":" + e.path, e.data, depth + 1) + elif e.data.size >= 8 and e.data.bytes(0, 8) == b"BOMStore": + inventories.append({"path": e.path, "entries": len(archive.bom(e.data))}) + return {"layer": "xar", "path": path, "entries": len(package.entries), + "members_verified": True, "toc_sha256": fingerprint(package.toc), + "payloads": payloads, "bom_inventories": inventories} + +def inspect_fat(path, file): + volume = filesystem.fat(file) + hashes = [] + directories = [volume["/"]] + for unused in range(1000000): + if not directories: + break + directory = directories.pop() + for name in directory.files: + child = volume[name] + if type(child) == "file": + hashes.append((name,child.size,fingerprint(child))) + else: + directories.append(child) + if directories: + fail("FAT directory count exceeded") + return {"layer":"fat", "path":path, "files":len(hashes),"all_files_read":True, + "inventory_sha256":fingerprint(json.encode(hashes))} + +def inspect_media(file, name): + queue = [(name, "udif", file, 0)] + layers = [] + for unused in range(100000): + if not queue: + break + path, kind, source, depth = queue.pop(0) + if kind == "xar": + layers.append(inspect_xar(path, source, queue, depth)) + continue + if kind == "im4p": + print("IM4P",path) + envelope = archive.im4p(source) + payload = envelope.payload + encoding = "stored" + if payload.bytes(0,min(4,payload.size)) in [b"bvx1",b"bvx2",b"bvxn",b"bvx-"]: + payload = archive.lzfse(payload) + encoding = "lzfse" + if envelope.expected_size and payload.size != envelope.expected_size: + fail("IM4P decoded length mismatch: " + path) + layers.append({"layer":"im4p","path":path,"payload_type":envelope.payload_type, + "encoding":encoding,"decoded_bytes":payload.size,"sha256":fingerprint(payload), + "extras_sha256":[fingerprint(e) for e in envelope.extras]}) + layers.append(inspect_volume(path + ":payload",payload,queue,depth)) + continue + print("UDIF", path) + image = archive.udif(source) + image.verify() + layout, partitions = inspect_partitions(image.disk) + inventory = [] + for p in partitions: + record = {k: v for k, v in p.items() if k != "data"} + partition = p["data"] + if partition != None and partition.size >= 1026 and partition.bytes(1024, 2) in [b"H+", b"HX", b"BD"]: + layers.append(inspect_volume(path + ":" + p["name"], partition, queue, depth)) + record["filesystem"] = "hfs" + elif p["type"].upper().strip("{}") == "C12A7328-F81F-11D2-BA4B-00A0C93EC93B": + layers.append(inspect_fat(path + ":" + p["name"],partition)) + record["filesystem"] = "fat" + else: + record["filesystem"] = None + inventory.append(record) + layers.append({"layer": "udif", "path": path, "logical_bytes": image.disk.size, + "layout": layout, "block_maps": len(image.tables), "partitions": inventory, + "declared_checksums_verified": True}) + if queue: + fail("container count exceeded") + return {"media": name, "stored_bytes": file.size, "complete": True, "layers": layers, + "scope": "All discovered DMG/XAR/CPIO/IM4P layers and HFS data/resource/xattr forks read; declared format checksums checked. No Apple signature authentication, boot or installation proof."} + +def main(args): + if len(args) != 2: + fail("usage: macos_layers.star MEDIA.dmg REPORT.json") + report = inspect_media(open(args[0]), args[0]) + write(args[1], json.encode(report, indent=" ")) + print("VERIFIED", args[0], "layers", len(report["layers"]), "report", args[1]) diff --git a/vmm/cc/acpi.go b/vmm/cc/acpi.go index ce99edc..3b8e781 100644 --- a/vmm/cc/acpi.go +++ b/vmm/cc/acpi.go @@ -126,7 +126,14 @@ func (p *pc) installACPI() error { if err != nil { return err } - pciAML, err := (acpi.PCIRoot{Legacy: true, MemoryBase: 0xc0000000, MemorySize: 0x3ec00000}).AML() + root := acpi.PCIRoot{Legacy: true, MemoryBase: 0xc0000000, MemorySize: 0x3ec00000} + if p.pciIDE != nil && p.pciIDE.ich7 { + root.Interrupts = []acpi.PCIInterrupt{{Device: 1, Pin: 0, Interrupt: 16}} + } + if p.uhci != nil { + root.Interrupts = append(root.Interrupts, acpi.PCIInterrupt{Device: 3, Pin: 0, Interrupt: 17}) + } + pciAML, err := root.AML() if err != nil { return err } @@ -153,6 +160,13 @@ func (p *pc) installACPI() error { } aml = append(aml, data...) } + if p.smc != nil { + data, err := (acpi.ISADevice{Name: "SMC0", ID: "APP0001", Ports: [][2]uint16{{smcBase, 32}}, IRQs: []uint8{6}}).AML() + if err != nil { + return err + } + aml = append(aml, data...) + } if p.hpet != nil { data, err := (acpi.ISADevice{Name: "HPET", ID: "PNP0103", Parent: `\_SB.PCI0`, Memory: [][2]uint32{{hpetAddress, 0x400}}}).AML() if err != nil { @@ -203,7 +217,7 @@ func (p *pc) installACPI() error { roots := []uint32{fadtAddr, madtAddr} if p.hpet != nil { body := make([]byte, 20) - binary.LittleEndian.PutUint32(body, uint32(hpetCapabilities&0xffffffff)) + binary.LittleEndian.PutUint32(body, uint32(p.hpet.read(0, p.now())&0xffffffff)) body[5] = 64 // System-memory GAS, 64-bit registers. binary.LittleEndian.PutUint64(body[8:], hpetAddress) binary.LittleEndian.PutUint16(body[17:], 128) diff --git a/vmm/cc/backend.go b/vmm/cc/backend.go index 6655b91..de6f807 100644 --- a/vmm/cc/backend.go +++ b/vmm/cc/backend.go @@ -18,13 +18,15 @@ type Backend struct { ACPI, PCIIDE, HPET, UEFI bool // PIOOnly disables the PCI IDE disk's DMA capability. PIOOnly bool + // IDEModel selects synthetic (default) or the native ich7-pata profile. + IDEModel string // OverlayLimit bounds dirty snapshot memory; zero uses 256 MiB. OverlayLimit int64 } func Available() bool { return runtime.GOOS == "linux" && runtime.GOARCH == "amd64" } func Capabilities() []string { - return []string{"disk", "disk.bus.ide", "disk.snapshot", "disk.geometry.chs", "display.capturable", "screenshot", "input.key", "input.pointer", "lifecycle.pause", "lifecycle.stop", "extension.cc.v1", "network.ethernet", "channel.shared-memory"} + return []string{"boot.darwin", "disk", "disk.bus.ide", "disk.snapshot", "disk.geometry.chs", "display.capturable", "screenshot", "input.key", "input.pointer", "lifecycle.pause", "lifecycle.stop", "extension.cc.v1", "network.ethernet", "channel.shared-memory"} } func (*Backend) ID() string { return "cc.v1" } func (*Backend) Capabilities() []string { return Capabilities() } @@ -33,6 +35,26 @@ func (b *Backend) Validate(m vmm.Machine) []vmm.ValidationIssue { add := func(field, message string) { issues = append(issues, vmm.ValidationIssue{Code: "cc.unsupported", Field: field, Message: message, Backend: b.ID()}) } + if b.IDEModel != "" && b.IDEModel != "synthetic" && b.IDEModel != "ich7-pata" { + add("ide_model", "IDE model must be synthetic or ich7-pata") + } + if b.IDEModel == "ich7-pata" && !b.PCIIDE && !b.UEFI { + add("ide_model", "ich7-pata requires PCI IDE") + } + if m.Boot != nil { + add("boot", "cc does not support Linux direct boot") + } + if m.DarwinBoot != nil { + if len(m.Networks) != 0 { + add("networks", "Darwin SMC and NE2000 require distinct I/O resources; networking is not supported in this profile") + } + if b.UEFI || m.Architecture != "x86_64" { + add("boot", "Darwin direct boot requires x86_64 without UEFI image execution") + } + if err := m.DarwinBoot.Validate(); err != nil { + add("boot", err.Error()) + } + } if b.UEFI && m.Architecture != "x86_64" { add("architecture", "cc UEFI requires x86_64") } @@ -132,7 +154,12 @@ func (b *Backend) Start(ctx context.Context, m vmm.Machine) (vmm.Driver, error) if err != nil { return nil, err } - if err = configureCPUArchitecture(cpu, m.Architecture == "x86_64"); err != nil { + if m.DarwinBoot != nil { + err = configureDarwinCPU(cpu) + } else { + err = configureCPUArchitecture(cpu, m.Architecture == "x86_64") + } + if err != nil { cpu.Close() return nil, err } @@ -147,7 +174,7 @@ func (b *Backend) Start(ctx context.Context, m vmm.Machine) (vmm.Driver, error) cpu.Close() return nil, err } - platform, err := newPC(cpu, ram[:m.Memory], disk, time.Now) + platform, err := newPCBoot(cpu, ram[:m.Memory], disk, time.Now, m.DarwinBoot == nil) if err != nil { cpu.Close() return nil, err @@ -155,16 +182,36 @@ func (b *Backend) Start(ctx context.Context, m vmm.Machine) (vmm.Driver, error) platform.framebuffer = ram[m.Memory : uint64(m.Memory)+ramfb.Size] if b.PCIIDE || b.UEFI { platform.pciIDE = newPCIIDE() - platform.ide.dmaEnabled = !b.PIOOnly - platform.ide.irq = func(irq uint32, level bool) error { - if level { - platform.pciIDE.bm[2] |= 4 - } - return cpu.SetIRQ(irq, level && platform.pciIDE.config[4]&1 != 0) + if b.IDEModel == "ich7-pata" { + platform.pciIDE = newICH7PATA() } + platform.ide.dmaEnabled = !b.PIOOnly + platform.ide.irq = platform.setIDEIRQ } if b.HPET { platform.hpet = newHPET(platform.now(), cpu.SetIRQ) + if m.DarwinBoot != nil { + if legacy, ok := cpu.(interface{ SetLegacyTimerReplacement(bool) error }); ok { + platform.hpet.setLegacy = func(enabled bool) error { + if err := legacy.SetLegacyTimerReplacement(enabled); err != nil { + return err + } + if enabled && platform.rtcIRQ { + if err := cpu.SetIRQ(8, false); err != nil { + return err + } + platform.rtcIRQ = false + } + platform.rtcNext = time.Time{} + return nil + } + } + } + } + if m.DarwinBoot != nil { + platform.smc = newSMC(m.DarwinBoot.SMCOSK) + platform.pciDisplay = newPCIDisplay() + platform.uhci = newUHCI(platform.inputMemory, cpu.SetIRQ) } if b.ACPI || b.PCIIDE || b.HPET || b.UEFI || m.Architecture == "x86_64" { if err = platform.installACPI(); err != nil { @@ -188,5 +235,11 @@ func (b *Backend) Start(ctx context.Context, m vmm.Machine) (vmm.Driver, error) return nil, err } } + if m.DarwinBoot != nil { + if err = platform.installDarwin(m.DarwinBoot); err != nil { + cpu.Close() + return nil, err + } + } return newDriver(ctx, platform, m.StartPaused), nil } diff --git a/vmm/cc/bios.go b/vmm/cc/bios.go index 69dab83..8f12416 100644 --- a/vmm/cc/bios.go +++ b/vmm/cc/bios.go @@ -18,11 +18,15 @@ const biosPort = 0xf1 // Interrupt entries execute small real-mode ROM routines; native execution // retains all mode switches and executes the original boot sector and loader. type pc struct { + darwin map[string]any + darwinSymbols map[string]uint64 + serial [8]byte efiTrace []any efi *uefi.Firmware efiMemory *efiMemory acpi *acpiPM hpet *hpet + smc *smc nic *ne2000 framebuffer []byte input *virtioInput @@ -52,9 +56,15 @@ type pc struct { videoPacket videoPacket pciAddress uint32 pciIDE *pciIDE + pciDisplay *pciDisplay + uhci *uhci } func newPC(cpu hypervisor.X86, ram []byte, disk vmm.Disk, now func() time.Time) (*pc, error) { + return newPCBoot(cpu, ram, disk, now, true) +} + +func newPCBoot(cpu hypervisor.X86, ram []byte, disk vmm.Disk, now func() time.Time, legacyBoot bool) (*pc, error) { p := &pc{cpu: cpu, ram: ram, disk: disk, now: now, started: now(), mode: 3, ports: make(map[uint16]byte)} p.cmos[0x0a], p.cmos[0x0b] = 0x26, 2 p.geometry = vmm.CHSGeometry{Cylinders: int(disk.Device.Geometry().Size / 512 / 16 / 63), Heads: 16, Sectors: 63} @@ -114,6 +124,9 @@ func newPC(cpu hypervisor.X86, ram []byte, disk vmm.Disk, now func() time.Time) ram[i+1] = 7 } p.vga.syncText() + if !legacyBoot { + return p, nil + } if n, err := disk.Device.ReadAt(ram[0x7c00:0x7e00], 0); err != nil || n != 512 { return nil, fmt.Errorf("read boot sector: %w", err) } diff --git a/vmm/cc/darwin.go b/vmm/cc/darwin.go new file mode 100644 index 0000000..36f7429 --- /dev/null +++ b/vmm/cc/darwin.go @@ -0,0 +1,172 @@ +package cc + +import ( + "crypto/rand" + "encoding/binary" + "fmt" + "github.com/tinyrange/trex/boot/darwin" + "github.com/tinyrange/trex/vmm" + "github.com/tinyrange/trex/vmm/ramfb" + "github.com/tinyrange/trex/windows/guid" + "hash/crc32" + "j5.nz/cc/hypervisor/x86state" + "strings" +) + +// installDarwin loads the supplied XNU image and constructs its firmware ABI. +// The original _pstart performs the 32->64-bit switch, using its own tables. +func (p *pc) installDarwin(boot *vmm.DarwinBoot) error { + clock, ok := p.cpu.(interface { + TSCFrequency() (uint64, error) + ReadMSR(uint32) (uint64, error) + }) + if !ok { + return fmt.Errorf("cc: Darwin requires guest TSC frequency and performance-ratio observations") + } + tscHz, err := clock.TSCFrequency() + if err != nil { + return err + } + perf, err := clock.ReadMSR(0x198) + if err != nil { + return err + } + ratio := ((perf>>40)&31)*2 + ((perf >> 46) & 1) + if tscHz == 0 || ratio == 0 { + return fmt.Errorf("cc: unusable guest TSC/performance ratio") + } + busHz := tscHz * 2 / ratio + img, err := darwin.Open(boot.Kernel) + if err != nil { + return err + } + align := func(v uint64) uint64 { return (v + 4095) &^ 4095 } + top := align(img.End) + argsAddr := top + mapAddr := top + 4096 + dtAddr := top + 8192 + runtimeAddr := top + 0x10000 + runtimeEnd := runtimeAddr + 0x2000 + ramEnd := uint64(len(p.ram)) + // The original bootstrap maps the low 1GiB and addresses metadata with + // 32-bit pointers. Check the complete arena before touching guest RAM. + loadedEnd := runtimeEnd + if img.Base < 0x100000 || runtimeEnd > 1<<30 || ramEnd%4096 != 0 || runtimeEnd > ramEnd || ramEnd-runtimeEnd < 128<<20 { + return fmt.Errorf("cc: Darwin handoff must fit low 1GiB and leave 128MiB free RAM") + } + if err = img.Load(p.ram); err != nil { + return err + } + // EFI system/configuration/runtime tables. Runtime functions return + // EFI_UNSUPPORTED rather than pretending a persistent variable store exists. + clear(p.ram[runtimeAddr:runtimeEnd]) + system, runtime, config, stub := runtimeAddr, runtimeAddr+256, runtimeAddr+512, runtimeAddr+4096 + put64 := func(addr, v uint64) { binary.LittleEndian.PutUint64(p.ram[addr:], v) } + put32 := func(addr uint64, v uint32) { binary.LittleEndian.PutUint32(p.ram[addr:], v) } + header := func(addr, signature uint64, size uint32) { + put64(addr, signature) + put32(addr+8, 0x2000a) + put32(addr+12, size) + put32(addr+16, 0) + put32(addr+16, crc32.ChecksumIEEE(p.ram[addr:addr+uint64(size)])) + } + // movabs EFI_UNSUPPORTED,%rax; ret -- Microsoft x64 ABI. + copy(p.ram[stub:], []byte{0x48, 0xb8, 3, 0, 0, 0, 0, 0, 0, 0x80, 0xc3}) + for i := uint64(0); i < 14; i++ { + put64(runtime+24+i*8, stub|0xffffff8000000000) + } + header(runtime, 0x56524553544e5552, 136) + acpi, _ := guid.Parse("{8868E871-E4F1-11D3-BC22-0080C73C8881}") + copy(p.ram[config:], acpi[:]) + put64(config+16, 0xe0000) + vendor := runtimeAddr + 768 + for i, c := range "TinyRangeX" { + binary.LittleEndian.PutUint16(p.ram[vendor+uint64(i)*2:], uint16(c)) + } + put64(system+24, vendor|0xffffff8000000000) + put32(system+32, 1) + put64(system+88, runtime|0xffffff8000000000) + put64(system+104, 1) + put64(system+112, config|0xffffff8000000000) + header(system, 0x5453595320494249, 120) + node := func(name string, props map[string][]byte, children ...darwin.Node) darwin.Node { + props["name"] = darwin.CString(name) + return darwin.Node{Properties: props, Children: children} + } + // XNU1699 registerNVRAMController derives IOPlatformUUID from this + // 16-byte EFI identity. Without it, gethostuuid blocks core services + // (including DiskArbitration and therefore kextd). This is a fresh virtual + // machine identity, not an Apple hardware UUID or serial number. + systemID := make([]byte, 16) + if _, err := rand.Read(systemID); err != nil { + return err + } + systemID[6] = (systemID[6] & 15) | 0x40 + systemID[8] = (systemID[8] & 63) | 0x80 + memoryProps := map[string][]byte{"Kernel-__TEXT": darwin.Range(uint32(img.Base), uint32(img.End-img.Base))} + tree := node("/", map[string][]byte{"compatible": darwin.CString("ACPI"), "model": darwin.CString("ACPI"), "board-id": darwin.CString("TREX-CCPC"), "#address-cells": darwin.U32(2), "#size-cells": darwin.U32(2)}, + node("chosen", map[string][]byte{"boot-args": darwin.CString(boot.CommandLine), "boot-file": darwin.CString("kernelcache")}, node("memory-map", memoryProps)), + node("efi", map[string][]byte{"firmware-abi": darwin.CString("EFI64"), "firmware-vendor": p.ram[vendor : vendor+20], "firmware-revision": darwin.U32(1)}, + node("platform", map[string][]byte{"FSBFrequency": darwin.U64(busHz), "system-id": systemID}), + node("runtime-services", map[string][]byte{"table": darwin.U64(runtime | 0xffffff8000000000)}), + node("configuration-table", map[string][]byte{}, node("8868E871-E4F1-11D3-BC22-0080C73C8881", map[string][]byte{"guid": acpi[:], "table": darwin.U64(0xe0000)})))) + dt, err := tree.Encode() + if err != nil { + return err + } + if len(dt) > 0xe000 { + return fmt.Errorf("cc: Darwin device tree exceeds metadata arena") + } + copy(p.ram[dtAddr:], dt) + ranges := []darwin.MemoryRange{ + {Type: 0, Physical: 0, Pages: 1}, + {Type: 7, Physical: 0x1000, Pages: 0x9f, Attributes: 8}, + {Type: 0, Physical: 0xa0000, Pages: 0x60}, + {Type: 2, Physical: 0x100000, Pages: (runtimeAddr - 0x100000) / 4096, Attributes: 8}, + {Type: 6, Physical: runtimeAddr, Virtual: runtimeAddr | 0xffffff8000000000, Pages: 1, Attributes: 0x8000000000000008}, + {Type: 5, Physical: stub, Virtual: stub | 0xffffff8000000000, Pages: 1, Attributes: 0x8000000000000008}, + } + ranges = append(ranges, darwin.MemoryRange{Type: 7, Physical: loadedEnd, Pages: (ramEnd - loadedEnd) / 4096, Attributes: 8}) + mmap, err := darwin.MemoryMap(ranges) + if err != nil { + return err + } + copy(p.ram[mapAddr:], mmap) + displayMode := uint32(1) // GRAPHICS_MODE + for _, arg := range strings.Fields(boot.CommandLine) { + if arg == "-v" || arg == "-s" { + displayMode = 2 + } // FB_TEXT_MODE + } + ba, err := (darwin.Arguments{CommandLine: boot.CommandLine, MemoryMap: uint32(mapAddr), MemoryMapSize: uint32(len(mmap)), DeviceTree: uint32(dtAddr), DeviceTreeSize: uint32(len(dt)), KernelAddress: uint32(img.Base), KernelSize: uint32(loadedEnd - img.Base), EFISystemTable: uint32(system), RuntimePageStart: uint32(runtimeAddr / 4096), RuntimePageCount: 2, RuntimeVirtualPage: (runtimeAddr | 0xffffff8000000000) / 4096, PhysicalMemorySize: uint64(len(p.ram)), FSBFrequency: busHz, Video: [6]uint32{ramfb.Address + ramfb.Pixels | 1, displayMode, 1280 * 4, 1280, 720, 32}}).Encode() + if err != nil { + return err + } + copy(p.ram[argsAddr:], ba) + for i, v := range []uint32{ramfb.Magic, 1, 1280, 720, 1280 * 4, 1} { + binary.LittleEndian.PutUint32(p.framebuffer[i*4:], v) + } + // Flat 32-bit protected mode, IF clear and paging off. XNU owns GDT and CR3. + binary.LittleEndian.PutUint64(p.ram[0x9008:], 0x00cf9b000000ffff) + binary.LittleEndian.PutUint64(p.ram[0x9010:], 0x00cf93000000ffff) + s, err := p.cpu.SystemRegisters() + if err != nil { + return err + } + seg := x86state.Segment{Limit: 0xffffffff, Selector: 16, Present: 1, S: 1, Type: 3, Db: 1, G: 1} + s.Cs, s.Ds, s.Es, s.Ss, s.Fs, s.Gs = seg, seg, seg, seg, seg, seg + s.Cs.Selector = 8 + s.Cs.Type = 11 + s.Cr0 = 0x11 + s.Cr3 = 0 + s.Cr4 = 0 + s.Efer = 0 + s.Gdt = x86state.DescriptorTable{Base: 0x9000, Limit: 23} + s.Idt = x86state.DescriptorTable{} + if err = p.cpu.SetSystemRegisters(s); err != nil { + return err + } + p.darwin = map[string]any{"entry": int64(img.Entry), "boot_args": int64(argsAddr), "kernel_base": int64(img.Base), "loaded_end": int64(loadedEnd)} + p.darwinSymbols = img.Symbols + return p.cpu.SetRegisters(x86state.Registers{Rip: img.Entry, Rax: argsAddr, Rsp: 0x90000, Rflags: 2}) +} diff --git a/vmm/cc/darwin_cpu.go b/vmm/cc/darwin_cpu.go new file mode 100644 index 0000000..c9149a8 --- /dev/null +++ b/vmm/cc/darwin_cpu.go @@ -0,0 +1,57 @@ +package cc + +import ( + "fmt" + "j5.nz/cc/hypervisor" +) + +// Penryn is an XNU-supported family that does not require newer topology or +// XSAVE state. Features remain intersected with accelerator support. +func configureDarwinCPU(cpu hypervisor.X86) error { + entries, err := cpu.SupportedCPUID() + if err != nil { + return err + } + filtered := entries[:0] + longMode, cache := false, false + for _, e := range entries { + if e.Function > 4 && (e.Function < 0x80000000 || e.Function > 0x80000008) { + continue + } + switch e.Function { + case 0: + e.Eax = 4 + e.Ebx, e.Edx, e.Ecx = 0x756e6547, 0x49656e69, 0x6c65746e + case 1: + e.Eax = 0x10676 + e.Ebx = e.Ebx&0xffff | 1<<16 + // CPUID.HYPERVISOR identifies the virtual CPU without claiming any + // extra instruction support. XNU's native power-management driver + // uses it to select its virtual-processor implementation. + e.Ecx &= 1<<0 | 1<<9 | 1<<13 | 1<<19 | 1<<31 + e.Ecx |= 1 << 31 + e.Edx &= (1 << 27) - 1 + case 4: + e.Eax &= (1 << 14) - 1 + if e.Eax&31 != 0 { + cache = true + } + case 0x80000000: + e.Eax = min(e.Eax, 0x80000008) + case 0x80000001: + e.Ecx &= 1 + e.Edx &= 0x2193fbff + longMode = e.Edx&(1<<29) != 0 + case 0x80000007: + e.Eax, e.Ebx, e.Ecx = 0, 0, 0 + e.Edx &= 1 << 8 + case 0x80000008: + e.Ebx, e.Ecx, e.Edx = 0, 0, 0 + } + filtered = append(filtered, e) + } + if !longMode || !cache { + return fmt.Errorf("cc Darwin requires accelerator long mode and deterministic cache enumeration") + } + return cpu.SetCPUID(filtered) +} diff --git a/vmm/cc/darwin_cpu_test.go b/vmm/cc/darwin_cpu_test.go new file mode 100644 index 0000000..b992ae5 --- /dev/null +++ b/vmm/cc/darwin_cpu_test.go @@ -0,0 +1,46 @@ +package cc + +import ( + "j5.nz/cc/hypervisor/x86state" + "testing" +) + +func TestDarwinCPUTruthfulVirtualProcessor(t *testing.T) { + c := &cpuidTestCPU{input: []x86state.CPUIDEntry{ + {Function: 0, Eax: 32}, + {Function: 1, Ecx: 1<<9 | 1<<26, Edx: 1<<4 | 1<<28, Ebx: 0xff080800}, + {Function: 4, Eax: 1 | 7<<26 | 3<<14}, + {Function: 7, Ebx: 0xffffffff}, + {Function: 0x80000000, Eax: 0x80000020}, + {Function: 0x80000001, Edx: 1 << 29}, + }} + if err := configureDarwinCPU(c); err != nil { + t.Fatal(err) + } + for _, e := range c.output { + switch e.Function { + case 1: + if e.Eax != 0x10676 || e.Ecx != 1<<9|1<<31 || e.Edx != 1<<4 || e.Ebx>>16 != 1 { + t.Fatalf("wrong virtual CPU %#v", e) + } + case 4: + if e.Eax != 1 { + t.Fatalf("invented cache topology %#v", e) + } + case 7: + t.Fatal("unconfigured feature leaf exposed") + } + } + // Missing accelerator prerequisites must fail before applying a profile. + for _, fn := range []uint32{4, 0x80000001} { + bad := &cpuidTestCPU{} + for _, e := range c.input { + if e.Function != fn { + bad.input = append(bad.input, e) + } + } + if configureDarwinCPU(bad) == nil || bad.output != nil { + t.Fatal("accepted missing long mode/cache") + } + } +} diff --git a/vmm/cc/darwin_inspect.go b/vmm/cc/darwin_inspect.go new file mode 100644 index 0000000..17dee6b --- /dev/null +++ b/vmm/cc/darwin_inspect.go @@ -0,0 +1,105 @@ +package cc + +import ( + "fmt" + "github.com/tinyrange/trex/emulator/cpu" + "github.com/tinyrange/trex/script/value" + "go.starlark.net/starlark" + "sort" + "strings" +) + +func (i inspection) symbol(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var name string + if err := starlark.UnpackArgs("cc.symbol", args, kwargs, "name", &name); err != nil { + return nil, err + } + var address uint64 + err := i.driver.call(i.ctx, func() error { + var ok bool + address, ok = i.driver.pc.darwinSymbols[name] + if !ok { + return fmt.Errorf("cc: kernel symbol %q not found", name) + } + return nil + }) + return starlark.MakeUint64(address), err +} + +// symbols lists a bounded selection of original kernel symbols for observations. +func (i inspection) symbols(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var prefix string + if err := starlark.UnpackArgs("cc.symbols", args, kwargs, "prefix", &prefix); err != nil { + return nil, err + } + var result starlark.Value + err := i.driver.call(i.ctx, func() error { + names := []string{} + for n := range i.driver.pc.darwinSymbols { + if strings.HasPrefix(n, prefix) { + names = append(names, n) + } + } + if len(names) > 128 { + return fmt.Errorf("cc: more than 128 symbols; narrow prefix") + } + sort.Strings(names) + out := starlark.NewDict(len(names)) + for _, n := range names { + if err := out.SetKey(starlark.String(n), starlark.MakeUint64(i.driver.pc.darwinSymbols[n])); err != nil { + return err + } + } + result = out + return nil + }) + return result, err +} +func (i inspection) symbolize(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var address uint64 + if err := starlark.UnpackArgs("cc.symbolize", args, kwargs, "address", &address); err != nil { + return nil, err + } + var result starlark.Value + err := i.driver.call(i.ctx, func() error { + var best uint64 + var name string + for n, a := range i.driver.pc.darwinSymbols { + if a <= address && (a > best || a == best && n < name) { + best, name = a, n + } + } + if name == "" { + result = starlark.None + return nil + } + var err error + result = value.NewRecord(starlark.StringDict{"symbol": starlark.String(name), "offset": starlark.MakeUint64(address - best)}) + return err + }) + return result, err +} +func (i inspection) readVirtual(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var address uint64 + var size int + if err := starlark.UnpackArgs("cc.read_virtual", args, kwargs, "address", &address, "size", &size); err != nil { + return nil, err + } + if size < 0 || size > 65536 || uint64(size) > ^uint64(0)-address { + return nil, fmt.Errorf("cc: read_virtual size must be 0..65536") + } + var result starlark.Value + err := i.driver.call(i.ctx, func() error { + s, err := i.driver.pc.cpu.SystemRegisters() + if err != nil { + return err + } + b := make([]byte, size) + if err = (efiMemory{p: i.driver.pc, system: &s}).ReadMemory(address, b, cpu.Read); err != nil { + return err + } + result = starlark.Bytes(string(b)) + return nil + }) + return result, err +} diff --git a/vmm/cc/darwin_test.go b/vmm/cc/darwin_test.go new file mode 100644 index 0000000..300d787 --- /dev/null +++ b/vmm/cc/darwin_test.go @@ -0,0 +1,104 @@ +package cc + +import ( + "bytes" + "encoding/binary" + "github.com/tinyrange/trex/storage" + "github.com/tinyrange/trex/vmm" + "github.com/tinyrange/trex/vmm/ramfb" + "hash/crc32" + "j5.nz/cc/hypervisor" + "j5.nz/cc/hypervisor/x86state" + "testing" +) + +type darwinTestSource struct{ *bytes.Reader } + +func (s darwinTestSource) Size() int64 { return s.Reader.Size() } + +type darwinTestCPU struct { + hypervisor.X86 + regs x86state.Registers + sys x86state.SystemRegisters +} + +func (c *darwinTestCPU) TSCFrequency() (uint64, error) { return 2400000000, nil } +func (c *darwinTestCPU) ReadMSR(uint32) (uint64, error) { return 12 << 40, nil } +func (c *darwinTestCPU) SystemRegisters() (x86state.SystemRegisters, error) { return c.sys, nil } +func (c *darwinTestCPU) SetSystemRegisters(s x86state.SystemRegisters) error { c.sys = s; return nil } +func (c *darwinTestCPU) SetRegisters(r x86state.Registers) error { c.regs = r; return nil } +func darwinKernel(base uint64) storage.Reader { + b := make([]byte, 4096) + p := binary.LittleEndian.PutUint32 + q := binary.LittleEndian.PutUint64 + p(b, 0xfeedfacf) + p(b[4:], 0x01000007) + p(b[12:], 2) + p(b[16:], 2) + p(b[20:], 256) + c := b[32:] + p(c, 0x19) + p(c[4:], 72) + copy(c[8:], "__TEXT") + q(c[24:], 0xffffff8000000000+base) + q(c[32:], 8192) + q(c[48:], 4096) + c = b[104:] + p(c, 5) + p(c[4:], 184) + p(c[8:], 4) + p(c[12:], 42) + q(c[144:], base+512) + return darwinTestSource{bytes.NewReader(b)} +} +func TestDarwinFirmwareHandoff(t *testing.T) { + c := &darwinTestCPU{} + p := &pc{cpu: c, ram: make([]byte, 160<<20), framebuffer: make([]byte, ramfb.Size)} + if err := p.installDarwin(&vmm.DarwinBoot{Kernel: darwinKernel(0x200000), CommandLine: "-v -s rd=disk0s3"}); err != nil { + t.Fatal(err) + } + a := p.ram[c.regs.Rax : c.regs.Rax+4096] + u := binary.LittleEndian.Uint32 + q := binary.LittleEndian.Uint64 + if c.regs.Rip != 0x200200 || c.regs.Rflags != 2 || c.sys.Cr0 != 0x11 || c.sys.Cr3 != 0 || c.sys.Efer != 0 || c.sys.Cs.Db != 1 || c.sys.Cs.Selector != 8 { + t.Fatal("XNU must enter original protected-mode pstart, not a loader") + } + if u(a[1048:]) != (ramfb.Address+ramfb.Pixels)|1 || u(a[1052:]) != 2 { + t.Fatal("physical text console flags lost") + } + mm := p.ram[u(a[1032:]) : u(a[1032:])+u(a[1036:])] + if u(mm[40:]) != 7 || q(mm[48:]) != 0x1000 || q(mm[64:]) != 0x9f { + t.Fatal("first usable region must be bounded conventional low memory") + } + system := uint64(u(a[1104:])) + runtime := q(p.ram[system+88:]) + if runtime>>32 != 0xffffff80 || q(a[1096:])<<12 != runtime&^0xfff { + t.Fatal("EFI runtime pointers must be XNU high virtual addresses") + } + for _, addr := range []uint64{system, runtime & 0xffffffff} { + n := uint64(u(p.ram[addr+12:])) + header := bytes.Clone(p.ram[addr : addr+n]) + sum := u(header[16:]) + clear(header[16:20]) + if crc32.ChecksumIEEE(header) != sum { + t.Fatal("EFI header CRC invalid") + } + } + dt := p.ram[u(a[1072:]) : u(a[1072:])+u(a[1076:])] + if !bytes.Contains(dt, []byte("8868E871-E4F1-11D3-BC22-0080C73C8881")) || bytes.Contains(dt, []byte("ACPI_20")) { + t.Fatal("original AppleACPIPlatform GUID path absent") + } + if q(a[1152:]) != 200000000 { + t.Fatal("FSB frequency not derived from guest TSC and ratio") + } +} +func TestDarwinRejectsBootstrapOverflowBeforeWrites(t *testing.T) { + c := &darwinTestCPU{} + p := &pc{cpu: c, ram: bytes.Repeat([]byte{0xcc}, 2<<20)} + before := bytes.Clone(p.ram) + for _, base := range []uint64{0x200000, 0x3ffff000, 0xfffff000} { + if p.installDarwin(&vmm.DarwinBoot{Kernel: darwinKernel(base)}) == nil || !bytes.Equal(p.ram, before) { + t.Fatal("unsafe handoff modified RAM") + } + } +} diff --git a/vmm/cc/disassemble.go b/vmm/cc/disassemble.go new file mode 100644 index 0000000..8bf8e59 --- /dev/null +++ b/vmm/cc/disassemble.go @@ -0,0 +1,56 @@ +package cc + +import ( + "fmt" + "github.com/tinyrange/trex/emulator/cpu" + "go.starlark.net/starlark" + "golang.org/x/arch/x86/x86asm" +) + +// disassemble is a read-only, bounded observation of the owning guest memory. +func (i inspection) disassemble(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var address uint64 + size, mode := 256, 64 + if err := starlark.UnpackArgs("cc.disassemble", args, kwargs, "address", &address, "size?", &size, "mode?", &mode); err != nil { + return nil, err + } + if size < 0 || size > 4096 || (mode != 16 && mode != 32 && mode != 64) || uint64(size) > ^uint64(0)-address { + return nil, fmt.Errorf("cc: disassemble requires size 0..4096, nonwrapping address and mode 16/32/64") + } + var result starlark.Value + err := i.driver.call(i.ctx, func() error { + s, err := i.driver.pc.cpu.SystemRegisters() + if err != nil { + return err + } + b := make([]byte, size) + if err = (efiMemory{p: i.driver.pc, system: &s}).ReadMemory(address, b, cpu.Read); err != nil { + return err + } + result, err = decodeInstructions(b, address, mode) + return err + }) + return result, err +} + +func decodeInstructions(b []byte, address uint64, mode int) (starlark.Value, error) { + out := []starlark.Value{} + for off := 0; off < len(b); { + inst, err := x86asm.Decode(b[off:], mode) + if err != nil || inst.Len == 0 || inst.Op == 0 { + row := starlark.NewDict(2) + _ = row.SetKey(starlark.String("address"), starlark.MakeUint64(address+uint64(off))) + _ = row.SetKey(starlark.String("error"), starlark.String("invalid or truncated instruction")) + out = append(out, row) + break + } + pc := address + uint64(off) + row := starlark.NewDict(3) + _ = row.SetKey(starlark.String("address"), starlark.MakeUint64(pc)) + _ = row.SetKey(starlark.String("size"), starlark.MakeInt(inst.Len)) + _ = row.SetKey(starlark.String("text"), starlark.String(x86asm.IntelSyntax(inst, pc, nil))) + out = append(out, row) + off += inst.Len + } + return starlark.NewList(out), nil +} diff --git a/vmm/cc/disassemble_test.go b/vmm/cc/disassemble_test.go new file mode 100644 index 0000000..4da25ac --- /dev/null +++ b/vmm/cc/disassemble_test.go @@ -0,0 +1,29 @@ +package cc + +import ( + "go.starlark.net/starlark" + "strings" + "testing" +) + +func TestDisassemblyRelativeAndTruncated(t *testing.T) { + v, err := decodeInstructions([]byte{0xe8, 1, 0, 0, 0, 0x90, 0x48, 0xb8, 1}, 0x1000, 64) + if err != nil { + t.Fatal(err) + } + rows := v.(*starlark.List) + if rows.Len() != 3 { + t.Fatal(v) + } + text, _, _ := rows.Index(0).(*starlark.Dict).Get(starlark.String("text")) + if !strings.Contains(text.(starlark.String).GoString(), "0x1006") { + t.Fatalf("wrong relative target %s", text) + } + last := rows.Index(2).(*starlark.Dict) + a, _, _ := last.Get(starlark.String("address")) + msg, found, _ := last.Get(starlark.String("error")) + n, _ := a.(starlark.Int).Uint64() + if n != 0x1006 || !found || msg == nil { + t.Fatalf("truncated terminal instruction not identified %v", last) + } +} diff --git a/vmm/cc/hpet.go b/vmm/cc/hpet.go index df990d4..4e2b08b 100644 --- a/vmm/cc/hpet.go +++ b/vmm/cc/hpet.go @@ -13,7 +13,8 @@ const hpetCapabilities uint64 = 10000000<<32 | 0x8086<<16 | 1<<13 | 2<<8 | 1 const hpetTimerCapabilities uint64 = 0x00f00000<<32 | 1<<5 | 1<<4 // hpet implements three 64-bit comparators at 100 MHz. Interrupts use the -// IOAPIC; legacy replacement and FSB delivery are not advertised. +// IOAPIC; FSB delivery is not advertised. Legacy replacement is advertised +// only when the accelerator can suppress its PIT output. // Register semantics follow Intel HPET specification 1.0a. type hpet struct { config, status uint64 @@ -21,6 +22,7 @@ type hpet struct { started time.Time timers [3]hpetTimer setIRQ func(uint32, bool) error + setLegacy func(bool) error } type hpetTimer struct { config, match, period uint64 @@ -95,12 +97,24 @@ func (h *hpet) expire(i int, now time.Time) { } } } + +// irq uses the accelerator's legacy input line numbers (IRQ0 is routed to +// IOAPIC pin 2 by the PC interrupt controller), not the programmable timer route. +func (h *hpet) irq(i int) uint32 { + if h.config&2 != 0 && i < 2 { + if i == 0 { + return 0 + } + return 8 + } + return uint32(h.timers[i].config >> 9 & 31) +} func (h *hpet) delivery(i int) func() error { t := h.timers[i] - irq := uint32(t.config >> 9 & 31) + irq := h.irq(i) shared := false for j, other := range h.timers { - if j != i && other.asserted && uint32(other.config>>9&31) == irq { + if j != i && other.asserted && h.irq(j) == irq { shared = true } } @@ -145,6 +159,9 @@ func (h *hpet) deadline() (int, time.Time) { func (h *hpet) read(reg uint64, now time.Time) uint64 { switch reg { case 0: + if h.setLegacy != nil { + return hpetCapabilities | 1<<15 + } return hpetCapabilities case 0x10: return h.config @@ -169,9 +186,9 @@ func (h *hpet) lower(i int) error { if !t.asserted { return nil } - irq := uint32(t.config >> 9 & 31) + irq := h.irq(i) for j, other := range h.timers { - if j != i && other.asserted && uint32(other.config>>9&31) == irq { + if j != i && other.asserted && h.irq(j) == irq { t.asserted = false return nil } @@ -187,9 +204,33 @@ func (h *hpet) write(reg, value, mask uint64, now time.Time) error { merged := old&^mask | value&mask switch reg { case 0x10: + config := merged & 1 + if h.setLegacy != nil { + config = merged & 3 + } + if (h.config^config)&2 != 0 { + // Drain latched outputs on their old routes before changing ownership. + for i := range h.timers { + if err := h.lower(i); err != nil { + return err + } + } + if err := h.setLegacy(config&2 != 0); err != nil { + return err + } + } h.base = h.counter(now) h.started = now - h.config = merged & 1 + h.config = config + // Pending level status follows the newly selected route. + for i := range h.timers { + if h.status&(1<>9&31), true); err != nil { + if err := h.setIRQ(h.irq(i), true); err != nil { return err } t.asserted = true diff --git a/vmm/cc/hpet_legacy_test.go b/vmm/cc/hpet_legacy_test.go new file mode 100644 index 0000000..49b95e8 --- /dev/null +++ b/vmm/cc/hpet_legacy_test.go @@ -0,0 +1,63 @@ +package cc + +import ( + "reflect" + "testing" + "time" +) + +func TestHPETLegacyOwnershipAndRouteChanges(t *testing.T) { + now := time.Unix(100, 0) + var edges [][2]uint32 + var modes []bool + h := newHPET(now, func(irq uint32, level bool) error { + v := uint32(0) + if level { + v = 1 + } + edges = append(edges, [2]uint32{irq, v}) + return nil + }) + if h.read(0, now)&(1<<15) != 0 { + t.Fatal("legacy advertised without accelerator control") + } + h.setLegacy = func(enabled bool) error { modes = append(modes, enabled); return nil } + if h.read(0, now)&(1<<15) == 0 { + t.Fatal("missing implemented legacy capability") + } + write := func(reg, value uint64) { + t.Helper() + if err := h.write(reg, value, ^uint64(0), now); err != nil { + t.Fatal(err) + } + } + for i := 0; i < 3; i++ { + write(uint64(0x100+i*32), uint64(20+i)<<9|6) + write(uint64(0x108+i*32), 100) + } + write(0x10, 3) + now = now.Add(time.Microsecond) + if err := h.poll(now); err != nil { + t.Fatal(err) + } + want := [][2]uint32{{0, 1}, {8, 1}, {22, 1}} + if !reflect.DeepEqual(edges, want) { + t.Fatalf("legacy routes %v", edges) + } + // Restore programmable routes with level status pending. Old routes must + // lower before new routes assert; timer2 remains on its programmed route. + write(0x10, 1) + want = append(want, [2]uint32{0, 0}, [2]uint32{8, 0}, [2]uint32{22, 0}, [2]uint32{20, 1}, [2]uint32{21, 1}, [2]uint32{22, 1}) + if !reflect.DeepEqual(edges, want) || !reflect.DeepEqual(modes, []bool{true, false}) { + t.Fatalf("ownership %v %v", edges, modes) + } + write(0x20, 7) + if h.status != 0 { + t.Fatal("legacy status not acknowledged") + } + for _, timer := range h.timers { + if timer.asserted { + t.Fatal("route still asserted") + } + } +} diff --git a/vmm/cc/ich7.go b/vmm/cc/ich7.go new file mode 100644 index 0000000..f6a8a4d --- /dev/null +++ b/vmm/cc/ich7.go @@ -0,0 +1,145 @@ +package cc + +import ( + "encoding/binary" + "j5.nz/cc/hypervisor" +) + +// ICH7 PATA register semantics are derived from Intel 307013, chapter 15. +// The board supplies one disk on the primary channel. The ICH7 secondary +// channel is unimplemented hardware: enabled reads return zero, not a disk. +// Electrical cable timing is recorded, not simulated; transfers use the same +// bounded native PIO/PRD block engine as the synthetic controller. +func newICH7PATA() *pciIDE { + d := newPCIIDE() + d.ich7 = true + binary.LittleEndian.PutUint16(d.config[0:], 0x8086) + binary.LittleEndian.PutUint16(d.config[2:], 0x27df) + binary.LittleEndian.PutUint16(d.config[6:], 0x280) + d.config[9] = 0x8a + for n, v := range []uint32{0xd001, 0xd009, 0xd011, 0xd019} { + binary.LittleEndian.PutUint32(d.config[0x10+n*4:], v) + } + d.config[0x3c], d.config[0x3d] = 16, 1 + d.config[0x41] = 0x80 // Firmware enables primary decode only. + return d +} + +func (d *pciIDE) writeICH7(offset int, value byte) bool { + switch { + case offset == 5: + d.config[5] = value & 4 // Interrupt disable. + case offset == 9: + d.config[9] = 0x8a | value&5 + case offset >= 0x10 && offset <= 0x23: + lane := (offset - 0x10) % 4 + if lane == 0 { + mask := byte(0xf8) + if offset == 0x14 || offset == 0x1c { + mask = 0xfc + } + if offset == 0x20 { + mask = 0xf0 + } + d.config[offset] = value&mask | 1 + } else if lane == 1 { + d.config[offset] = value + } + // The upper 16 address bits are reserved. + case offset >= 0x40 && offset <= 0x44: + mask := byte(0xff) + if offset == 0x41 { + mask = 0xf3 + } + if offset == 0x43 { + mask = 0xf7 + } + d.config[offset] = value & mask + case offset == 0x48: + d.config[offset] = value & 0xf + case offset == 0x4a || offset == 0x4b: + d.config[offset] = value & 0x33 + case offset >= 0x54 && offset <= 0x57: + masks := [4]byte{0xff, 0xf0, 0xff, 0} + d.config[offset] = value & masks[offset-0x54] + default: + return false + } + return true +} + +func (d *pciIDE) primaryEnabled() bool { + return d.config[4]&1 != 0 && (!d.ich7 || d.config[0x41]&0x80 != 0 && d.config[0x56]&3 == 0) +} +func (d *pciIDE) commandBase(channel int) uint16 { + if d.ich7 && d.config[9]&(1<= scmd && uint32(ex.Port) < uint32(scmd)+8 || ex.Port == sctrl) { + if !ex.Write { + v := byte(0xff) + if d.config[4]&1 != 0 && d.config[0x43]&0x80 != 0 { + v = 0 + } + for n := range ex.Data { + ex.Data[n] = v + } + } + return true, nil + } + } + } + if cmd != 0 && ex.Port >= cmd && uint32(ex.Port) < uint32(cmd)+8 { + ex.Port = 0x1f0 + (ex.Port - cmd) + return true, p.ideIO(ex) + } + if cmd != 0 && ex.Port == ctrl { + ex.Port = 0x3f6 + return true, p.ideIO(ex) + } + return false, nil +} + +// Compatibility interrupts are hardwired to IRQ14. Native INTA has its own +// PCI route (GSI16); never alias the two in firmware or the controller. +func (d *pciIDE) primaryIRQ() uint32 { + if d.ich7 && d.config[9]&1 != 0 { + return 16 + } + return 14 +} +func (p *pc) setIDEIRQ(_ uint32, level bool) error { + d := p.pciIDE + if level { + d.bm[2] |= 4 + } + return p.cpu.SetIRQ(d.primaryIRQ(), level && d.primaryEnabled() && (d.primaryIRQ() == 14 || d.config[5]&4 == 0)) +} diff --git a/vmm/cc/ich7_test.go b/vmm/cc/ich7_test.go new file mode 100644 index 0000000..ee192b3 --- /dev/null +++ b/vmm/cc/ich7_test.go @@ -0,0 +1,121 @@ +package cc + +import ( + "bytes" + "encoding/binary" + "j5.nz/cc/hypervisor" + "testing" +) + +type ideIRQCPU struct { + hypervisor.X86 + levels map[uint32]bool +} + +func (c *ideIRQCPU) SetIRQ(n uint32, v bool) error { c.levels[n] = v; return nil } + +func TestICH7RelocatedPIOAndRouting(t *testing.T) { + p, data := dmaTestPC(t) + p.pciIDE = newICH7PATA() + c := &ideIRQCPU{levels: map[uint32]bool{}} + p.cpu = c + p.ide.irq = p.setIDEIRQ + io := func(port uint16, write bool, b []byte) { + t.Helper() + if err := p.handleIO(hypervisor.X86Exit{Port: port, Size: 1, Count: uint32(len(b)), Write: write, Data: b}); err != nil { + t.Fatal(err) + } + } + config := func(offset uint32, b []byte) { + t.Helper() + a := make([]byte, 4) + binary.LittleEndian.PutUint32(a, 0x80000800|offset&^3) + if err := p.pciIO(hypervisor.X86Exit{Port: 0xcf8, Size: 4, Count: 1, Write: true, Data: a}); err != nil { + t.Fatal(err) + } + if err := p.pciIO(hypervisor.X86Exit{Port: 0xcfc + uint16(offset&3), Size: uint8(len(b)), Count: 1, Write: true, Data: b}); err != nil { + t.Fatal(err) + } + } + // Firmware's compatibility IRQ must not alias the native PCI route. + if p.pciIDE.config[0x3c] != 16 || p.pciIDE.primaryIRQ() != 14 { + t.Fatal("wrong compatibility/native split") + } + config(0x10, []byte{1, 0xe0, 0, 0}) + config(0x14, []byte{9, 0xe0, 0, 0}) + config(9, []byte{1}) + if p.pciIDE.commandBase(0) != 0xe000 || p.pciIDE.controlPort(0) != 0xe00a { + t.Fatal("BAR relocation ignored") + } + for n, v := range []byte{0, 1, 2, 0, 0, 0xe0, 0x20} { + io(0xe001+uint16(n), true, []byte{v}) + } + if !c.levels[16] || c.levels[14] { + t.Fatal("native transfer did not assert only INTA") + } + sector := make([]byte, 512) + if err := p.handleIO(hypervisor.X86Exit{Port: 0xe000, Size: 2, Count: 256, Data: sector}); err != nil { + t.Fatal(err) + } + if !bytes.Equal(sector, data[1024:1536]) { + t.Fatal("relocated PIO did not read original disk") + } + // Status acknowledges the interrupt; decode and interrupt-disable gates work. + io(0xe007, false, []byte{0}) + if c.levels[16] { + t.Fatal("status did not acknowledge native interrupt") + } + p.ide.signal(true) + config(9, []byte{0}) + if c.levels[16] || !c.levels[14] { + t.Fatal("mode switch left stale native interrupt") + } + config(0x41, []byte{0}) + if c.levels[14] { + t.Fatal("channel decode disable left asserted IRQ") + } + disabled := []byte{0} + io(0x1f7, false, disabled) + if disabled[0] != 0xff { + t.Fatal("disabled primary decoded") + } + config(0x41, []byte{0x80}) + config(9, []byte{1}) + config(5, []byte{4}) + if c.levels[16] { + t.Fatal("PCI interrupt-disable ignored in native mode") + } + absent := []byte{0} + io(0x177, false, absent) + if absent[0] != 0xff { + t.Fatal("disabled secondary decoded") + } + config(0x43, []byte{0x80}) + io(0x177, false, absent) + if absent[0] != 0 { + t.Fatal("unimplemented enabled secondary pretended to be a disk") + } + for n := 0; n < 40; n++ { + config(0x48, []byte{byte(n)}) + } + if len(p.pciIDE.trace) != 32 { + t.Fatal("configuration trace unbounded") + } +} + +func TestICH7RelocatedDMA(t *testing.T) { + p, data := dmaTestPC(t) + p.pciIDE = newICH7PATA() + p.pciIDE.config[4] = 5 + binary.LittleEndian.PutUint32(p.pciIDE.bm[4:], 0x1000) + p.pciIDE.write(0x20, 0x01) + p.pciIDE.write(0x21, 0xe1) + dmaPRD(p, 0, 0x20000, 512, true) + dmaCommand(t, p, false, 1) + if err := p.handleIO(hypervisor.X86Exit{Port: 0xe100, Size: 1, Count: 1, Write: true, Data: []byte{9}}); err != nil { + t.Fatal(err) + } + if !bytes.Equal(p.ram[0x20000:0x20200], data[1024:1536]) { + t.Fatal("relocated bus-master DMA failed") + } +} diff --git a/vmm/cc/ide.go b/vmm/cc/ide.go index 0c209bc..072a21b 100644 --- a/vmm/cc/ide.go +++ b/vmm/cc/ide.go @@ -138,6 +138,14 @@ func (d *ide) command(command byte) error { word(63, modes) } word(53, 1) + if d.dmaEnabled { + // ATA field-validity bit 1 covers the multiword DMA timing data. + // Advertising word 63 without it makes original IOATADevConfig + // (and other standards-compliant clients) select PIO instead. + word(53, 3) + word(65, 120) // Minimum MWDMA cycle time, modes 0..2 supported. + word(66, 120) // Recommended cycle time, in nanoseconds. + } word(54, uint16(d.geometry.Cylinders)) word(55, uint16(d.geometry.Heads)) word(56, uint16(d.geometry.Sectors)) diff --git a/vmm/cc/ide_dma.go b/vmm/cc/ide_dma.go index 9c0cde6..6c9acdf 100644 --- a/vmm/cc/ide_dma.go +++ b/vmm/cc/ide_dma.go @@ -54,7 +54,7 @@ func (p *pc) busMasterIO(ex hypervisor.X86Exit) error { // or disk, and each buffer must stay within its specified 64 KiB window. func (p *pc) tryIDEDMA() error { d, ata := p.pciIDE, p.ide - if d == nil || ata == nil || !ata.dmaPending || d.bm[0]&1 == 0 || d.config[4]&5 != 5 { + if d == nil || ata == nil || !ata.dmaPending || d.bm[0]&1 == 0 || d.config[4]&5 != 5 || !d.primaryEnabled() { return nil } d.bm[2] |= 1 diff --git a/vmm/cc/ide_identify_test.go b/vmm/cc/ide_identify_test.go new file mode 100644 index 0000000..5d469a1 --- /dev/null +++ b/vmm/cc/ide_identify_test.go @@ -0,0 +1,48 @@ +package cc + +import ( + "bytes" + "encoding/binary" + "j5.nz/cc/hypervisor" + "testing" +) + +func TestIdentifyNegotiatesImplementedMultiwordDMA(t *testing.T) { + p, data := dmaTestPC(t) + p.pciIDE = newICH7PATA() + p.pciIDE.config[4] = 5 + if err := p.ide.command(0xec); err != nil { + t.Fatal(err) + } + word := func(i int) uint16 { return binary.LittleEndian.Uint16(p.ide.buffer[i*2:]) } + // Independent ATA client: validity gates supported modes and cycle times. + if word(49)&0x100 == 0 || word(53)&2 == 0 || word(63)&7 != 7 || word(65) != 120 || word(66) != 120 { + t.Fatal("incomplete DMA identify contract") + } + p.ide.task[1], p.ide.task[2] = 3, 0x22 + if err := p.ide.command(0xef); err != nil { + t.Fatal(err) + } + if err := p.ide.command(0xec); err != nil { + t.Fatal(err) + } + if word(63)&0x700 != 0x400 { + t.Fatal("selected MWDMA2 not reported") + } + binary.LittleEndian.PutUint32(p.pciIDE.bm[4:], 0x1000) + dmaPRD(p, 0, 0x20000, 512, true) + dmaCommand(t, p, false, 1) + if err := p.handleIO(hypervisor.X86Exit{Port: uint16(p.pciIDE.busMasterBase()), Size: 1, Count: 1, Write: true, Data: []byte{9}}); err != nil { + t.Fatal(err) + } + if !bytes.Equal(p.ram[0x20000:0x20200], data[1024:1536]) { + t.Fatal("negotiated DMA did not deliver original bytes") + } + p.ide.dmaEnabled = false + if err := p.ide.command(0xec); err != nil { + t.Fatal(err) + } + if word(49)&0x100 != 0 || word(63) != 0 || word(53)&2 != 0 { + t.Fatal("PIO-only disk falsely advertises DMA") + } +} diff --git a/vmm/cc/inspect.go b/vmm/cc/inspect.go index d12993d..978b2f8 100644 --- a/vmm/cc/inspect.go +++ b/vmm/cc/inspect.go @@ -27,6 +27,11 @@ func (d *driver) Extension(ctx context.Context, name string) (starlark.Value, er "read_physical": starlark.NewBuiltin("cc.read_physical", i.readPhysical), "disk": starlark.NewBuiltin("cc.disk", i.disk), "state": starlark.NewBuiltin("cc.state", i.state), + "symbol": starlark.NewBuiltin("cc.symbol", i.symbol), + "symbols": starlark.NewBuiltin("cc.symbols", i.symbols), + "symbolize": starlark.NewBuiltin("cc.symbolize", i.symbolize), + "read_virtual": starlark.NewBuiltin("cc.read_virtual", i.readVirtual), + "disassemble": starlark.NewBuiltin("cc.disassemble", i.disassemble), }), nil } @@ -139,10 +144,22 @@ func (p *pc) inspectState() (starlark.Value, error) { } timer = map[string]any{"config": int64(h.config), "counter": int64(h.counter(p.now())), "status": int64(h.status), "timers": comparators} } + var smcState any + if s := p.smc; s != nil { + smcState = map[string]any{"status": int(s.status), "result": int(s.result), "command": int(s.command), "argument_bytes": s.argumentSize, "requests": s.trace} + } + var usbState any + if u := p.uhci; u != nil { + devices := []any{} + for n, d := range u.devices { + devices = append(devices, map[string]any{"port": n, "port_status": int(u.ports[n]), "address": int(d.address), "configuration": int(d.configuration), "controls": int64(d.controls), "reports": int64(d.reports), "pending": len(d.pending), "requests": d.trace}) + } + usbState = map[string]any{"command": int(u.command), "status": int(u.status), "interrupt": int(u.interrupt), "frame": int(u.frame), "base": int64(u.base), "transactions": int64(u.transactions), "frames": int64(u.frames), "fault": u.fault, "config": fmt.Sprintf("%x", u.config[:]), "devices": devices} + } var network any var pciIDE any if d := p.pciIDE; d != nil { - pciIDE = map[string]any{"config": fmt.Sprintf("%x", d.config[:64]), "read_dwords": fmt.Sprintf("%016x", d.reads), "write_dwords": fmt.Sprintf("%016x", d.writes), "bus_master": fmt.Sprintf("%x", d.bm)} + pciIDE = map[string]any{"config": fmt.Sprintf("%x", d.config[:]), "read_dwords": fmt.Sprintf("%016x", d.reads), "write_dwords": fmt.Sprintf("%016x", d.writes), "bus_master": fmt.Sprintf("%x", d.bm), "configuration_accesses": d.trace} } if n := p.nic; n != nil { network = map[string]any{"tx": int64(n.tx), "rx": int64(n.rx), "command": int(n.command), "isr": int(n.isr), "imr": int(n.imr), "start": int(n.start), "stop": int(n.stop), "current": int(n.current), "boundary": int(n.boundary), "mac": fmt.Sprintf("%x", n.physical)} @@ -155,10 +172,11 @@ func (p *pc) inspectState() (starlark.Value, error) { "virtio_input": map[string]any{"status": int(p.input.status), "reports": int64(p.input.reports), "queue_ready": p.input.queues[0].ready, "used": int(p.input.queues[0].written), "descriptor": int64(p.input.queues[0].desc), "available": int64(p.input.queues[0].avail), "pending": len(p.input.pending)}, "ata_failures": failures, "ata_last_command": int(p.ide.lastCommand), "ata_dma_pending": p.ide.dmaPending, "ata_irq_pending": p.ide.pending, "pci_address": int64(p.pciAddress), "pci_ide": pciIDE, - "network": network, "hpet": timer, + "network": network, "hpet": timer, "smc": smcState, "usb": usbState, "pc": int64(s.Cs.Base + r.Rip), "cr0": int64(s.Cr0), "cr3": int64(s.Cr3), - "cr2": int64(s.Cr2), "cr4": int64(s.Cr4), + "cr2": int64(s.Cr2), "cr4": int64(s.Cr4), "efer": int64(s.Efer), "gs_base": int64(s.Gs.Base), "registers": registers, "idt_base": int64(s.Idt.Base), + "console": string(p.console), "darwin": p.darwin, "efi": p.efiTrace, "last_bios": p.lastService, "ata_commands": int64(p.ide.commands), "ata_task": fmt.Sprintf("%x", p.ide.task), "vga_accesses": int64(p.vga.accesses), "keyboard_commands": keyboard, "rtc": p.rtcTrace, "interrupts": interrupts, diff --git a/vmm/cc/keyboard.go b/vmm/cc/keyboard.go index ff65151..7fc7a30 100644 --- a/vmm/cc/keyboard.go +++ b/vmm/cc/keyboard.go @@ -18,7 +18,10 @@ type ps2byte struct { func (d *driver) AbsolutePointer(ctx context.Context) (bool, error) { var active bool - err := d.call(ctx, func() error { active = d.pc.input.active(); return nil }) + err := d.call(ctx, func() error { + active = d.pc.input.active() || (d.pc.uhci != nil && d.pc.uhci.devices[1].configuration != 0) + return nil + }) return active, err } @@ -259,6 +262,9 @@ func (d *driver) Input(ctx context.Context, input vmm.Input) error { } } return d.call(ctx, func() error { + if d.pc.uhci != nil { + return d.pc.uhci.devices[1].point(uint16(input.X), uint16(input.Y), buttons, input.Wheel) + } return d.pc.input.pointer(uint32(input.X*65535/32767), uint32(input.Y*65535/32767), buttons, int32(input.Wheel)) }) } @@ -284,7 +290,7 @@ func (d *driver) Input(ctx context.Context, input vmm.Input) error { return d.call(ctx, func() error { return d.pc.keyboard.mouseMove(int(input.X), -int(input.Y), buttons) }) } if input.Kind == "key" { - return d.call(ctx, func() error { return d.pc.keyboard.key(input.Key, input.Down) }) + return d.call(ctx, func() error { return d.pc.keyEvent(input.Key, input.Down) }) } if input.Kind != "keys" { return &vmm.Error{Code: vmm.ErrorUnsupported, Message: "cc input supports key transitions and key chords"} @@ -296,7 +302,7 @@ func (d *driver) Input(ctx context.Context, input vmm.Input) error { } if err := d.call(ctx, func() error { for _, key := range input.Keys { - if err := d.pc.keyboard.key(key, true); err != nil { + if err := d.pc.keyEvent(key, true); err != nil { return err } } @@ -315,7 +321,7 @@ func (d *driver) Input(ctx context.Context, input vmm.Input) error { remaining := len(input.Keys) releaseKeys := func() error { for remaining > 0 { - if err := d.pc.keyboard.key(input.Keys[remaining-1], false); err != nil { + if err := d.pc.keyEvent(input.Keys[remaining-1], false); err != nil { return err } remaining-- diff --git a/vmm/cc/pc_io.go b/vmm/cc/pc_io.go index 87cdfc8..0309db1 100644 --- a/vmm/cc/pc_io.go +++ b/vmm/cc/pc_io.go @@ -20,8 +20,8 @@ func (p *pc) runDevices(ctx context.Context) (hypervisor.X86Exit, error) { // Bulk disk data can use hundreds of native exits per sector. // Leave control-port writes to the outer loop so timer changes // immediately recompute their native execution deadline. - if ex.Port == 0x1f0 { - return true, p.ideIO(ex) + if ex.Port == p.ideDataPort() { + return p.routeIDE(ex) } return false, nil case hypervisor.X86ExitMMIO: @@ -41,6 +41,11 @@ func (p *pc) runWithHandler(ctx context.Context, handle func(hypervisor.X86Exit) return hypervisor.X86Exit{}, err } now := p.now() + if p.uhci != nil { + if err := p.uhci.poll(now); err != nil { + return hypervisor.X86Exit{}, err + } + } if p.acpi != nil { if err := p.acpi.poll(now); err != nil { return hypervisor.X86Exit{}, err @@ -60,7 +65,7 @@ func (p *pc) runWithHandler(ctx context.Context, handle func(hypervisor.X86Exit) var deliver func() error var commit func() rate := p.cmos[0xa] & 15 - if p.cmos[0xb]&0x40 == 0 || rate < 3 { + if p.cmos[0xb]&0x40 == 0 || rate < 3 || (p.hpet != nil && p.hpet.config&2 != 0) { p.rtcNext = time.Time{} } else { period := time.Second * time.Duration(uint64(1)<<(rate-1)) / 32768 @@ -113,12 +118,18 @@ func (p *pc) runWithHandler(ctx context.Context, handle func(hypervisor.X86Exit) } func (p *pc) handleIO(ex hypervisor.X86Exit) error { + if ex.Port >= 0x3f8 && ex.Port <= 0x3ff { + return p.serialIO(ex) + } if p.efi != nil && ex.Port == efiPort && ex.Write && ex.Size == 1 && ex.Count == 1 { return p.efiCall() } if p.acpi != nil && ex.Port >= 0xcf8 && ex.Port <= 0xcff { return p.pciIO(ex) } + if p.uhci != nil && p.uhci.config[4]&1 != 0 && uint32(ex.Port) >= p.uhci.ioBase() && uint32(ex.Port) < p.uhci.ioBase()+32 { + return p.uhci.io(ex, p.now()) + } if p.pciIDE != nil && p.pciIDE.config[4]&1 != 0 { base := p.pciIDE.busMasterBase() if uint32(ex.Port) >= base && uint32(ex.Port) < base+16 { @@ -131,6 +142,9 @@ func (p *pc) handleIO(ex hypervisor.X86Exit) error { if p.acpi != nil && ex.Port >= acpiPMBase && ex.Port < acpiPMBase+12 { return p.acpi.io(ex, p.now()) } + if p.smc != nil && ex.Port >= smcBase && ex.Port < smcBase+32 { + return p.smc.io(ex) + } if p.nic != nil && ex.Port >= 0x300 && ex.Port <= 0x31f { return p.nic.io(ex) } @@ -146,8 +160,8 @@ func (p *pc) handleIO(ex hypervisor.X86Exit) error { if ex.Port >= 0x3b0 && ex.Port <= 0x3df { return p.vga.io(ex) } - if ex.Port >= 0x1f0 && ex.Port <= 0x1f7 || ex.Port == 0x3f6 { - return p.ideIO(ex) + if handled, err := p.routeIDE(ex); handled { + return err } if ex.Port == biosPort && ex.Write && ex.Size == 1 && ex.Count == 1 { return p.bios() diff --git a/vmm/cc/pci.go b/vmm/cc/pci.go index 4f84f55..aa9a4ce 100644 --- a/vmm/cc/pci.go +++ b/vmm/cc/pci.go @@ -2,6 +2,8 @@ package cc import ( "encoding/binary" + "fmt" + "github.com/tinyrange/trex/emulator/cpu" "j5.nz/cc/hypervisor" ) @@ -12,8 +14,10 @@ import ( // BARs and a PCI interrupt pin are not advertised. // The synthetic identity selects the guest's generic PCI IDE class driver. type pciIDE struct { + ich7 bool config [256]byte reads, writes uint64 // Dwords touched, for bounded device inspection. + trace []any // Last 32 configuration transfers, including stopped guest PC. bm [16]byte } @@ -32,6 +36,9 @@ func newPCIIDE() *pciIDE { } func (d *pciIDE) write(offset int, value byte) { + if d.ich7 && d.writeICH7(offset, value) { + return + } switch offset { case 4: d.config[offset] = value & 5 // I/O decoding and bus mastering. @@ -45,9 +52,9 @@ func (d *pciIDE) write(offset int, value byte) { } func (p *pc) pciIO(ex hypervisor.X86Exit) error { - oldCommand := byte(0) + oldConfig := [256]byte{} if p.pciIDE != nil { - oldCommand = p.pciIDE.config[4] + oldConfig = p.pciIDE.config } for i := uint32(0); i < ex.Count; i++ { data := ex.Data[int(i)*int(ex.Size) : int(i+1)*int(ex.Size)] @@ -72,12 +79,73 @@ func (p *pc) pciIO(ex hypervisor.X86Exit) error { } value = p.pciIDE.config[offset] } + if port >= 0xcfc && port <= 0xcff && p.pciDisplay != nil && p.pciAddress&0xffffff00 == 0x80001000 { + offset := int(p.pciAddress&0xfc) + port - 0xcfc + if ex.Write { + p.pciDisplay.write(offset, data[j]) + } + value = p.pciDisplay.config[offset] + } + if port >= 0xcfc && port <= 0xcff && p.uhci != nil && p.pciAddress&0xffffff00 == 0x80001800 { + offset := int(p.pciAddress&0xfc) + port - 0xcfc + if ex.Write { + p.uhci.writeConfig(offset, data[j]) + } + value = p.uhci.config[offset] + } if !ex.Write { data[j] = value } } } - if p.ide != nil && p.pciIDE != nil && oldCommand != p.pciIDE.config[4] { + if p.pciIDE != nil && ex.Port >= 0xcfc && ex.Port <= 0xcff && p.pciAddress&0xffffff00 == 0x80000800 { + row := map[string]any{"offset": int64(p.pciAddress&0xfc) + int64(ex.Port-0xcfc), "write": ex.Write, "data": fmt.Sprintf("%x", ex.Data)} + if p.cpu != nil && p.darwin != nil { + r, err := p.cpu.Registers() + if err != nil { + return err + } + row["pc"] = int64(r.Rip) + row["sp"] = int64(r.Rsp) + row["bp"] = int64(r.Rbp) + if p.darwin != nil { + s, err := p.cpu.SystemRegisters() + if err != nil { + return err + } + mem := efiMemory{p: p, system: &s} + frames := []any{} + bp := r.Rbp + for n := 0; n < 6 && bp != 0; n++ { + var frame [16]byte + if err := mem.ReadMemory(bp, frame[:], cpu.Read); err != nil { + break + } + ret := binary.LittleEndian.Uint64(frame[8:]) + frames = append(frames, map[string]any{"bp": int64(bp), "return": int64(ret)}) + next := binary.LittleEndian.Uint64(frame[:]) + if next <= bp { + break + } + bp = next + } + row["frames"] = frames + } + } + if len(p.pciIDE.trace) == 32 { + copy(p.pciIDE.trace, p.pciIDE.trace[1:]) + p.pciIDE.trace = p.pciIDE.trace[:31] + } + p.pciIDE.trace = append(p.pciIDE.trace, row) + } + if p.ide != nil && p.pciIDE != nil && oldConfig != p.pciIDE.config { + old := *p.pciIDE + old.config = oldConfig + if old.primaryIRQ() != p.pciIDE.primaryIRQ() && p.cpu != nil { + if err := p.cpu.SetIRQ(old.primaryIRQ(), false); err != nil { + return err + } + } if err := p.ide.signal(p.ide.pending); err != nil { return err } @@ -87,7 +155,7 @@ func (p *pc) pciIO(ex hypervisor.X86Exit) error { } func (p *pc) ideIO(ex hypervisor.X86Exit) error { - if p.pciIDE != nil && p.pciIDE.config[4]&1 == 0 { + if p.pciIDE != nil && !p.pciIDE.primaryEnabled() { if !ex.Write { for i := range ex.Data { ex.Data[i] = 0xff diff --git a/vmm/cc/pci_display.go b/vmm/cc/pci_display.go new file mode 100644 index 0000000..153431f --- /dev/null +++ b/vmm/cc/pci_display.go @@ -0,0 +1,42 @@ +package cc + +import ( + "encoding/binary" + "github.com/tinyrange/trex/vmm/ramfb" +) + +// pciDisplay owns the firmware's fixed shared framebuffer aperture at 00:02.0. +// It is an unaccelerated synthetic display, not an emulated Apple GPU. The +// aperture is firmware-reserved RAM: relocation and decode disabling are not +// supported, and config readback must never claim an unmapped address. BAR0 +// nevertheless implements the standard all-ones resource size probe. +type pciDisplay struct { + config [256]byte + barWrite [4]byte +} + +func newPCIDisplay() *pciDisplay { + d := &pciDisplay{} + binary.LittleEndian.PutUint16(d.config[0:], 0x1234) + binary.LittleEndian.PutUint16(d.config[2:], 0xcc02) + d.config[4] = 2 // Fixed firmware memory aperture; no I/O or bus mastering. + d.config[8], d.config[11] = 1, 3 // VGA-compatible display class. + binary.LittleEndian.PutUint32(d.config[0x10:], ramfb.Address) + binary.LittleEndian.PutUint16(d.config[0x2c:], 0x1234) + binary.LittleEndian.PutUint16(d.config[0x2e:], 0xcc02) + d.config[0x3c] = 0xff // No interrupt pin. + copy(d.barWrite[:], d.config[0x10:0x14]) + return d +} + +func (d *pciDisplay) write(offset int, value byte) { + if offset < 0x10 || offset >= 0x14 { + return + } + d.barWrite[offset-0x10] = value + bar := uint32(ramfb.Address) + if binary.LittleEndian.Uint32(d.barWrite[:]) == 0xffffffff { + bar = ^uint32(ramfb.Size-1) & 0xfffffff0 + } + binary.LittleEndian.PutUint32(d.config[0x10:], bar) +} diff --git a/vmm/cc/pci_display_test.go b/vmm/cc/pci_display_test.go new file mode 100644 index 0000000..7f0003f --- /dev/null +++ b/vmm/cc/pci_display_test.go @@ -0,0 +1,63 @@ +package cc + +import ( + "encoding/binary" + "github.com/tinyrange/trex/vmm/ramfb" + "j5.nz/cc/hypervisor" + "testing" +) + +// Exercise firmware/OS resource enumeration through configuration mechanism 1, +// including byte-lane writes, absent functions and fixed aperture readback. +func TestPCIDisplayEnumeration(t *testing.T) { + p := &pc{pciDisplay: newPCIDisplay(), pciIDE: newPCIIDE()} + config := func(address uint32, write bool, data []byte) { + t.Helper() + a := make([]byte, 4) + binary.LittleEndian.PutUint32(a, address&^3) + if err := p.pciIO(hypervisor.X86Exit{Port: 0xcf8, Size: 4, Count: 1, Write: true, Data: a}); err != nil { + t.Fatal(err) + } + if err := p.pciIO(hypervisor.X86Exit{Port: 0xcfc + uint16(address&3), Size: uint8(len(data)), Count: 1, Write: write, Data: data}); err != nil { + t.Fatal(err) + } + } + read := func(address uint32) uint32 { + b := make([]byte, 4) + config(address, false, b) + return binary.LittleEndian.Uint32(b) + } + if read(0x80001000) != 0xcc021234 || read(0x80001008) != 0x03000001 { + t.Fatal("wrong synthetic display identity") + } + if read(0x80001100) != 0xffffffff || read(0x80001800) != 0xffffffff { + t.Fatal("absent function/device decoded") + } + if read(0x80000800) != 0xcc011234 { + t.Fatal("display collided with IDE") + } + for n := uint32(0); n < 4; n++ { + config(0x80001010+n, true, []byte{0xff}) + } + mask := read(0x80001010) + if ^(mask&0xfffffff0)+1 != ramfb.Size { + t.Fatalf("BAR size differs from actual mapping: %#x", mask) + } + config(0x80001010, true, []byte{0, 0, 0, 0xd0}) + if read(0x80001010) != ramfb.Address { + t.Fatal("fixed BAR claimed unmapped relocation") + } + config(0x80001004, true, []byte{0, 0, 0xff, 0xff}) + if read(0x80001004) != 2 { + t.Fatal("fixed decode or unsupported features advertised") + } + for offset := uint32(0x14); offset < 0x28; offset += 4 { + config(0x80001000+offset, true, []byte{0xff, 0xff, 0xff, 0xff}) + if read(0x80001000+offset) != 0 { + t.Fatal("unimplemented BAR claimed memory") + } + } + if read(0x8000103c) != 0xff { + t.Fatal("display claimed interrupt") + } +} diff --git a/vmm/cc/serial.go b/vmm/cc/serial.go new file mode 100644 index 0000000..d76c68e --- /dev/null +++ b/vmm/cc/serial.go @@ -0,0 +1,50 @@ +package cc + +import "j5.nz/cc/hypervisor" + +// Polled 16550-compatible COM1. No incoming data and no UART IRQ are advertised. +// Divisor-latch accesses and loopback are kept distinct from console output. +func (p *pc) serialIO(ex hypervisor.X86Exit) error { + offset := int(ex.Port - 0x3f8) + for i := uint32(0); i < ex.Count; i++ { + data := ex.Data[int(i)*int(ex.Size) : int(i+1)*int(ex.Size)] + if ex.Size != 1 { + if !ex.Write { + for j := range data { + data[j] = 0xff + } + } + continue + } + if ex.Write { + if offset == 0 && p.serial[3]&0x80 == 0 && p.serial[4]&0x10 == 0 { + p.console = append(p.console, data[0]) + if len(p.console) > 65536 { + p.console = p.console[len(p.console)-65536:] + } + } + p.serial[offset] = data[0] + } else { + switch offset { + case 0: + if p.serial[3]&0x80 != 0 || p.serial[4]&0x10 != 0 { + data[0] = p.serial[0] + } else { + data[0] = 0 + } + case 2: + data[0] = 1 // no interrupt + case 5: + data[0] = 0x60 + if p.serial[4]&0x10 != 0 { + data[0] |= 1 + } + case 6: + data[0] = 0xb0 + default: + data[0] = p.serial[offset] + } + } + } + return nil +} diff --git a/vmm/cc/smc.go b/vmm/cc/smc.go new file mode 100644 index 0000000..b16a477 --- /dev/null +++ b/vmm/cc/smc.go @@ -0,0 +1,200 @@ +package cc + +import ( + "encoding/binary" + "j5.nz/cc/hypervisor" + "sort" +) + +// smc implements the byte-wide SMC key protocol, not a guest decryptor. +// Wire facts: Linux drivers/hwmon/applesmc.c (ports, commands, status bits, +// big-endian indices and six-byte key metadata). No driver code is copied. +// Original AppleSMC matches the ACPI APP0001 device. The optional caller's +// OSK is never synthesized, logged, or included in inspection results. +const smcBase = 0x300 + +type smcKey struct { + data []byte + kind string + writable bool +} +type smc struct { + keys map[string]smcKey + names []string + command, status, result byte + argument [4]byte + argumentSize int + length int + payload [32]byte + position int + trace []any +} + +func newSMC(osk []byte) *smc { + s := &smc{keys: map[string]smcKey{ + "#KEY": {data: make([]byte, 4), kind: "ui32"}, + "REV ": {data: []byte{1, 0, 0, 0, 0, 0}, kind: "{rev"}, + "FNum": {data: []byte{0}, kind: "ui8 "}, + "MSTS": {data: []byte{0}, kind: "ui8 "}, + }} + if len(osk) == 64 { + s.keys["OSK0"] = smcKey{data: append([]byte(nil), osk[:32]...), kind: "ch8*"} + s.keys["OSK1"] = smcKey{data: append([]byte(nil), osk[32:]...), kind: "ch8*"} + } + for name := range s.keys { + s.names = append(s.names, name) + } + sort.Strings(s.names) + binary.BigEndian.PutUint32(s.keys["#KEY"].data, uint32(len(s.names))) + return s +} + +func (s *smc) record(key string) { + s.trace = append(s.trace, map[string]any{"command": int(s.command), "key": key, "length": s.length, "result": int(s.result)}) + if len(s.trace) > 64 { + s.trace = s.trace[len(s.trace)-64:] + } +} +func (s *smc) fail(result byte) { s.result = result; s.status = 0; s.position = 0 } +func (s *smc) begin(command byte) { + s.command = command + s.status = 0x0c + s.result = 0 + s.argumentSize = 0 + s.position = 0 + s.length = 0 + clear(s.payload[:]) + if command < 0x10 || command > 0x13 { + s.fail(0x82) + } +} +func (s *smc) input(value byte) { + if s.status&4 == 0 || s.status&1 != 0 { + return + } + if s.argumentSize < 4 { + s.argument[s.argumentSize] = value + s.argumentSize++ + s.status = 4 + // Lion's original client omits the redundant length byte for + // fixed-width index/info commands; accept Linux's extra byte too. + if s.argumentSize == 4 && (s.command == 0x12 || s.command == 0x13) { + s.length = 4 + if s.command == 0x13 { + s.length = 6 + } + s.prepare() + } + return + } + if s.length == 0 { + s.length = int(value) + s.prepare() + return + } + if s.command == 0x11 && s.position < s.length { + s.payload[s.position] = value + s.position++ + if s.position == s.length { + entry := s.keys[string(s.argument[:])] + copy(entry.data, s.payload[:s.length]) + s.status = 0 + } + } +} +func (s *smc) output() byte { + if s.status&1 == 0 || s.position >= s.length { + return 0 + } + value := s.payload[s.position] + s.position++ + if s.position == s.length { + s.status = 0 + } + return value +} +func (s *smc) io(ex hypervisor.X86Exit) error { + for i := uint32(0); i < ex.Count; i++ { + for j := uint16(0); j < uint16(ex.Size); j++ { + pos := int(i)*int(ex.Size) + int(j) + port := ex.Port + j + value := byte(0) + switch port { + case smcBase: + if ex.Write { + s.input(ex.Data[pos]) + } else { + value = s.output() + } + case smcBase + 4: + if ex.Write { + s.begin(ex.Data[pos]) + } else { + value = s.status + } + case smcBase + 0x1e: + value = s.result + } + if !ex.Write { + ex.Data[pos] = value + } + } + } + return nil +} + +func (s *smc) prepare() { + name := string(s.argument[:]) + if s.length == 0 || s.length > len(s.payload) { + s.fail(0x85) + s.record(name) + return + } + entry, found := s.keys[name] + switch s.command { + case 0x10: + if !found { + s.fail(0x84) + } else if s.length != len(entry.data) { + s.fail(0x85) + } else { + copy(s.payload[:], entry.data) + s.status = 5 + } + case 0x11: + if !found { + s.fail(0x84) + } else if !entry.writable { + s.fail(0x86) + } else if s.length != len(entry.data) { + s.fail(0x85) + } else { + s.status = 4 + } + case 0x12: + index := binary.BigEndian.Uint32(s.argument[:]) + if index >= uint32(len(s.names)) { + s.fail(0x84) + } else if s.length != 4 { + s.fail(0x85) + } else { + copy(s.payload[:], s.names[index]) + s.status = 5 + } + case 0x13: + if !found { + s.fail(0x84) + } else if s.length != 6 { + s.fail(0x85) + } else { + s.payload[0] = byte(len(entry.data)) + copy(s.payload[1:5], entry.kind) + s.payload[5] = 0x80 + if entry.writable { + s.payload[5] |= 0x40 + } + s.status = 5 + } + } + s.record(name) +} diff --git a/vmm/cc/smc_lion_test.go b/vmm/cc/smc_lion_test.go new file mode 100644 index 0000000..32b272e --- /dev/null +++ b/vmm/cc/smc_lion_test.go @@ -0,0 +1,20 @@ +package cc + +import "testing" + +func TestSMCLionFixedWidthCommandsWithoutLength(t *testing.T) { + s := newSMC(nil) + // Unlike Linux, original Lion omits the length byte for fixed-width requests. + smcPort(t, s, 0x304, true, 0x12) + smcPort(t, s, 0x300, true, 0, 0, 0, 0) + key := smcRead(t, s, 4) + if string(key) != "#KEY" { + t.Fatalf("bad index zero: %q", key) + } + smcPort(t, s, 0x304, true, 0x13) + smcPort(t, s, 0x300, true, key...) + info := smcRead(t, s, 6) + if info[0] != 4 || string(info[1:5]) != "ui32" { + t.Fatalf("bad key metadata: %x", info) + } +} diff --git a/vmm/cc/smc_test.go b/vmm/cc/smc_test.go new file mode 100644 index 0000000..136b4d2 --- /dev/null +++ b/vmm/cc/smc_test.go @@ -0,0 +1,113 @@ +package cc + +import ( + "bytes" + "encoding/binary" + "fmt" + "j5.nz/cc/hypervisor" + "testing" +) + +// Exercise the consumer's wire contract through native exit routing, rather +// than poking internal transaction fields. Byte-status assertions distinguish +// a real complete read from a permanently-ready/dummy-data device. +func smcPort(t *testing.T, s *smc, port uint16, write bool, data ...byte) []byte { + t.Helper() + if len(data) == 0 { + data = []byte{0} + } + ex := hypervisor.X86Exit{Port: port, Write: write, Size: 1, Count: uint32(len(data)), Data: data} + if err := s.io(ex); err != nil { + t.Fatal(err) + } + return ex.Data +} +func smcRequest(t *testing.T, s *smc, command byte, key []byte, length byte) { + t.Helper() + smcPort(t, s, 0x304, true, command) + if status := smcPort(t, s, 0x304, false)[0]; status&6 != 4 { + t.Fatalf("command not accepting input: %x", status) + } + smcPort(t, s, 0x300, true, key...) + smcPort(t, s, 0x300, true, length) +} +func smcRead(t *testing.T, s *smc, length int) []byte { + t.Helper() + out := make([]byte, length) + for i := range out { + if status := smcPort(t, s, 0x304, false)[0]; status&7 != 5 { + t.Fatalf("byte%d not ready: %x", i, status) + } + out[i] = smcPort(t, s, 0x300, false)[0] + } + if status := smcPort(t, s, 0x304, false)[0]; status&5 != 0 { + t.Fatalf("read never completes: %x", status) + } + return out +} +func TestSMCConsumerEnumerationAndMetadata(t *testing.T) { + s := newSMC(nil) + smcRequest(t, s, 0x10, []byte("#KEY"), 4) + count := binary.BigEndian.Uint32(smcRead(t, s, 4)) + if count != 4 { + t.Fatalf("unexpected key count %d", count) + } + for i := uint32(0); i < count; i++ { + index := make([]byte, 4) + binary.BigEndian.PutUint32(index, i) + smcRequest(t, s, 0x12, index, 4) + key := smcRead(t, s, 4) + smcRequest(t, s, 0x13, key, 6) + info := smcRead(t, s, 6) + if info[5] != 0x80 || info[0] == 0 || info[0] > 32 { + t.Fatalf("bad metadata for %q: %x", key, info) + } + smcRequest(t, s, 0x10, key, info[0]) + smcRead(t, s, int(info[0])) + } +} +func TestSMCMissingOSKAndErrorRecovery(t *testing.T) { + s := newSMC(nil) + for _, request := range []struct { + cmd byte + key string + length, result byte + }{ + {0x10, "OSK0", 32, 0x84}, {0x10, "OSK1", 32, 0x84}, + {0x10, "#KEY", 255, 0x85}, {0x11, "#KEY", 4, 0x86}, + {0x13, "NONE", 6, 0x84}, {0x10, "#KEY", 3, 0x85}, + } { + smcRequest(t, s, request.cmd, []byte(request.key), request.length) + if result := smcPort(t, s, 0x31e, false)[0]; result != request.result { + t.Fatalf("%q result %x", request.key, result) + } + if status := smcPort(t, s, 0x304, false)[0]; status != 0 { + t.Fatalf("error still busy: %x", status) + } + smcRequest(t, s, 0x10, []byte("#KEY"), 4) + if binary.BigEndian.Uint32(smcRead(t, s, 4)) != 4 { + t.Fatal("error poisoned next transaction") + } + } +} +func TestSMCExplicitKeyOwnershipAndTraceRedaction(t *testing.T) { + key := bytes.Repeat([]byte{0xa5}, 64) // synthetic test bytes, not Apple material + s := newSMC(key) + clear(key) + for _, name := range []string{"OSK0", "OSK1"} { + smcRequest(t, s, 0x10, []byte(name), 32) + if !bytes.Equal(smcRead(t, s, 32), bytes.Repeat([]byte{0xa5}, 32)) { + t.Fatal("did not retain caller key") + } + } + for i := 0; i < 100; i++ { + smcRequest(t, s, 0x10, []byte("OSK0"), 32) + smcRead(t, s, 32) + } + if len(s.trace) != 64 { + t.Fatal("unbounded trace") + } + if bytes.Contains([]byte(fmt.Sprint(s.trace)), []byte("165")) { + t.Fatal("key bytes disclosed by trace") + } +} diff --git a/vmm/cc/star.go b/vmm/cc/star.go index d528dcb..e7ce756 100644 --- a/vmm/cc/star.go +++ b/vmm/cc/star.go @@ -21,13 +21,14 @@ func Builtins() starlark.StringDict { return starlark.StringDict{"backend": starlark.NewBuiltin("cc.backend", func(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { acpi, pciIDE, hpet, uefi := false, false, false, false ideDMA := true + ideModel := "synthetic" overlayLimit := int64(256 << 20) - if err := starlark.UnpackArgs("cc.backend", args, kwargs, "acpi?", &acpi, "uefi?", &uefi, "hpet?", &hpet, "pci_ide?", &pciIDE, "ide_dma?", &ideDMA, "overlay_limit?", &overlayLimit); err != nil { + if err := starlark.UnpackArgs("cc.backend", args, kwargs, "acpi?", &acpi, "uefi?", &uefi, "hpet?", &hpet, "pci_ide?", &pciIDE, "ide_model?", &ideModel, "ide_dma?", &ideDMA, "overlay_limit?", &overlayLimit); err != nil { return nil, err } if overlayLimit <= 0 { return nil, fmt.Errorf("cc.backend: overlay_limit must be positive") } - return &Backend{UEFI: uefi, ACPI: acpi, HPET: hpet, PCIIDE: pciIDE, PIOOnly: !ideDMA, OverlayLimit: overlayLimit}, nil + return &Backend{UEFI: uefi, ACPI: acpi, HPET: hpet, PCIIDE: pciIDE, IDEModel: ideModel, PIOOnly: !ideDMA, OverlayLimit: overlayLimit}, nil })} } diff --git a/vmm/cc/uhci.go b/vmm/cc/uhci.go new file mode 100644 index 0000000..fe0c731 --- /dev/null +++ b/vmm/cc/uhci.go @@ -0,0 +1,340 @@ +package cc + +import ( + "encoding/binary" + "fmt" + "j5.nz/cc/hypervisor" + "time" +) + +// UHCI 1.1 transport: PCI 00:03.0, BAR4 I/O, two low-speed root ports. +// Native descriptor DMA is bounded to ordinary guest RAM. No guest USB driver +// or accelerator is injected. Queue processing stays on the owning CPU loop. +type uhci struct { + config [256]byte + command, status, interrupt, frame uint16 + base uint32 + sof byte + ports [2]uint16 + devices [2]*usbHID + last time.Time + memory func(uint64, uint64) ([]byte, error) + irq func(uint32, bool) error + transactions, frames uint64 + fault string +} + +func newUHCI(memory func(uint64, uint64) ([]byte, error), irq func(uint32, bool) error) *uhci { + u := &uhci{memory: memory, irq: irq, status: 0x20, sof: 64} + binary.LittleEndian.PutUint16(u.config[:], 0x1234) + binary.LittleEndian.PutUint16(u.config[2:], 0xcc03) + u.config[4] = 5 + u.config[8] = 1 + u.config[10] = 3 + u.config[11] = 12 + binary.LittleEndian.PutUint32(u.config[0x20:], 0xc101) + binary.LittleEndian.PutUint16(u.config[0x2c:], 0x1234) + binary.LittleEndian.PutUint16(u.config[0x2e:], 0xcc03) + u.config[0x3c], u.config[0x3d] = 17, 1 + u.config[0xc1] = 0x20 + for n := range u.ports { + u.ports[n] = 0x1a3 + u.devices[n] = newUSBHID(n == 1) + } + return u +} +func (u *uhci) writeConfig(offset int, v byte) { + switch offset { + case 4: + u.config[offset] = v & 5 + case 5: + u.config[offset] = v & 4 // INTx disable. + case 0x20: + u.config[offset] = v&0xe0 | 1 + case 0x21, 0x22, 0x23: + u.config[offset] = v + case 0xc1: + u.config[offset] = v & 0x20 + } +} +func (u *uhci) ioBase() uint32 { return binary.LittleEndian.Uint32(u.config[0x20:]) & 0xffffffe0 } +func (u *uhci) updateIRQ() error { + enabled := u.config[4]&1 != 0 && u.config[5]&4 == 0 + pending := (u.status&1 != 0 && u.interrupt&12 != 0) || (u.status&2 != 0 && u.interrupt&1 != 0) || (u.status&4 != 0 && u.interrupt&2 != 0) || u.status&0x18 != 0 + return u.irq(17, enabled && pending) +} +func (u *uhci) io(ex hypervisor.X86Exit, now time.Time) error { + for n := uint32(0); n < ex.Count; n++ { + data := ex.Data[int(n)*int(ex.Size) : int(n+1)*int(ex.Size)] + off := int(uint32(ex.Port) - u.ioBase()) + if off < 0 || off+len(data) > 32 { + return fmt.Errorf("UHCI register transfer outside aperture") + } + var regs [32]byte + binary.LittleEndian.PutUint16(regs[:], u.command) + binary.LittleEndian.PutUint16(regs[2:], u.status) + binary.LittleEndian.PutUint16(regs[4:], u.interrupt) + binary.LittleEndian.PutUint16(regs[6:], u.frame) + binary.LittleEndian.PutUint32(regs[8:], u.base) + regs[12] = u.sof + for i, p := range u.ports { + binary.LittleEndian.PutUint16(regs[16+i*2:], p) + } + if !ex.Write { + copy(data, regs[off:]) + continue + } + old := regs + copy(regs[off:], data) + touched := func(start, size int) bool { return off < start+size && off+len(data) > start } + if touched(0, 2) { + cmd := binary.LittleEndian.Uint16(regs[:]) & 0xff + if cmd&2 != 0 { + u.command = 0 + u.status = 0x20 + u.interrupt = 0 + u.frame = 0 + u.base = 0 + u.sof = 64 + u.last = now + } else { + if cmd&4 != 0 && u.command&4 == 0 { + for i := range u.devices { + u.devices[i] = newUSBHID(i == 1) + u.ports[i] = 0x1a3 + } + } + if (cmd^u.command)&1 != 0 { + u.last = now + } + u.command = cmd + if cmd&1 != 0 { + u.status &^= 0x20 + } else { + u.status |= 0x20 + } + } + } + if touched(2, 2) { + var mask uint16 + for i := range data { + if off+i >= 2 && off+i < 4 { + mask |= uint16(data[i]) << ((off + i - 2) * 8) + } + } + u.status &^= mask & 0x1f + } + if touched(4, 2) { + u.interrupt = binary.LittleEndian.Uint16(regs[4:]) & 15 + } + if touched(6, 2) { + u.frame = binary.LittleEndian.Uint16(regs[6:]) & 0x7ff + } + if touched(8, 4) { + u.base = binary.LittleEndian.Uint32(regs[8:]) & 0xfffff000 + } + if touched(12, 1) { + u.sof = regs[12] & 0x7f + } + for i := range u.ports { + at := 16 + i*2 + if !touched(at, 2) { + continue + } + v, previous := binary.LittleEndian.Uint16(regs[at:]), binary.LittleEndian.Uint16(old[at:]) + // Connection/line status are read-only; CSC/PEC are W1C. + changed := previous & 0xa &^ (v & 0xa) + control := v & 0x1244 + if v&0x200 != 0 && previous&0x200 == 0 { + u.devices[i] = newUSBHID(i == 1) + control &^= 4 + } + if (control^previous)&4 != 0 { + changed |= 8 + } + u.ports[i] = 0x1a1 | changed | control + } + } + return u.updateIRQ() +} +func (u *uhci) fail(message string, process bool) error { + u.fault = message + if process { + u.status |= 0x10 + } else { + u.status |= 8 + } + u.command &^= 1 + u.status |= 0x20 + return u.updateIRQ() +} +func (u *uhci) poll(now time.Time) error { + if u.command&1 == 0 || u.command&0xc != 0 || u.config[4]&5 != 5 { + u.last = now + return u.updateIRQ() + } + if u.last.IsZero() { + u.last = now + return nil + } + elapsed := int(now.Sub(u.last) / time.Millisecond) + // Host suspension must not replay an unbounded number of stale schedules. + if elapsed > 1024 { + u.frame = (u.frame + uint16((elapsed-1024)&0x7ff)) & 0x7ff + elapsed = 1024 + u.last = now.Add(-1024 * time.Millisecond) + } + for n := 0; n < elapsed; n++ { + frame, err := u.memory(uint64(u.base)+uint64(u.frame&1023)*4, 4) + if err != nil { + return u.fail(err.Error(), false) + } + if err := u.schedule(binary.LittleEndian.Uint32(frame)); err != nil { + return u.fail(err.Error(), true) + } + u.frame = (u.frame + 1) & 0x7ff + u.frames++ + u.last = u.last.Add(time.Millisecond) + } + return u.updateIRQ() +} + +func (u *uhci) schedule(link uint32) error { + // Horizontal QH links can legally cycle for bandwidth reclamation. Process + // each QH once per frame; descriptor cycles and bounds never hang the host. + seen := map[uint32]bool{} + stack := []uint32{} + qh := uint32(0) + for budget := 0; budget < 2048; budget++ { + if link&1 != 0 { + if len(stack) == 0 { + return nil + } + link = stack[len(stack)-1] + stack = stack[:len(stack)-1] + qh = 0 + continue + } + address := link & 0xfffffff0 + if link&2 != 0 { + if seen[address] { + link = 1 + continue + } + seen[address] = true + data, err := u.memory(uint64(address), 8) + if err != nil { + return err + } + if len(stack) >= 128 { + return fmt.Errorf("UHCI QH nesting exceeds bound") + } + stack = append(stack, binary.LittleEndian.Uint32(data)) + qh = address + link = binary.LittleEndian.Uint32(data[4:]) + continue + } + data, err := u.memory(uint64(address), 16) + if err != nil { + return err + } + next := binary.LittleEndian.Uint32(data) + // An inactive queue TD is the software tail sentinel. Hardware must + // leave the QH element on it, not consume its link and lose the + // controller driver's completion boundary. + if binary.LittleEndian.Uint32(data[4:])&(1<<23) == 0 { + if qh != 0 { + link = 1 + } else { + link = next + } + continue + } + done, short, err := u.transfer(data) + if err != nil { + return err + } + if !done || short { + link = 1 + continue + } + if qh != 0 { + head, err := u.memory(uint64(qh)+4, 4) + if err != nil { + return err + } + binary.LittleEndian.PutUint32(head, next) + } + link = next + // Breadth-first TDs return to the horizontal schedule after completion. + if qh != 0 && next&4 == 0 { + link = 1 + } + } + return fmt.Errorf("UHCI descriptor traversal exceeds bound") +} +func (u *uhci) transfer(td []byte) (bool, bool, error) { + status := binary.LittleEndian.Uint32(td[4:]) + if status&(1<<23) == 0 { + return true, false, nil + } + token := binary.LittleEndian.Uint32(td[8:]) + pid, address, ep := byte(token), byte(token>>8)&127, byte(token>>15)&15 + maximum := int((token>>21)+1) & 0x7ff + if maximum > 8 { + status &^= 1 << 23 + status |= 1 << 22 + binary.LittleEndian.PutUint32(td[4:], status) + u.status |= 2 + return false, false, nil + } + buffer, err := u.memory(uint64(binary.LittleEndian.Uint32(td[12:])), uint64(maximum)) + if err != nil { + return false, false, err + } + var device *usbHID + for i, d := range u.devices { + if u.ports[i]&0x204 == 4 && d.address == address { + device = d + break + } + } + n, result := 0, 3 + if device != nil { + n, result = device.transaction(pid, ep, buffer) + } + u.transactions++ + status &^= 0x007f0000 // Old transaction errors. + switch result { + case 1: + status |= 1 << 19 + binary.LittleEndian.PutUint32(td[4:], status) + return false, false, nil + case 2: + status |= 1 << 22 + status &^= 1 << 23 + u.status |= 2 + case 3: + status |= 1 << 18 + retry := (status >> 27) & 3 + if retry > 0 { + retry-- + status = status&^(3<<27) | retry<<27 + } + if retry == 0 { + status &^= 1 << 23 + u.status |= 2 + } + binary.LittleEndian.PutUint32(td[4:], status) + return false, false, nil + default: + status &^= 1 << 23 + } + status = status&^0x7ff | uint32(n-1)&0x7ff + binary.LittleEndian.PutUint32(td[4:], status) + short := n < maximum && status&(1<<29) != 0 + if status&(1<<24) != 0 || short { + u.status |= 1 + } + return result == 0, short, nil +} diff --git a/vmm/cc/uhci_tail_test.go b/vmm/cc/uhci_tail_test.go new file mode 100644 index 0000000..9b98884 --- /dev/null +++ b/vmm/cc/uhci_tail_test.go @@ -0,0 +1,31 @@ +package cc + +import ( + "encoding/binary" + "testing" +) + +func TestUHCIKeepsInactiveQueueTail(t *testing.T) { + u, ram, _ := usbTestController() + u.ports[0] |= 4 + u.devices[0].configuration = 1 + if err := u.devices[0].key("a", true); err != nil { + t.Fatal(err) + } + binary.LittleEndian.PutUint32(ram[0x1000:], 1) + binary.LittleEndian.PutUint32(ram[0x1004:], 0x2000) + usbTestTD(ram, 0x2000, 0x2014, 0x69, 0, 1, 8) + binary.LittleEndian.PutUint32(ram[0x2010:], 1) // Inactive software tail. + if err := u.schedule(0x1002); err != nil { + t.Fatal(err) + } + if element := binary.LittleEndian.Uint32(ram[0x1004:]); element != 0x2014 { + t.Fatalf("hardware consumed inactive tail: %#x", element) + } + if err := u.schedule(0x1002); err != nil { + t.Fatal(err) + } + if binary.LittleEndian.Uint32(ram[0x1004:]) != 0x2014 || u.devices[0].reports != 1 { + t.Fatal("tail changed or report replayed") + } +} diff --git a/vmm/cc/uhci_test.go b/vmm/cc/uhci_test.go new file mode 100644 index 0000000..af10d3b --- /dev/null +++ b/vmm/cc/uhci_test.go @@ -0,0 +1,184 @@ +package cc + +import ( + "bytes" + "encoding/binary" + "fmt" + "j5.nz/cc/hypervisor" + "testing" + "time" +) + +func usbTestController() (*uhci, []byte, *bool) { + ram := make([]byte, 0x10000) + level := false + memory := func(a, n uint64) ([]byte, error) { + if a > uint64(len(ram)) || n > uint64(len(ram))-a { + return nil, fmt.Errorf("DMA outside guest RAM") + } + return ram[a : a+n], nil + } + return newUHCI(memory, func(irq uint32, v bool) error { + if irq != 17 { + panic("wrong route") + } + level = v + return nil + }), ram, &level +} +func usbTestTD(ram []byte, at, next uint32, pid, address, endpoint byte, length int) { + binary.LittleEndian.PutUint32(ram[at:], next) + binary.LittleEndian.PutUint32(ram[at+4:], 1<<23|1<<24|3<<27|1<<26) + binary.LittleEndian.PutUint32(ram[at+8:], uint32(pid)|uint32(address)<<8|uint32(endpoint)<<15|uint32((length-1)&2047)<<21) + binary.LittleEndian.PutUint32(ram[at+12:], 0x8000) +} + +func TestUHCIEnumerationAndHIDTransitions(t *testing.T) { + u, ram, _ := usbTestController() + u.ports[0] |= 4 + packet := func(pid, address, ep byte, payload []byte, length int) []byte { + t.Helper() + copy(ram[0x8000:], payload) + usbTestTD(ram, 0x2000, 1, pid, address, ep, length) + binary.LittleEndian.PutUint32(ram[0x1000:], 1) + binary.LittleEndian.PutUint32(ram[0x1004:], 0x2000) + if err := u.schedule(0x1002); err != nil { + t.Fatal(err) + } + status := binary.LittleEndian.Uint32(ram[0x2004:]) + if status&(1<<23|0x7f0000) != 0 { + t.Fatalf("packet did not ACK: %#x", status) + } + n := int(status+1) & 2047 + return append([]byte(nil), ram[0x8000:0x8000+n]...) + } + // Firmware learns endpoint-zero max packet size, then assigns address. + packet(0x2d, 0, 0, []byte{0x80, 6, 0, 1, 0, 0, 8, 0}, 8) + first := packet(0x69, 0, 0, nil, 8) + if !bytes.Equal(first, []byte{18, 1, 0x10, 1, 0, 0, 0, 8}) { + t.Fatal("wrong device prefix", first) + } + packet(0xe1, 0, 0, nil, 0) + packet(0x2d, 0, 0, []byte{0, 5, 7, 0, 0, 0, 0, 0}, 8) + if u.devices[0].address != 0 { + t.Fatal("address applied before status") + } + packet(0x69, 0, 0, nil, 0) + if u.devices[0].address != 7 { + t.Fatal("address missing after status") + } + packet(0x2d, 7, 0, []byte{0x80, 6, 0, 1, 0, 0, 18, 0}, 8) + full := packet(0x69, 7, 0, nil, 8) + full = append(full, packet(0x69, 7, 0, nil, 8)...) + full = append(full, packet(0x69, 7, 0, nil, 8)...) + if len(full) != 18 || binary.LittleEndian.Uint16(full[8:]) != 0x1234 { + t.Fatal("multi-packet descriptor corrupted", full) + } + packet(0xe1, 7, 0, nil, 0) + packet(0x2d, 7, 0, []byte{0, 9, 1, 0, 0, 0, 0, 0}, 8) + packet(0x69, 7, 0, nil, 0) + packet(0x69, 7, 1, nil, 8) // Initial idle report. + if err := u.devices[0].key("shift", true); err != nil { + t.Fatal(err) + } + if err := u.devices[0].key("a", true); err != nil { + t.Fatal(err) + } + if err := u.devices[0].key("a", false); err != nil { + t.Fatal(err) + } + a, b, c := packet(0x69, 7, 1, nil, 8), packet(0x69, 7, 1, nil, 8), packet(0x69, 7, 1, nil, 8) + if a[0] != 2 || a[2] != 0 || b[0] != 2 || b[2] != 4 || c[0] != 2 || c[2] != 0 { + t.Fatal("keyboard transition lost", a, b, c) + } + usbTestTD(ram, 0x2000, 1, 0x69, 7, 1, 8) + if _, _, err := u.transfer(ram[0x2000:0x2010]); err != nil { + t.Fatal(err) + } + if s := binary.LittleEndian.Uint32(ram[0x2004:]); s&(1<<23|1<<19) != (1<<23 | 1<<19) { + t.Fatal("idle interrupt endpoint must NAK and remain active") + } +} + +func TestUHCIScheduleIRQAndReset(t *testing.T) { + u, ram, level := usbTestController() + now := time.Unix(1, 0) + u.command = 1 + u.status = 0 + u.interrupt = 4 + u.last = now + u.base = 0x4000 + u.ports[1] |= 4 + u.devices[1].address = 9 + u.devices[1].configuration = 1 + if err := u.devices[1].point(1234, 2345, 1, -2); err != nil { + t.Fatal(err) + } + for n := 0; n < 1024; n++ { + binary.LittleEndian.PutUint32(ram[0x4000+n*4:], 0x1002) + } + binary.LittleEndian.PutUint32(ram[0x1000:], 0x1002) // Legal reclamation loop. + binary.LittleEndian.PutUint32(ram[0x1004:], 0x2000) + usbTestTD(ram, 0x2000, 1, 0x69, 9, 1, 6) + if err := u.poll(now.Add(time.Millisecond)); err != nil { + t.Fatal(err) + } + if !*level || u.frame != 1 || u.devices[1].reports != 1 || binary.LittleEndian.Uint16(ram[0x8001:]) != 1234 || ram[0x8005] != 254 { + t.Fatal("frame DMA or interrupt/report failed") + } + ack := []byte{1, 0} + if err := u.io(hypervisor.X86Exit{Port: 0xc102, Size: 2, Count: 1, Write: true, Data: ack}, now); err != nil { + t.Fatal(err) + } + if *level || u.status&1 != 0 { + t.Fatal("W1C completion did not deassert INTx") + } + reset := []byte{2, 0} + if err := u.io(hypervisor.X86Exit{Port: 0xc100, Size: 2, Count: 1, Write: true, Data: reset}, now); err != nil { + t.Fatal(err) + } + if u.command != 0 || u.status != 0x20 || u.base != 0 || u.frame != 0 { + t.Fatal("HC reset not self-clearing/halted") + } +} +func TestUHCIMalformedDMAAndCycles(t *testing.T) { + u, ram, _ := usbTestController() + if err := u.schedule(0xfffffff2); err == nil { + t.Fatal("invalid QH DMA accepted") + } + binary.LittleEndian.PutUint32(ram[0x2000:], 0x2004) + if err := u.schedule(0x2000); err == nil { + t.Fatal("TD cycle did not reach bounded failure") + } + u.command = 1 + u.last = time.Unix(1, 0) + u.base = 0xfffff000 + if err := u.poll(u.last.Add(time.Millisecond)); err != nil { + t.Fatal(err) + } + if u.fault == "" || u.command&1 != 0 || u.status&0x28 != 0x28 { + t.Fatal("invalid frame DMA did not halt controller") + } +} +func TestUSBHIDStallRecoveryAndBounds(t *testing.T) { + d := newUSBHID(false) + if d.setup([]byte{0x80, 6, 0, 0xff, 0, 0, 8, 0}) { + t.Fatal("unknown descriptor did not stall") + } + if !d.setup([]byte{0x80, 6, 0, 1, 0, 0, 8, 0}) || len(d.reply) != 8 { + t.Fatal("new SETUP did not clear stall/limit length") + } + if d.setup([]byte{0, 5, 128, 0, 0, 0, 0, 0}) { + t.Fatal("out-of-range address accepted") + } + d.configuration = 1 + for n := 0; n < 1024; n++ { + if err := d.key("a", n%2 == 0); err != nil { + t.Fatal(err) + } + } + before := append([]byte(nil), d.report...) + if err := d.key("b", true); err == nil || !bytes.Equal(before, d.report) { + t.Fatal("queue overflow changed live key state") + } +} diff --git a/vmm/cc/usb_hid.go b/vmm/cc/usb_hid.go new file mode 100644 index 0000000..c118b4f --- /dev/null +++ b/vmm/cc/usb_hid.go @@ -0,0 +1,326 @@ +package cc + +import ( + "encoding/binary" + "fmt" + "strings" + "unicode/utf16" +) + +// USB 1.1 low-speed HID devices. Descriptor/control handling is independent of +// the host-controller transport. Identities name synthetic trex devices. +type usbHID struct { + pointer bool + address, configuration, protocol, idle byte + pendingAddress byte + addressPending bool + request [8]byte + reply []byte + replyOffset int + output []byte + stalled bool + report []byte + pending [][]byte + controls, reports uint64 + trace []any +} + +func newUSBHID(pointer bool) *usbHID { + d := &usbHID{pointer: pointer, protocol: 1} + d.report = make([]byte, 8) + if pointer { + d.report = make([]byte, 6) + } + return d +} + +// Standard boot keyboard and an absolute pointing device with wheel. The +// pointer uses generic report protocol, not a misleading boot-mouse protocol. +var usbKeyboardDescriptor = []byte{ + 0x05, 1, 0x09, 6, 0xa1, 1, 0x05, 7, 0x19, 0xe0, 0x29, 0xe7, 0x15, 0, 0x25, 1, + 0x75, 1, 0x95, 8, 0x81, 2, 0x95, 1, 0x75, 8, 0x81, 1, + 0x95, 5, 0x75, 1, 0x05, 8, 0x19, 1, 0x29, 5, 0x91, 2, + 0x95, 1, 0x75, 3, 0x91, 1, 0x95, 6, 0x75, 8, 0x15, 0, 0x25, 0x65, + 0x05, 7, 0x19, 0, 0x29, 0x65, 0x81, 0, 0xc0, +} +var usbPointerDescriptor = []byte{ + 0x05, 1, 0x09, 2, 0xa1, 1, 0x09, 1, 0xa1, 0, + 0x05, 9, 0x19, 1, 0x29, 3, 0x15, 0, 0x25, 1, 0x95, 3, 0x75, 1, 0x81, 2, + 0x95, 1, 0x75, 5, 0x81, 1, 0x05, 1, 0x09, 0x30, 0x09, 0x31, + 0x15, 0, 0x26, 0xff, 0x7f, 0x75, 16, 0x95, 2, 0x81, 2, + 0x09, 0x38, 0x15, 0x81, 0x25, 0x7f, 0x75, 8, 0x95, 1, 0x81, 6, 0xc0, 0xc0, +} + +func (d *usbHID) reportDescriptor() []byte { + if d.pointer { + return usbPointerDescriptor + } + return usbKeyboardDescriptor +} +func usbString(s string) []byte { + units := utf16.Encode([]rune(s)) + out := []byte{byte(2 + len(units)*2), 3} + for _, v := range units { + out = binary.LittleEndian.AppendUint16(out, v) + } + return out +} +func (d *usbHID) descriptor(kind, index byte) []byte { + hid := []byte{9, 0x21, 0x11, 1, 0, 1, 0x22, byte(len(d.reportDescriptor())), 0} + switch kind { + case 1: + product := byte(0x10) + if d.pointer { + product = 0x11 + } + return []byte{18, 1, 0x10, 1, 0, 0, 0, 8, 0x34, 0x12, product, 0xcc, 0, 1, 1, 2, 0, 1} + case 2: + sub, proto := byte(1), byte(1) + if d.pointer { + sub, proto = 0, 0 + } + out := []byte{9, 2, 34, 0, 1, 1, 0, 0x80, 50, 9, 4, 0, 0, 1, 3, sub, proto, 0} + out = append(out, hid...) + return append(out, 7, 5, 0x81, 3, byte(len(d.report)), 0, 10) + case 3: + switch index { + case 0: + return []byte{4, 3, 9, 4} + case 1: + return usbString("TinyRangeX") + case 2: + if d.pointer { + return usbString("trex absolute pointer") + } + return usbString("trex USB keyboard") + } + case 0x21: + return hid + case 0x22: + return d.reportDescriptor() + } + return nil +} + +func (d *usbHID) setup(data []byte) bool { + if len(data) != 8 { + return false + } + copy(d.request[:], data) + d.reply = nil + d.replyOffset = 0 + d.output = nil + d.stalled = false + d.addressPending = false + typ, request := data[0], data[1] + value, index, length := binary.LittleEndian.Uint16(data[2:]), binary.LittleEndian.Uint16(data[4:]), binary.LittleEndian.Uint16(data[6:]) + d.controls++ + row := map[string]any{"type": int(typ), "request": int(request), "value": int(value), "index": int(index), "length": int(length)} + if len(d.trace) == 32 { + d.trace = d.trace[1:] + } + d.trace = append(d.trace, row) + switch typ & 0x60 { + case 0: + switch request { + case 6: + if typ&0x80 == 0 { + d.stalled = true + break + } + d.reply = d.descriptor(byte(value>>8), byte(value)) + d.stalled = d.reply == nil + case 5: + if typ != 0 || value > 127 || index != 0 || length != 0 { + d.stalled = true + break + } + d.pendingAddress = byte(value) + d.addressPending = true + case 9: + if typ != 0 || value > 1 || index != 0 || length != 0 { + d.stalled = true + break + } + d.configuration = byte(value) + d.pending = nil + if value != 0 { + d.pending = append(d.pending, append([]byte(nil), d.report...)) + } + case 8: + d.reply = []byte{d.configuration} + case 0: + d.reply = []byte{0, 0} + case 1: // Endpoint CLEAR_FEATURE(HALT). + d.stalled = typ != 2 || value != 0 + case 10: + d.reply = []byte{0} // GET_INTERFACE + case 11: + d.stalled = value != 0 || index != 0 // SET_INTERFACE + default: + d.stalled = true + } + case 0x20: + if index != 0 { + d.stalled = true + break + } + switch request { + case 1: + d.reply = append([]byte(nil), d.report...) // GET_REPORT + case 2: + d.reply = []byte{d.idle} + case 3: + d.reply = []byte{d.protocol} + case 9: + d.stalled = length > 8 // SET_REPORT (keyboard LEDs). + case 10: + d.idle = byte(value >> 8) + case 11: + if value > 1 { + d.stalled = true + } else { + d.protocol = byte(value) + } + default: + d.stalled = true + } + default: + d.stalled = true + } + if len(d.reply) > int(length) { + d.reply = d.reply[:length] + } + return !d.stalled +} + +// transaction returns USB ACK/NAK/STALL as 0/1/2. Control status applies the +// pending address only after successful completion, so address zero remains +// valid throughout SET_ADDRESS. Interrupt queues preserve every transition. +func (d *usbHID) transaction(pid byte, endpoint byte, data []byte) (int, int) { + if pid == 0x2d { + if endpoint == 0 && d.setup(data) { + return len(data), 0 + } + return 0, 2 + } + if endpoint == 1 && pid == 0x69 { + if d.configuration == 0 || len(d.pending) == 0 { + return 0, 1 + } + if len(data) < len(d.pending[0]) { + return 0, 2 + } + n := copy(data, d.pending[0]) + d.pending = d.pending[1:] + d.reports++ + return n, 0 + } + if endpoint != 0 || d.stalled { + return 0, 2 + } + input := d.request[0]&0x80 != 0 + length := int(binary.LittleEndian.Uint16(d.request[6:])) + if pid == 0x69 && (input && length != 0) { + n := copy(data, d.reply[d.replyOffset:]) + d.replyOffset += n + return n, 0 + } + if pid == 0xe1 && !input && length != 0 { + if len(d.output)+len(data) > length || len(d.output)+len(data) > 8 { + return 0, 2 + } + d.output = append(d.output, data...) + return len(data), 0 + } + if (pid == 0x69 && !input) || (pid == 0xe1 && input && len(data) == 0) { + if d.addressPending { + d.address = d.pendingAddress + d.addressPending = false + } + return 0, 0 + } + return 0, 2 +} +func (d *usbHID) enqueue(report []byte) error { + if d.configuration == 0 { + return fmt.Errorf("USB HID device is not configured") + } + if len(d.pending) >= 1024 { + return fmt.Errorf("USB HID report queue full") + } + d.report = append(d.report[:0], report...) + d.pending = append(d.pending, append([]byte(nil), report...)) + return nil +} + +var usbKeys = map[string]byte{ + "enter": 40, "ret": 40, "esc": 41, "escape": 41, "backspace": 42, "tab": 43, "space": 44, "spc": 44, + "minus": 45, "equal": 46, "bracket_left": 47, "bracket_right": 48, "backslash": 49, + "semicolon": 51, "apostrophe": 52, "grave_accent": 53, "comma": 54, "dot": 55, "slash": 56, "caps_lock": 57, + "insert": 73, "home": 74, "pgup": 75, "delete": 76, "end": 77, "pgdn": 78, "right": 79, "left": 80, "down": 81, "up": 82, + "ctrl": 224, "control": 224, "shift": 225, "alt": 226, "meta_l": 227, "ctrl_r": 228, "shift_r": 229, "alt_r": 230, "meta_r": 231, +} + +func usbKey(name string) (byte, bool) { + name = strings.ToLower(name) + if len(name) == 1 && name[0] >= 'a' && name[0] <= 'z' { + return name[0] - 'a' + 4, true + } + if len(name) == 1 && name[0] >= '1' && name[0] <= '9' { + return name[0] - '1' + 30, true + } + if name == "0" { + return 39, true + } + for n := 1; n <= 12; n++ { + if name == fmt.Sprintf("f%d", n) { + return byte(57 + n), true + } + } + v, ok := usbKeys[name] + return v, ok +} +func (d *usbHID) key(name string, down bool) error { + code, ok := usbKey(name) + if !ok { + return fmt.Errorf("unknown USB keyboard key %q", name) + } + r := append([]byte(nil), d.report...) + if code >= 224 { + bit := byte(1 << (code - 224)) + if down { + r[0] |= bit + } else { + r[0] &^= bit + } + } else { + found := false + for i := 2; i < 8; i++ { + if r[i] == code { + found = true + if !down { + r[i] = 0 + } + } + } + if down && !found { + inserted := false + for i := 2; i < 8; i++ { + if r[i] == 0 { + r[i] = code + inserted = true + break + } + } + if !inserted { + return fmt.Errorf("USB boot keyboard supports six simultaneous keys") + } + } + } + return d.enqueue(r) +} +func (d *usbHID) point(x, y uint16, buttons byte, wheel int) error { + r := []byte{buttons, byte(x), byte(x >> 8), byte(y), byte(y >> 8), byte(wheel)} + return d.enqueue(r) +} diff --git a/vmm/cc/usb_input.go b/vmm/cc/usb_input.go new file mode 100644 index 0000000..1079b65 --- /dev/null +++ b/vmm/cc/usb_input.go @@ -0,0 +1,8 @@ +package cc + +func (p *pc) keyEvent(name string, down bool) error { + if p.uhci != nil { + return p.uhci.devices[0].key(name, down) + } + return p.keyboard.key(name, down) +} diff --git a/vmm/darwin.go b/vmm/darwin.go new file mode 100644 index 0000000..2f879a2 --- /dev/null +++ b/vmm/darwin.go @@ -0,0 +1,30 @@ +package vmm + +import ( + "fmt" + "github.com/tinyrange/trex/boot/darwin" + "github.com/tinyrange/trex/storage" + "strings" +) + +// DarwinBoot requests the unslid amd64 XNU v2 handoff, not UEFI image execution. +type DarwinBoot struct { + Kernel storage.Reader + CommandLine string + // SMCOSK is optional caller-supplied 64-byte SMC key material. Nil leaves + // OSK0/OSK1 absent; the platform never manufactures these keys. + SMCOSK []byte +} + +func (b DarwinBoot) Validate() error { + if b.Kernel == nil || b.Kernel.Size() < 4 || b.Kernel.Size() > darwin.MaxKernelSize { + return fmt.Errorf("darwin_boot: kernel size outside bounds") + } + if len(b.CommandLine) > 1023 || strings.ContainsRune(b.CommandLine, 0) { + return fmt.Errorf("darwin_boot: invalid command line") + } + if len(b.SMCOSK) != 0 && len(b.SMCOSK) != 64 { + return fmt.Errorf("darwin_boot: smc_osk must be empty or exactly 64 bytes") + } + return nil +} diff --git a/vmm/qemu/config.go b/vmm/qemu/config.go index 4844a0d..1b18724 100644 --- a/vmm/qemu/config.go +++ b/vmm/qemu/config.go @@ -117,6 +117,9 @@ func (b *qemuBackend) ID() string { return "qemu.v1" } func (b *qemuBackend) Capabilities() []string { return append([]string(nil), b.capabilities...) } func (b *qemuBackend) Validate(machine vmmapi.Machine) []vmmapi.ValidationIssue { var issues []vmmapi.ValidationIssue + if machine.DarwinBoot != nil { + issues = append(issues, vmmapi.ValidationIssue{Code: "qemu.darwin_boot", Field: "boot", Message: "Darwin direct boot is not supported by this backend"}) + } if machine.Boot != nil { if runtime.GOOS != "linux" || (machine.Architecture != "i386" && machine.Architecture != "x86_64") { issues = append(issues, vmmapi.ValidationIssue{Code: "qemu.linux_boot", Field: "boot", Message: "direct Linux boot currently requires a Linux host and x86 guest"}) diff --git a/vmm/star/backend.go b/vmm/star/backend.go index 6ef5fcd..8397489 100644 --- a/vmm/star/backend.go +++ b/vmm/star/backend.go @@ -115,6 +115,9 @@ func validateVMM(machine VMMMachine, backend VMMBackend) []VMMValidationIssue { if machine.Boot != nil { require("boot.linux", "boot", "direct Linux boot", true) } + if machine.DarwinBoot != nil { + require("boot.darwin", "boot", "direct Darwin boot", true) + } if machine.StartPaused { require("lifecycle.pause", "start_paused", "paused startup", true) } @@ -169,6 +172,14 @@ func validateVMMCore(machine VMMMachine) []VMMValidationIssue { if machine.CPUs <= 0 { add("machine.cpus", "cpus", "CPU count must be positive") } + if machine.Boot != nil && machine.DarwinBoot != nil { + add("machine.boot", "boot", "Linux and Darwin boot requests are mutually exclusive") + } + if machine.DarwinBoot != nil { + if err := machine.DarwinBoot.Validate(); err != nil { + add("machine.boot", "boot", err.Error()) + } + } if err := machine.Boot.Validate(); err != nil { add("machine.boot", "boot", err.Error()) } diff --git a/vmm/star/boot_test.go b/vmm/star/boot_test.go index e859859..f7c776e 100644 --- a/vmm/star/boot_test.go +++ b/vmm/star/boot_test.go @@ -66,3 +66,38 @@ assertion = "boot" in dir(machine) and machine.boot.command_line == boot.command t.Fatal("boot intent lost during composition or introspection") } } + +func TestDarwinBootValidationBeforeLaunch(t *testing.T) { + kernel := &starfile.Bytes{Name: "kernel", Data: []byte("kernel")} + for _, tc := range []struct { + name string + boot *vmmapi.DarwinBoot + linux *vmmapi.LinuxBoot + caps []string + valid bool + }{ + {"supported", &vmmapi.DarwinBoot{Kernel: kernel}, nil, []string{"boot.darwin"}, true}, + {"missing-capability", &vmmapi.DarwinBoot{Kernel: kernel}, nil, nil, false}, + {"missing-kernel", &vmmapi.DarwinBoot{}, nil, []string{"boot.darwin"}, false}, + {"bad-smc-key-length", &vmmapi.DarwinBoot{Kernel: kernel, SMCOSK: make([]byte, 63)}, nil, []string{"boot.darwin"}, false}, + {"explicit-smc-key", &vmmapi.DarwinBoot{Kernel: kernel, SMCOSK: make([]byte, 64)}, nil, []string{"boot.darwin"}, true}, + {"nul-command-line", &vmmapi.DarwinBoot{Kernel: kernel, CommandLine: "-v\x00-s"}, nil, []string{"boot.darwin"}, false}, + {"conflicting-intents", &vmmapi.DarwinBoot{Kernel: kernel}, &vmmapi.LinuxBoot{Kernel: kernel}, []string{"boot.darwin", "boot.linux"}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + thread, _, _ := newStarlarkRuntime("-") + resources, _ := lifecycle.ForThread(thread) + defer resources.Close() + backend := &fakeVMMBackend{capabilities: tc.caps} + machine := &vmmMachineValue{machine: VMMMachine{Architecture: "x86_64", Memory: 256 << 20, CPUs: 1, Display: VMMDisplay{Mode: "none"}, Boot: tc.linux, DarwinBoot: tc.boot}} + _, err := vmmStartBuiltin(thread, nil, starlark.Tuple{machine, &fakeVMMBackendValue{backend: backend}}, nil) + if tc.valid { + if err != nil || backend.starts != 1 { + t.Fatal(err, backend.starts) + } + } else if err == nil || backend.starts != 0 { + t.Fatal("invalid boot reached backend", err, backend.starts) + } + }) + } +} diff --git a/vmm/star/darwin.go b/vmm/star/darwin.go new file mode 100644 index 0000000..e84d667 --- /dev/null +++ b/vmm/star/darwin.go @@ -0,0 +1,45 @@ +package star + +import ( + "fmt" + + "github.com/tinyrange/trex/storage" + starfile "github.com/tinyrange/trex/storage/star" + vmmapi "github.com/tinyrange/trex/vmm" + "go.starlark.net/starlark" +) + +type darwinBootValue struct{ boot vmmapi.DarwinBoot } + +func (v *darwinBootValue) String() string { return "" } +func (v *darwinBootValue) Type() string { return "vmm_darwin_boot" } +func (v *darwinBootValue) Freeze() {} +func (v *darwinBootValue) Truth() starlark.Bool { return starlark.True } +func (v *darwinBootValue) Hash() (uint32, error) { return 0, fmt.Errorf("unhashable: vmm_darwin_boot") } +func (v *darwinBootValue) AttrNames() []string { return []string{"kernel", "command_line"} } +func (v *darwinBootValue) Attr(name string) (starlark.Value, error) { + switch name { + case "kernel": + return starfile.NewReader("Darwin kernel", v.boot.Kernel), nil + case "command_line": + return starlark.String(v.boot.CommandLine), nil + } + return nil, nil +} +func darwinBootBuiltin(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var kernel starlark.Value + commandLine := "" + var osk starlark.Bytes + if err := starlark.UnpackArgs("darwin_boot", args, kwargs, "kernel", &kernel, "command_line?", &commandLine, "smc_osk?", &osk); err != nil { + return nil, err + } + file, ok := kernel.(storage.Reader) + if !ok { + return nil, fmt.Errorf("darwin_boot: kernel must be a file") + } + boot := vmmapi.DarwinBoot{Kernel: file, CommandLine: commandLine, SMCOSK: []byte(osk)} + if err := boot.Validate(); err != nil { + return nil, err + } + return &darwinBootValue{boot: boot}, nil +} diff --git a/vmm/star/value.go b/vmm/star/value.go index 994cf98..3be1a4c 100644 --- a/vmm/star/value.go +++ b/vmm/star/value.go @@ -58,17 +58,18 @@ func RegisterBackend(id string, capabilities []string) { func Builtins() starlark.StringDict { return starlark.StringDict{ - "backends": starlark.NewBuiltin("backends", vmmBackendsBuiltin), - "channel": starlark.NewBuiltin("channel", vmmChannelBuiltin), - "disk": starlark.NewBuiltin("disk", vmmDiskBuiltin), - "display": starlark.NewBuiltin("display", vmmDisplayBuiltin), - "machine": starlark.NewBuiltin("machine", vmmMachineBuiltin), - "linux_boot": starlark.NewBuiltin("linux_boot", linuxBootBuiltin), - "network": starlark.NewBuiltin("network", vmmNetworkBuiltin), - "switch": starlark.NewBuiltin("switch", switchBuiltin), - "workspace": starlark.NewBuiltin("workspace", workspaceBuiltin), - "start": starlark.NewBuiltin("start", vmmStartBuiltin), - "validate": starlark.NewBuiltin("validate", vmmValidateBuiltin), + "backends": starlark.NewBuiltin("backends", vmmBackendsBuiltin), + "channel": starlark.NewBuiltin("channel", vmmChannelBuiltin), + "disk": starlark.NewBuiltin("disk", vmmDiskBuiltin), + "display": starlark.NewBuiltin("display", vmmDisplayBuiltin), + "machine": starlark.NewBuiltin("machine", vmmMachineBuiltin), + "linux_boot": starlark.NewBuiltin("linux_boot", linuxBootBuiltin), + "darwin_boot": starlark.NewBuiltin("darwin_boot", darwinBootBuiltin), + "network": starlark.NewBuiltin("network", vmmNetworkBuiltin), + "switch": starlark.NewBuiltin("switch", switchBuiltin), + "workspace": starlark.NewBuiltin("workspace", workspaceBuiltin), + "start": starlark.NewBuiltin("start", vmmStartBuiltin), + "validate": starlark.NewBuiltin("validate", vmmValidateBuiltin), } } @@ -189,6 +190,9 @@ func (v *vmmMachineValue) Hash() (uint32, error) { return 0, fmt.Errorf("unhasha func (v *vmmMachineValue) Attr(name string) (starlark.Value, error) { switch name { case "boot": + if v.machine.DarwinBoot != nil { + return &darwinBootValue{boot: *v.machine.DarwinBoot}, nil + } if v.machine.Boot == nil { return starlark.None, nil } @@ -400,12 +404,16 @@ func vmmMachineBuiltin(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tu } machine := VMMMachine{Architecture: architecture, Memory: memory, CPUs: cpus, Display: display.display, StartPaused: startPaused} if bootValue != starlark.None { - boot, ok := bootValue.(*linuxBootValue) - if !ok { - return nil, fmt.Errorf("machine: boot must be vmm_linux_boot or None") + switch boot := bootValue.(type) { + case *linuxBootValue: + copy := boot.boot + machine.Boot = © + case *darwinBootValue: + copy := boot.boot + machine.DarwinBoot = © + default: + return nil, fmt.Errorf("machine: boot must be vmm_linux_boot, vmm_darwin_boot or None") } - copy := boot.boot - machine.Boot = © } if err := appendTypedList(disksValue, "disks", func(index int, value starlark.Value) error { disk, ok := value.(*vmmDiskValue) diff --git a/vmm/types.go b/vmm/types.go index 185e45e..4364dda 100644 --- a/vmm/types.go +++ b/vmm/types.go @@ -72,6 +72,7 @@ type Channel struct { type Machine struct { Boot *LinuxBoot + DarwinBoot *DarwinBoot Architecture string Memory int64 CPUs int From 851d934108a168782137a0f3007d7179aec5e886 Mon Sep 17 00:00:00 2001 From: Joshua Scarsbrook Date: Wed, 7 Oct 2026 11:21:03 +1000 Subject: [PATCH 2/2] Pin cc to merged native timing and legacy timer support --- cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cc b/cc index e0e7f59..69e22fb 160000 --- a/cc +++ b/cc @@ -1 +1 @@ -Subproject commit e0e7f59c269b6f7269b8eb662651ee21237d5208 +Subproject commit 69e22fb67f7a907428cc32093ed662081128a634