Add native macOS media construction and original Lion desktop support

archive/bom/bom.go

@@ -30,7 +30,31 @@ func Open(prefix []byte, source storage.Reader, o auto.Options) (auto.View, erro
 	if !bytes.HasPrefix(prefix, []byte("BOMStore")) {
 		return nil, auto.ErrNoMatch
 	}
-	limit := o.MaxEntries
+	entries, err := Read(source, o.MaxEntries)
+	if err != nil {
+		return nil, err
+	}
+	seen := map[string]bool{}
+	var treeEntries []auto.Entry
+	for i := range entries {
+		e := &entries[i]
+		if e.Name == "." {
+			continue
+		}
+		if seen[e.Name] {
+			e.Attributes["bom_path"] = e.Name
+			e.Name = fmt.Sprintf("%%00bom-variants/%v", e.Attributes["bom_id"])
+		} else {
+			seen[e.Name] = true
+		}
+		treeEntries = append(treeEntries, *e)
+	}
+	return auto.Tree(treeEntries, o)
+}
+
+// Read returns every flat inventory record, including architecture variants
+// sharing a path. A BOM is metadata, never a substitute for payload contents.
+func Read(source storage.Reader, limit int) ([]auto.Entry, error) {
 	if limit <= 0 {
 		limit = 100000
 	}
@@ -46,7 +70,7 @@ func Open(prefix []byte, source storage.Reader, o auto.Options) (auto.View, erro
 	if err != nil {
 		return nil, err
 	}
-	if be.Uint32(h[8:]) != 1 {
+	if string(h[:8]) != "BOMStore" || be.Uint32(h[8:]) != 1 {
 		return nil, fmt.Errorf("bom: unsupported version")
 	}
 	index, err := read(uint64(be.Uint32(h[16:])), uint64(be.Uint32(h[20:])))
@@ -121,7 +145,23 @@ func Open(prefix []byte, source storage.Reader, o auto.Options) (auto.View, erro
 	records := map[uint32]*record{}
 	var order []uint32
 	var keyCount uint64
-	for len(pending) > 0 {
+	leafLinks := map[uint32][2]uint32{}
+	followingLinks := false
+	for {
+		if len(pending) == 0 {
+			// Apple's branch index can omit the final partial leaf. The leaf
+			// chain is authoritative; retain branch validation, then follow
+			// its missing tails rather than dropping inventory records.
+			followingLinks = true
+			for _, link := range leafLinks {
+				if link[0] != 0 && !visited[link[0]] {
+					pending = append(pending, link[0])
+				}
+			}
+			if len(pending) == 0 {
+				break
+			}
+		}
 		id := pending[len(pending)-1]
 		pending = pending[:len(pending)-1]
 		if visited[id] {
@@ -139,6 +179,12 @@ func Open(prefix []byte, source storage.Reader, o auto.Options) (auto.View, erro
 		if leaf > 1 || n > (len(b)-12)/8 {
 			return nil, fmt.Errorf("bom: invalid tree node")
 		}
+		if followingLinks && leaf != 1 {
+			return nil, fmt.Errorf("bom: leaf chain points to branch")
+		}
+		if leaf == 1 {
+			leafLinks[id] = [2]uint32{be.Uint32(b[4:]), be.Uint32(b[8:])}
+		}
 		keyCount += uint64(n)
 		for i := 0; i < n; i++ {
 			pair := b[12+i*8:]
@@ -204,6 +250,34 @@ func Open(prefix []byte, source storage.Reader, o auto.Options) (auto.View, erro
 			order = append(order, r.id)
 		}
 	}
+	// Validate the complete doubly-linked leaf chain, including tails missing
+	// from the branch index. Multiple heads and cycles are malformed.
+	var head uint32
+	for id, link := range leafLinks {
+		if link[1] == 0 {
+			if head != 0 {
+				return nil, fmt.Errorf("bom: multiple leaf chain heads")
+			}
+			head = id
+		} else if back, ok := leafLinks[link[1]]; !ok || back[0] != id {
+			return nil, fmt.Errorf("bom: inconsistent leaf backlink")
+		}
+		if link[0] != 0 {
+			if next, ok := leafLinks[link[0]]; !ok || next[1] != id {
+				return nil, fmt.Errorf("bom: inconsistent leaf forward link")
+			}
+		}
+	}
+	seenLeaves := map[uint32]bool{}
+	for id := head; id != 0; id = leafLinks[id][0] {
+		if seenLeaves[id] {
+			return nil, fmt.Errorf("bom: cyclic leaf chain")
+		}
+		seenLeaves[id] = true
+	}
+	if len(seenLeaves) != len(leafLinks) {
+		return nil, fmt.Errorf("bom: disconnected leaf chain")
+	}
 	// Apple also counts internal separator keys in some BOM generations.
 	if uint32(len(records)) != expected && keyCount != uint64(expected) {
 		return nil, fmt.Errorf("bom: path count mismatch: %d != %d", len(records), expected)
@@ -240,7 +314,6 @@ func Open(prefix []byte, source storage.Reader, o auto.Options) (auto.View, erro
 		return name, nil
 	}
 	var entries []auto.Entry
-	seenPaths := map[string]bool{}
 	for _, id := range order {
 		r := records[id]
 		name, err := resolve(id, map[uint32]bool{})
@@ -251,14 +324,9 @@ func Open(prefix []byte, source storage.Reader, o auto.Options) (auto.View, erro
 			if r.entry.Kind != "directory" {
 				return nil, fmt.Errorf("bom: invalid root")
 			}
-			continue
-		}
-		if seenPaths[name] {
-			return nil, fmt.Errorf("bom: duplicate inventory path")
 		}
-		seenPaths[name] = true
 		r.entry.Name = name
 		entries = append(entries, r.entry)
 	}
-	return auto.Tree(entries, o)
+	return entries, nil
 }

archive/bom/bom_test.go

@@ -6,7 +6,8 @@ import (
 	"testing"
 )
 
-func fixture(branch bool) []byte {
+func fixture(branch bool) []byte { return variantFixture(branch, false, false) }
+func variantFixture(branch, variants, tail bool) []byte {
 	b := make([]byte, 32)
 	pointers := []pointer{{}}
 	add := func(data []byte) uint32 {
@@ -16,7 +17,11 @@ func fixture(branch bool) []byte {
 		return id
 	}
 	var pairs [][2]uint32
-	for i, name := range []string{".", "hello", "link"} {
+	names := []string{".", "hello", "link"}
+	if variants {
+		names = append(names, "hello")
+	}
+	for i, name := range names {
 		meta := make([]byte, 27)
 		meta[0] = 1
 		if i == 0 {
@@ -29,6 +34,7 @@ func fixture(branch bool) []byte {
 			be.PutUint32(meta[27:], 6)
 			copy(meta[31:], "hello")
 		}
+		be.PutUint16(meta[2:], uint16(i))
 		be.PutUint16(meta[4:], 0644)
 		be.PutUint32(meta[6:], 501)
 		be.PutUint32(meta[18:], 17)
@@ -46,6 +52,10 @@ func fixture(branch bool) []byte {
 		kid := add(key)
 		pairs = append(pairs, [2]uint32{iid, kid})
 	}
+	allPairs := pairs
+	if tail {
+		pairs = pairs[:len(pairs)-1]
+	}
 	leaf := make([]byte, 12+8*len(pairs))
 	be.PutUint16(leaf, 1)
 	be.PutUint16(leaf[2:], uint16(len(pairs)))
@@ -54,7 +64,18 @@ func fixture(branch bool) []byte {
 		be.PutUint32(leaf[16+i*8:], p[1])
 	}
 	child := add(leaf)
-	count := uint32(3)
+	count := uint32(len(allPairs))
+	if tail {
+		last := allPairs[len(allPairs)-1]
+		node := make([]byte, 20)
+		be.PutUint16(node, 1)
+		be.PutUint16(node[2:], 1)
+		be.PutUint32(node[8:], child)
+		be.PutUint32(node[12:], last[0])
+		be.PutUint32(node[16:], last[1])
+		next := add(node)
+		be.PutUint32(b[pointers[child].off+4:], next)
+	}
 	if branch {
 		node := make([]byte, 20)
 		be.PutUint16(node[2:], 1)
@@ -132,3 +153,39 @@ func TestMalformedBOM(t *testing.T) {
 		}
 	}
 }
+
+func TestTrailingLeafAndArchitectureVariants(t *testing.T) {
+	for _, tail := range []bool{false, true} {
+		b := variantFixture(true, true, tail)
+		records, err := Read(&starfile.Bytes{Data: b}, 100)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(records) != 4 {
+			t.Fatalf("dropped records: %d", len(records))
+		}
+		var variants int
+		for _, e := range records {
+			if e.Name == "hello" {
+				variants++
+			}
+		}
+		if variants != 2 {
+			t.Fatal("architecture variants lost")
+		}
+		root := auto.Open(&starfile.Bytes{Data: b}, "Bom", auto.Options{})
+		n, err := root.Resolve("%00bom-variants/4")
+		if err != nil || n.Summary().Attributes["bom_path"] != "hello" {
+			t.Fatalf("variant tree %v %v", n, err)
+		}
+		if tail {
+			// The partial final leaf must point back to its predecessor.
+			index := be.Uint32(b[16:])
+			last := be.Uint32(b[index+4+14*8:]) // leaf after four sets of metadata blocks
+			be.PutUint32(b[last+8:], 0)
+			if _, err := Read(&starfile.Bytes{Data: b}, 100); err == nil {
+				t.Fatal("accepted disconnected leaf chain")
+			}
+		}
+	}
+}

archive/bom/star.go

@@ -0,0 +1,42 @@
+package bom
+
+import (
+	"fmt"
+	"github.com/tinyrange/trex/storage"
+	starfile "github.com/tinyrange/trex/storage/star"
+	"go.starlark.net/starlark"
+)
+
+func Builtin(_ *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	var value starlark.Value
+	maximum := 1000000
+	if err := starlark.UnpackArgs("bom", args, kwargs, "file", &value, "maximum_entries?", &maximum); err != nil {
+		return nil, err
+	}
+	source, ok := value.(storage.Reader)
+	if !ok || maximum <= 0 {
+		return nil, fmt.Errorf("bom: expected file and positive maximum_entries")
+	}
+	entries, err := Read(source, maximum)
+	if err != nil {
+		return nil, err
+	}
+	result := make([]starlark.Value, len(entries))
+	for i, e := range entries {
+		record := starlark.StringDict{"path": starlark.String(e.Name), "kind": starlark.String(e.Kind)}
+		for k, v := range e.Attributes {
+			switch v := v.(type) {
+			case uint32:
+				record[k] = starlark.MakeUint64(uint64(v))
+			case uint16:
+				record[k] = starlark.MakeUint64(uint64(v))
+			case string:
+				record[k] = starlark.String(v)
+			case bool:
+				record[k] = starlark.Bool(v)
+			}
+		}
+		result[i] = starfile.NewRecord(record)
+	}
+	return starlark.NewList(result), nil
+}

archive/gzip/materialize.go

@@ -0,0 +1,42 @@
+package gzip
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/tinyrange/trex/storage"
+)
+
+// Materialize decodes and validates the entire stream into bounded volatile
+// pages. Unlike Open's replaying window, the returned immutable reader supports
+// arbitrary access without consulting the compressed source again. No result
+// is returned after an integrity failure or limit overflow.
+func Materialize(source storage.Reader, maximum int64) (storage.Reader, error) {
+	if maximum <= 0 {
+		return nil, fmt.Errorf("gzip: materialize requires positive maximum_bytes")
+	}
+	f, err := Open(source, maximum)
+	if err != nil {
+		return nil, err
+	}
+	defer f.decoder.Close()
+	store := storage.NewMemoryStore(maximum)
+	defer store.Close()
+	var buf [64 << 10]byte
+	var off int64
+	for {
+		n, err := f.read(buf[:])
+		if err != nil && err != io.EOF {
+			return nil, err
+		}
+		if n != 0 {
+			if _, e := store.WriteAt(buf[:n], off); e != nil {
+				return nil, e
+			}
+			off += int64(n)
+		}
+		if err == io.EOF {
+			return store.Snapshot()
+		}
+	}
+}

archive/gzip/materialize_test.go

@@ -0,0 +1,64 @@
+package gzip
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"testing"
+
+	"github.com/tinyrange/trex/auto"
+	starfile "github.com/tinyrange/trex/storage/star"
+)
+
+type guardedSource struct {
+	data      []byte
+	forbidden bool
+	reads     int
+}
+
+func (s *guardedSource) Size() int64 { return int64(len(s.data)) }
+func (s *guardedSource) ReadAt(p []byte, off int64) (int, error) {
+	s.reads++
+	if s.forbidden {
+		return 0, errors.New("compressed source consulted after materialization")
+	}
+	return bytes.NewReader(s.data).ReadAt(p, off)
+}
+func TestMaterializeIndependentRandomAccessAndIntegrity(t *testing.T) {
+	data := bytes.Repeat([]byte("borrowed random-access package bytes"), 100000)
+	source := &guardedSource{data: encode(t, data)}
+	source.data = append(source.data, encode(t, []byte("tail"))...)
+	source.data = append(source.data, 0, 0, 0)
+	r, err := Materialize(source, int64(len(data)+4))
+	if err != nil {
+		t.Fatal(err)
+	}
+	source.forbidden = true
+	reads := source.reads
+	want := append(data, []byte("tail")...)
+	for _, off := range []int64{int64(len(data)) - 10, 0, 1000000, 13} {
+		var got [14]byte
+		if _, err := r.ReadAt(got[:], off); err != nil || !bytes.Equal(got[:], want[off:off+14]) {
+			t.Fatal(off, err)
+		}
+	}
+	if source.reads != reads || r.Size() != int64(len(want)) {
+		t.Fatal("replayed source or incorrect size")
+	}
+	if n, err := r.ReadAt(make([]byte, 8), r.Size()-4); n != 4 || err != io.EOF {
+		t.Fatal(n, err)
+	}
+	bad := bytes.Clone(encode(t, data))
+	bad[len(bad)-8] ^= 1
+	for _, encoded := range [][]byte{bad, bad[:len(bad)-3]} {
+		if r, err := Materialize(&starfile.Bytes{Data: encoded}, int64(len(data))); r != nil || err == nil {
+			t.Fatal("invalid stream returned", r, err)
+		}
+	}
+	if r, err := Materialize(&starfile.Bytes{Data: encode(t, data)}, int64(len(data)-1)); r != nil || !errors.Is(err, auto.ErrLimit) {
+		t.Fatal(r, err)
+	}
+	if r, err := Materialize(source, 0); r != nil || err == nil {
+		t.Fatal("unbounded cache", r, err)
+	}
+}