diff --git a/cmd/syft/internal/options/catalog.go b/cmd/syft/internal/options/catalog.go index 0d7e94b5022..3ff68ecbd57 100644 --- a/cmd/syft/internal/options/catalog.go +++ b/cmd/syft/internal/options/catalog.go @@ -199,7 +199,8 @@ func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config { WithFromLDFlags(cfg.Golang.MainModuleVersion.FromLDFlags), ). WithUsePackagesLib(*multiLevelOption(true, enrichmentEnabled(cfg.Enrich, task.Go, task.Golang), cfg.Golang.UsePackagesLib)). - WithCaptureSymbols(cfg.Golang.CaptureSymbols), + WithCaptureSymbols(cfg.Golang.CaptureSymbols). + WithCaptureSymbolsModules(cfg.Golang.CaptureSymbolsModules), JavaScript: javascript.DefaultCatalogerConfig(). WithIncludeDevDependencies(*multiLevelOption(false, cfg.JavaScript.IncludeDevDependencies)). WithSearchRemoteLicenses(*multiLevelOption(false, enrichmentEnabled(cfg.Enrich, task.JavaScript, task.Node, task.NPM), cfg.JavaScript.SearchRemoteLicenses)). diff --git a/cmd/syft/internal/options/golang.go b/cmd/syft/internal/options/golang.go index c811a34c307..e8147a06d93 100644 --- a/cmd/syft/internal/options/golang.go +++ b/cmd/syft/internal/options/golang.go @@ -4,6 +4,7 @@ import ( "strings" "github.com/anchore/clio" + "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/cataloging" "github.com/anchore/syft/syft/pkg/cataloger/golang" ) @@ -19,6 +20,7 @@ type golangConfig struct { MainModuleVersion golangMainModuleVersionConfig `json:"main-module-version" yaml:"main-module-version" mapstructure:"main-module-version"` UsePackagesLib *bool `json:"use-packages-lib" yaml:"use-packages-lib" mapstructure:"use-packages-lib"` CaptureSymbols cataloging.SymbolScope `json:"capture-symbols" yaml:"capture-symbols" mapstructure:"capture-symbols"` + CaptureSymbolsModules []string `json:"capture-symbols-modules" yaml:"capture-symbols-modules" mapstructure:"capture-symbols-modules"` } var _ interface { @@ -42,8 +44,16 @@ if unset this defaults to $GONOPROXY`) always show (devel) as the version. Use these options to control heuristics to guess a more accurate version from the binary.`) descriptions.Add(&o.UsePackagesLib, `use the golang.org/x/tools/go/packages library, which executes golang tooling found on the path in addition to potential network access to get the most accurate results`) + // note: descriptions must be static string literals; the app config discovery that generates the + // capability docs reads them straight out of the AST descriptions.Add(&o.CaptureSymbols, `capture function symbols from the binary symbol table (pclntab). valid values are: -"none" (disabled), "stdlib" (only the synthetic stdlib package), and "all" (all module packages plus stdlib)`) +"none" (disabled), "stdlib" (only the synthetic stdlib package), "extended-stdlib" (stdlib plus every +module under golang.org/x/), and "all" (all module packages plus stdlib)`) + descriptions.Add(&o.CaptureSymbolsModules, `glob patterns matched against go module paths (e.g. github.com/klauspost/**) that should have symbols +captured in addition to whatever capture-symbols selects. ** crosses path separators, * does not. +a trailing major version suffix is ignored when matching, so github.com/foo/* covers github.com/foo/bar/v2; +spelling a suffix out in the pattern selects only that major version. +this can only widen the selection, never narrow it, and is inert when capture-symbols is none`) descriptions.Add(&o.MainModuleVersion.FromLDFlags, `look for LD flags that appear to be setting a version (e.g. -X main.version=1.0.0)`) descriptions.Add(&o.MainModuleVersion.FromBuildSettings, `use the build settings (e.g. vcs.version & vcs.time) to craft a v0 pseudo version (e.g. v0.0.0-20220308212642-53e6d0aaf6fb) when a more accurate version cannot be found otherwise`) @@ -51,7 +61,24 @@ a more accurate version from the binary.`) } func (o *golangConfig) PostLoad() error { + raw := strings.TrimSpace(string(o.CaptureSymbols)) o.CaptureSymbols = o.CaptureSymbols.Parse() + + // an unrecognized value still resolves to "none", but say so rather than silently capturing nothing. + // stay quiet for unset and an explicit "none", which is the default and would otherwise warn on nearly + // every scan. + if o.CaptureSymbols == cataloging.SymbolScopeNone && raw != "" && !strings.EqualFold(raw, string(cataloging.SymbolScopeNone)) { + log.Warnf("unknown golang.capture-symbols value %q, defaulting to %q (valid values: none, stdlib, extended-stdlib, all)", raw, cataloging.SymbolScopeNone) + } + + // trim only, deliberately not Flatten: viper already splits a comma-separated scalar (env var or a bare + // yaml string) into a slice before this point, and there is no CLI flag feeding this key. The one thing + // Flatten would add is splitting commas *inside* a list entry, which silently breaks doublestar brace + // alternation like github.com/{foo,bar}/**. Viper's split does not trim, so that part is still needed. + for i, pattern := range o.CaptureSymbolsModules { + o.CaptureSymbolsModules[i] = strings.TrimSpace(pattern) + } + return nil } @@ -76,7 +103,8 @@ func defaultGolangConfig() golangConfig { FromContents: def.MainModuleVersion.FromContents, FromBuildSettings: def.MainModuleVersion.FromBuildSettings, }, - UsePackagesLib: nil, // this defaults to true, which is the API default - CaptureSymbols: def.CaptureSymbols, + UsePackagesLib: nil, // this defaults to true, which is the API default + CaptureSymbols: def.CaptureSymbols, + CaptureSymbolsModules: def.CaptureSymbolsModules, } } diff --git a/cmd/syft/internal/options/golang_test.go b/cmd/syft/internal/options/golang_test.go index 64334990491..b8c1f13709c 100644 --- a/cmd/syft/internal/options/golang_test.go +++ b/cmd/syft/internal/options/golang_test.go @@ -10,10 +10,11 @@ import ( func Test_golangConfig_PostLoad(t *testing.T) { tests := []struct { - name string - cfg golangConfig - expected cataloging.SymbolScope - wantErr assert.ErrorAssertionFunc + name string + cfg golangConfig + expected cataloging.SymbolScope + expectedModules []string + wantErr assert.ErrorAssertionFunc }{ { name: "normalize all", @@ -25,6 +26,32 @@ func Test_golangConfig_PostLoad(t *testing.T) { cfg: golangConfig{CaptureSymbols: "stdlib"}, expected: cataloging.SymbolScopeStdlib, }, + { + name: "normalize extended-stdlib", + cfg: golangConfig{CaptureSymbols: " Extended-Stdlib "}, + expected: cataloging.SymbolScopeExtendedStdlib, + }, + { + name: "module patterns keep embedded commas", + cfg: golangConfig{ + CaptureSymbols: "stdlib", + // brace alternation contains a comma; splitting on it would corrupt the pattern + CaptureSymbolsModules: []string{"github.com/{foo,bar}/**", "golang.org/x/**"}, + }, + expected: cataloging.SymbolScopeStdlib, + expectedModules: []string{"github.com/{foo,bar}/**", "golang.org/x/**"}, + }, + { + // viper splits a comma-separated scalar (env var or bare yaml string) but does not trim, + // so a leading space would otherwise survive into a pattern that silently matches nothing + name: "module patterns are trimmed", + cfg: golangConfig{ + CaptureSymbols: "stdlib", + CaptureSymbolsModules: []string{"golang.org/x/**", " github.com/foo/** "}, + }, + expected: cataloging.SymbolScopeStdlib, + expectedModules: []string{"golang.org/x/**", "github.com/foo/**"}, + }, { name: "empty defaults to none", cfg: golangConfig{CaptureSymbols: ""}, @@ -32,7 +59,7 @@ func Test_golangConfig_PostLoad(t *testing.T) { }, { name: "invalid value defaults to none", - cfg: golangConfig{CaptureSymbols: "bogus"}, + cfg: golangConfig{CaptureSymbols: "stdlbi"}, expected: cataloging.SymbolScopeNone, }, { @@ -40,6 +67,16 @@ func Test_golangConfig_PostLoad(t *testing.T) { cfg: golangConfig{CaptureSymbols: "true"}, expected: cataloging.SymbolScopeNone, }, + { + name: "explicit none resolves to none", + cfg: golangConfig{CaptureSymbols: "none"}, + expected: cataloging.SymbolScopeNone, + }, + { + name: "explicit none is not case sensitive", + cfg: golangConfig{CaptureSymbols: " NONE "}, + expected: cataloging.SymbolScopeNone, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -52,6 +89,7 @@ func Test_golangConfig_PostLoad(t *testing.T) { return } assert.Equal(t, tt.expected, tt.cfg.CaptureSymbols) + assert.Equal(t, tt.expectedModules, tt.cfg.CaptureSymbolsModules) }) } } diff --git a/internal/capabilities/appconfig.yaml b/internal/capabilities/appconfig.yaml index 295df39df21..e15fdde0f10 100644 --- a/internal/capabilities/appconfig.yaml +++ b/internal/capabilities/appconfig.yaml @@ -15,7 +15,9 @@ application: # AUTO-GENERATED - application-level config keys - key: dotnet.relax-dll-claims-when-bundling-detected description: show all packages from the deps.json if bundling tooling is present as a dependency (e.g. ILRepack) - key: golang.capture-symbols - description: 'capture function symbols from the binary symbol table (pclntab). valid values are: "none" (disabled), "stdlib" (only the synthetic stdlib package), and "all" (all module packages plus stdlib)' + description: 'capture function symbols from the binary symbol table (pclntab). valid values are: "none" (disabled), "stdlib" (only the synthetic stdlib package), "extended-stdlib" (stdlib plus every module under golang.org/x/), and "all" (all module packages plus stdlib)' + - key: golang.capture-symbols-modules + description: glob patterns matched against go module paths (e.g. github.com/klauspost/**) that should have symbols captured in addition to whatever capture-symbols selects. ** crosses path separators, * does not. a trailing major version suffix is ignored when matching, so github.com/foo/* covers github.com/foo/bar/v2; spelling a suffix out in the pattern selects only that major version. this can only widen the selection, never narrow it, and is inert when capture-symbols is none - key: golang.local-mod-cache-dir description: specify an explicit go mod cache directory, if unset this defaults to $GOPATH/pkg/mod or $HOME/go/pkg/mod - key: golang.local-vendor-dir diff --git a/schema/json/README.md b/schema/json/README.md index 52fb639f56f..37fd50b0c54 100644 --- a/schema/json/README.md +++ b/schema/json/README.md @@ -35,3 +35,24 @@ Create the new schema by running `make generate-json-schema` from the root of th - If there is an existing schema for the given version and the new schema **does not** match the existing schema, an error is shown indicating to increment the version appropriately (see the "Versioning" section) ***Note: never delete a JSON schema and never change an existing JSON schema once it has been published in a release!*** Only add new schemas with a newly incremented version. All previous schema files must be stored in the `schema/json/` directory. + +### Exception: `description`-only corrections + +A published schema may be amended in place for one narrow case: the change touches only `description` text and leaves the data shape identical. Descriptions are documentation carried alongside the schema rather than constraints a validator evaluates, so correcting one cannot invalidate a document that already validated against that version. Minting a new version instead would leave the old one permanently describing the tool incorrectly, and spend a version number on no semantic change. + +This applies when **every** one of the following holds: + +- the only differences are `description` values +- no field, type, enum, `required` entry, or `$ref` is added, removed, or altered +- the `$id` version is unchanged + +Anything else, including adding a field that happens to be optional, is a schema change and needs a version bump per the "Versioning" section above. + +The generator blocks an in-place edit by design, since it refuses to overwrite a file that differs from what it would produce. To amend one, delete the schema file and regenerate so it is rewritten from the current Go doc comments: + +```bash +rm schema/json/schema-$VERSION.json +make generate-json-schema +``` + +The result is byte-identical to what the generator produces, so do not hand-edit the JSON. Re-running `make generate-json-schema` afterwards should report `No change to the existing schema!`, and `make check-json-schema-drift` should pass. Confirm with `git diff` that the only changes are the intended description lines. diff --git a/schema/json/schema-16.1.10.json b/schema/json/schema-16.1.10.json index 87756c843a9..37a6582388f 100644 --- a/schema/json/schema-16.1.10.json +++ b/schema/json/schema-16.1.10.json @@ -1662,7 +1662,7 @@ "type": "array" }, "type": "object", - "description": "Symbols are the function symbols from this module that are compiled into the binary, extracted from\nthe binary symbol table (pclntab) and grouped by the import path of the package that owns them. Each\nvalue is the sorted, deduplicated list of symbol names local to that package, i.e. with the import\npath prefix stripped (e.g. import path \"github.com/foo/bar\" -\u003e \"(*Type).Method\"). The fully qualified\nname is the import path, a \".\", and the local name. One exception: the binary's main package appears\nunder the key \"main\" (the name the linker assigns), not its original source import path, which is not\nrecoverable from the binary. Populated only when the golang cataloger's capture-symbols scope covers\nthis package: the \"all\" scope populates every module package plus the synthetic stdlib package, while\nthe \"stdlib\" scope populates only the stdlib package." + "description": "Symbols are the function symbols from this module that are compiled into the binary, extracted from\nthe binary symbol table (pclntab) and grouped by the import path of the package that owns them. Each\nvalue is the sorted, deduplicated list of symbol names local to that package, i.e. with the import\npath prefix stripped (e.g. import path \"github.com/foo/bar\" -\u003e \"(*Type).Method\"). The fully qualified\nname is the import path, a \".\", and the local name. One exception: the binary's main package appears\nunder the key \"main\" (the name the linker assigns), not its original source import path, which is not\nrecoverable from the binary. Populated only when the golang cataloger's capture-symbols scope covers\nthis package: the \"all\" scope populates every module package plus the synthetic stdlib package, the\n\"extended-stdlib\" scope populates the stdlib package plus every module under golang.org/x/, and the\n\"stdlib\" scope populates only the stdlib package. The capture-symbols-modules glob patterns populate\nany additional modules they match." } }, "type": "object", diff --git a/schema/json/schema-latest.json b/schema/json/schema-latest.json index 87756c843a9..37a6582388f 100644 --- a/schema/json/schema-latest.json +++ b/schema/json/schema-latest.json @@ -1662,7 +1662,7 @@ "type": "array" }, "type": "object", - "description": "Symbols are the function symbols from this module that are compiled into the binary, extracted from\nthe binary symbol table (pclntab) and grouped by the import path of the package that owns them. Each\nvalue is the sorted, deduplicated list of symbol names local to that package, i.e. with the import\npath prefix stripped (e.g. import path \"github.com/foo/bar\" -\u003e \"(*Type).Method\"). The fully qualified\nname is the import path, a \".\", and the local name. One exception: the binary's main package appears\nunder the key \"main\" (the name the linker assigns), not its original source import path, which is not\nrecoverable from the binary. Populated only when the golang cataloger's capture-symbols scope covers\nthis package: the \"all\" scope populates every module package plus the synthetic stdlib package, while\nthe \"stdlib\" scope populates only the stdlib package." + "description": "Symbols are the function symbols from this module that are compiled into the binary, extracted from\nthe binary symbol table (pclntab) and grouped by the import path of the package that owns them. Each\nvalue is the sorted, deduplicated list of symbol names local to that package, i.e. with the import\npath prefix stripped (e.g. import path \"github.com/foo/bar\" -\u003e \"(*Type).Method\"). The fully qualified\nname is the import path, a \".\", and the local name. One exception: the binary's main package appears\nunder the key \"main\" (the name the linker assigns), not its original source import path, which is not\nrecoverable from the binary. Populated only when the golang cataloger's capture-symbols scope covers\nthis package: the \"all\" scope populates every module package plus the synthetic stdlib package, the\n\"extended-stdlib\" scope populates the stdlib package plus every module under golang.org/x/, and the\n\"stdlib\" scope populates only the stdlib package. The capture-symbols-modules glob patterns populate\nany additional modules they match." } }, "type": "object", diff --git a/syft/cataloging/symbols.go b/syft/cataloging/symbols.go index 4e892fceed3..2b7907eda62 100644 --- a/syft/cataloging/symbols.go +++ b/syft/cataloging/symbols.go @@ -12,6 +12,10 @@ const ( // SymbolScopeStdlib captures symbols only for the synthetic "stdlib" package, leaving module packages without symbols. SymbolScopeStdlib SymbolScope = "stdlib" + // SymbolScopeExtendedStdlib captures symbols for the synthetic "stdlib" package as well as every module + // under golang.org/x/ (the extended standard library). + SymbolScopeExtendedStdlib SymbolScope = "extended-stdlib" + // SymbolScopeAll captures symbols for all module packages as well as the synthetic "stdlib" package. SymbolScopeAll SymbolScope = "all" ) @@ -21,6 +25,8 @@ func (s SymbolScope) Parse() SymbolScope { switch strings.ToLower(strings.TrimSpace(string(s))) { case string(SymbolScopeAll): return SymbolScopeAll + case string(SymbolScopeExtendedStdlib): + return SymbolScopeExtendedStdlib case string(SymbolScopeStdlib): return SymbolScopeStdlib } diff --git a/syft/cataloging/symbols_test.go b/syft/cataloging/symbols_test.go index 307119a1544..8d8d12717be 100644 --- a/syft/cataloging/symbols_test.go +++ b/syft/cataloging/symbols_test.go @@ -16,6 +16,11 @@ func Test_SymbolScope_Parse(t *testing.T) { {" all ", SymbolScopeAll}, {"stdlib", SymbolScopeStdlib}, {"Stdlib", SymbolScopeStdlib}, + {"extended-stdlib", SymbolScopeExtendedStdlib}, + {"Extended-Stdlib", SymbolScopeExtendedStdlib}, + {"EXTENDED-STDLIB", SymbolScopeExtendedStdlib}, + {" extended-stdlib ", SymbolScopeExtendedStdlib}, + {"extended_stdlib", SymbolScopeNone}, {"none", SymbolScopeNone}, {"", SymbolScopeNone}, {"true", SymbolScopeNone}, diff --git a/syft/pkg/cataloger/golang/capabilities.yaml b/syft/pkg/cataloger/golang/capabilities.yaml index 01e24b146f6..c9c314b191b 100644 --- a/syft/pkg/cataloger/golang/capabilities.yaml +++ b/syft/pkg/cataloger/golang/capabilities.yaml @@ -25,8 +25,11 @@ configs: # AUTO-GENERATED - config structs and their fields description: NoProxy is a list of glob patterns that match go module names that should not be fetched from the go proxy. When not set, syft will use the GOPRIVATE and GONOPROXY env vars. app_key: golang.no-proxy - key: CaptureSymbols - description: CaptureSymbols controls extracting function symbols from the binary symbol table (pclntab). Valid values are "none" (disabled), "stdlib" (only the synthetic stdlib package), and "all" (all module packages plus stdlib). + description: CaptureSymbols controls extracting function symbols from the binary symbol table (pclntab). Valid values are "none" (disabled), "stdlib" (only the synthetic stdlib package), "extended-stdlib" (stdlib plus every module under golang.org/x/), and "all" (all module packages plus stdlib). app_key: golang.capture-symbols + - key: CaptureSymbolsModules + description: 'CaptureSymbolsModules is a list of glob patterns (doublestar syntax, where ** crosses path separators and * does not) matched against go module paths. Matching modules get symbols in addition to whatever CaptureSymbols selects, so this can only widen the selection and never narrow it. It has no effect under the "none" scope. A trailing major version suffix is not part of a module''s identity, so a pattern matches with or without it: both "github.com/foo/bar" and "github.com/foo/*" select "github.com/foo/bar/v2". A pattern that spells out a suffix selects only that major version.' + app_key: golang.capture-symbols-modules catalogers: - ecosystem: go # MANUAL name: go-module-binary-cataloger # AUTO-GENERATED diff --git a/syft/pkg/cataloger/golang/config.go b/syft/pkg/cataloger/golang/config.go index 228cd67b18e..3d69bd8d7b0 100644 --- a/syft/pkg/cataloger/golang/config.go +++ b/syft/pkg/cataloger/golang/config.go @@ -51,10 +51,20 @@ type CatalogerConfig struct { MainModuleVersion MainModuleVersionConfig `yaml:"main-module-version" json:"main-module-version" mapstructure:"main-module-version"` // CaptureSymbols controls extracting function symbols from the binary symbol table (pclntab). Valid values are - // "none" (disabled), "stdlib" (only the synthetic stdlib package), and "all" (all module packages plus stdlib). + // "none" (disabled), "stdlib" (only the synthetic stdlib package), "extended-stdlib" (stdlib plus every module + // under golang.org/x/), and "all" (all module packages plus stdlib). // app-config: golang.capture-symbols CaptureSymbols cataloging.SymbolScope `yaml:"capture-symbols" json:"capture-symbols" mapstructure:"capture-symbols"` + // CaptureSymbolsModules is a list of glob patterns (doublestar syntax, where ** crosses path separators + // and * does not) matched against go module paths. Matching modules get symbols in addition to whatever + // CaptureSymbols selects, so this can only widen the selection and never narrow it. It has no effect + // under the "none" scope. A trailing major version suffix is not part of a module's identity, so a + // pattern matches with or without it: both "github.com/foo/bar" and "github.com/foo/*" select + // "github.com/foo/bar/v2". A pattern that spells out a suffix selects only that major version. + // app-config: golang.capture-symbols-modules + CaptureSymbolsModules []string `yaml:"capture-symbols-modules,omitempty" json:"capture-symbols-modules,omitempty" mapstructure:"capture-symbols-modules"` + // Whether to use the golang.org/x/tools/go/packages, which executes golang tooling found on the path in addition to potential network access UsePackagesLib bool `json:"use-packages-lib" yaml:"use-packages-lib" mapstructure:"use-packages-lib"` } @@ -196,6 +206,11 @@ func (g CatalogerConfig) WithCaptureSymbols(input cataloging.SymbolScope) Catalo return g } +func (g CatalogerConfig) WithCaptureSymbolsModules(input []string) CatalogerConfig { + g.CaptureSymbolsModules = input + return g +} + func (g CatalogerConfig) WithUsePackagesLib(useLib bool) CatalogerConfig { g.UsePackagesLib = useLib return g diff --git a/syft/pkg/cataloger/golang/parse_go_binary.go b/syft/pkg/cataloger/golang/parse_go_binary.go index a4d24c918e4..f1a905673f9 100644 --- a/syft/pkg/cataloger/golang/parse_go_binary.go +++ b/syft/pkg/cataloger/golang/parse_go_binary.go @@ -21,7 +21,6 @@ import ( "github.com/anchore/syft/internal" "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" - "github.com/anchore/syft/syft/cataloging" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/internal/unionreader" "github.com/anchore/syft/syft/pkg" @@ -51,7 +50,7 @@ const devel = "(devel)" type goBinaryCataloger struct { licenseResolver goLicenseResolver mainModuleVersion MainModuleVersionConfig - symbolScope cataloging.SymbolScope + symbolSelector symbolSelector // stdlibSymbols holds the standard-library function symbols discovered per binary (keyed by the // binary's location), grouped by import path, populated during parsing and consumed by stdlibProcessor @@ -65,7 +64,7 @@ func newGoBinaryCataloger(opts CatalogerConfig) *goBinaryCataloger { return &goBinaryCataloger{ licenseResolver: newGoLicenseResolver(binaryCatalogerName, opts), mainModuleVersion: opts.MainModuleVersion, - symbolScope: opts.CaptureSymbols, + symbolSelector: newSymbolSelector(opts.CaptureSymbols, opts.CaptureSymbolsModules), stdlibSymbols: make(map[file.Coordinates]map[string][]string), } } @@ -121,7 +120,7 @@ func (c *goBinaryCataloger) parseGoBinary(ctx context.Context, resolver file.Res } defer internal.CloseAndLogError(reader.ReadCloser, reader.RealPath) - mods, errs := scanFile(reader.Location, unionReader, c.symbolScope != cataloging.SymbolScopeNone) + mods, errs := scanFile(reader.Location, unionReader, c.symbolSelector.enabled()) var rels []artifact.Relationship for _, mod := range mods { @@ -184,11 +183,10 @@ func (c *goBinaryCataloger) buildGoPkgInfo(ctx context.Context, resolver file.Re symbolsByModule, stdlibSymbols := moduleSymbols(mod.symbols, &mod.Main, mod.Deps) c.recordStdlibSymbols(location.Coordinates, stdlibSymbols) - if c.symbolScope != cataloging.SymbolScopeAll { - // only the "all" scope attaches per-module symbols; for the "stdlib" scope we keep just the - // recorded stdlib symbols. nil map lookups below then yield nil symbol lists for each module. - symbolsByModule = nil - } + // keep only the modules the selector covers; the main module goes through the same map, so this + // incidentally decides the main module too (which is intended: it is treated like any dependency). + // unselected modules fall out entirely, so the lookups below yield nil rather than an empty map. + symbolsByModule = c.symbolSelector.filter(symbolsByModule) var pkgs []pkg.Package for _, dep := range mod.Deps { diff --git a/syft/pkg/cataloger/golang/parse_go_binary_test.go b/syft/pkg/cataloger/golang/parse_go_binary_test.go index 9881f91cc0b..14d03e206da 100644 --- a/syft/pkg/cataloger/golang/parse_go_binary_test.go +++ b/syft/pkg/cataloger/golang/parse_go_binary_test.go @@ -4,12 +4,14 @@ import ( "bufio" "bytes" "context" + "encoding/json" "errors" "io" "os" "os/exec" "path/filepath" "runtime/debug" + "slices" "strconv" "strings" "syscall" @@ -1441,9 +1443,12 @@ func Test_buildGoPkgInfo_symbolScope(t *testing.T) { tests := []struct { name string scope cataloging.SymbolScope + modules []string + extraDeps []*debug.Module symbols []binarySymbol wantMainSyms map[string][]string wantDepSyms map[string][]string + wantExtraSyms []map[string][]string wantStdlibSyms map[string][]string }{ { @@ -1468,6 +1473,84 @@ func Test_buildGoPkgInfo_symbolScope(t *testing.T) { }, wantStdlibSyms: map[string][]string{"net/http": {"(*Client).Do"}}, }, + { + // golang.org/x/net is reached through a vendored import path here, which must still be + // attributed to (and selected by) the module that owns it + name: "extended-stdlib captures golang.org/x modules and stdlib only", + scope: cataloging.SymbolScopeExtendedStdlib, + extraDeps: extendedDeps, + symbols: slices.Concat(populatedSymbols, extendedSymbols), + wantExtraSyms: []map[string][]string{ + {"golang.org/x/net/http2": {"NewClientConn"}}, + nil, + }, + wantStdlibSyms: map[string][]string{"net/http": {"(*Client).Do"}}, + }, + { + name: "module patterns widen extended-stdlib", + scope: cataloging.SymbolScopeExtendedStdlib, + modules: []string{"github.com/klauspost/**"}, + extraDeps: extendedDeps, + symbols: slices.Concat(populatedSymbols, extendedSymbols), + wantExtraSyms: []map[string][]string{ + {"golang.org/x/net/http2": {"NewClientConn"}}, + {"github.com/klauspost/compress/zstd": {"NewReader"}}, + }, + wantStdlibSyms: map[string][]string{"net/http": {"(*Client).Do"}}, + }, + { + name: "module patterns can select the main module", + scope: cataloging.SymbolScopeStdlib, + modules: []string{"github.com/anchore/**"}, + extraDeps: extendedDeps, + symbols: slices.Concat(populatedSymbols, extendedSymbols), + // the main package is keyed by the "main" import path the linker assigns, not its real path + wantMainSyms: map[string][]string{"main": {"main"}}, + wantExtraSyms: []map[string][]string{nil, nil}, + wantStdlibSyms: map[string][]string{"net/http": {"(*Client).Do"}}, + }, + { + name: "module patterns are inert under the none scope", + scope: cataloging.SymbolScopeNone, + modules: []string{"github.com/**", "golang.org/x/**"}, + extraDeps: extendedDeps, + // scanFile never runs under "none", so the build info carries no symbols to begin with + symbols: nil, + wantExtraSyms: []map[string][]string{nil, nil}, + }, + { + // all short-circuits ahead of the pattern walk, so a narrow module pattern list cannot subtract + // from it. this is also the guard on that short-circuit still existing. + name: "module patterns cannot narrow the all scope", + scope: cataloging.SymbolScopeAll, + modules: []string{"golang.org/x/**"}, + extraDeps: extendedDeps, + symbols: slices.Concat(populatedSymbols, extendedSymbols), + wantMainSyms: map[string][]string{"main": {"main"}}, + wantDepSyms: map[string][]string{ + "github.com/foo/bar": {"Parse"}, + "github.com/foo/bar/baz": {"Helper"}, + }, + wantExtraSyms: []map[string][]string{ + {"golang.org/x/net/http2": {"NewClientConn"}}, + {"github.com/klauspost/compress/zstd": {"NewReader"}}, + }, + wantStdlibSyms: map[string][]string{"net/http": {"(*Client).Do"}}, + }, + { + // a module pattern that restates what the preset already covers must not duplicate or drop anything: + // selection is a per-module boolean, so overlap is idempotent + name: "a module pattern overlapping the preset changes nothing", + scope: cataloging.SymbolScopeExtendedStdlib, + modules: []string{"golang.org/x/**"}, + extraDeps: extendedDeps, + symbols: slices.Concat(populatedSymbols, extendedSymbols), + wantExtraSyms: []map[string][]string{ + {"golang.org/x/net/http2": {"NewClientConn"}}, + nil, + }, + wantStdlibSyms: map[string][]string{"net/http": {"(*Client).Do"}}, + }, } for _, tt := range tests { @@ -1476,27 +1559,52 @@ func Test_buildGoPkgInfo_symbolScope(t *testing.T) { BuildInfo: &debug.BuildInfo{ GoVersion: "go1.22.0", Main: debug.Module{Path: "github.com/anchore/syft", Version: "v1.0.0"}, - Deps: []*debug.Module{{Path: "github.com/foo/bar", Version: "v1.2.3"}}, + Deps: append([]*debug.Module{{Path: "github.com/foo/bar", Version: "v1.2.3"}}, tt.extraDeps...), }, arch: "amd64", symbols: tt.symbols, } - c := newGoBinaryCataloger(CatalogerConfig{CaptureSymbols: tt.scope}) + c := newGoBinaryCataloger(CatalogerConfig{CaptureSymbols: tt.scope, CaptureSymbolsModules: tt.modules}) reader, err := unionreader.GetUnionReader(io.NopCloser(strings.NewReader(""))) require.NoError(t, err) mainPkg, pkgs := c.buildGoPkgInfo(context.Background(), fileresolver.Empty{}, location, mod, mod.arch, reader) require.NotNil(t, mainPkg) - require.Len(t, pkgs, 1) + require.Len(t, pkgs, 1+len(tt.extraDeps)) assert.Equal(t, tt.wantMainSyms, mainPkg.Metadata.(pkg.GolangBinaryBuildinfoEntry).Symbols, "main module symbols") assert.Equal(t, tt.wantDepSyms, pkgs[0].Metadata.(pkg.GolangBinaryBuildinfoEntry).Symbols, "dependency symbols") + for i, want := range tt.wantExtraSyms { + assert.Equal(t, want, pkgs[1+i].Metadata.(pkg.GolangBinaryBuildinfoEntry).Symbols, "symbols for %s", pkgs[1+i].Name) + } assert.Equal(t, tt.wantStdlibSyms, c.stdlibSymbolsFor(location.Coordinates), "recorded stdlib symbols") + + // a module selected by nothing must carry no symbols key at all: assert on the serialized form, + // since an empty (non-nil) map would still emit "symbols":{} despite the omitempty tag + for _, p := range append([]pkg.Package{*mainPkg}, pkgs...) { + encoded, err := json.Marshal(p.Metadata) + require.NoError(t, err) + if p.Metadata.(pkg.GolangBinaryBuildinfoEntry).Symbols == nil { + assert.NotContains(t, string(encoded), `"symbols"`, "expected no symbols key for %s", p.Name) + } + } }) } } +var ( + extendedDeps = []*debug.Module{ + {Path: "golang.org/x/net", Version: "v0.30.0"}, + {Path: "github.com/klauspost/compress", Version: "v1.17.0"}, + } + + extendedSymbols = []binarySymbol{ + {packagePath: "vendor/golang.org/x/net/http2", name: "vendor/golang.org/x/net/http2.NewClientConn"}, + {packagePath: "github.com/klauspost/compress/zstd", name: "github.com/klauspost/compress/zstd.NewReader"}, + } +) + // Test_recordStdlibSymbols_merge covers the merge path where the same binary location records stdlib // symbols more than once. This happens in production for universal/fat Mach-O binaries: scanFile yields // one build info per architecture and each is recorded under the same location coordinates. diff --git a/syft/pkg/cataloger/golang/symbol_selector.go b/syft/pkg/cataloger/golang/symbol_selector.go new file mode 100644 index 00000000000..59333a9c53f --- /dev/null +++ b/syft/pkg/cataloger/golang/symbol_selector.go @@ -0,0 +1,113 @@ +package golang + +import ( + "slices" + + "github.com/bmatcuk/doublestar/v4" + "golang.org/x/mod/module" + + "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/cataloging" +) + +// scopePatterns maps a capture-symbols scope onto the go module path globs it selects. The "none" and +// "all" scopes are intentionally absent: both are answered without matching. The "stdlib" scope selects +// no modules at all (the synthetic stdlib package is not a module and is handled separately). +var scopePatterns = map[cataloging.SymbolScope][]string{ + cataloging.SymbolScopeExtendedStdlib: {"golang.org/x/**"}, +} + +// symbolSelector decides which go module paths get function symbols attached to their metadata. The scope +// preset and any user-supplied module patterns are compiled into a single glob list so there is exactly +// one matcher answering "does this module path get symbols" (two matchers over the same subject drift). +type symbolSelector struct { + scope cataloging.SymbolScope + patterns []string +} + +func newSymbolSelector(scope cataloging.SymbolScope, modules []string) symbolSelector { + // normalize here rather than trusting the caller: the CLI runs Parse in PostLoad, but a library + // consumer setting CaptureSymbols directly (or via WithCaptureSymbols) does not, and an unnormalized + // value falls through both switches below into stdlib-only capture rather than the intended scope. + scope = scope.Parse() + + var patterns []string + for _, pattern := range slices.Concat(scopePatterns[scope], modules) { + if !doublestar.ValidatePattern(pattern) { + // a typo in a filter that decides what gets vulnerability-scanned must not pass quietly: + // someone would believe they captured symbols they did not. Drop just this pattern and keep + // the rest of the selection in force. + log.WithFields("pattern", pattern).Warn("ignoring malformed golang capture-symbols-modules pattern") + continue + } + patterns = append(patterns, pattern) + } + return symbolSelector{scope: scope, patterns: patterns} +} + +// enabled reports whether symbols should be extracted from the binary at all. +func (s symbolSelector) enabled() bool { + return s.scope != cataloging.SymbolScopeNone +} + +// selects reports whether the given go module path gets symbols. The binary's own main module is treated +// exactly like a dependency. +func (s symbolSelector) selects(modulePath string) bool { + switch s.scope { + case cataloging.SymbolScopeNone: + return false + case cataloging.SymbolScopeAll: + return true + } + + // a major version suffix is part of a module's path but not part of its identity, so patterns are matched + // against the path both with and without it: `github.com/foo/bar` and `github.com/foo/*` each select + // `github.com/foo/bar/v2`, which is what whoever wrote either one meant, and a config does not quietly + // stop covering a module the day it bumps a major version. Spelling a suffix out in the pattern still + // selects that major version alone, since the unsuffixed path cannot match a pattern carrying one. + // Only a trailing suffix is a version: in `github.com/foo/v2/bar` the `v2` is an ordinary path element, + // and SplitPathVersion leaves it there. + unversioned, major, ok := module.SplitPathVersion(modulePath) + versioned := ok && major != "" + + for _, pattern := range s.patterns { + if globMatches(pattern, modulePath) { + return true + } + if versioned && globMatches(pattern, unversioned) { + return true + } + } + return false +} + +// globMatches treats a pattern that cannot compile as no match, which newSymbolSelector has already warned about. +func globMatches(pattern, modulePath string) bool { + matched, err := doublestar.Match(pattern, modulePath) + if err != nil { + // unreachable: newSymbolSelector rejects (and warns about) patterns that cannot compile + return false + } + return matched +} + +// filter drops the entries of a module-path-keyed symbol map that the selector does not select. It returns +// nil when nothing is selected so callers attach no symbols map at all (rather than an empty one, which +// would defeat the omitempty JSON tag). +func (s symbolSelector) filter(byModule map[string]map[string][]string) map[string]map[string][]string { + switch s.scope { + case cataloging.SymbolScopeNone: + return nil + case cataloging.SymbolScopeAll: + return byModule + } + for modulePath := range byModule { + if !s.selects(modulePath) { + delete(byModule, modulePath) + } + } + if len(byModule) == 0 { + return nil + } + return byModule +} diff --git a/syft/pkg/cataloger/golang/symbol_selector_test.go b/syft/pkg/cataloger/golang/symbol_selector_test.go new file mode 100644 index 00000000000..b9f992465a6 --- /dev/null +++ b/syft/pkg/cataloger/golang/symbol_selector_test.go @@ -0,0 +1,280 @@ +package golang + +import ( + "maps" + "slices" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/anchore/syft/syft/cataloging" +) + +func Test_symbolSelector_selects(t *testing.T) { + tests := []struct { + name string + scope cataloging.SymbolScope + modules []string + modulePath string + want bool + }{ + { + name: "none selects nothing", + scope: cataloging.SymbolScopeNone, + modulePath: "github.com/foo/bar", + }, + { + name: "none is inert even with module patterns", + scope: cataloging.SymbolScopeNone, + modules: []string{"github.com/foo/**"}, + modulePath: "github.com/foo/bar", + }, + { + name: "all selects everything", + scope: cataloging.SymbolScopeAll, + modulePath: "github.com/foo/bar", + want: true, + }, + { + name: "stdlib selects no modules", + scope: cataloging.SymbolScopeStdlib, + modulePath: "golang.org/x/crypto", + }, + { + name: "extended-stdlib matches a golang.org/x module", + scope: cataloging.SymbolScopeExtendedStdlib, + modulePath: "golang.org/x/crypto", + want: true, + }, + { + // ** crosses path separators, which is what lets one pattern cover the whole subtree + name: "extended-stdlib matches a nested golang.org/x module", + scope: cataloging.SymbolScopeExtendedStdlib, + modulePath: "golang.org/x/tools/gopls", + want: true, + }, + { + // the pattern must not match on a bare prefix, only at a path boundary + name: "extended-stdlib does not match golang.org/xtra", + scope: cataloging.SymbolScopeExtendedStdlib, + modulePath: "golang.org/xtra", + }, + { + name: "extended-stdlib does not match an unrelated module", + scope: cataloging.SymbolScopeExtendedStdlib, + modulePath: "github.com/klauspost/compress", + }, + { + name: "single star matches one path segment", + scope: cataloging.SymbolScopeStdlib, + modules: []string{"github.com/klauspost/*"}, + modulePath: "github.com/klauspost/compress", + want: true, + }, + { + // a single star does not cross "/", but a major version suffix is not a path segment for this + // purpose: the module is matched with the suffix stripped as well, so a config written before a + // major bump keeps covering the module after it + name: "single star reaches across a major version suffix", + scope: cataloging.SymbolScopeStdlib, + modules: []string{"github.com/klauspost/*"}, + modulePath: "github.com/klauspost/compress/v2", + want: true, + }, + { + name: "single star does not cross a separator that is not a version suffix", + scope: cataloging.SymbolScopeStdlib, + modules: []string{"github.com/klauspost/*"}, + modulePath: "github.com/klauspost/compress/internal/thing", + }, + { + // only a trailing suffix is a version. here "v2" is an ordinary path element naming the major + // subdirectory a nested module lives in, so it is matched literally and ** is the way to reach it + name: "a mid-path version-like element is an ordinary segment", + scope: cataloging.SymbolScopeStdlib, + modules: []string{"github.com/anchore/*/thing"}, + modulePath: "github.com/anchore/syft/v2/thing", + }, + { + name: "doublestar reaches a mid-path version-like element", + scope: cataloging.SymbolScopeStdlib, + modules: []string{"github.com/anchore/**/thing"}, + modulePath: "github.com/anchore/syft/v2/thing", + want: true, + }, + { + name: "doublestar crosses a separator", + scope: cataloging.SymbolScopeStdlib, + modules: []string{"github.com/klauspost/**"}, + modulePath: "github.com/klauspost/compress/v2", + want: true, + }, + { + name: "an exact path covers every major version of that module", + scope: cataloging.SymbolScopeStdlib, + modules: []string{"github.com/klauspost/compress"}, + modulePath: "github.com/klauspost/compress/v2", + want: true, + }, + { + // spelling the suffix out is how a single major version is targeted: v1's path carries no suffix, + // so there is nothing for a suffixed pattern to match + name: "a pattern naming a major version selects only that one", + scope: cataloging.SymbolScopeStdlib, + modules: []string{"github.com/klauspost/compress/v2"}, + modulePath: "github.com/klauspost/compress", + }, + { + name: "a pattern naming a major version selects it", + scope: cataloging.SymbolScopeStdlib, + modules: []string{"github.com/klauspost/compress/v2"}, + modulePath: "github.com/klauspost/compress/v2", + want: true, + }, + { + // gopkg.in spells the major version as a .vN suffix on the last element, which SplitPathVersion + // understands, so it strips the same way + name: "gopkg.in style suffixes strip too", + scope: cataloging.SymbolScopeStdlib, + modules: []string{"gopkg.in/yaml"}, + modulePath: "gopkg.in/yaml.v2", + want: true, + }, + { + // /v1 and /v0 are not valid major version suffixes, so this is not a versioned path at all and + // nothing is stripped from it + name: "a v1 element is not a version suffix", + scope: cataloging.SymbolScopeStdlib, + modules: []string{"github.com/klauspost/compress"}, + modulePath: "github.com/klauspost/compress/v1", + }, + { + name: "module patterns widen a preset", + scope: cataloging.SymbolScopeExtendedStdlib, + modules: []string{"github.com/klauspost/**"}, + modulePath: "github.com/klauspost/compress", + want: true, + }, + { + name: "module patterns cannot narrow a preset", + scope: cataloging.SymbolScopeExtendedStdlib, + modules: []string{"github.com/klauspost/**"}, + modulePath: "golang.org/x/net", + want: true, + }, + { + name: "exact module path", + scope: cataloging.SymbolScopeStdlib, + modules: []string{"google.golang.org/grpc"}, + modulePath: "google.golang.org/grpc", + want: true, + }, + { + name: "an empty module pattern list is a no-op", + scope: cataloging.SymbolScopeStdlib, + modules: []string{}, + modulePath: "github.com/foo/bar", + }, + { + // all short-circuits before any matching, so a module pattern list cannot subtract from it + name: "module patterns cannot narrow all", + scope: cataloging.SymbolScopeAll, + modules: []string{"github.com/klauspost/**"}, + modulePath: "github.com/foo/bar", + want: true, + }, + { + // selection is a per-module boolean, so a pattern overlapping the preset is idempotent + name: "a module pattern overlapping the preset is idempotent", + scope: cataloging.SymbolScopeExtendedStdlib, + modules: []string{"golang.org/x/**"}, + modulePath: "golang.org/x/net", + want: true, + }, + { + // library consumers set CaptureSymbols directly without going through PostLoad, so the + // selector normalizes rather than falling through to stdlib-only capture + name: "unnormalized scope is parsed", + scope: "All", + modulePath: "github.com/foo/bar", + want: true, + }, + { + name: "unrecognized scope resolves to none", + scope: "bogus", + modules: []string{"github.com/foo/**"}, + modulePath: "github.com/foo/bar", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, newSymbolSelector(tt.scope, tt.modules).selects(tt.modulePath)) + }) + } +} + +// a malformed pattern must be dropped at construction, leaving the remaining patterns in force rather +// than aborting the selection or silently matching nothing +func Test_symbolSelector_malformedPattern(t *testing.T) { + s := newSymbolSelector(cataloging.SymbolScopeExtendedStdlib, []string{"[", "github.com/klauspost/**"}) + + require.Equal(t, []string{"golang.org/x/**", "github.com/klauspost/**"}, s.patterns) + assert.True(t, s.selects("golang.org/x/net"), "preset still applies") + assert.True(t, s.selects("github.com/klauspost/compress"), "valid sibling pattern still applies") + assert.False(t, s.selects("github.com/foo"), "malformed pattern selects nothing") +} + +func Test_symbolSelector_filter(t *testing.T) { + symbols := func() map[string]map[string][]string { + return map[string]map[string][]string{ + "golang.org/x/net": {"golang.org/x/net/http2": {"NewClientConn"}}, + "github.com/klauspost/compress": {"github.com/klauspost/compress/zstd": {"NewReader"}}, + } + } + + tests := []struct { + name string + scope cataloging.SymbolScope + modules []string + want []string // remaining module paths + }{ + { + name: "none drops everything", + scope: cataloging.SymbolScopeNone, + }, + { + name: "stdlib drops every module", + scope: cataloging.SymbolScopeStdlib, + }, + { + name: "extended-stdlib keeps only golang.org/x", + scope: cataloging.SymbolScopeExtendedStdlib, + want: []string{"golang.org/x/net"}, + }, + { + name: "module patterns widen the selection", + scope: cataloging.SymbolScopeExtendedStdlib, + modules: []string{"github.com/klauspost/**"}, + want: []string{"golang.org/x/net", "github.com/klauspost/compress"}, + }, + { + name: "all keeps everything", + scope: cataloging.SymbolScopeAll, + want: []string{"golang.org/x/net", "github.com/klauspost/compress"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := newSymbolSelector(tt.scope, tt.modules).filter(symbols()) + if len(tt.want) == 0 { + // nil, not an empty map, so the omitempty JSON tag keeps the field out of output + assert.Nil(t, got) + return + } + assert.ElementsMatch(t, tt.want, slices.Collect(maps.Keys(got))) + }) + } +} diff --git a/syft/pkg/golang.go b/syft/pkg/golang.go index 8b133a8de72..60f374d6f75 100644 --- a/syft/pkg/golang.go +++ b/syft/pkg/golang.go @@ -30,8 +30,10 @@ type GolangBinaryBuildinfoEntry struct { // name is the import path, a ".", and the local name. One exception: the binary's main package appears // under the key "main" (the name the linker assigns), not its original source import path, which is not // recoverable from the binary. Populated only when the golang cataloger's capture-symbols scope covers - // this package: the "all" scope populates every module package plus the synthetic stdlib package, while - // the "stdlib" scope populates only the stdlib package. + // this package: the "all" scope populates every module package plus the synthetic stdlib package, the + // "extended-stdlib" scope populates the stdlib package plus every module under golang.org/x/, and the + // "stdlib" scope populates only the stdlib package. The capture-symbols-modules glob patterns populate + // any additional modules they match. Symbols map[string][]string `json:"symbols,omitempty"` }