feat(golang): add extended-stdlib scope and module patterns for symbol capture

cmd/syft/internal/options/catalog.go

@@ -199,7 +199,8 @@ func (cfg Catalog) ToPackagesConfig() pkgcataloging.Config {
 					WithFromLDFlags(cfg.Golang.MainModuleVersion.FromLDFlags),
 			).
 			WithUsePackagesLib(*multiLevelOption(true, enrichmentEnabled(cfg.Enrich, task.Go, task.Golang), cfg.Golang.UsePackagesLib)).
-			WithCaptureSymbols(cfg.Golang.CaptureSymbols),
+			WithCaptureSymbols(cfg.Golang.CaptureSymbols).
+			WithCaptureSymbolsInclude(cfg.Golang.CaptureSymbolsInclude),
 		JavaScript: javascript.DefaultCatalogerConfig().
 			WithIncludeDevDependencies(*multiLevelOption(false, cfg.JavaScript.IncludeDevDependencies)).
 			WithSearchRemoteLicenses(*multiLevelOption(false, enrichmentEnabled(cfg.Enrich, task.JavaScript, task.Node, task.NPM), cfg.JavaScript.SearchRemoteLicenses)).

cmd/syft/internal/options/golang.go

@@ -4,6 +4,7 @@ import (
 	"strings"
 
 	"github.com/anchore/clio"
+	"github.com/anchore/syft/internal/log"
 	"github.com/anchore/syft/syft/cataloging"
 	"github.com/anchore/syft/syft/pkg/cataloger/golang"
 )
@@ -19,6 +20,7 @@ type golangConfig struct {
 	MainModuleVersion           golangMainModuleVersionConfig `json:"main-module-version" yaml:"main-module-version" mapstructure:"main-module-version"`
 	UsePackagesLib              *bool                         `json:"use-packages-lib" yaml:"use-packages-lib" mapstructure:"use-packages-lib"`
 	CaptureSymbols              cataloging.SymbolScope        `json:"capture-symbols" yaml:"capture-symbols" mapstructure:"capture-symbols"`
+	CaptureSymbolsInclude       []string                      `json:"capture-symbols-include" yaml:"capture-symbols-include" mapstructure:"capture-symbols-include"`
 }
 
 var _ interface {
@@ -42,16 +44,39 @@ if unset this defaults to $GONOPROXY`)
 always show (devel) as the version. Use these options to control heuristics to guess
 a more accurate version from the binary.`)
 	descriptions.Add(&o.UsePackagesLib, `use the golang.org/x/tools/go/packages library, which executes golang tooling found on the path in addition to potential network access to get the most accurate results`)
+	// note: descriptions must be static string literals; the app config discovery that generates the
+	// capability docs reads them straight out of the AST
 	descriptions.Add(&o.CaptureSymbols, `capture function symbols from the binary symbol table (pclntab). valid values are:
-"none" (disabled), "stdlib" (only the synthetic stdlib package), and "all" (all module packages plus stdlib)`)
+"none" (disabled), "stdlib" (only the synthetic stdlib package), "extended-stdlib" (stdlib plus every
+module under golang.org/x/), and "all" (all module packages plus stdlib)`)
+	descriptions.Add(&o.CaptureSymbolsInclude, `glob patterns matched against go module paths (e.g. github.com/klauspost/**) that should have symbols
+captured in addition to whatever capture-symbols selects. ** crosses path separators, * does not.
+this can only widen the selection, never narrow it, and is inert when capture-symbols is none`)
 	descriptions.Add(&o.MainModuleVersion.FromLDFlags, `look for LD flags that appear to be setting a version (e.g. -X main.version=1.0.0)`)
 	descriptions.Add(&o.MainModuleVersion.FromBuildSettings, `use the build settings (e.g. vcs.version & vcs.time) to craft a v0 pseudo version
 (e.g. v0.0.0-20220308212642-53e6d0aaf6fb) when a more accurate version cannot be found otherwise`)
 	descriptions.Add(&o.MainModuleVersion.FromContents, `search for semver-like strings in the binary contents`)
 }
 
 func (o *golangConfig) PostLoad() error {
+	raw := strings.TrimSpace(string(o.CaptureSymbols))
 	o.CaptureSymbols = o.CaptureSymbols.Parse()
+
+	// an unrecognized value still resolves to "none", but say so rather than silently capturing nothing.
+	// stay quiet for unset and an explicit "none", which is the default and would otherwise warn on nearly
+	// every scan.
+	if o.CaptureSymbols == cataloging.SymbolScopeNone && raw != "" && !strings.EqualFold(raw, string(cataloging.SymbolScopeNone)) {
+		log.Warnf("unknown golang.capture-symbols value %q, defaulting to %q (valid values: none, stdlib, extended-stdlib, all)", raw, cataloging.SymbolScopeNone)
+	}
+
+	// trim only, deliberately not Flatten: viper already splits a comma-separated scalar (env var or a bare
+	// yaml string) into a slice before this point, and there is no CLI flag feeding this key. The one thing
+	// Flatten would add is splitting commas *inside* a list entry, which silently breaks doublestar brace
+	// alternation like github.com/{foo,bar}/**. Viper's split does not trim, so that part is still needed.
+	for i, pattern := range o.CaptureSymbolsInclude {
+		o.CaptureSymbolsInclude[i] = strings.TrimSpace(pattern)
+	}
+
 	return nil
 }
 
@@ -76,7 +101,8 @@ func defaultGolangConfig() golangConfig {
 			FromContents:      def.MainModuleVersion.FromContents,
 			FromBuildSettings: def.MainModuleVersion.FromBuildSettings,
 		},
-		UsePackagesLib: nil, // this defaults to true, which is the API default
-		CaptureSymbols: def.CaptureSymbols,
+		UsePackagesLib:        nil, // this defaults to true, which is the API default
+		CaptureSymbols:        def.CaptureSymbols,
+		CaptureSymbolsInclude: def.CaptureSymbolsInclude,
 	}
 }

cmd/syft/internal/options/golang_test.go

@@ -10,10 +10,11 @@ import (
 
 func Test_golangConfig_PostLoad(t *testing.T) {
 	tests := []struct {
-		name     string
-		cfg      golangConfig
-		expected cataloging.SymbolScope
-		wantErr  assert.ErrorAssertionFunc
+		name            string
+		cfg             golangConfig
+		expected        cataloging.SymbolScope
+		expectedInclude []string
+		wantErr         assert.ErrorAssertionFunc
 	}{
 		{
 			name:     "normalize all",
@@ -25,21 +26,57 @@ func Test_golangConfig_PostLoad(t *testing.T) {
 			cfg:      golangConfig{CaptureSymbols: "stdlib"},
 			expected: cataloging.SymbolScopeStdlib,
 		},
+		{
+			name:     "normalize extended-stdlib",
+			cfg:      golangConfig{CaptureSymbols: " Extended-Stdlib "},
+			expected: cataloging.SymbolScopeExtendedStdlib,
+		},
+		{
+			name: "include patterns keep embedded commas",
+			cfg: golangConfig{
+				CaptureSymbols: "stdlib",
+				// brace alternation contains a comma; splitting on it would corrupt the pattern
+				CaptureSymbolsInclude: []string{"github.com/{foo,bar}/**", "golang.org/x/**"},
+			},
+			expected:        cataloging.SymbolScopeStdlib,
+			expectedInclude: []string{"github.com/{foo,bar}/**", "golang.org/x/**"},
+		},
+		{
+			// viper splits a comma-separated scalar (env var or bare yaml string) but does not trim,
+			// so a leading space would otherwise survive into a pattern that silently matches nothing
+			name: "include patterns are trimmed",
+			cfg: golangConfig{
+				CaptureSymbols:        "stdlib",
+				CaptureSymbolsInclude: []string{"golang.org/x/**", " github.com/foo/** "},
+			},
+			expected:        cataloging.SymbolScopeStdlib,
+			expectedInclude: []string{"golang.org/x/**", "github.com/foo/**"},
+		},
 		{
 			name:     "empty defaults to none",
 			cfg:      golangConfig{CaptureSymbols: ""},
 			expected: cataloging.SymbolScopeNone,
 		},
 		{
 			name:     "invalid value defaults to none",
-			cfg:      golangConfig{CaptureSymbols: "bogus"},
+			cfg:      golangConfig{CaptureSymbols: "stdlbi"},
 			expected: cataloging.SymbolScopeNone,
 		},
 		{
 			name:     "boolean spellings default to none",
 			cfg:      golangConfig{CaptureSymbols: "true"},
 			expected: cataloging.SymbolScopeNone,
 		},
+		{
+			name:     "explicit none resolves to none",
+			cfg:      golangConfig{CaptureSymbols: "none"},
+			expected: cataloging.SymbolScopeNone,
+		},
+		{
+			name:     "explicit none is not case sensitive",
+			cfg:      golangConfig{CaptureSymbols: " NONE "},
+			expected: cataloging.SymbolScopeNone,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -52,6 +89,7 @@ func Test_golangConfig_PostLoad(t *testing.T) {
 				return
 			}
 			assert.Equal(t, tt.expected, tt.cfg.CaptureSymbols)
+			assert.Equal(t, tt.expectedInclude, tt.cfg.CaptureSymbolsInclude)
 		})
 	}
 }

internal/capabilities/appconfig.yaml

@@ -15,7 +15,9 @@ application: # AUTO-GENERATED - application-level config keys
   - key: dotnet.relax-dll-claims-when-bundling-detected
     description: show all packages from the deps.json if bundling tooling is present as a dependency (e.g. ILRepack)
   - key: golang.capture-symbols
-    description: 'capture function symbols from the binary symbol table (pclntab). valid values are: "none" (disabled), "stdlib" (only the synthetic stdlib package), and "all" (all module packages plus stdlib)'
+    description: 'capture function symbols from the binary symbol table (pclntab). valid values are: "none" (disabled), "stdlib" (only the synthetic stdlib package), "extended-stdlib" (stdlib plus every module under golang.org/x/), and "all" (all module packages plus stdlib)'
+  - key: golang.capture-symbols-include
+    description: glob patterns matched against go module paths (e.g. github.com/klauspost/**) that should have symbols captured in addition to whatever capture-symbols selects. ** crosses path separators, * does not. this can only widen the selection, never narrow it, and is inert when capture-symbols is none
   - key: golang.local-mod-cache-dir
     description: specify an explicit go mod cache directory, if unset this defaults to $GOPATH/pkg/mod or $HOME/go/pkg/mod
   - key: golang.local-vendor-dir

schema/json/README.md

@@ -35,3 +35,24 @@ Create the new schema by running `make generate-json-schema` from the root of th
 - If there is an existing schema for the given version and the new schema **does not** match the existing schema, an error is shown indicating to increment the version appropriately (see the "Versioning" section)
 
 ***Note: never delete a JSON schema and never change an existing JSON schema once it has been published in a release!*** Only add new schemas with a newly incremented version. All previous schema files must be stored in the `schema/json/` directory.
+
+### Exception: `description`-only corrections
+
+A published schema may be amended in place for one narrow case: the change touches only `description` text and leaves the data shape identical. Descriptions are documentation carried alongside the schema rather than constraints a validator evaluates, so correcting one cannot invalidate a document that already validated against that version. Minting a new version instead would leave the old one permanently describing the tool incorrectly, and spend a version number on no semantic change.
+
+This applies when **every** one of the following holds:
+
+- the only differences are `description` values
+- no field, type, enum, `required` entry, or `$ref` is added, removed, or altered
+- the `$id` version is unchanged
+
+Anything else, including adding a field that happens to be optional, is a schema change and needs a version bump per the "Versioning" section above.
+
+The generator blocks an in-place edit by design, since it refuses to overwrite a file that differs from what it would produce. To amend one, delete the schema file and regenerate so it is rewritten from the current Go doc comments:
+
+```bash
+rm schema/json/schema-$VERSION.json
+make generate-json-schema
+```
+
+The result is byte-identical to what the generator produces, so do not hand-edit the JSON. Re-running `make generate-json-schema` afterwards should report `No change to the existing schema!`, and `make check-json-schema-drift` should pass. Confirm with `git diff` that the only changes are the intended description lines.

schema/json/schema-16.1.10.json

@@ -1662,7 +1662,7 @@
             "type": "array"
           },
           "type": "object",
-          "description": "Symbols are the function symbols from this module that are compiled into the binary, extracted from\nthe binary symbol table (pclntab) and grouped by the import path of the package that owns them. Each\nvalue is the sorted, deduplicated list of symbol names local to that package, i.e. with the import\npath prefix stripped (e.g. import path \"github.com/foo/bar\" -\u003e \"(*Type).Method\"). The fully qualified\nname is the import path, a \".\", and the local name. One exception: the binary's main package appears\nunder the key \"main\" (the name the linker assigns), not its original source import path, which is not\nrecoverable from the binary. Populated only when the golang cataloger's capture-symbols scope covers\nthis package: the \"all\" scope populates every module package plus the synthetic stdlib package, while\nthe \"stdlib\" scope populates only the stdlib package."
+          "description": "Symbols are the function symbols from this module that are compiled into the binary, extracted from\nthe binary symbol table (pclntab) and grouped by the import path of the package that owns them. Each\nvalue is the sorted, deduplicated list of symbol names local to that package, i.e. with the import\npath prefix stripped (e.g. import path \"github.com/foo/bar\" -\u003e \"(*Type).Method\"). The fully qualified\nname is the import path, a \".\", and the local name. One exception: the binary's main package appears\nunder the key \"main\" (the name the linker assigns), not its original source import path, which is not\nrecoverable from the binary. Populated only when the golang cataloger's capture-symbols scope covers\nthis package: the \"all\" scope populates every module package plus the synthetic stdlib package, the\n\"extended-stdlib\" scope populates the stdlib package plus every module under golang.org/x/, and the\n\"stdlib\" scope populates only the stdlib package. The capture-symbols-include glob patterns populate\nany additional modules they match."
         }
       },
       "type": "object",

schema/json/schema-latest.json

@@ -1662,7 +1662,7 @@
             "type": "array"
           },
           "type": "object",
-          "description": "Symbols are the function symbols from this module that are compiled into the binary, extracted from\nthe binary symbol table (pclntab) and grouped by the import path of the package that owns them. Each\nvalue is the sorted, deduplicated list of symbol names local to that package, i.e. with the import\npath prefix stripped (e.g. import path \"github.com/foo/bar\" -\u003e \"(*Type).Method\"). The fully qualified\nname is the import path, a \".\", and the local name. One exception: the binary's main package appears\nunder the key \"main\" (the name the linker assigns), not its original source import path, which is not\nrecoverable from the binary. Populated only when the golang cataloger's capture-symbols scope covers\nthis package: the \"all\" scope populates every module package plus the synthetic stdlib package, while\nthe \"stdlib\" scope populates only the stdlib package."
+          "description": "Symbols are the function symbols from this module that are compiled into the binary, extracted from\nthe binary symbol table (pclntab) and grouped by the import path of the package that owns them. Each\nvalue is the sorted, deduplicated list of symbol names local to that package, i.e. with the import\npath prefix stripped (e.g. import path \"github.com/foo/bar\" -\u003e \"(*Type).Method\"). The fully qualified\nname is the import path, a \".\", and the local name. One exception: the binary's main package appears\nunder the key \"main\" (the name the linker assigns), not its original source import path, which is not\nrecoverable from the binary. Populated only when the golang cataloger's capture-symbols scope covers\nthis package: the \"all\" scope populates every module package plus the synthetic stdlib package, the\n\"extended-stdlib\" scope populates the stdlib package plus every module under golang.org/x/, and the\n\"stdlib\" scope populates only the stdlib package. The capture-symbols-include glob patterns populate\nany additional modules they match."
         }
       },
       "type": "object",

syft/cataloging/symbols.go

@@ -12,6 +12,10 @@ const (
 	// SymbolScopeStdlib captures symbols only for the synthetic "stdlib" package, leaving module packages without symbols.
 	SymbolScopeStdlib SymbolScope = "stdlib"
 
+	// SymbolScopeExtendedStdlib captures symbols for the synthetic "stdlib" package as well as every module
+	// under golang.org/x/ (the extended standard library).
+	SymbolScopeExtendedStdlib SymbolScope = "extended-stdlib"
+
 	// SymbolScopeAll captures symbols for all module packages as well as the synthetic "stdlib" package.
 	SymbolScopeAll SymbolScope = "all"
 )
@@ -21,6 +25,8 @@ func (s SymbolScope) Parse() SymbolScope {
 	switch strings.ToLower(strings.TrimSpace(string(s))) {
 	case string(SymbolScopeAll):
 		return SymbolScopeAll
+	case string(SymbolScopeExtendedStdlib):
+		return SymbolScopeExtendedStdlib
 	case string(SymbolScopeStdlib):
 		return SymbolScopeStdlib
 	}

syft/cataloging/symbols_test.go

@@ -16,6 +16,11 @@ func Test_SymbolScope_Parse(t *testing.T) {
 		{"  all  ", SymbolScopeAll},
 		{"stdlib", SymbolScopeStdlib},
 		{"Stdlib", SymbolScopeStdlib},
+		{"extended-stdlib", SymbolScopeExtendedStdlib},
+		{"Extended-Stdlib", SymbolScopeExtendedStdlib},
+		{"EXTENDED-STDLIB", SymbolScopeExtendedStdlib},
+		{"  extended-stdlib  ", SymbolScopeExtendedStdlib},
+		{"extended_stdlib", SymbolScopeNone},
 		{"none", SymbolScopeNone},
 		{"", SymbolScopeNone},
 		{"true", SymbolScopeNone},

syft/pkg/cataloger/golang/capabilities.yaml

@@ -25,8 +25,11 @@ configs: # AUTO-GENERATED - config structs and their fields
         description: NoProxy is a list of glob patterns that match go module names that should not be fetched from the go proxy. When not set, syft will use the GOPRIVATE and GONOPROXY env vars.
         app_key: golang.no-proxy
       - key: CaptureSymbols
-        description: CaptureSymbols controls extracting function symbols from the binary symbol table (pclntab). Valid values are "none" (disabled), "stdlib" (only the synthetic stdlib package), and "all" (all module packages plus stdlib).
+        description: CaptureSymbols controls extracting function symbols from the binary symbol table (pclntab). Valid values are "none" (disabled), "stdlib" (only the synthetic stdlib package), "extended-stdlib" (stdlib plus every module under golang.org/x/), and "all" (all module packages plus stdlib).
         app_key: golang.capture-symbols
+      - key: CaptureSymbolsInclude
+        description: CaptureSymbolsInclude is a list of glob patterns (doublestar syntax, where ** crosses path separators and * does not) matched against go module paths. Matching modules get symbols in addition to whatever CaptureSymbols selects, so this can only widen the selection and never narrow it. It has no effect under the "none" scope.
+        app_key: golang.capture-symbols-include
 catalogers:
   - ecosystem: go # MANUAL
     name: go-module-binary-cataloger # AUTO-GENERATED