diff --git a/src/Microsoft.ComponentDetection.Detectors/uv/UvDependency.cs b/src/Microsoft.ComponentDetection.Detectors/uv/UvDependency.cs index 9d05954a2..e9b408500 100644 --- a/src/Microsoft.ComponentDetection.Detectors/uv/UvDependency.cs +++ b/src/Microsoft.ComponentDetection.Detectors/uv/UvDependency.cs @@ -1,8 +1,12 @@ namespace Microsoft.ComponentDetection.Detectors.Uv; +using System.Text.Json.Serialization; + internal class UvDependency { - public required string Name { get; init; } + [JsonPropertyName("name")] + public string Name { get; set; } = string.Empty; + [JsonPropertyName("specifier")] public string? Specifier { get; set; } } diff --git a/src/Microsoft.ComponentDetection.Detectors/uv/UvLock.cs b/src/Microsoft.ComponentDetection.Detectors/uv/UvLock.cs index db7d52024..e589be524 100644 --- a/src/Microsoft.ComponentDetection.Detectors/uv/UvLock.cs +++ b/src/Microsoft.ComponentDetection.Detectors/uv/UvLock.cs @@ -1,146 +1,40 @@ namespace Microsoft.ComponentDetection.Detectors.Uv; -using System; using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Text.Json.Serialization; using Tomlyn; -using Tomlyn.Model; internal class UvLock { - // a list of packages with their dependencies + // A list of packages with their dependencies. + [JsonPropertyName("package")] public List Packages { get; set; } = []; - // static method to parse the TOML stream into a UvLock model public static UvLock Parse(Stream tomlStream) { using var reader = new StreamReader(tomlStream); var tomlContent = reader.ReadToEnd(); - var model = Toml.ToModel(tomlContent); - return new UvLock + var options = new TomlModelOptions { - Packages = ParsePackagesFromModel(model), + IgnoreMissingProperties = true, }; - } - - internal static List ParsePackagesFromModel(object? model) - { - if (model is not TomlTable table) - { - throw new InvalidOperationException("TOML root is not a table"); - } - - if (!table.TryGetValue("package", out var packagesObj) || packagesObj is not TomlTableArray packages) - { - return []; - } - - var result = new List(); - foreach (var pkg in packages) - { - var parsed = ParsePackage(pkg); - if (parsed is not null) - { - result.Add(parsed); - } - } - - return result; - } - - internal static UvPackage? ParsePackage(object? pkg) - { - if (pkg is not TomlTable pkgTable) - { - return null; - } - - if (pkgTable.TryGetValue("name", out var nameObj) && nameObj is string name && - pkgTable.TryGetValue("version", out var versionObj) && versionObj is string version) - { - var uvPackage = new UvPackage - { - Name = name, - Version = version, - Dependencies = [], - MetadataRequiresDist = [], - MetadataRequiresDev = [], - }; - if (pkgTable.TryGetValue("dependencies", out var depsObj) && depsObj is TomlArray depsArray) - { - uvPackage.Dependencies = ParseDependenciesArray(depsArray); - } - - if (pkgTable.TryGetValue("metadata", out var metadataObj) && metadataObj is TomlTable metadataTable) - { - ParseMetadata(metadataTable, uvPackage); - } - - // Parse source - if (pkgTable.TryGetValue("source", out var sourceObj) && sourceObj is TomlTable sourceTable) - { - var source = new UvSource - { - Registry = sourceTable.TryGetValue("registry", out var regObj) && regObj is string reg ? reg : null, - Virtual = sourceTable.TryGetValue("virtual", out var virtObj) && virtObj is string virt ? virt : null, - Git = sourceTable.TryGetValue("git", out var gitObj) && gitObj is string git ? git : null, - }; - uvPackage.Source = source; - } - - return uvPackage; - } - - return null; + var parsed = Toml.ToModel(tomlContent, options: options); + parsed.Normalize(); + return parsed; } - internal static List ParseDependenciesArray(TomlArray? depsArray) + private void Normalize() { - var deps = new List(); - if (depsArray is null) - { - return deps; - } - - foreach (var dep in depsArray) - { - if (dep is TomlTable depTable && - depTable.TryGetValue("name", out var depNameObj) && depNameObj is string depName) - { - var depSpec = depTable.TryGetValue("specifier", out var specObj) && specObj is string s ? s : null; - deps.Add(new UvDependency - { - Name = depName, - Specifier = depSpec, - }); - } - } - - return deps; - } - - internal static void ParseMetadata(TomlTable? metadataTable, UvPackage uvPackage) - { - if (metadataTable is null) - { - return; - } - - if (metadataTable.TryGetValue("requires-dist", out var requiresDistObj) && requiresDistObj is TomlArray requiresDistArr) - { - uvPackage.MetadataRequiresDist = ParseDependenciesArray(requiresDistArr); - } + this.Packages = this.Packages + .Where(p => !string.IsNullOrWhiteSpace(p.Name) && !string.IsNullOrWhiteSpace(p.Version)) + .ToList(); - if (metadataTable.TryGetValue("requires-dev", out var requiresDevObj) && requiresDevObj is TomlTable requiresDevTable) + foreach (var package in this.Packages) { - foreach (var kvp in requiresDevTable) - { - if (kvp.Value is TomlArray groupArr) - { - uvPackage.MetadataRequiresDev.AddRange(ParseDependenciesArray(groupArr)); - } - } + package.Normalize(); } } } diff --git a/src/Microsoft.ComponentDetection.Detectors/uv/UvMetadata.cs b/src/Microsoft.ComponentDetection.Detectors/uv/UvMetadata.cs new file mode 100644 index 000000000..3fe43dfbd --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/uv/UvMetadata.cs @@ -0,0 +1,31 @@ +namespace Microsoft.ComponentDetection.Detectors.Uv; + +using System.Collections.Generic; +using System.Linq; +using System.Text.Json.Serialization; + +internal class UvMetadata +{ + [JsonPropertyName("requires-dist")] + public List RequiresDist { get; set; } = []; + + [JsonPropertyName("requires-dev")] + public Dictionary> RequiresDev { get; set; } = []; + + public void Normalize() + { + this.RequiresDist = this.RequiresDist + .Where(d => !string.IsNullOrWhiteSpace(d.Name)) + .ToList(); + + var normalizedDev = new Dictionary>(this.RequiresDev.Count, this.RequiresDev.Comparer); + foreach (var (group, dependencies) in this.RequiresDev) + { + normalizedDev[group] = dependencies + .Where(d => !string.IsNullOrWhiteSpace(d.Name)) + .ToList(); + } + + this.RequiresDev = normalizedDev; + } +} diff --git a/src/Microsoft.ComponentDetection.Detectors/uv/UvPackage.cs b/src/Microsoft.ComponentDetection.Detectors/uv/UvPackage.cs index d9d7443c4..94b82a11c 100644 --- a/src/Microsoft.ComponentDetection.Detectors/uv/UvPackage.cs +++ b/src/Microsoft.ComponentDetection.Detectors/uv/UvPackage.cs @@ -2,25 +2,36 @@ namespace Microsoft.ComponentDetection.Detectors.Uv; using System; using System.Collections.Generic; +using System.Linq; +using System.Text.Json.Serialization; using Microsoft.ComponentDetection.Contracts.TypedComponent; internal class UvPackage { - public required string Name { get; init; } + [JsonPropertyName("name")] + public string Name { get; set; } = string.Empty; - public required string Version { get; init; } + [JsonPropertyName("version")] + public string Version { get; set; } = string.Empty; + [JsonPropertyName("dependencies")] public List Dependencies { get; set; } = []; - // Metadata dependencies (requires-dist) - public List MetadataRequiresDist { get; set; } = []; + [JsonPropertyName("metadata")] + public UvMetadata? Metadata { get; set; } - // Metadata dev dependencies (requires-dev) - public List MetadataRequiresDev { get; set; } = []; - - // Source property for uv.lock + [JsonPropertyName("source")] public UvSource? Source { get; set; } + [JsonIgnore] + public List MetadataRequiresDist => this.Metadata?.RequiresDist ?? []; + + [JsonIgnore] + public List MetadataRequiresDev => this.Metadata?.RequiresDev?.Values + .Where(group => group != null) + .SelectMany(group => group!) + .ToList() ?? []; + public TypedComponent ToTypedComponent() { if (this.Source?.Git != null) @@ -32,6 +43,15 @@ public TypedComponent ToTypedComponent() return new PipComponent(this.Name, this.Version); } + public void Normalize() + { + this.Dependencies = this.Dependencies + .Where(d => !string.IsNullOrWhiteSpace(d.Name)) + .ToList(); + + this.Metadata?.Normalize(); + } + private static (Uri RepositoryUrl, string CommitHash) ParseGitUrl(string gitUrl) { var uri = new Uri(gitUrl); diff --git a/src/Microsoft.ComponentDetection.Detectors/uv/UvSource.cs b/src/Microsoft.ComponentDetection.Detectors/uv/UvSource.cs index b17cae64e..8605feec3 100644 --- a/src/Microsoft.ComponentDetection.Detectors/uv/UvSource.cs +++ b/src/Microsoft.ComponentDetection.Detectors/uv/UvSource.cs @@ -1,10 +1,15 @@ namespace Microsoft.ComponentDetection.Detectors.Uv; +using System.Text.Json.Serialization; + internal class UvSource { + [JsonPropertyName("registry")] public string? Registry { get; set; } + [JsonPropertyName("virtual")] public string? Virtual { get; set; } + [JsonPropertyName("git")] public string? Git { get; set; } } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/UvLockTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/UvLockTests.cs index 589fa8234..8c6e3d144 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/UvLockTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/UvLockTests.cs @@ -8,7 +8,6 @@ namespace Microsoft.ComponentDetection.Detectors.Tests; using AwesomeAssertions; using Microsoft.ComponentDetection.Detectors.Uv; using Microsoft.VisualStudio.TestTools.UnitTesting; -using Tomlyn.Model; [TestClass] public class UvLockTests @@ -79,6 +78,8 @@ public void Parse_ParsesPackagesAndDependencies() uvLock.Packages.Should().HaveCount(2); uvLock.Packages.First().Name.Should().Be("foo"); uvLock.Packages.First().Dependencies.Should().ContainSingle(d => d.Name == "bar" && d.Specifier == ">=2.0.0"); + uvLock.Packages.Last().Name.Should().Be("bar"); + uvLock.Packages.Last().Dependencies.Should().BeEmpty(); } [TestMethod] @@ -108,12 +109,12 @@ public void Parse_NoPackageKey_ReturnsNoPackages() } [TestMethod] - public void Parse_PackageKeyNotArray_ReturnsNoPackages() + public void Parse_PackageKeyNotArray_Throws() { var toml = "package = 42"; using var ms = new MemoryStream(Encoding.UTF8.GetBytes(toml)); - var uvLock = UvLock.Parse(ms); - uvLock.Packages.Should().BeEmpty(); + var act = () => UvLock.Parse(ms); + act.Should().Throw(); } [TestMethod] @@ -130,200 +131,6 @@ public void Parse_PackageMissingNameOrVersion_IgnoresPackage() uvLock.Packages.Should().BeEmpty(); } - [TestMethod] - public void Parse_PackageWithMalformedDependencies_IgnoresMalformed() - { - var toml = @" -[[package]] -name = 'foo' -version = '1.2.3' -dependencies = [42, { name = 'bar' }] -"; - using var ms = new MemoryStream(Encoding.UTF8.GetBytes(toml)); - var uvLock = UvLock.Parse(ms); - uvLock.Packages.Should().ContainSingle(); - var pkg = uvLock.Packages.First(); - pkg.Dependencies.Should().ContainSingle(d => d.Name == "bar"); - } - - [TestMethod] - public void Parse_PackageWithMalformedMetadata_IgnoresMalformed() - { - var toml = @" -[[package]] -name = 'foo' -version = '1.2.3' -[package.metadata] -requires-dist = [42, { name = 'bar' }] -[package.metadata.requires-dev] -dev = [42, { name = 'baz' }] -"; - using var ms = new MemoryStream(Encoding.UTF8.GetBytes(toml)); - var uvLock = UvLock.Parse(ms); - uvLock.Packages.Should().ContainSingle(); - var pkg = uvLock.Packages.First(); - pkg.MetadataRequiresDist.Should().ContainSingle(d => d.Name == "bar"); - pkg.MetadataRequiresDev.Should().ContainSingle(d => d.Name == "baz"); - } - - [TestMethod] - public void ParsePackagesFromModel_InvalidRoot_Throws() - { - Action act = () => UvLock.ParsePackagesFromModel(42); - act.Should().Throw(); - } - - [TestMethod] - public void ParsePackagesFromModel_NoPackages_ReturnsEmpty() - { - var table = new TomlTable(); - var result = UvLock.ParsePackagesFromModel(table); - result.Should().BeEmpty(); - } - - [TestMethod] - public void ParsePackage_ValidPackage_ParsesCorrectly() - { - var pkg = new TomlTable - { - ["name"] = "foo", - ["version"] = "1.0.0", - ["dependencies"] = new TomlArray { new TomlTable { ["name"] = "bar", ["specifier"] = ">=2.0.0" } }, - }; - var result = UvLock.ParsePackage(pkg); - result.Should().NotBeNull(); - result.Name.Should().Be("foo"); - result.Version.Should().Be("1.0.0"); - result.Dependencies.Should().ContainSingle(d => d.Name == "bar" && d.Specifier == ">=2.0.0"); - } - - [TestMethod] - public void ParsePackage_MissingNameOrVersion_ReturnsNull() - { - var pkg1 = new TomlTable { ["version"] = "1.0.0" }; - var pkg2 = new TomlTable { ["name"] = "foo" }; - UvLock.ParsePackage(pkg1).Should().BeNull(); - UvLock.ParsePackage(pkg2).Should().BeNull(); - } - - [TestMethod] - public void ParsePackage_NullOrNonTable_ReturnsNull() - { - UvLock.ParsePackage(null).Should().BeNull(); - UvLock.ParsePackage(42).Should().BeNull(); - } - - [TestMethod] - public void ParsePackage_BranchCoverage_AllPaths() - { - // Path: pkg is TomlTable, but missing name - var pkgMissingName = new TomlTable { ["version"] = "1.0.0" }; - UvLock.ParsePackage(pkgMissingName).Should().BeNull(); - - // Path: pkg is TomlTable, but missing version - var pkgMissingVersion = new TomlTable { ["name"] = "foo" }; - UvLock.ParsePackage(pkgMissingVersion).Should().BeNull(); - } - - [TestMethod] - public void ParseDependenciesArray_ParsesValidDepsAndSkipsMalformed() - { - var arr = new TomlArray { 42, new TomlTable { ["name"] = "bar", ["specifier"] = "==1.2.3" }, new TomlTable { ["name"] = "baz" } }; - var result = UvLock.ParseDependenciesArray(arr); - result.Should().Contain(d => d.Name == "bar" && d.Specifier == "==1.2.3"); - result.Should().Contain(d => d.Name == "baz" && d.Specifier == null); - result.Should().HaveCount(2); - } - - [TestMethod] - public void ParseDependenciesArray_NullOrNoValidDeps_ReturnsEmpty() - { - UvLock.ParseDependenciesArray(null).Should().BeEmpty(); - var arr = new TomlArray { 42, "foo", 3.14 }; - UvLock.ParseDependenciesArray(arr).Should().BeEmpty(); - } - - [TestMethod] - public void ParseDependenciesArray_BranchCoverage_AllPaths() - { - // Path: dep is TomlTable but missing name - var arr = new TomlArray { new TomlTable { ["specifier"] = "==1.2.3" } }; - UvLock.ParseDependenciesArray(arr).Should().BeEmpty(); - } - - [TestMethod] - public void ParseMetadata_ParsesRequiresDistAndDev() - { - var pkg = new UvPackage { Name = "foo", Version = "1.0.0" }; - var metadata = new TomlTable - { - ["requires-dist"] = new TomlArray { new TomlTable { ["name"] = "bar", ["specifier"] = ">=2.0.0" } }, - ["requires-dev"] = new TomlTable { ["dev"] = new TomlArray { new TomlTable { ["name"] = "baz" } } }, - }; - UvLock.ParseMetadata(metadata, pkg); - pkg.MetadataRequiresDist.Should().ContainSingle(d => d.Name == "bar" && d.Specifier == ">=2.0.0"); - pkg.MetadataRequiresDev.Should().ContainSingle(d => d.Name == "baz" && d.Specifier == null); - } - - [TestMethod] - public void ParseMetadata_NullOrNoRelevantKeys_DoesNothing() - { - var pkg = new UvPackage { Name = "foo", Version = "1.0.0" }; - UvLock.ParseMetadata(null, pkg); // Should not throw - var emptyTable = new TomlTable(); - UvLock.ParseMetadata(emptyTable, pkg); // Should not throw or set anything - pkg.MetadataRequiresDist.Should().BeEmpty(); - pkg.MetadataRequiresDev.Should().BeEmpty(); - } - - [TestMethod] - public void ParseMetadata_BranchCoverage_RequiresDistOnly() - { - var pkg = new UvPackage { Name = "foo", Version = "1.0.0" }; - var metadata = new TomlTable - { - ["requires-dist"] = new TomlArray { new TomlTable { ["name"] = "bar" } }, - }; - UvLock.ParseMetadata(metadata, pkg); - pkg.MetadataRequiresDist.Should().ContainSingle(d => d.Name == "bar"); - pkg.MetadataRequiresDev.Should().BeEmpty(); - } - - [TestMethod] - public void ParseMetadata_BranchCoverage_RequiresDevOnly() - { - var pkg = new UvPackage { Name = "foo", Version = "1.0.0" }; - var metadata = new TomlTable - { - ["requires-dev"] = new TomlTable { ["dev"] = new TomlArray { new TomlTable { ["name"] = "baz" } } }, - }; - UvLock.ParseMetadata(metadata, pkg); - pkg.MetadataRequiresDist.Should().BeEmpty(); - pkg.MetadataRequiresDev.Should().ContainSingle(d => d.Name == "baz"); - } - - [TestMethod] - public void ParseMetadata_RequiresDevTableWithoutDevArray_DoesNotThrowOrSet() - { - var pkg = new UvPackage { Name = "foo", Version = "1.0.0" }; - - // requires-dev exists but no "dev" key - var metadata = new TomlTable - { - ["requires-dev"] = new TomlTable { ["notdev"] = 42 }, - }; - UvLock.ParseMetadata(metadata, pkg); - pkg.MetadataRequiresDev.Should().BeEmpty(); - - // requires-dev exists, "dev" is not a TomlArray - metadata = new TomlTable - { - ["requires-dev"] = new TomlTable { ["dev"] = 42 }, - }; - UvLock.ParseMetadata(metadata, pkg); - pkg.MetadataRequiresDev.Should().BeEmpty(); - } - [TestMethod] public void ParsePackage_ParsesSourceRegistryAndVirtual() { @@ -338,7 +145,7 @@ public void ParsePackage_ParsesSourceRegistryAndVirtual() uvLock.Packages.Should().ContainSingle(); var pkg = uvLock.Packages.First(); pkg.Source.Should().NotBeNull(); - pkg.Source!.Registry.Should().Be("https://example.com/"); + pkg.Source.Registry.Should().Be("https://example.com/"); pkg.Source.Virtual.Should().Be("."); } @@ -356,7 +163,7 @@ public void ParsePackage_ParsesSource_RegistryOnly() uvLock.Packages.Should().ContainSingle(); var pkg = uvLock.Packages.First(); pkg.Source.Should().NotBeNull(); - pkg.Source!.Registry.Should().Be("https://example.com/"); + pkg.Source.Registry.Should().Be("https://example.com/"); pkg.Source.Virtual.Should().BeNull(); } @@ -374,7 +181,7 @@ public void ParsePackage_ParsesSource_VirtualOnly() uvLock.Packages.Should().ContainSingle(); var pkg = uvLock.Packages.First(); pkg.Source.Should().NotBeNull(); - pkg.Source!.Registry.Should().BeNull(); + pkg.Source.Registry.Should().BeNull(); pkg.Source.Virtual.Should().Be("."); }