From 2ae34559b5f8cb7fa8d2ac927cbb018a29d4f315 Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Thu, 5 Feb 2026 15:32:24 +0200 Subject: [PATCH 01/21] Uses string delimeter safe value list parsing for validation (cherry picked from commit f4a4a1abc5f583f422974e0a6f2f6b036ec056d8) --- .../ContentValidator.ValidationEntryFunctions.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs b/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs index d78a8682..0de2425a 100644 --- a/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs +++ b/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs @@ -1,4 +1,4 @@ -using Px.Utils.Validation.SyntaxValidation; +using Px.Utils.Validation.SyntaxValidation; using System.Globalization; namespace Px.Utils.Validation.ContentValidation @@ -399,7 +399,7 @@ validator._dimensionValueNames is null || return null; } - string[] codes = entry.Value.Split(validator.Conf.Symbols.Value.ListSeparator); + string[] codes = [.. SyntaxValidationUtilityMethods.GetListItemsFromString(entry.Value, validator.Conf.Symbols.Value.ListSeparator, validator.Conf.Symbols.Value.StringDelimeter)]; string defaultLanguage = validator._defaultLanguage ?? string.Empty; string lang = entry.Key.Language ?? defaultLanguage; if (codes.Length != validator._dimensionValueNames[new(lang, entry.Key.FirstSpecifier)].Length) From f6e1248e78f29a3cf53bd505884035c65d44b91b Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Thu, 5 Feb 2026 15:32:41 +0200 Subject: [PATCH 02/21] Accepts 1 length time val series (cherry picked from commit 1f423fcfebc3baec5d7f1ecdc8fd55b4f850d2f8) --- .../SyntaxValidation/SyntaxValidationUtilityMethods.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Px.Utils/Validation/SyntaxValidation/SyntaxValidationUtilityMethods.cs b/Px.Utils/Validation/SyntaxValidation/SyntaxValidationUtilityMethods.cs index 86156b2c..035b050e 100644 --- a/Px.Utils/Validation/SyntaxValidation/SyntaxValidationUtilityMethods.cs +++ b/Px.Utils/Validation/SyntaxValidation/SyntaxValidationUtilityMethods.cs @@ -458,8 +458,8 @@ private static bool GetTimeValueFormat(string input, out ValueType? valueFormat, } // Remove preceding list separator from input - input = input.Remove(0, 1); - if (!IsStringListFormat(input, listSeparator, conf.Symbols.Value.StringDelimeter)) + input = input[1..]; + if (!IsStringListFormat(input, listSeparator, conf.Symbols.Value.StringDelimeter) && !IsStringFormat(input, conf.Symbols.Value.StringDelimeter)) { return null; } From ca6e002d58f75e45b573fa7579c15ba401626097 Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Thu, 5 Feb 2026 15:44:11 +0200 Subject: [PATCH 03/21] Tests to cover aforementioned cases (cherry picked from commit 49c2979f71e4d294d3ea2aa6d0f6d515e4acf014) --- .../ContentValidationTests.cs | 21 +++++++++++++++++++ .../Fixtures/ContentValidationFixtures.cs | 9 ++++++++ .../StreamSyntaxValidationTests.cs | 3 ++- 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs b/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs index ad7b7d55..7bab2518 100644 --- a/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs +++ b/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs @@ -559,6 +559,27 @@ public void ValidateValueAmountsCalledWithUnmatchingAmountOfElementsReturnsWithE Assert.AreEqual(ValidationFeedbackRule.UnmatchingValueAmount, result.First().Key.Rule); } + [TestMethod] + public void ValidateValueAmountsCalledWithMatchingAmountOfElementsWithListDelimetersReturnsNoError() + { + // Arrange + ValidationStructuredEntry[] entries = [ContentValidationFixtures.StructuredEntryWithMatchingAmountOfElementsWithListDelimeters]; + ContentValidator validator = new(filename, encoding, entries); + SetValidatorField(validator, "_defaultLanguage", defaultLanguage); + SetValidatorField(validator, "_availableLanguages", availableLanguages); + SetValidatorField(validator, "_stubDimensionNames", stubDimensionNames); + SetValidatorField(validator, "_dimensionValueNames", dimensionValueNames); + + // Act + ValidationFeedback? result = ContentValidator.ValidateValueAmounts( + entries[0], + validator + ); + + // Assert + Assert.IsNull(result); + } + [TestMethod] public void ValidateValueUppercaseRecommendationsCalledWithLowerCaseEntryReturnsWithWarning() { diff --git a/Px.Utils.UnitTests/Validation/Fixtures/ContentValidationFixtures.cs b/Px.Utils.UnitTests/Validation/Fixtures/ContentValidationFixtures.cs index b220a5e3..67d36528 100644 --- a/Px.Utils.UnitTests/Validation/Fixtures/ContentValidationFixtures.cs +++ b/Px.Utils.UnitTests/Validation/Fixtures/ContentValidationFixtures.cs @@ -761,6 +761,15 @@ internal static class ContentValidationFixtures codesBarEntry.ValueStartIndex, Utils.Validation.ValueType.ListOfStrings); + internal static ValidationStructuredEntry StructuredEntryWithMatchingAmountOfElementsWithListDelimeters => + new(filename, + codesBarEntry.Key, + "\"foo,bar\"", // foo,bar should be treated as one element due to the quotes + codesBarEntry.ValueStartIndex, + codesBarEntry.LineChangeIndexes, + codesBarEntry.ValueStartIndex, + Utils.Validation.ValueType.ListOfStrings); + internal static ValidationStructuredEntry StructuredEntryWithLowerCaseValue => new(filename, codepageEntry.Key, diff --git a/Px.Utils.UnitTests/Validation/SyntaxValidationTests/StreamSyntaxValidationTests.cs b/Px.Utils.UnitTests/Validation/SyntaxValidationTests/StreamSyntaxValidationTests.cs index 709ac01e..3a5f2814 100644 --- a/Px.Utils.UnitTests/Validation/SyntaxValidationTests/StreamSyntaxValidationTests.cs +++ b/Px.Utils.UnitTests/Validation/SyntaxValidationTests/StreamSyntaxValidationTests.cs @@ -1,4 +1,4 @@ -using Px.Utils.Validation.SyntaxValidation; +using Px.Utils.Validation.SyntaxValidation; using Px.Utils.UnitTests.Validation.Fixtures; using System.Text; using System.Reflection; @@ -419,6 +419,7 @@ public void GetValueTypeFromStringCalledWithValidListsValuesCorrectValueType(str [TestMethod] [DataRow("TLIST(A1),\"2000\",\"2001\",\"2003\"", Utils.Validation.ValueType.TimeValSeries)] + [DataRow("TLIST(A1),\"2000\"", Utils.Validation.ValueType.TimeValSeries)] // Only one value - not a list [DataRow("TLIST(H1),\"20001\", \"20002\", \"20011\", \"20012\"", Utils.Validation.ValueType.TimeValSeries)] // Has spaces between values [DataRow("TLIST(T1),\"20001\",\"20002\",\"20003\"", Utils.Validation.ValueType.TimeValSeries)] [DataRow("TLIST(Q1),\"20001\",\"20002\",\"20003\",\"20004\"", Utils.Validation.ValueType.TimeValSeries)] From 741d9b7b35fdb4e6e6c773d5fd729ee1fa36002c Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Fri, 6 Feb 2026 11:37:23 +0200 Subject: [PATCH 04/21] Use safer list item extraction method for language validation --- .../ContentValidator.ValidationFindKeywordFunctions.cs | 7 +++++-- .../Validation/DatabaseValidation/DatabaseValidator.cs | 7 +++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationFindKeywordFunctions.cs b/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationFindKeywordFunctions.cs index 5dbbae0f..f48839d8 100644 --- a/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationFindKeywordFunctions.cs +++ b/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationFindKeywordFunctions.cs @@ -1,4 +1,4 @@ -using Px.Utils.Validation.SyntaxValidation; +using Px.Utils.Validation.SyntaxValidation; using System; using System.Collections.Generic; using System.Linq; @@ -106,7 +106,10 @@ public sealed partial class ContentValidator if (availableLanguageEntries.Length == 1) { - List languages = availableLanguageEntries[0].Value.Split(validator.Conf.Symbols.Value.ListSeparator).ToList(); + List languages = SyntaxValidationUtilityMethods.GetListItemsFromString( + availableLanguageEntries[0].Value, + validator.Conf.Symbols.Value.ListSeparator, + validator.Conf.Symbols.Value.StringDelimeter); validator._availableLanguages = [.. languages.Select(lang => SyntaxValidationUtilityMethods.CleanString(lang, validator.Conf))]; return null; } diff --git a/Px.Utils/Validation/DatabaseValidation/DatabaseValidator.cs b/Px.Utils/Validation/DatabaseValidation/DatabaseValidator.cs index 9cd9ac70..3a164955 100644 --- a/Px.Utils/Validation/DatabaseValidation/DatabaseValidator.cs +++ b/Px.Utils/Validation/DatabaseValidation/DatabaseValidator.cs @@ -1,4 +1,4 @@ -using Px.Utils.Exceptions; +using Px.Utils.Exceptions; using Px.Utils.PxFile; using Px.Utils.PxFile.Metadata; using Px.Utils.Validation.SyntaxValidation; @@ -373,7 +373,10 @@ private void ProcessEntry(string entry, ref string defaultLanguage, ref string[] } else if (splitEntry[0] == _conf.Tokens.KeyWords.AvailableLanguages) { - languages = splitEntry[1].Split(_conf.Symbols.Value.ListSeparator); + languages = [.. SyntaxValidationUtilityMethods.GetListItemsFromString( + splitEntry[1], + _conf.Symbols.Value.ListSeparator, + _conf.Symbols.Value.StringDelimeter)]; for (int j = 0; j < languages.Length; j++) { languages[j] = languages[j].Trim(_conf.Symbols.Key.StringDelimeter); From bc6870caa103d66932d0ef6b11cb5fa9d62e6a2c Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Thu, 16 Apr 2026 13:40:05 +0300 Subject: [PATCH 05/21] Allows missing value encoding characters as accepted data section start characters --- .../DataValidationTests/DataValidationTest.cs | 76 ++++++++++++++++++- .../Validation/Fixtures/DataStreamContents.cs | 30 +++++++- Px.Utils/Px.Utils.csproj | 2 +- .../DataValidation/DataValidator.cs | 15 +++- 4 files changed, 116 insertions(+), 7 deletions(-) diff --git a/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs b/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs index b83652ce..7b9c4333 100644 --- a/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs +++ b/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs @@ -112,7 +112,7 @@ public void TestValidateWithErrors() } Assert.HasCount(7, validationFeedbacks); // Unique feedbacks - Assert.HasCount(13, validationFeedbacks.Values.SelectMany(f => f)); // Total feedbacks including duplicates + Assert.HasCount(12, validationFeedbacks.Values.SelectMany(f => f)); // Total feedbacks including duplicates } [TestMethod] @@ -135,7 +135,7 @@ public async Task TestValidateAsyncWithErrors() } Assert.HasCount(7, validationFeedbacks); // Unique feedbacks - Assert.HasCount(13, validationFeedbacks.Values.SelectMany(f => f)); // Total feedbacks including duplicates + Assert.HasCount(12, validationFeedbacks.Values.SelectMany(f => f)); // Total feedbacks including duplicates } [TestMethod] @@ -221,5 +221,77 @@ public async Task ValidateAsyncWithDataOnSingleRowReturnsErrors() Assert.HasCount(2, validationFeedbacks);// Unique feedbacks Assert.HasCount(6, validationFeedbacks.Values.SelectMany(f => f)); // Total feedbacks including duplicates } + + [TestMethod] + public void ValidateDataStartingWithEnclosedMissingValueReturnsWithoutErrors() + { + using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.DATA_STARTING_WITH_ENCLOSED_MISSING_VALUE)); + stream.Seek(6, 0); + DataValidator validator = new(5, 4, 1); + + ValidationFeedback validationFeedbacks = validator.Validate(stream, "foo", Encoding.UTF8).FeedbackItems; + + Assert.HasCount(0, validationFeedbacks); + } + + [TestMethod] + public void ValidateDataStartingWithUnenclosedMissingValueReturnsWithoutErrors() + { + using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.DATA_STARTING_WITH_UNENCLOSED_MISSING_VALUE)); + stream.Seek(6, 0); + DataValidator validator = new(5, 4, 1); + + ValidationFeedback validationFeedbacks = validator.Validate(stream, "foo", Encoding.UTF8).FeedbackItems; + + Assert.HasCount(0, validationFeedbacks); + } + + [TestMethod] + public void ValidateDataStartingWithNilValueReturnsWithoutErrors() + { + using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.DATA_STARTING_WITH_NIL_VALUE)); + stream.Seek(6, 0); + DataValidator validator = new(5, 4, 1); + + ValidationFeedback validationFeedbacks = validator.Validate(stream, "foo", Encoding.UTF8).FeedbackItems; + + Assert.HasCount(0, validationFeedbacks); + } + + [TestMethod] + public void ValidateDataStartingWithUnenclosedNilValueReturnsWithoutErrors() + { + using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.DATA_STARTING_WITH_UNENCLOSED_NIL_VALUE)); + stream.Seek(6, 0); + DataValidator validator = new(5, 4, 1); + + ValidationFeedback validationFeedbacks = validator.Validate(stream, "foo", Encoding.UTF8).FeedbackItems; + + Assert.HasCount(0, validationFeedbacks); + } + + [TestMethod] + public async Task ValidateAsyncDataStartingWithEnclosedMissingValueReturnsWithoutErrors() + { + using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.DATA_STARTING_WITH_ENCLOSED_MISSING_VALUE)); + stream.Seek(6, 0); + DataValidator validator = new(5, 4, 1); + + ValidationResult result = await validator.ValidateAsync(stream, "foo", Encoding.UTF8, cancellationToken: TestContext.CancellationToken); + + Assert.HasCount(0, result.FeedbackItems); + } + + [TestMethod] + public async Task ValidateAsyncDataStartingWithUnenclosedMissingValueReturnsWithoutErrors() + { + using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.DATA_STARTING_WITH_UNENCLOSED_MISSING_VALUE)); + stream.Seek(6, 0); + DataValidator validator = new(5, 4, 1); + + ValidationResult result = await validator.ValidateAsync(stream, "foo", Encoding.UTF8, cancellationToken: TestContext.CancellationToken); + + Assert.HasCount(0, result.FeedbackItems); + } } } \ No newline at end of file diff --git a/Px.Utils.UnitTests/Validation/Fixtures/DataStreamContents.cs b/Px.Utils.UnitTests/Validation/Fixtures/DataStreamContents.cs index 65c6d8b8..01a566ec 100644 --- a/Px.Utils.UnitTests/Validation/Fixtures/DataStreamContents.cs +++ b/Px.Utils.UnitTests/Validation/Fixtures/DataStreamContents.cs @@ -1,4 +1,4 @@ -namespace Px.Utils.UnitTests.Validation.Fixtures +namespace Px.Utils.UnitTests.Validation.Fixtures { internal static class DataStreamContents { @@ -29,5 +29,33 @@ internal static class DataStreamContents internal static string DATA_ON_SINGLE_ROW => "DATA=1 2 3 4 5 6 7 8 9 10;"; + + internal static string DATA_STARTING_WITH_ENCLOSED_MISSING_VALUE => + "DATA=\n" + + "\".\" 2 3 4 5 \r\n" + + "6 7 8 9 10 \n\r" + + "\".\" \"..\" \"...\" \"....\" \".....\" \r" + + "\"......\" \"-\" -1 1.2 -1.3; \r\n"; + + internal static string DATA_STARTING_WITH_UNENCLOSED_MISSING_VALUE => + "DATA=\n" + + ". 2 3 4 5 \r\n" + + "6 7 8 9 10 \n\r" + + ". .. ... .... ..... \r" + + "...... - -1 1.2 -1.3; \r\n"; + + internal static string DATA_STARTING_WITH_NIL_VALUE => + "DATA=\n" + + "\"-\" 2 3 4 5 \r\n" + + "6 7 8 9 10 \n\r" + + "\".\" \"..\" \"...\" \"....\" \".....\" \r" + + "\"......\" \"-\" -1 1.2 -1.3; \r\n"; + + internal static string DATA_STARTING_WITH_UNENCLOSED_NIL_VALUE => + "DATA=\n" + + "- 2 3 4 5 \r\n" + + "6 7 8 9 10 \n\r" + + ". .. ... .... ..... \r" + + "...... - -1 1.2 -1.3; \r\n"; } } \ No newline at end of file diff --git a/Px.Utils/Px.Utils.csproj b/Px.Utils/Px.Utils.csproj index dfac49e2..2930d85b 100644 --- a/Px.Utils/Px.Utils.csproj +++ b/Px.Utils/Px.Utils.csproj @@ -2,7 +2,7 @@ Px.Utils - 1.4.0 + 1.4.1 net10.0 enable enable diff --git a/Px.Utils/Validation/DataValidation/DataValidator.cs b/Px.Utils/Validation/DataValidation/DataValidator.cs index 6025418e..f726cec2 100644 --- a/Px.Utils/Validation/DataValidation/DataValidator.cs +++ b/Px.Utils/Validation/DataValidation/DataValidator.cs @@ -1,4 +1,4 @@ -using System.Runtime.CompilerServices; +using System.Runtime.CompilerServices; using System.Text; using Px.Utils.PxFile; using Px.Utils.Validation.DatabaseValidation; @@ -15,6 +15,7 @@ namespace Px.Utils.Validation.DataValidation public class DataValidator(int rowLen, int numOfRows, int startRow, PxFileConfiguration? conf = null) : IPxFileStreamValidator, IPxFileStreamValidatorAsync { private const int _streamBufferSize = 4096; + private static ReadOnlySpan MissingValueStartBytes => [CharacterConstants.QUOTATIONMARK, (byte)'-', (byte)'.']; private readonly PxFileConfiguration _conf = conf ?? PxFileConfiguration.Default; @@ -267,11 +268,13 @@ private int GetStreamIndexOfFirstDataValue(Stream stream, ref ValidationFeedback bytesRead = stream.Read(buffer, 0, buffer.Length); for (int i = 0; i < bytesRead; i++) { - if (buffer[i] >= CharacterConstants.Zero && buffer[i] <= CharacterConstants.Nine) + byte currentByte = buffer[i]; + char currentChar = (char)currentByte; + if (IsDataValueStartByte(currentByte)) { return (int)stream.Position - bytesRead + i; } - else if (!CharacterConstants.WhitespaceCharacters.Contains((char)buffer[i])) + else if (!CharacterConstants.WhitespaceCharacters.Contains(currentChar)) { feedbacks.Add(new( new(ValidationFeedbackLevel.Error, @@ -284,6 +287,12 @@ private int GetStreamIndexOfFirstDataValue(Stream stream, ref ValidationFeedback return -1; } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool IsDataValueStartByte(byte currentByte) + { + return currentByte >= CharacterConstants.Zero && currentByte <= CharacterConstants.Nine || MissingValueStartBytes.Contains(currentByte); + } } /// From 71c019878190ae3b7a74370a1b84a4b24eb8c6b3 Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Thu, 16 Apr 2026 14:02:10 +0300 Subject: [PATCH 06/21] Generate documentation file on build --- Px.Utils/Px.Utils.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/Px.Utils/Px.Utils.csproj b/Px.Utils/Px.Utils.csproj index 2930d85b..bd2ca8d5 100644 --- a/Px.Utils/Px.Utils.csproj +++ b/Px.Utils/Px.Utils.csproj @@ -19,6 +19,7 @@ true snupkg LICENSE.md + true From 4dc9654a127d330864652ed5d9ed254199be05d5 Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Thu, 16 Apr 2026 14:30:51 +0300 Subject: [PATCH 07/21] Architecture documentation --- .github/copilot-instructions.md | 10 +++ docs/architecture.md | 15 ++++ docs/architecture.models.md | 86 +++++++++++++++++++++++ docs/architecture.operations.md | 20 ++++++ docs/architecture.overview.md | 70 ++++++++++++++++++ docs/architecture.readers.md | 58 +++++++++++++++ docs/architecture.serializers.md | 17 +++++ docs/architecture.testing.md | 104 +++++++++++++++++++++++++++ docs/architecture.validation.md | 117 +++++++++++++++++++++++++++++++ 9 files changed, 497 insertions(+) create mode 100644 docs/architecture.md create mode 100644 docs/architecture.models.md create mode 100644 docs/architecture.operations.md create mode 100644 docs/architecture.overview.md create mode 100644 docs/architecture.readers.md create mode 100644 docs/architecture.serializers.md create mode 100644 docs/architecture.testing.md create mode 100644 docs/architecture.validation.md diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index d86b596a..72fae3e1 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -24,5 +24,15 @@ Unit testing: - Use Moq for mocking dependencies in unit tests. - Use PascalCase for naming test methods, following the pattern: MethodNameStateUnderTestExpectedBehavior. +Architecture: +- See `docs/architecture.md` for the architecture index. It links to focused sub-files — read only the one relevant to your task: + - `docs/architecture.overview.md` — solution structure, config, patterns, file map + - `docs/architecture.validation.md` — validation pipeline, interfaces, extensibility + - `docs/architecture.models.md` — models, metadata, dimensions, builders + - `docs/architecture.readers.md` — metadata/data readers, binary codecs + - `docs/architecture.operations.md` — matrix operations + - `docs/architecture.serializers.md` — JSON converters + - `docs/architecture.testing.md` — unit tests, mocks, benchmarks + Other: - When refactoring code, update the documentation comments to reflect the changes made. \ No newline at end of file diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 00000000..63cdeabd --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,15 @@ +# Architecture Reference — Px.Utils + +> Machine-readable architecture index for AI coding agents. Read only the sub-file relevant to your task. + +## Sub-files + +| File | Scope | When to read | +|---|---|---| +| [architecture.overview.md](architecture.overview.md) | Solution structure, configuration, patterns, dependencies, file map | Onboarding, cross-cutting changes, understanding project layout | +| [architecture.validation.md](architecture.validation.md) | Validation pipeline, interfaces, result types, extensibility | Working on any validator, feedback types, or custom validation | +| [architecture.models.md](architecture.models.md) | Matrix, metadata, dimensions, properties, data values, model builders | Working on data models, metadata, or model building | +| [architecture.readers.md](architecture.readers.md) | Metadata reader, data reader, binary data codecs | Working on file reading, stream processing, or binary I/O | +| [architecture.operations.md](architecture.operations.md) | Matrix aggregation and arithmetic extensions | Working on sum/multiply/divide or adding new operations | +| [architecture.serializers.md](architecture.serializers.md) | JSON converters for model types | Working on JSON serialization/deserialization | +| [architecture.testing.md](architecture.testing.md) | Unit tests, mocks, benchmarking CLI | Writing tests, finding test coverage, or running benchmarks | diff --git a/docs/architecture.models.md b/docs/architecture.models.md new file mode 100644 index 00000000..7705f8cc --- /dev/null +++ b/docs/architecture.models.md @@ -0,0 +1,86 @@ +# Architecture — Models + +> Matrix, metadata, dimensions, properties, data values, and model builders. + +## Core Types + +| Type | Purpose | +|---|---| +| `Matrix` | Container for metadata + data array; supports `GetTransform(IMatrixMap)` | +| `IReadOnlyMatrixMetadata` | Read-only metadata: languages, dimensions, additional properties | +| `IMatrixMap` / `IDimensionMap` | Structural map of matrix dimensions and value codes | +| `MatrixMetadata` | Mutable metadata implementation of `IReadOnlyMatrixMetadata` | +| `MatrixMap` | Implementation of `IMatrixMap` | +| `MultilanguageString` | Dictionary-backed multilingual string with JSON serialization support | + +## Dimensions + +| Type | Purpose | +|---|---| +| `Dimension` | Base dimension with code, name, and `ValueList` | +| `ContentDimension` | Dimension with `ContentDimensionValue` entries (units, precision) | +| `TimeDimension` | Dimension with time interval information | +| `DimensionValue` | Single value in a dimension | +| `ContentDimensionValue` | Value with unit and decimal precision | +| `ValueList` | Ordered list of dimension values | + +Enums: `DimensionType`, `TimeDimensionInterval` + +## Meta Properties + +Abstract base: `MetaProperty` (`MetaPropertyType` enum). + +| Subtype | Value type | +|---|---| +| `StringProperty` | `string` | +| `NumericProperty` | `decimal` | +| `BooleanProperty` | `bool` | +| `StringListProperty` | `IReadOnlyList` | +| `MultilanguageStringProperty` | `MultilanguageString` | +| `MultilanguageStringListProperty` | `IReadOnlyList` | + +## Data Values + +| Type | Purpose | +|---|---| +| `IDataValue` | Marker interface | +| `DoubleDataValue` | `double` value + `DataValueType` | +| `DecimalDataValue` | `decimal` value + `DataValueType` | +| `DataValueType` | Enum classifying value type | + +## Model Building + +| Interface | Implementation | +|---|---| +| `IMatrixMetadataBuilder` | `MatrixMetadataBuilder` | + +``` +MatrixMetadata Build(IEnumerable> metadataInput) +MatrixMetadata Build(IReadOnlyDictionary metadataInput) +Task BuildAsync(IAsyncEnumerable> metadataInput) +``` + +Helpers: `MetadataEntryKeyBuilder`, `ValueParserUtilities`, `MetadataEntryKey`. + +## Extension Methods + +| File | Purpose | +|---|---| +| `MatrixMapExtensions` | Size calculations, dimension collapsing | +| `MatrixMetadataExtensions` | Metadata transforms and queries | +| `DimensionMapExtensions` | Dimension map utilities | +| `PropertyUtilities` | Property parsing and cleaning | + +## Dependency Graph + +``` +Matrix +├── IReadOnlyMatrixMetadata → MatrixMetadata +│ ├── Dimension / ContentDimension / TimeDimension +│ │ └── ValueList → DimensionValue / ContentDimensionValue +│ └── MetaProperty subtypes +└── DataIndexer (for GetTransform) + +MatrixMetadataBuilder : IMatrixMetadataBuilder +└── PxFileConfiguration +``` diff --git a/docs/architecture.operations.md b/docs/architecture.operations.md new file mode 100644 index 00000000..0925290f --- /dev/null +++ b/docs/architecture.operations.md @@ -0,0 +1,20 @@ +# Architecture — Operations + +> Matrix aggregation and arithmetic extension methods. + +## Extension Methods on `Matrix` + +| Class | File | Purpose | +|---|---|---| +| `MatrixFunctionExtensions` | `Operations/MatrixFunctionExtensions.cs` | `ApplyOverDimension` — abstract aggregation over a dimension | +| `SumMatrixFunctionExtensions` | `Operations/SumMatrixFunctionExtensions.cs` | Sum operations | +| `MultiplicationMatrixFunction` | `Operations/MultiplicationMatrixFunction.cs` | Multiplication operations | +| `DivisionMatrixFunctionExtensions` | `Operations/DivisionMatrixFunctionExtensions.cs` | Division operations | + +## Pattern + +All operations are extension methods on `Matrix`. They use `ApplyOverDimension` which: + +1. Takes a source dimension map, an aggregation function, and a function identity value +2. Adds a new `DimensionValue` to the target dimension holding the result +3. Returns a new `Matrix` with the aggregated data diff --git a/docs/architecture.overview.md b/docs/architecture.overview.md new file mode 100644 index 00000000..3940ea1c --- /dev/null +++ b/docs/architecture.overview.md @@ -0,0 +1,70 @@ +# Architecture — Overview + +> Solution structure, configuration, patterns, dependencies, and file map. + +## Solution + +Library for reading, processing, and validating PX (PC-Axis) statistical file data and metadata. + +| Project | Type | Target | Description | +|---|---|---|---| +| `Px.Utils` | Class library (NuGet) | `net10.0` | Core library — validation, models, readers, serializers, operations | +| `Px.Utils.TestingApp` | Console app | `net10.0` | Benchmarking CLI for performance testing | +| `Px.Utils.UnitTests` | Test project (MSTest) | `net10.0` | Unit tests for `Px.Utils` | + +No DI container — all types instantiated directly via constructors. Configuration via `PxFileConfiguration`. + +## Configuration + +`PxFileConfiguration` — configurable symbols and tokens for PX file format. Access default via `PxFileConfiguration.Default`. + +| Property path | Purpose | Default | +|---|---|---| +| `Symbols.Key.ListSeparator` | Separator in keys | `,` | +| `Symbols.Key.StringDelimeter` | String delimiter in keys | `"` | +| `Symbols.Value.StringDelimeter` | String delimiter in values | `"` | +| `Symbols.Value.ListSeparator` | Separator in values | `,` | +| `Symbols.KeywordSeparator` | Keyword = value separator | `=` | +| `Symbols.EntrySeparator` | Entry separator | `;` | +| `Tokens.KeyWords.Data` | Data section keyword | `DATA` | + +## Key Patterns + +- **No DI container** — manual wiring via constructors; primary constructors used extensively. +- **Configuration object** — `PxFileConfiguration` passed through constructors; `.Default` as fallback. +- **Sync + async pairs** — most validators and readers expose both sync and async APIs. +- **Partial classes** — `ContentValidator` and `SyntaxValidationFunctions` split across multiple files by concern. +- **Primary constructors** — used for validators, readers, builders, and model types. +- **Stream-oriented** — PX file processing operates on `Stream` inputs throughout. +- **`IFileSystem` abstraction** — enables testing without real file I/O. +- **InternalsVisibleTo** — `Px.Utils` exposes internals to `Px.Utils.UnitTests`. + +## External Dependencies (Px.Utils only) + +| Dependency | Purpose | +|---|---| +| `System.Text.Json` | JSON serialization of metadata models | + +No third-party NuGet dependencies in the main library. + +## File Map + +``` +Px.Utils/ +├── BinaryData/ -- Binary data reading and codec implementations +├── Exceptions/ -- Custom exceptions +├── Language/ -- MultilanguageString and extensions +├── ModelBuilders/ -- IMatrixMetadataBuilder, MetadataEntryKeyBuilder, ValueParserUtilities +├── Models/ +│ ├── Matrix.cs -- Matrix container +│ ├── Data/ -- IDataValue, DataValueType, DoubleDataValue, DecimalDataValue +│ └── Metadata/ -- Metadata interfaces, implementations, dimensions, enums, properties, extensions +├── Operations/ -- Matrix aggregation/sum/multiply/divide extensions +├── PxFile/ +│ ├── PxFileConfiguration.cs -- Format configuration +│ ├── CharacterConstants.cs -- Byte constants +│ ├── Data/ -- IPxFileStreamDataReader, PxFileStreamDataReader, DataIndexer, parsers +│ └── Metadata/ -- IPxFileMetadataReader, PxFileMetadataReader +├── Serializers/Json/ -- JSON converters for models +└── Validation/ -- Validation pipeline (syntax → content → data → database) +``` diff --git a/docs/architecture.readers.md b/docs/architecture.readers.md new file mode 100644 index 00000000..635b2e81 --- /dev/null +++ b/docs/architecture.readers.md @@ -0,0 +1,58 @@ +# Architecture — Readers & Binary Data + +> Metadata reading, data reading, and binary data codecs. + +## Metadata Reading + +| Interface | Implementation | File | +|---|---|---| +| `IPxFileMetadataReader` | `PxFileMetadataReader` | `PxFile/Metadata/PxFileMetadataReader.cs` | + +``` +IEnumerable> ReadMetadata(Stream, Encoding, int bufferSize = 4096) +Encoding GetEncoding(Stream) +Task GetEncodingAsync(Stream, CancellationToken) +IAsyncEnumerable> ReadMetadataAsync(Stream, Encoding, int bufferSize = 4096) +``` + +Depends on: `PxFileConfiguration`. + +## Data Reading + +| Interface | Implementation | File | +|---|---|---| +| `IPxFileStreamDataReader` | `PxFileStreamDataReader` | `PxFile/Data/PxFileStreamDataReader.cs` | + +``` +void ReadUnsafeDoubles(double[] buffer, int offset, IMatrixMap target, IMatrixMap complete, double[] missingValueEncodings) +void ReadDoubleDataValues(DoubleDataValue[] buffer, int offset, IMatrixMap target, IMatrixMap complete) +void ReadDecimalDataValues(DecimalDataValue[] buffer, int offset, IMatrixMap target, IMatrixMap complete) +``` + +Async variants available. Implements `IDisposable`. +Depends on: `PxFileConfiguration`. + +### Helpers + +| File | Purpose | +|---|---| +| `DataIndexer.cs` | Index mapping between source and target `IMatrixMap` | +| `DataValueParsers.cs` | Parse raw data values from byte spans | +| `StreamUtilities.cs` | Stream helper methods | + +## Binary Data + +| Type | File | Purpose | +|---|---|---| +| `BinaryDataReader` | `BinaryData/BinaryDataReader.cs` | Generic windowed reader for binary-encoded matrix data | +| `BinaryDataReaderStatic` | `BinaryData/BinaryDataReaderStatic.cs` | Static helper methods | +| `IBinaryValueCodec` | `BinaryData/ValueConverters/IBinaryValueCodec.cs` | Static interface for encode/decode | +| `BinaryValueCodecSelector` | `BinaryData/ValueConverters/BinaryValueCodecSelector.cs` | Codec factory by `BinaryValueCodecType` | + +### Codecs + +All implement `IBinaryValueCodec` (static abstract `ByteCount`): + +`DoubleCodec`, `FloatCodec`, `Int16Codec`, `Int24Codec`, `Int32Codec`, `UInt16Codec`, `UInt24Codec`, `UInt32Codec` + +Pattern: static interface members (`static abstract`) for zero-allocation encode/decode on spans. diff --git a/docs/architecture.serializers.md b/docs/architecture.serializers.md new file mode 100644 index 00000000..7a17251b --- /dev/null +++ b/docs/architecture.serializers.md @@ -0,0 +1,17 @@ +# Architecture — JSON Serializers + +> Custom `System.Text.Json` converters for model types. + +## Converters + +| Type | File | Serializes | +|---|---|---| +| `MultilanguageStringConverter` | `Serializers/Json/MultilanguageStringConverter.cs` | `MultilanguageString` | +| `DimensionConverter` | `Serializers/Json/DimensionConverter.cs` | `Dimension` and subtypes | +| `MetaPropertyConverter` | `Serializers/Json/MetaPropertyConverter.cs` | `MetaProperty` subtypes | +| `ContentValueListConverter` | `Serializers/Json/ContentValueListConverter.cs` | Content dimension value lists | +| `ValueListConverter` | `Serializers/Json/ValueListConverter.cs` | Generic value lists | + +### Helpers + +`ExtensionMethods.cs` — `JsonElement.GetProperty` with case-insensitive support. diff --git a/docs/architecture.testing.md b/docs/architecture.testing.md new file mode 100644 index 00000000..b3305d6b --- /dev/null +++ b/docs/architecture.testing.md @@ -0,0 +1,104 @@ +# Architecture — Testing + +> Unit tests, test mocks, and benchmarking CLI. + +## Unit Tests (`Px.Utils.UnitTests`) + +Framework: MSTest + Moq. Naming: `MethodNameStateUnderTestExpectedBehavior`. + +### Validation Tests + +| Test class | File | Covers | +|---|---|---| +| `StreamSyntaxValidationTests` | `Validation/SyntaxValidationTests/StreamSyntaxValidationTests.cs` | `SyntaxValidator` sync | +| `StreamSyntaxValidationAsyncTests` | `Validation/SyntaxValidationTests/StreamSyntaxValidationAsyncTests.cs` | `SyntaxValidator` async | +| `ContentValidationTests` | `Validation/ContentValidationTests/ContentValidationTests.cs` | `ContentValidator` | +| `DataValidationTest` | `Validation/DataValidationTests/DataValidationTest.cs` | `DataValidator` integration | +| `DataNumberValueValidatorTest` | `Validation/DataValidationTests/DataNumberValueValidatorTest.cs` | Numeric data validation | +| `DataStringValueValidatorTests` | `Validation/DataValidationTests/DataStringValueValidatorTests.cs` | String data validation | +| `DataSeparatorValidatorTest` | `Validation/DataValidationTests/DataSeparatorValidatorTest.cs` | Separator validation | +| `DataStructureValidationTests` | `Validation/DataValidationTests/DataStructureValidationTests.cs` | Data structure validation | +| `DatabaseValidatorTests` | `Validation/DatabaseValidation/DatabaseValidatorTests.cs` | `DatabaseValidator` | +| `DatabaseValidatorFunctionTests` | `Validation/DatabaseValidation/DatabaseValidatorFunctionTests.cs` | Database validator functions | +| `PxFileValidationTests` | `Validation/PxFileValidationTests/PxFileValidationTests.cs` | `PxFileValidator` | + +### Reader Tests + +| Test class | File | Covers | +|---|---|---| +| `ReadMetadataTests` | `PxFileTests/PxFileMetadataReaderTests/ReadMetadataTests.cs` | `PxFileMetadataReader` | +| `ReadMetadataAsyncTests` | `PxFileTests/PxFileMetadataReaderTests/ReadMetadataAsyncTests.cs` | `PxFileMetadataReader` async | +| `GetEncodingTests` | `PxFileTests/PxFileMetadataReaderTests/GetEncodingTests.cs` | Encoding detection | +| `DataReaderTests` | `PxFileTests/DataTests/PxFileStreamDataReaderTests/DataReaderTests.cs` | `PxFileStreamDataReader` | +| `AsyncDataReaderTests` | `PxFileTests/DataTests/PxFileStreamDataReaderTests/AsyncDataReaderTests.cs` | Async data reading | +| `MultiPartReadingTests` | `PxFileTests/DataTests/PxFileStreamDataReaderTests/MultiPartReadingTests.cs` | Multi-part reads | +| `DataIndexerTests` | `PxFileTests/DataTests/DataIndexerTests.cs` | `DataIndexer` | +| `DataValueParserTests` | `PxFileTests/DataTests/DataValueParserTests.cs` | `DataValueParsers` | + +### Model & Builder Tests + +| Test class | File | Covers | +|---|---|---| +| `MatrixMetadataBuilderTests` | `ModelBuilderTests/MatrixMetadataBuilderTests.cs` | `MatrixMetadataBuilder` | +| `MetadataEntryKeyBuilderTests` | `ModelBuilderTests/MetadataEntryKeyBuilderTests.cs` | `MetadataEntryKeyBuilder` | +| `MultilanguageStringTests` | `LanguageTests/MultilanguageStringTests.cs` | `MultilanguageString` | +| `MatrixTransformationsTests` | `ModelTests/MatrixTransformationsTests.cs` | `Matrix.GetTransform` | +| `ValueListTests` | `ModelTests/ValueListTests.cs` | `ValueList` | +| `PropertyTests` | `ModelTests/PropertyTests.cs` | `MetaProperty` subtypes | +| `DataValueTests` | `ModelTests/DataValueTests.cs` | Data value types | + +### Operations Tests + +| Test class | File | Covers | +|---|---|---| +| `SumMatrixFunctionExtensionTests` | `OperationsTests/SumMatrixFunctionExtensionTests.cs` | Sum | +| `MultiplicationMatrixFunctionExtensionTests` | `OperationsTests/MultiplicationMatrixFunctionExtensionTests.cs` | Multiplication | +| `DivisionMatrixFunctionExtensionTests` | `OperationsTests/DivisionMatrixFunctionExtensionTests.cs` | Division | + +### Binary Data Tests + +| Test class | File | Covers | +|---|---|---| +| `BinaryDataReaderCreateTests` | `BinaryData/BinaryDataReaderCreateTests.cs` | Reader creation | +| `BinaryDataReaderStreamTests` | `BinaryData/BinaryDataReaderStreamTests.cs` | Stream reading | +| `BinaryDataReaderChunkTests` | `BinaryData/BinaryDataReaderChunkTests.cs` | Chunk reading | +| `*CodecTests` | `BinaryData/ValueConverters/*CodecTests.cs` | Individual codecs | + +### Serializer Tests + +| Test class | File | Covers | +|---|---|---| +| `MatrixMetadataConverterSerializeTests` | `SerializerTests/MatrixMetadataConverterSerializeTests.cs` | JSON round-trip | +| `DimensionConverterTests` | `SerializerTests/DimensionConverterTests.cs` | Dimension JSON | +| `MetaPropertyConverterTests` | `SerializerTests/MetaPropertyConverterTests.cs` | MetaProperty JSON | +| `MultilanguageStringConverterTests` | `SerializerTests/MultilanguageStringConverterTests.cs` | MultilanguageString JSON | +| `ValueListConverterTests` | `SerializerTests/ValueListConverterTests.cs` | ValueList JSON | + +### Mocks + +`MockFileSystem`, `MockCustomValidatorFunctions`, `MockCustomSyntaxValidationFunctions`, `MockCustomContentValidationFunctions`, `MockDatabaseFileStreams`. + +--- + +## Benchmarking CLI (`Px.Utils.TestingApp`) + +Entry point: `TestingApp.Main(args)` → no args starts `InteractiveFlow`; batch mode not yet implemented. + +All commands inherit `Command`. Benchmarks inherit `Benchmark`. + +| Command | Purpose | +|---|---| +| `PxFileValidationBenchmark` | Full PX file validation | +| `DatabaseValidationBenchmark` | Database validation | +| `MetadataSyntaxValidationBenchmark` | Syntax validation | +| `MetadataContentValidationBenchmark` | Content validation | +| `DataValidationBenchmark` | Data validation | +| `MetadataReaderBenchmark` | Metadata reading | +| `MetadataBuilderBenchmark` | Metadata building | +| `DataReadBenchmark` | Data reading | +| `FileBenchmark` | File I/O | +| `ComputationBenchmark` | Matrix operations | +| `BinaryReadBenchmark` | Binary read | +| `BinaryWriteBenchmark` | Binary write | + +Supporting: `BenchmarkRunner.cs`, `TestAppConsole.cs`, `InteractiveFlow.cs`, `TestDataGenerator/`. diff --git a/docs/architecture.validation.md b/docs/architecture.validation.md new file mode 100644 index 00000000..1873220f --- /dev/null +++ b/docs/architecture.validation.md @@ -0,0 +1,117 @@ +# Architecture — Validation + +> Validation pipeline, interfaces, result types, and extensibility. + +## Interfaces + +| Interface | Implementations | Purpose | +|---|---|---| +| `IValidator` | `ContentValidator`, `DatabaseValidator` | Sync validation → `ValidationResult` | +| `IValidatorAsync` | `DatabaseValidator` | Async validation → `Task` | +| `IPxFileStreamValidator` | `SyntaxValidator`, `DataValidator`, `PxFileValidator` | Sync stream-based PX file validation | +| `IPxFileStreamValidatorAsync` | `SyntaxValidator`, `DataValidator`, `PxFileValidator` | Async stream-based PX file validation | +| `IDatabaseValidator` | — (consumer-provided) | Extension point for database-level custom validators | +| `IFileSystem` | `LocalFileSystem` | File system abstraction for database validation | + +## Pipeline + +`PxFileValidator` orchestrates: `SyntaxValidator` → `ContentValidator` → `DataValidator` → custom validators. + +``` +Validate(stream, filename, encoding?, fileSystem?) +ValidateAsync(stream, filename, encoding?, fileSystem?, cancellationToken) +``` + +### SyntaxValidator + +Validates PX file metadata syntax (key-value structure, encoding, characters). +File: `Validation/SyntaxValidation/SyntaxValidator.cs` +Partial helpers: `SyntaxValidationFunctions.StringValidationFunctions.cs`, `KeyValueValidationFunctions.cs`, `StructuredValidationFunctions.cs` + +### ContentValidator + +Validates metadata content (required keys, language definitions, dimension consistency). +File: `Validation/ContentValidation/ContentValidator.cs` +Partial files: `ValidationEntryFunctions.cs`, `ValidationFindKeywordFunctions.cs`, `UtilityMethods.cs` + +### DataValidator + +Validates data section (row counts, row lengths, value types, separators). +File: `Validation/DataValidation/DataValidator.cs` + +### DatabaseValidator + +Validates entire PX database directory (all `.px` files, alias files, directory structure). +File: `Validation/DatabaseValidation/DatabaseValidator.cs` + +## Extensibility + +- `CustomSyntaxValidationFunctions` — inject custom string/key-value/structured validation functions. +- `CustomContentValidationFunctions` — inject custom content validation functions. +- `PxFileValidator.SetCustomValidators(...)` — inject `IPxFileStreamValidator[]`, `IPxFileStreamValidatorAsync[]`, `IValidator[]`, `IValidatorAsync[]`. +- `DatabaseValidator` accepts `IDatabaseValidator[]` arrays for px files, alias files, and directories. + +## Result Types + +- `ValidationResult` — wraps `ValidationFeedback` +- `ValidationFeedback` — `ConcurrentDictionary>` +- `ValidationFeedbackKey` — (`ValidationFeedbackLevel`, `ValidationFeedbackRule`) +- `ValidationFeedbackLevel` — `Warning` | `Error` +- `ValidationFeedbackRule` — enum with 59 rule codes (see `Validation/Enums.cs`) + +## Dependency Graph + +``` +PxFileValidator +├── SyntaxValidator : IPxFileStreamValidator +│ ├── PxFileConfiguration +│ ├── CustomSyntaxValidationFunctions (optional) +│ └── IFileSystem → LocalFileSystem +├── ContentValidator : IValidator +│ ├── PxFileConfiguration +│ ├── ValidationStructuredEntry[] (from SyntaxValidator) +│ └── CustomContentValidationFunctions (optional) +├── DataValidator : IPxFileStreamValidator +│ └── PxFileConfiguration +└── Custom validators (optional) + +DatabaseValidator : IValidator, IValidatorAsync +├── IFileSystem → LocalFileSystem +├── PxFileConfiguration +├── PxFileMetadataReader +├── SyntaxValidator +└── IDatabaseValidator[] (optional custom) +``` + +## File Map + +``` +Validation/ +├── IValidator.cs -- IValidator, IValidatorAsync +├── IPxFileStreamValidator.cs -- IPxFileStreamValidator, IPxFileStreamValidatorAsync +├── IValidationResult.cs -- ValidationResult +├── ValidationFeedback.cs -- ValidationFeedbackKey, ValidationFeedbackValue, ValidationFeedback +├── ValidationObject.cs -- Validation context +├── Enums.cs -- ValidationFeedbackLevel, ValidationFeedbackRule, ValueType +├── PxFileValidator.cs -- Orchestrator +├── ContentValidation/ +│ ├── ContentValidator.cs -- + partial files +│ ├── ContentValidationResult.cs +│ └── CustomContentValidationFunctions.cs +├── DataValidation/ +│ ├── DataValidator.cs +│ └── DataValidatorFunctions.cs +├── DatabaseValidation/ +│ ├── DatabaseValidator.cs +│ ├── DatabaseValidatorFunctions.cs +│ ├── IFileSystem.cs +│ └── LocalFileSystem.cs +└── SyntaxValidation/ + ├── SyntaxValidator.cs + ├── SyntaxValidationFunctions.*.cs + ├── SyntaxValidationResult.cs + ├── SyntaxValidationUtilityMethods.cs + ├── CustomSyntaxValidationFunctions.cs + ├── ValidationEntry.cs / ValidationStructuredEntry.cs / ValidationKeyValuePair.cs + └── Bcp47Codes.cs +``` From 2a5a7e992f8eb549b18684e32429368a49fb80f9 Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Fri, 17 Apr 2026 10:46:02 +0300 Subject: [PATCH 08/21] Test cases for inconsistent linebreaks + simplified data validation tests --- .../DataValidationTests/DataValidationTest.cs | 320 +++--------------- .../Validation/Fixtures/DataStreamContents.cs | 25 +- 2 files changed, 68 insertions(+), 277 deletions(-) diff --git a/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs b/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs index 7b9c4333..66846a97 100644 --- a/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs +++ b/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs @@ -1,5 +1,5 @@ +using System.Linq; using System.Text; -using Microsoft.VisualStudio.TestTools.UnitTesting.Logging; using Px.Utils.UnitTests.Validation.Fixtures; using Px.Utils.Validation; using Px.Utils.Validation.DataValidation; @@ -11,287 +11,71 @@ public class DataValidationTest { public TestContext TestContext { get; set; } - [TestMethod] - public void TestValidateWithoutErrors() - { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.SIMPLE_VALID_DATA)); - stream.Seek(6, 0); - DataValidator validator = new(5, 4, 1); - - ValidationFeedback validationFeedbacks = validator.Validate(stream, "foo", Encoding.UTF8).FeedbackItems; - - foreach (KeyValuePair> validationFeedback in validationFeedbacks) - { - foreach (ValidationFeedbackValue instance in validationFeedback.Value) - { - Logger.LogMessage($"Line {instance.Line}, Char {instance.Character}: " - + $"{validationFeedback.Key.Rule} {instance.AdditionalInfo}"); - } - } - Assert.HasCount(0, validationFeedbacks); - } - - [TestMethod] - public void TestValidateWithoutMissingCodeDelimetersReturnsWithoutErrors() - { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.SIMPLE_VALID_DATA_WITHOUT_MISISNG_CODE_DELIMETERS)); - stream.Seek(6, 0); - DataValidator validator = new(5, 4, 1); - - ValidationFeedback validationFeedbacks = validator.Validate(stream, "foo", Encoding.UTF8).FeedbackItems; - - foreach (KeyValuePair> validationFeedback in validationFeedbacks) - { - foreach (ValidationFeedbackValue instance in validationFeedback.Value) - { - Logger.LogMessage($"Line {instance.Line}, Char {instance.Character}: " - + $"{validationFeedback.Key.Rule} {instance.AdditionalInfo}"); - } - } - Assert.HasCount(0, validationFeedbacks); - } - - [TestMethod] - public async Task TestValidateAsyncWithoutErrors() - { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.SIMPLE_VALID_DATA)); - stream.Seek(6, 0); - DataValidator validator = new(5, 4, 1); - - ValidationResult result = await validator.ValidateAsync(stream, "foo", Encoding.UTF8, cancellationToken: TestContext.CancellationToken); - ValidationFeedback validationFeedbacks = result.FeedbackItems; - - foreach (KeyValuePair> validationFeedback in validationFeedbacks) - { - foreach (ValidationFeedbackValue instance in validationFeedback.Value) - { - Logger.LogMessage($"Line {instance.Line}, Char {instance.Character}: " - + $"{validationFeedback.Key.Rule} {instance.AdditionalInfo}"); - } - } - Assert.HasCount(0, validationFeedbacks); - } - - [TestMethod] - public async Task TestValidateAsyncWithoutMissingCodeDelimetersReturnsWithoutErrors() - { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.SIMPLE_VALID_DATA_WITHOUT_MISISNG_CODE_DELIMETERS)); - stream.Seek(6, 0); - DataValidator validator = new(5, 4, 1); - - ValidationResult result = await validator.ValidateAsync(stream, "foo", Encoding.UTF8, cancellationToken: TestContext.CancellationToken); - ValidationFeedback validationFeedbacks = result.FeedbackItems; - - foreach (KeyValuePair> validationFeedback in validationFeedbacks) - { - foreach (ValidationFeedbackValue instance in validationFeedback.Value) - { - Logger.LogMessage($"Line {instance.Line}, Char {instance.Character}: " - + $"{validationFeedback.Key.Rule} {instance.AdditionalInfo}"); - } - } - Assert.HasCount(0, validationFeedbacks); - } - - [TestMethod] - public void TestValidateWithErrors() - { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.SIMPLE_INVALID_DATA)); - stream.Seek(6, 0); - DataValidator validator = new(5, 4, 1); - - ValidationFeedback validationFeedbacks = validator.Validate(stream, "foo", Encoding.UTF8).FeedbackItems; - - foreach (KeyValuePair> validationFeedback in validationFeedbacks) - { - foreach (ValidationFeedbackValue instance in validationFeedback.Value) - { - Logger.LogMessage($"Line {instance.Line}, Char {instance.Character}: " - + $"{validationFeedback.Key.Rule} {instance.AdditionalInfo}"); - } - } - - Assert.HasCount(7, validationFeedbacks); // Unique feedbacks - Assert.HasCount(12, validationFeedbacks.Values.SelectMany(f => f)); // Total feedbacks including duplicates - } - - [TestMethod] - public async Task TestValidateAsyncWithErrors() - { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.SIMPLE_INVALID_DATA)); - stream.Seek(6, 0); - DataValidator validator = new(5, 4, 1); - - ValidationResult result = await validator.ValidateAsync(stream, "foo", Encoding.UTF8, cancellationToken: TestContext.CancellationToken); - ValidationFeedback validationFeedbacks = result.FeedbackItems; - - foreach (KeyValuePair> validationFeedback in validationFeedbacks) + private const int DataStartOffset = 6; + + [TestMethod] + [DataRow(DataStreamContents.SIMPLE_VALID_DATA, 0, 0)] + [DataRow(DataStreamContents.SIMPLE_VALID_DATA_WITH_INCONSISTENT_LINEBREAKS, 0, 0)] + [DataRow(DataStreamContents.SIMPLE_VALID_DATA_WITHOUT_MISISNG_CODE_DELIMETERS, 0, 0)] + [DataRow(DataStreamContents.SIMPLE_INVALID_DATA, 7, 12)] + [DataRow(DataStreamContents.NO_DATA, 2, 6, false)] + [DataRow(DataStreamContents.DATA_ON_SINGLE_ROW, 2, 6, false)] + [DataRow(DataStreamContents.DATA_STARTING_WITH_ENCLOSED_MISSING_VALUE, 0, 0)] + [DataRow(DataStreamContents.DATA_STARTING_WITH_UNENCLOSED_MISSING_VALUE, 0, 0)] + [DataRow(DataStreamContents.DATA_STARTING_WITH_NIL_VALUE, 0, 0)] + [DataRow(DataStreamContents.DATA_STARTING_WITH_UNENCLOSED_NIL_VALUE, 0, 0)] + public void ValidateDataReturnsExpectedErrorCount( + string dataContents, + int expectedUniqueErrorCount, + int expectedTotalErrorCount, + bool shouldSeekDataStart = true) + { + using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(dataContents)); + if (shouldSeekDataStart) { - foreach (ValidationFeedbackValue instance in validationFeedback.Value) - { - Logger.LogMessage($"Line {instance.Line}, Char {instance.Character}: " - + $"{validationFeedback.Key.Rule} {instance.AdditionalInfo}"); - } + stream.Seek(DataStartOffset, SeekOrigin.Begin); } - Assert.HasCount(7, validationFeedbacks); // Unique feedbacks - Assert.HasCount(12, validationFeedbacks.Values.SelectMany(f => f)); // Total feedbacks including duplicates - } - - [TestMethod] - public void ValidateWithoutDataReturnsErrors() - { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.NO_DATA)); DataValidator validator = new(5, 4, 1); ValidationFeedback validationFeedbacks = validator.Validate(stream, "foo", Encoding.UTF8).FeedbackItems; - - foreach (KeyValuePair> validationFeedback in validationFeedbacks) + int actualErrorCount = validationFeedbacks.Values.SelectMany(f => f).Count(); + + Assert.HasCount(expectedUniqueErrorCount, validationFeedbacks); + Assert.AreEqual(expectedTotalErrorCount, actualErrorCount); + } + + [TestMethod] + [DataRow(DataStreamContents.SIMPLE_VALID_DATA, 0, 0)] + [DataRow(DataStreamContents.SIMPLE_VALID_DATA_WITH_INCONSISTENT_LINEBREAKS, 0, 0)] + [DataRow(DataStreamContents.SIMPLE_VALID_DATA_WITHOUT_MISISNG_CODE_DELIMETERS, 0, 0)] + [DataRow(DataStreamContents.SIMPLE_INVALID_DATA, 7, 12)] + [DataRow(DataStreamContents.NO_DATA, 2, 6, false)] + [DataRow(DataStreamContents.DATA_ON_SINGLE_ROW, 2, 6, false)] + [DataRow(DataStreamContents.DATA_STARTING_WITH_ENCLOSED_MISSING_VALUE, 0, 0)] + [DataRow(DataStreamContents.DATA_STARTING_WITH_UNENCLOSED_MISSING_VALUE, 0, 0)] + [DataRow(DataStreamContents.DATA_STARTING_WITH_NIL_VALUE, 0, 0)] + [DataRow(DataStreamContents.DATA_STARTING_WITH_UNENCLOSED_NIL_VALUE, 0, 0)] + public async Task ValidateAsyncDataReturnsExpectedErrorCount( + string dataContents, + int expectedUniqueErrorCount, + int expectedTotalErrorCount, + bool shouldSeekToDataStart = true) + { + using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(dataContents)); + if (shouldSeekToDataStart) { - foreach (ValidationFeedbackValue instance in validationFeedback.Value) - { - Logger.LogMessage($"Line {instance.Line}, Char {instance.Character}: " - + $"{validationFeedback.Key.Rule} {instance.AdditionalInfo}"); - } + stream.Seek(DataStartOffset, SeekOrigin.Begin); } - Assert.HasCount(2, validationFeedbacks); - } - - [TestMethod] - public async Task ValidateAsyncWithoutDataReturnsErrors() - { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.NO_DATA)); DataValidator validator = new(5, 4, 1); ValidationResult result = await validator.ValidateAsync(stream, "foo", Encoding.UTF8, cancellationToken: TestContext.CancellationToken); ValidationFeedback validationFeedbacks = result.FeedbackItems; + int actualErrorCount = validationFeedbacks.Values.SelectMany(f => f).Count(); - foreach (KeyValuePair> validationFeedback in validationFeedbacks) - { - foreach (ValidationFeedbackValue instance in validationFeedback.Value) - { - Logger.LogMessage($"Line {instance.Line}, Char {instance.Character}: " - + $"{validationFeedback.Key.Rule} {instance.AdditionalInfo}"); - } - } - - Assert.HasCount(2, validationFeedbacks); - } - - [TestMethod] - public void ValidateWithDataOnOnSingleRowReturnsErrors() - { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.DATA_ON_SINGLE_ROW)); - DataValidator validator = new(5, 4, 1); - - ValidationFeedback validationFeedbacks = validator.Validate(stream, "foo", Encoding.UTF8).FeedbackItems; - - foreach (KeyValuePair> validationFeedback in validationFeedbacks) - { - foreach (ValidationFeedbackValue instance in validationFeedback.Value) - { - Logger.LogMessage($"Line {instance.Line}, Char {instance.Character}: " - + $"{validationFeedback.Key.Rule} {instance.AdditionalInfo}"); - } - } - - Assert.HasCount(2, validationFeedbacks); // Unique feedbacks - Assert.HasCount(6, validationFeedbacks.Values.SelectMany(f => f)); // Total feedbacks including duplicates - } - - [TestMethod] - public async Task ValidateAsyncWithDataOnSingleRowReturnsErrors() - { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.DATA_ON_SINGLE_ROW)); - DataValidator validator = new(5, 4, 1); - - ValidationResult result = await validator.ValidateAsync(stream, "foo", Encoding.UTF8, cancellationToken: TestContext.CancellationToken); - ValidationFeedback validationFeedbacks = result.FeedbackItems; - - foreach (KeyValuePair> validationFeedback in validationFeedbacks) - { - foreach (ValidationFeedbackValue instance in validationFeedback.Value) - { - Logger.LogMessage($"Line {instance.Line}, Char {instance.Character}: " - + $"{validationFeedback.Key.Rule} {instance.AdditionalInfo}"); - } - } - - Assert.HasCount(2, validationFeedbacks);// Unique feedbacks - Assert.HasCount(6, validationFeedbacks.Values.SelectMany(f => f)); // Total feedbacks including duplicates - } - - [TestMethod] - public void ValidateDataStartingWithEnclosedMissingValueReturnsWithoutErrors() - { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.DATA_STARTING_WITH_ENCLOSED_MISSING_VALUE)); - stream.Seek(6, 0); - DataValidator validator = new(5, 4, 1); - - ValidationFeedback validationFeedbacks = validator.Validate(stream, "foo", Encoding.UTF8).FeedbackItems; - - Assert.HasCount(0, validationFeedbacks); - } - - [TestMethod] - public void ValidateDataStartingWithUnenclosedMissingValueReturnsWithoutErrors() - { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.DATA_STARTING_WITH_UNENCLOSED_MISSING_VALUE)); - stream.Seek(6, 0); - DataValidator validator = new(5, 4, 1); - - ValidationFeedback validationFeedbacks = validator.Validate(stream, "foo", Encoding.UTF8).FeedbackItems; - - Assert.HasCount(0, validationFeedbacks); - } - - [TestMethod] - public void ValidateDataStartingWithNilValueReturnsWithoutErrors() - { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.DATA_STARTING_WITH_NIL_VALUE)); - stream.Seek(6, 0); - DataValidator validator = new(5, 4, 1); - - ValidationFeedback validationFeedbacks = validator.Validate(stream, "foo", Encoding.UTF8).FeedbackItems; - - Assert.HasCount(0, validationFeedbacks); - } - - [TestMethod] - public void ValidateDataStartingWithUnenclosedNilValueReturnsWithoutErrors() - { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.DATA_STARTING_WITH_UNENCLOSED_NIL_VALUE)); - stream.Seek(6, 0); - DataValidator validator = new(5, 4, 1); - - ValidationFeedback validationFeedbacks = validator.Validate(stream, "foo", Encoding.UTF8).FeedbackItems; - - Assert.HasCount(0, validationFeedbacks); - } - - [TestMethod] - public async Task ValidateAsyncDataStartingWithEnclosedMissingValueReturnsWithoutErrors() - { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.DATA_STARTING_WITH_ENCLOSED_MISSING_VALUE)); - stream.Seek(6, 0); - DataValidator validator = new(5, 4, 1); - - ValidationResult result = await validator.ValidateAsync(stream, "foo", Encoding.UTF8, cancellationToken: TestContext.CancellationToken); - - Assert.HasCount(0, result.FeedbackItems); - } - - [TestMethod] - public async Task ValidateAsyncDataStartingWithUnenclosedMissingValueReturnsWithoutErrors() - { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.DATA_STARTING_WITH_UNENCLOSED_MISSING_VALUE)); - stream.Seek(6, 0); - DataValidator validator = new(5, 4, 1); - - ValidationResult result = await validator.ValidateAsync(stream, "foo", Encoding.UTF8, cancellationToken: TestContext.CancellationToken); - - Assert.HasCount(0, result.FeedbackItems); + Assert.HasCount(expectedUniqueErrorCount, validationFeedbacks); + Assert.AreEqual(expectedTotalErrorCount, actualErrorCount); } } } \ No newline at end of file diff --git a/Px.Utils.UnitTests/Validation/Fixtures/DataStreamContents.cs b/Px.Utils.UnitTests/Validation/Fixtures/DataStreamContents.cs index 01a566ec..f0ccdfda 100644 --- a/Px.Utils.UnitTests/Validation/Fixtures/DataStreamContents.cs +++ b/Px.Utils.UnitTests/Validation/Fixtures/DataStreamContents.cs @@ -2,21 +2,28 @@ namespace Px.Utils.UnitTests.Validation.Fixtures { internal static class DataStreamContents { - internal static string SIMPLE_VALID_DATA => + internal const string SIMPLE_VALID_DATA = "DATA=\n"+ "1 2 3 4 5 \r\n" + "6 7 8 9 10 \n\r" + "\".\" \"..\" \"...\" \"....\" \".....\" \r"+ "\"......\" \"-\" -1 1.2 -1.3; \r\n"; - internal static string SIMPLE_VALID_DATA_WITHOUT_MISISNG_CODE_DELIMETERS => + internal const string SIMPLE_VALID_DATA_WITH_INCONSISTENT_LINEBREAKS = + "DATA=\r\n" + + "1 2 3 4 5 \n" + + "6 7 8 9 10 \r\n" + + "\".\" \"..\" \"...\" \"....\" \".....\" \n" + + "\"......\" \"-\" -1 1.2 -1.3; \r\n"; + + internal const string SIMPLE_VALID_DATA_WITHOUT_MISISNG_CODE_DELIMETERS = "DATA=\n" + "1 2 3 4 5 \r\n" + "6 7 8 9 10 \n\r" + ". .. ... .... ..... \r" + "...... - -1 1.2 -1.3; \r\n"; - internal static string SIMPLE_INVALID_DATA => + internal const string SIMPLE_INVALID_DATA = "DATA=a\n" + "\"b\" 1 2. 3 4 5 \r\n" + "6 7 +8\t9 10 \n\r" + @@ -24,34 +31,34 @@ internal static class DataStreamContents "\"dots\" \"-\" \"..123\" -1 1.2 -1.3 \r\n" + "1 2 \0 4 5 \r\n;"; - internal static string NO_DATA => + internal const string NO_DATA = "DATA=\n"; - internal static string DATA_ON_SINGLE_ROW => + internal const string DATA_ON_SINGLE_ROW = "DATA=1 2 3 4 5 6 7 8 9 10;"; - internal static string DATA_STARTING_WITH_ENCLOSED_MISSING_VALUE => + internal const string DATA_STARTING_WITH_ENCLOSED_MISSING_VALUE = "DATA=\n" + "\".\" 2 3 4 5 \r\n" + "6 7 8 9 10 \n\r" + "\".\" \"..\" \"...\" \"....\" \".....\" \r" + "\"......\" \"-\" -1 1.2 -1.3; \r\n"; - internal static string DATA_STARTING_WITH_UNENCLOSED_MISSING_VALUE => + internal const string DATA_STARTING_WITH_UNENCLOSED_MISSING_VALUE = "DATA=\n" + ". 2 3 4 5 \r\n" + "6 7 8 9 10 \n\r" + ". .. ... .... ..... \r" + "...... - -1 1.2 -1.3; \r\n"; - internal static string DATA_STARTING_WITH_NIL_VALUE => + internal const string DATA_STARTING_WITH_NIL_VALUE = "DATA=\n" + "\"-\" 2 3 4 5 \r\n" + "6 7 8 9 10 \n\r" + "\".\" \"..\" \"...\" \"....\" \".....\" \r" + "\"......\" \"-\" -1 1.2 -1.3; \r\n"; - internal static string DATA_STARTING_WITH_UNENCLOSED_NIL_VALUE => + internal const string DATA_STARTING_WITH_UNENCLOSED_NIL_VALUE = "DATA=\n" + "- 2 3 4 5 \r\n" + "6 7 8 9 10 \n\r" + From b75da90f179155f9ae9dfe7f0ab26f298d106cf6 Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Wed, 29 Apr 2026 14:57:50 +0300 Subject: [PATCH 09/21] Fix known issues with validation false positives when using known alias for dimension types --- .../ContentValidationTests.cs | 21 +++++++++++++++++++ .../Fixtures/ContentValidationFixtures.cs | 9 ++++++++ .../ModelBuilders/ValueParserUtilities.cs | 4 +++- Px.Utils/PxFile/PxFileConfiguration.cs | 6 +++++- ...ntentValidator.ValidationEntryFunctions.cs | 16 +++++++++++--- 5 files changed, 51 insertions(+), 5 deletions(-) diff --git a/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs b/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs index c3b32180..cb596c6b 100644 --- a/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs +++ b/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs @@ -536,6 +536,27 @@ public void ValidateValueTypesCalledWithStructuredEntryArrayWithWrongValuesRetur } } + [TestMethod] + public void ValidateValueTypesCalledWithStructuredEntryArrayWithKnownDimensionTypeAliasesReturnsWithWarnings() + { + // Arrange + ValidationStructuredEntry[] entries = ContentValidationFixtures.STRUCTURED_ENTRY_ARRAY_WITH_KNOWN_DIMENSIONTYPE_ALIASES; + ContentValidator validator = new(filename, encoding, entries); + // Act + foreach (ValidationStructuredEntry entry in entries) + { + ValidationFeedback? result = ContentValidator.ValidateValueContents( + entry, + validator + ); + // Assert + Assert.IsNotNull(result); + Assert.HasCount(1, result); + Assert.AreEqual(ValidationFeedbackRule.InvalidValueFound, result.First().Key.Rule); + Assert.AreEqual(ValidationFeedbackLevel.Warning, result.First().Key.Level); + } + } + [TestMethod] public void ValidateValueAmountsCalledWithUnmatchingAmountOfElementsReturnsWithError() { diff --git a/Px.Utils.UnitTests/Validation/Fixtures/ContentValidationFixtures.cs b/Px.Utils.UnitTests/Validation/Fixtures/ContentValidationFixtures.cs index 67d36528..ae4ee466 100644 --- a/Px.Utils.UnitTests/Validation/Fixtures/ContentValidationFixtures.cs +++ b/Px.Utils.UnitTests/Validation/Fixtures/ContentValidationFixtures.cs @@ -429,6 +429,9 @@ internal static class ContentValidationFixtures 17, Utils.Validation.ValueType.StringValue); + private static readonly ValidationStructuredEntryKey dimensionTypeEntryKey = + new ("VARIABLE-TYPE", "fi", "foo"); + internal static ValidationStructuredEntry[] MINIMAL_STRUCTURED_ENTRY_ARRAY => [ charsetEntry, @@ -752,6 +755,12 @@ internal static class ContentValidationFixtures Utils.Validation.ValueType.StringValue), ]; + internal static ValidationStructuredEntry[] STRUCTURED_ENTRY_ARRAY_WITH_KNOWN_DIMENSIONTYPE_ALIASES => + [ + new ValidationStructuredEntry(filename, dimensionTypeEntryKey, "Contents", 0, [], 0, Utils.Validation.ValueType.StringValue), + new ValidationStructuredEntry(filename, dimensionTypeEntryKey, "Region", 1, [], 0, Utils.Validation.ValueType.StringValue), + ]; + internal static ValidationStructuredEntry StructuredEntryWithUnmatchingAmountOfElements => new(filename, codesBarEntry.Key, diff --git a/Px.Utils/ModelBuilders/ValueParserUtilities.cs b/Px.Utils/ModelBuilders/ValueParserUtilities.cs index 33cbe847..ced54567 100644 --- a/Px.Utils/ModelBuilders/ValueParserUtilities.cs +++ b/Px.Utils/ModelBuilders/ValueParserUtilities.cs @@ -1,4 +1,4 @@ -using Px.Utils.Models.Metadata.Enums; +using Px.Utils.Models.Metadata.Enums; using Px.Utils.Models.Metadata.ExtensionMethods; using Px.Utils.PxFile; @@ -113,10 +113,12 @@ public static DimensionType StringToDimensionType(string input, PxFileConfigurat Dictionary map = new() { {conf.Tokens.VariableTypes.Content, DimensionType.Content}, + {conf.Tokens.VariableTypes.Contents, DimensionType.Content}, // Known alias for content dimension type {conf.Tokens.VariableTypes.Time, DimensionType.Time}, {conf.Tokens.VariableTypes.Ordinal, DimensionType.Ordinal}, {conf.Tokens.VariableTypes.Nominal, DimensionType.Nominal}, {conf.Tokens.VariableTypes.Geographical, DimensionType.Geographical}, + {conf.Tokens.VariableTypes.Region, DimensionType.Geographical}, // Known alias for geographical dimension type {conf.Tokens.VariableTypes.Other, DimensionType.Other}, {conf.Tokens.VariableTypes.Unknown, DimensionType.Unknown} }; diff --git a/Px.Utils/PxFile/PxFileConfiguration.cs b/Px.Utils/PxFile/PxFileConfiguration.cs index f1cbf897..ce244a54 100644 --- a/Px.Utils/PxFile/PxFileConfiguration.cs +++ b/Px.Utils/PxFile/PxFileConfiguration.cs @@ -1,4 +1,4 @@ -using Px.Utils.Models.Metadata.Enums; +using Px.Utils.Models.Metadata.Enums; namespace Px.Utils.PxFile { @@ -110,19 +110,23 @@ private TimeValue() { } public class VariableTypeTokens { private const string CONTENT = "Content"; + private const string CONTENTS = "Contents"; private const string TIME = "Time"; private const string ORDINAL = "Ordinal"; private const string NOMINAL = "Nominal"; private const string GEOGRAPHICAL = "Geographical"; + private const string REGION = "Region"; private const string OTHER = "Other"; private const string UNKNOWN = "Unknown"; private const string CLASSIFICATORY = "Classificatory"; public string Content { get; set; } = CONTENT; + public string Contents { get; set; } = CONTENTS; public string Time { get; set; } = TIME; public string Ordinal { get; set; } = ORDINAL; public string Nominal { get; set; } = NOMINAL; public string Geographical { get; set; } = GEOGRAPHICAL; + public string Region { get; set; } = REGION; public string Other { get; set; } = OTHER; public string Unknown { get; set; } = UNKNOWN; public string Classificatory { get; set; } = CLASSIFICATORY; diff --git a/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs b/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs index 0de2425a..8f71281d 100644 --- a/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs +++ b/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs @@ -323,7 +323,7 @@ public sealed partial class ContentValidator string[] allowedCharsets = ["ANSI", "Unicode"]; string[] dimensionTypes = [ - validator.Conf.Tokens.VariableTypes.Content, + validator.Conf.Tokens.VariableTypes.Content, validator.Conf.Tokens.VariableTypes.Time, validator.Conf.Tokens.VariableTypes.Geographical, validator.Conf.Tokens.VariableTypes.Ordinal, @@ -331,20 +331,30 @@ public sealed partial class ContentValidator validator.Conf.Tokens.VariableTypes.Other, validator.Conf.Tokens.VariableTypes.Unknown, validator.Conf.Tokens.VariableTypes.Classificatory - ]; + ]; + + string[] knownDimensionTypeAliases = [ + validator.Conf.Tokens.VariableTypes.Contents, + validator.Conf.Tokens.VariableTypes.Region, + ]; string value = SyntaxValidationUtilityMethods.CleanString(entry.Value, validator.Conf); if ((entry.Key.Keyword == validator.Conf.Tokens.KeyWords.Charset && !allowedCharsets.Contains(value)) || (entry.Key.Keyword == validator.Conf.Tokens.KeyWords.CodePage && !value.Equals(validator._encoding.BodyName, StringComparison.OrdinalIgnoreCase)) || (entry.Key.Keyword == validator.Conf.Tokens.KeyWords.DimensionType && !dimensionTypes.Contains(value))) { + // If the value is included in known aliases, set level to warning instead of error + ValidationFeedbackLevel level = entry.Key.Keyword == validator.Conf.Tokens.KeyWords.DimensionType && knownDimensionTypeAliases.Contains(value) ? + ValidationFeedbackLevel.Warning : + ValidationFeedbackLevel.Error; + KeyValuePair feedbackIndexes = SyntaxValidationUtilityMethods.GetLineAndCharacterIndex( entry.KeyStartLineIndex, entry.ValueStartIndex, entry.LineChangeIndexes); KeyValuePair feedback = new( - new(ValidationFeedbackLevel.Error, + new(level, ValidationFeedbackRule.InvalidValueFound), new(validator._filename, feedbackIndexes.Key, From 31ab9a5e86e1d5439c4d35fd4138782a0a6b2972 Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Thu, 30 Apr 2026 08:17:50 +0300 Subject: [PATCH 10/21] Rename test --- .../Validation/ContentValidationTests/ContentValidationTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs b/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs index cb596c6b..e647a6fc 100644 --- a/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs +++ b/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs @@ -537,7 +537,7 @@ public void ValidateValueTypesCalledWithStructuredEntryArrayWithWrongValuesRetur } [TestMethod] - public void ValidateValueTypesCalledWithStructuredEntryArrayWithKnownDimensionTypeAliasesReturnsWithWarnings() + public void ValidateValueContentsCalledWithStructuredEntryArrayWithKnownDimensionTypeAliasesReturnsWithWarnings() { // Arrange ValidationStructuredEntry[] entries = ContentValidationFixtures.STRUCTURED_ENTRY_ARRAY_WITH_KNOWN_DIMENSIONTYPE_ALIASES; From c2d5794ce08e89c3b6071ea40d04db1f209d9940 Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Thu, 30 Apr 2026 09:12:46 +0300 Subject: [PATCH 11/21] Support for custom dimension type aliases --- .../StringToDimensionTypeTests.cs | 26 +++++++- .../ContentValidationTests.cs | 29 +++++++++ .../ModelBuilders/ValueParserUtilities.cs | 45 +++++++++---- Px.Utils/PxFile/PxFileConfiguration.cs | 26 ++++---- ...ntentValidator.ValidationEntryFunctions.cs | 64 +++++++++++++------ docs/architecture.models.md | 2 + docs/architecture.validation.md | 2 + 7 files changed, 147 insertions(+), 47 deletions(-) diff --git a/Px.Utils.UnitTests/ModelBuilderTests/ValueParserUtilitiesTests/StringToDimensionTypeTests.cs b/Px.Utils.UnitTests/ModelBuilderTests/ValueParserUtilitiesTests/StringToDimensionTypeTests.cs index c059b6d9..e7d4db91 100644 --- a/Px.Utils.UnitTests/ModelBuilderTests/ValueParserUtilitiesTests/StringToDimensionTypeTests.cs +++ b/Px.Utils.UnitTests/ModelBuilderTests/ValueParserUtilitiesTests/StringToDimensionTypeTests.cs @@ -1,7 +1,8 @@ -using Px.Utils.ModelBuilders; +using Px.Utils.ModelBuilders; using Px.Utils.Models.Metadata.Enums; +using Px.Utils.PxFile; -namespace ModelBuilderTests.ValueParserUtilitiesTests +namespace Px.Utils.UnitTests.ModelBuilderTests.ValueParserUtilitiesTests { [TestClass] public class StringToDimensionTypeTests @@ -24,6 +25,27 @@ public void ContentTest() Assert.AreEqual(expected, actual); } + [TestMethod] + public void ContentAliasTest() + { + string input = "Contents"; + DimensionType expected = DimensionType.Content; + DimensionType actual = ValueParserUtilities.StringToDimensionType(input); + Assert.AreEqual(expected, actual); + } + + [TestMethod] + public void CustomAliasTest() + { + string input = "Ranking"; + PxFileConfiguration conf = PxFileConfiguration.Default; + conf.Tokens.VariableTypes.Ordinal = ["Ordinal", "Ranking"]; + + DimensionType expected = DimensionType.Ordinal; + DimensionType actual = ValueParserUtilities.StringToDimensionType(input, conf); + Assert.AreEqual(expected, actual); + } + [TestMethod] public void DefaultValueTest() { diff --git a/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs b/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs index e647a6fc..4054f81f 100644 --- a/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs +++ b/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs @@ -2,6 +2,7 @@ using Px.Utils.Validation; using Px.Utils.Validation.ContentValidation; using Px.Utils.Validation.SyntaxValidation; +using Px.Utils.PxFile; using System.Text; using System.Reflection; @@ -557,6 +558,34 @@ public void ValidateValueContentsCalledWithStructuredEntryArrayWithKnownDimensio } } + [TestMethod] + public void ValidateValueContentsCalledWithConfiguredDimensionTypeAliasReturnsWithWarning() + { + // Arrange + PxFileConfiguration conf = PxFileConfiguration.Default; + conf.Tokens.VariableTypes.Ordinal = ["Ordinal", "Ranking"]; + + ValidationStructuredEntry entry = new( + filename, + new ValidationStructuredEntryKey("VARIABLE-TYPE", "fi", "foo"), + "Ranking", + 0, + [], + 0, + Px.Utils.Validation.ValueType.StringValue); + + ContentValidator validator = new(filename, encoding, [entry], conf: conf); + + // Act + ValidationFeedback? result = ContentValidator.ValidateValueContents(entry, validator); + + // Assert + Assert.IsNotNull(result); + Assert.HasCount(1, result); + Assert.AreEqual(ValidationFeedbackRule.InvalidValueFound, result.First().Key.Rule); + Assert.AreEqual(ValidationFeedbackLevel.Warning, result.First().Key.Level); + } + [TestMethod] public void ValidateValueAmountsCalledWithUnmatchingAmountOfElementsReturnsWithError() { diff --git a/Px.Utils/ModelBuilders/ValueParserUtilities.cs b/Px.Utils/ModelBuilders/ValueParserUtilities.cs index ced54567..1a9792a5 100644 --- a/Px.Utils/ModelBuilders/ValueParserUtilities.cs +++ b/Px.Utils/ModelBuilders/ValueParserUtilities.cs @@ -102,6 +102,7 @@ public static string GetTimeValValueRangeString(string input, PxFileConfiguratio /// /// Parses a string into a enumeration value. /// This method maps the input string to a enumeration value based on the provided or default PxFileConfiguration configuration. + /// The first configured token for a dimension type is treated as the primary value and any additional configured tokens are treated as aliases. /// If the input string does not map to a known , the method returns . /// /// The string to parse into a enumeration value. @@ -110,22 +111,40 @@ public static string GetTimeValValueRangeString(string input, PxFileConfiguratio public static DimensionType StringToDimensionType(string input, PxFileConfiguration? conf = null) { conf ??= PxFileConfiguration.Default; - Dictionary map = new() + Dictionary map = GetDimensionTypeTokenMap(conf); + + string cleanString = input.CleanStringDelimeters(conf.Symbols.Value.StringDelimeter); + if (map.TryGetValue(cleanString, out DimensionType value)) { - {conf.Tokens.VariableTypes.Content, DimensionType.Content}, - {conf.Tokens.VariableTypes.Contents, DimensionType.Content}, // Known alias for content dimension type - {conf.Tokens.VariableTypes.Time, DimensionType.Time}, - {conf.Tokens.VariableTypes.Ordinal, DimensionType.Ordinal}, - {conf.Tokens.VariableTypes.Nominal, DimensionType.Nominal}, - {conf.Tokens.VariableTypes.Geographical, DimensionType.Geographical}, - {conf.Tokens.VariableTypes.Region, DimensionType.Geographical}, // Known alias for geographical dimension type - {conf.Tokens.VariableTypes.Other, DimensionType.Other}, - {conf.Tokens.VariableTypes.Unknown, DimensionType.Unknown} + return value; + } + + return DimensionType.Unknown; + } + + private static Dictionary GetDimensionTypeTokenMap(PxFileConfiguration conf) + { + Dictionary map = []; + Dictionary tokensByType = new() + { + { DimensionType.Content, conf.Tokens.VariableTypes.Content }, + { DimensionType.Time, conf.Tokens.VariableTypes.Time }, + { DimensionType.Ordinal, conf.Tokens.VariableTypes.Ordinal }, + { DimensionType.Nominal, conf.Tokens.VariableTypes.Nominal }, + { DimensionType.Geographical, conf.Tokens.VariableTypes.Geographical }, + { DimensionType.Other, conf.Tokens.VariableTypes.Other }, + { DimensionType.Unknown, conf.Tokens.VariableTypes.Unknown } }; - string cleanString = input.CleanStringDelimeters(conf.Symbols.Value.StringDelimeter); - if (map.TryGetValue(cleanString, out DimensionType value)) return value; - else return DimensionType.Unknown; + foreach (KeyValuePair tokenSet in tokensByType) + { + foreach (string token in tokenSet.Value) + { + map[token] = tokenSet.Key; + } + } + + return map; } } } diff --git a/Px.Utils/PxFile/PxFileConfiguration.cs b/Px.Utils/PxFile/PxFileConfiguration.cs index ce244a54..da2ba25c 100644 --- a/Px.Utils/PxFile/PxFileConfiguration.cs +++ b/Px.Utils/PxFile/PxFileConfiguration.cs @@ -107,29 +107,31 @@ private TimeValue() { } public static TimeValue DefaultTimeValue => new(); } + /// + /// Defines accepted tokens for parsing and validating dimension types. + /// The first token in each array is treated as the primary value, while additional tokens are treated as aliases. + /// public class VariableTypeTokens { private const string CONTENT = "Content"; - private const string CONTENTS = "Contents"; + private const string CONTENTS_ALIAS = "Contents"; private const string TIME = "Time"; private const string ORDINAL = "Ordinal"; private const string NOMINAL = "Nominal"; private const string GEOGRAPHICAL = "Geographical"; - private const string REGION = "Region"; + private const string REGION_ALIAS = "Region"; private const string OTHER = "Other"; private const string UNKNOWN = "Unknown"; private const string CLASSIFICATORY = "Classificatory"; - public string Content { get; set; } = CONTENT; - public string Contents { get; set; } = CONTENTS; - public string Time { get; set; } = TIME; - public string Ordinal { get; set; } = ORDINAL; - public string Nominal { get; set; } = NOMINAL; - public string Geographical { get; set; } = GEOGRAPHICAL; - public string Region { get; set; } = REGION; - public string Other { get; set; } = OTHER; - public string Unknown { get; set; } = UNKNOWN; - public string Classificatory { get; set; } = CLASSIFICATORY; + public string[] Content { get; set; } = [CONTENT, CONTENTS_ALIAS]; + public string[] Time { get; set; } = [TIME]; + public string[] Ordinal { get; set; } = [ORDINAL]; + public string[] Nominal { get; set; } = [NOMINAL]; + public string[] Geographical { get; set; } = [GEOGRAPHICAL, REGION_ALIAS]; + public string[] Other { get; set; } = [OTHER]; + public string[] Unknown { get; set; } = [UNKNOWN]; + public string[] Classificatory { get; set; } = [CLASSIFICATORY]; private VariableTypeTokens() { } diff --git a/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs b/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs index 8f71281d..c7926312 100644 --- a/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs +++ b/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs @@ -1,3 +1,4 @@ +using Px.Utils.PxFile; using Px.Utils.Validation.SyntaxValidation; using System.Globalization; @@ -322,31 +323,19 @@ public sealed partial class ContentValidator { string[] allowedCharsets = ["ANSI", "Unicode"]; - string[] dimensionTypes = [ - validator.Conf.Tokens.VariableTypes.Content, - validator.Conf.Tokens.VariableTypes.Time, - validator.Conf.Tokens.VariableTypes.Geographical, - validator.Conf.Tokens.VariableTypes.Ordinal, - validator.Conf.Tokens.VariableTypes.Nominal, - validator.Conf.Tokens.VariableTypes.Other, - validator.Conf.Tokens.VariableTypes.Unknown, - validator.Conf.Tokens.VariableTypes.Classificatory - ]; - - string[] knownDimensionTypeAliases = [ - validator.Conf.Tokens.VariableTypes.Contents, - validator.Conf.Tokens.VariableTypes.Region, - ]; + HashSet primaryDimensionTypes = GetPrimaryDimensionTypeTokens(validator.Conf); + HashSet aliasDimensionTypes = GetAliasDimensionTypeTokens(validator.Conf); string value = SyntaxValidationUtilityMethods.CleanString(entry.Value, validator.Conf); if ((entry.Key.Keyword == validator.Conf.Tokens.KeyWords.Charset && !allowedCharsets.Contains(value)) || (entry.Key.Keyword == validator.Conf.Tokens.KeyWords.CodePage && !value.Equals(validator._encoding.BodyName, StringComparison.OrdinalIgnoreCase)) || - (entry.Key.Keyword == validator.Conf.Tokens.KeyWords.DimensionType && !dimensionTypes.Contains(value))) + (entry.Key.Keyword == validator.Conf.Tokens.KeyWords.DimensionType && !primaryDimensionTypes.Contains(value) && !aliasDimensionTypes.Contains(value)) || + (entry.Key.Keyword == validator.Conf.Tokens.KeyWords.DimensionType && aliasDimensionTypes.Contains(value))) { - // If the value is included in known aliases, set level to warning instead of error - ValidationFeedbackLevel level = entry.Key.Keyword == validator.Conf.Tokens.KeyWords.DimensionType && knownDimensionTypeAliases.Contains(value) ? - ValidationFeedbackLevel.Warning : - ValidationFeedbackLevel.Error; + // Using a known alias dimension type as a value for DimensionType is not invalid but is not recommended, so it is reported as a warning instead of an error + ValidationFeedbackLevel level = entry.Key.Keyword == validator.Conf.Tokens.KeyWords.DimensionType && aliasDimensionTypes.Contains(value) + ? ValidationFeedbackLevel.Warning + : ValidationFeedbackLevel.Error; KeyValuePair feedbackIndexes = SyntaxValidationUtilityMethods.GetLineAndCharacterIndex( entry.KeyStartLineIndex, @@ -473,5 +462,40 @@ validator._dimensionValueNames is null || } return null; } + + private static HashSet GetPrimaryDimensionTypeTokens(PxFileConfiguration conf) + { + IEnumerable primaryTokens = GetDimensionTypeTokenSets(conf) + .Select(tokens => tokens.FirstOrDefault()) + .Where(token => !string.IsNullOrWhiteSpace(token)) + .Cast(); + + return [.. primaryTokens]; + } + + private static HashSet GetAliasDimensionTypeTokens(PxFileConfiguration conf) + { + IEnumerable aliasTokens = GetDimensionTypeTokenSets(conf) + .Where(tokens => tokens.Length > 1) + .SelectMany(tokens => tokens.Skip(1)) + .Where(token => !string.IsNullOrWhiteSpace(token)); + + return [.. aliasTokens]; + } + + private static IEnumerable GetDimensionTypeTokenSets(PxFileConfiguration conf) + { + return + [ + conf.Tokens.VariableTypes.Content, + conf.Tokens.VariableTypes.Time, + conf.Tokens.VariableTypes.Ordinal, + conf.Tokens.VariableTypes.Nominal, + conf.Tokens.VariableTypes.Geographical, + conf.Tokens.VariableTypes.Other, + conf.Tokens.VariableTypes.Unknown, + conf.Tokens.VariableTypes.Classificatory + ]; + } } } diff --git a/docs/architecture.models.md b/docs/architecture.models.md index 7705f8cc..9c6143bc 100644 --- a/docs/architecture.models.md +++ b/docs/architecture.models.md @@ -62,6 +62,8 @@ Task BuildAsync(IAsyncEnumerable> m Helpers: `MetadataEntryKeyBuilder`, `ValueParserUtilities`, `MetadataEntryKey`. +`PxFileConfiguration.TokenDefinitions.VariableTypeTokens` exposes configurable string arrays for each dimension type. The first value is the primary token used by builders, and any additional values are treated as aliases during parsing. + ## Extension Methods | File | Purpose | diff --git a/docs/architecture.validation.md b/docs/architecture.validation.md index 1873220f..84736d06 100644 --- a/docs/architecture.validation.md +++ b/docs/architecture.validation.md @@ -34,6 +34,8 @@ Validates metadata content (required keys, language definitions, dimension consi File: `Validation/ContentValidation/ContentValidator.cs` Partial files: `ValidationEntryFunctions.cs`, `ValidationFindKeywordFunctions.cs`, `UtilityMethods.cs` +Dimension type validation uses `PxFileConfiguration.TokenDefinitions.VariableTypeTokens`. The first configured token for each dimension type is treated as the recommended primary value, while additional configured tokens are accepted as aliases and reported as warnings. + ### DataValidator Validates data section (row counts, row lengths, value types, separators). From a509e408881319916dbb314f3781ae30ad84a2a6 Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Thu, 30 Apr 2026 10:19:09 +0300 Subject: [PATCH 12/21] Cleanup --- ...ntentValidator.ValidationEntryFunctions.cs | 155 ++++++++++-------- docs/architecture.models.md | 2 +- 2 files changed, 85 insertions(+), 72 deletions(-) diff --git a/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs b/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs index c7926312..99cc464e 100644 --- a/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs +++ b/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs @@ -1,5 +1,6 @@ using Px.Utils.PxFile; using Px.Utils.Validation.SyntaxValidation; +using System.ComponentModel.DataAnnotations; using System.Globalization; namespace Px.Utils.Validation.ContentValidation @@ -318,71 +319,81 @@ public sealed partial class ContentValidator /// /// Entry in the Px file metadata. Represented by a object /// object that stores information that is gathered during the validation process - /// Key value pair containing information about the rule violation is returned if an unexpected value is detected + /// Key value pair containing information about the rule violation is returned if an unexpected or unrecommended value is detected from the entry public static ValidationFeedback? ValidateValueContents(ValidationStructuredEntry entry, ContentValidator validator) { - string[] allowedCharsets = ["ANSI", "Unicode"]; - - HashSet primaryDimensionTypes = GetPrimaryDimensionTypeTokens(validator.Conf); - HashSet aliasDimensionTypes = GetAliasDimensionTypeTokens(validator.Conf); - + string[] recommendedValueContents = []; + string[] knownAliases = []; + StringComparer comparer = StringComparer.Ordinal; + string keyword = entry.Key.Keyword; string value = SyntaxValidationUtilityMethods.CleanString(entry.Value, validator.Conf); - if ((entry.Key.Keyword == validator.Conf.Tokens.KeyWords.Charset && !allowedCharsets.Contains(value)) || - (entry.Key.Keyword == validator.Conf.Tokens.KeyWords.CodePage && !value.Equals(validator._encoding.BodyName, StringComparison.OrdinalIgnoreCase)) || - (entry.Key.Keyword == validator.Conf.Tokens.KeyWords.DimensionType && !primaryDimensionTypes.Contains(value) && !aliasDimensionTypes.Contains(value)) || - (entry.Key.Keyword == validator.Conf.Tokens.KeyWords.DimensionType && aliasDimensionTypes.Contains(value))) - { - // Using a known alias dimension type as a value for DimensionType is not invalid but is not recommended, so it is reported as a warning instead of an error - ValidationFeedbackLevel level = entry.Key.Keyword == validator.Conf.Tokens.KeyWords.DimensionType && aliasDimensionTypes.Contains(value) - ? ValidationFeedbackLevel.Warning - : ValidationFeedbackLevel.Error; - - KeyValuePair feedbackIndexes = SyntaxValidationUtilityMethods.GetLineAndCharacterIndex( - entry.KeyStartLineIndex, - entry.ValueStartIndex, - entry.LineChangeIndexes); - KeyValuePair feedback = new( - new(level, - ValidationFeedbackRule.InvalidValueFound), - new(validator._filename, - feedbackIndexes.Key, - feedbackIndexes.Value, - $"{entry.Key.Keyword}: {entry.Value}") - ); - - return new(feedback); + if (keyword == validator.Conf.Tokens.KeyWords.Charset) + { + recommendedValueContents = ["ANSI", "Unicode"]; + } + else if (keyword == validator.Conf.Tokens.KeyWords.CodePage) + { + recommendedValueContents = [validator._encoding.BodyName]; + comparer = StringComparer.OrdinalIgnoreCase; + } + else if (keyword == validator.Conf.Tokens.KeyWords.DimensionType) + { + (recommendedValueContents, knownAliases) = BuildAllowedDimensionTypes(validator.Conf); } - else if (entry.Key.Keyword == validator.Conf.Tokens.KeyWords.ContentVariableIdentifier) + else if (keyword == validator.Conf.Tokens.KeyWords.ContentVariableIdentifier) { string defaultLanguage = validator._defaultLanguage ?? string.Empty; string lang = entry.Key.Language ?? defaultLanguage; - if (validator._stubDimensionNames is not null && validator._stubDimensionNames.TryGetValue(lang, out string[]? stubValues) && - !Array.Exists(stubValues, d => d == value) && - (validator._headingDimensionNames is not null && validator._headingDimensionNames.TryGetValue(lang, out string[]? headingValues) && - !Array.Exists(headingValues, d => d == value))) - { - KeyValuePair feedbackIndexes = SyntaxValidationUtilityMethods.GetLineAndCharacterIndex( - entry.KeyStartLineIndex, - entry.ValueStartIndex, - entry.LineChangeIndexes); + List dimensionNames = []; - KeyValuePair feedback = new( - new(ValidationFeedbackLevel.Error, - ValidationFeedbackRule.InvalidValueFound), - new(validator._filename, - feedbackIndexes.Key, - feedbackIndexes.Value, - $"{entry.Key.Keyword}: {entry.Value}") - ); + if (validator._stubDimensionNames is not null && validator._stubDimensionNames.TryGetValue(lang, out string[]? stubValues)) + { + dimensionNames.AddRange(stubValues); + } - return new(feedback); + if (validator._headingDimensionNames is not null && validator._headingDimensionNames.TryGetValue(lang, out string[]? headingValues)) + { + dimensionNames.AddRange(headingValues); } + + recommendedValueContents = [.. dimensionNames]; + } + + if (recommendedValueContents.Length > 0 && !recommendedValueContents.Contains(value, comparer)) + { + ValidationFeedbackLevel level = knownAliases.Contains(value, comparer) + ? ValidationFeedbackLevel.Warning + : ValidationFeedbackLevel.Error; + + return CreateInvalidValueFeedback(entry, validator, level); } return null; } + private static ValidationFeedback CreateInvalidValueFeedback( + ValidationStructuredEntry entry, + ContentValidator validator, + ValidationFeedbackLevel level) + { + KeyValuePair feedbackIndexes = SyntaxValidationUtilityMethods.GetLineAndCharacterIndex( + entry.KeyStartLineIndex, + entry.ValueStartIndex, + entry.LineChangeIndexes); + + KeyValuePair feedback = new( + new(level, + ValidationFeedbackRule.InvalidValueFound), + new(validator._filename, + feedbackIndexes.Key, + feedbackIndexes.Value, + $"{entry.Key.Keyword}: {entry.Value}") + ); + + return new(feedback); + } + /// /// Validates that entries with specific keywords have the correct amount of values in the Px file metadata entry /// @@ -463,29 +474,11 @@ validator._dimensionValueNames is null || return null; } - private static HashSet GetPrimaryDimensionTypeTokens(PxFileConfiguration conf) + private static (string[] RecommendedValueContents, string[] KnownAliases) BuildAllowedDimensionTypes(PxFileConfiguration conf) { - IEnumerable primaryTokens = GetDimensionTypeTokenSets(conf) - .Select(tokens => tokens.FirstOrDefault()) - .Where(token => !string.IsNullOrWhiteSpace(token)) - .Cast(); - - return [.. primaryTokens]; - } - - private static HashSet GetAliasDimensionTypeTokens(PxFileConfiguration conf) - { - IEnumerable aliasTokens = GetDimensionTypeTokenSets(conf) - .Where(tokens => tokens.Length > 1) - .SelectMany(tokens => tokens.Skip(1)) - .Where(token => !string.IsNullOrWhiteSpace(token)); - - return [.. aliasTokens]; - } - - private static IEnumerable GetDimensionTypeTokenSets(PxFileConfiguration conf) - { - return + List primaryDimensionTypes = []; + List aliasDimensionTypes = []; + string[][] dimensionTypeTokenSets = [ conf.Tokens.VariableTypes.Content, conf.Tokens.VariableTypes.Time, @@ -496,6 +489,26 @@ private static IEnumerable GetDimensionTypeTokenSets(PxFileConfigurati conf.Tokens.VariableTypes.Unknown, conf.Tokens.VariableTypes.Classificatory ]; + + foreach (string[] tokenSet in dimensionTypeTokenSets) + { + if (tokenSet.Length == 0 || string.IsNullOrWhiteSpace(tokenSet[0])) + { + continue; + } + + primaryDimensionTypes.Add(tokenSet[0]); + + for (int i = 1; i < tokenSet.Length; i++) + { + if (!string.IsNullOrWhiteSpace(tokenSet[i])) + { + aliasDimensionTypes.Add(tokenSet[i]); + } + } + } + + return ([.. primaryDimensionTypes], [.. aliasDimensionTypes]); } } } diff --git a/docs/architecture.models.md b/docs/architecture.models.md index 9c6143bc..2ebfaa53 100644 --- a/docs/architecture.models.md +++ b/docs/architecture.models.md @@ -62,7 +62,7 @@ Task BuildAsync(IAsyncEnumerable> m Helpers: `MetadataEntryKeyBuilder`, `ValueParserUtilities`, `MetadataEntryKey`. -`PxFileConfiguration.TokenDefinitions.VariableTypeTokens` exposes configurable string arrays for each dimension type. The first value is the primary token used by builders, and any additional values are treated as aliases during parsing. +`PxFileConfiguration.TokenDefinitions.VariableTypeTokens` exposes configurable string arrays for each dimension type. The first value is the primary recommended token for validation and parsing, and any additional values are treated as aliases during parsing. ## Extension Methods From fff7b9d5dab64c624f8fb3e2dafec842cd177e54 Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Thu, 30 Apr 2026 10:36:45 +0300 Subject: [PATCH 13/21] More cleanup --- .../ContentValidator.ValidationEntryFunctions.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs b/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs index 99cc464e..77c17aee 100644 --- a/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs +++ b/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs @@ -1,6 +1,5 @@ using Px.Utils.PxFile; using Px.Utils.Validation.SyntaxValidation; -using System.ComponentModel.DataAnnotations; using System.Globalization; namespace Px.Utils.Validation.ContentValidation From 59688b00eb1299acf080087a3ef6afee3728f096 Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Mon, 1 Jun 2026 11:26:39 +0300 Subject: [PATCH 14/21] Simpler implementation for accepted dimension type values --- .../StringToDimensionTypeTests.cs | 2 +- .../ContentValidationTests.cs | 65 +++++++++++++------ .../Fixtures/ContentValidationFixtures.cs | 6 -- .../ModelBuilders/ValueParserUtilities.cs | 30 +-------- Px.Utils/PxFile/PxFileConfiguration.cs | 37 +++++------ ...ntentValidator.ValidationEntryFunctions.cs | 51 ++++----------- 6 files changed, 77 insertions(+), 114 deletions(-) diff --git a/Px.Utils.UnitTests/ModelBuilderTests/ValueParserUtilitiesTests/StringToDimensionTypeTests.cs b/Px.Utils.UnitTests/ModelBuilderTests/ValueParserUtilitiesTests/StringToDimensionTypeTests.cs index e7d4db91..a48fa7df 100644 --- a/Px.Utils.UnitTests/ModelBuilderTests/ValueParserUtilitiesTests/StringToDimensionTypeTests.cs +++ b/Px.Utils.UnitTests/ModelBuilderTests/ValueParserUtilitiesTests/StringToDimensionTypeTests.cs @@ -39,7 +39,7 @@ public void CustomAliasTest() { string input = "Ranking"; PxFileConfiguration conf = PxFileConfiguration.Default; - conf.Tokens.VariableTypes.Ordinal = ["Ordinal", "Ranking"]; + conf.Tokens.VariableTypes.Mappings["Ranking"] = DimensionType.Ordinal; DimensionType expected = DimensionType.Ordinal; DimensionType actual = ValueParserUtilities.StringToDimensionType(input, conf); diff --git a/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs b/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs index 4054f81f..9b13e776 100644 --- a/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs +++ b/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs @@ -3,6 +3,7 @@ using Px.Utils.Validation.ContentValidation; using Px.Utils.Validation.SyntaxValidation; using Px.Utils.PxFile; +using Px.Utils.Models.Metadata.Enums; using System.Text; using System.Reflection; @@ -538,32 +539,59 @@ public void ValidateValueTypesCalledWithStructuredEntryArrayWithWrongValuesRetur } [TestMethod] - public void ValidateValueContentsCalledWithStructuredEntryArrayWithKnownDimensionTypeAliasesReturnsWithWarnings() + public void ValidateValueContentsCalledWithDimensionTypeEnumValueReturnsWithoutFeedback() { // Arrange - ValidationStructuredEntry[] entries = ContentValidationFixtures.STRUCTURED_ENTRY_ARRAY_WITH_KNOWN_DIMENSIONTYPE_ALIASES; - ContentValidator validator = new(filename, encoding, entries); + PxFileConfiguration conf = PxFileConfiguration.Default; + + ValidationStructuredEntry entry = new( + filename, + new ValidationStructuredEntryKey("VARIABLE-TYPE", "fi", "foo"), + "Content", + 0, + [], + 0, + Utils.Validation.ValueType.StringValue); + + ContentValidator validator = new(filename, encoding, [entry], conf: conf); + // Act - foreach (ValidationStructuredEntry entry in entries) - { - ValidationFeedback? result = ContentValidator.ValidateValueContents( - entry, - validator - ); - // Assert - Assert.IsNotNull(result); - Assert.HasCount(1, result); - Assert.AreEqual(ValidationFeedbackRule.InvalidValueFound, result.First().Key.Rule); - Assert.AreEqual(ValidationFeedbackLevel.Warning, result.First().Key.Level); - } + ValidationFeedback? result = ContentValidator.ValidateValueContents(entry, validator); + + // Assert + Assert.IsNull(result); + } + + [TestMethod] + public void ValidateValueContentsCalledWithCustomDimensionTypeValueReturnsWithoutFeedback() + { + // Arrange + PxFileConfiguration conf = PxFileConfiguration.Default; + conf.Tokens.VariableTypes.Mappings["Ranking"] = DimensionType.Ordinal; + + ValidationStructuredEntry entry = new( + filename, + new ValidationStructuredEntryKey("VARIABLE-TYPE", "fi", "foo"), + "Ranking", + 0, + [], + 0, + Utils.Validation.ValueType.StringValue); + + ContentValidator validator = new(filename, encoding, [entry], conf: conf); + + // Act + ValidationFeedback? result = ContentValidator.ValidateValueContents(entry, validator); + + // Assert + Assert.IsNull(result); } [TestMethod] - public void ValidateValueContentsCalledWithConfiguredDimensionTypeAliasReturnsWithWarning() + public void ValidateValueContentsCalledWithUnknownDimensionTypeValueReturnsWithoutFeedback() { // Arrange PxFileConfiguration conf = PxFileConfiguration.Default; - conf.Tokens.VariableTypes.Ordinal = ["Ordinal", "Ranking"]; ValidationStructuredEntry entry = new( filename, @@ -572,7 +600,7 @@ public void ValidateValueContentsCalledWithConfiguredDimensionTypeAliasReturnsWi 0, [], 0, - Px.Utils.Validation.ValueType.StringValue); + Utils.Validation.ValueType.StringValue); ContentValidator validator = new(filename, encoding, [entry], conf: conf); @@ -583,7 +611,6 @@ public void ValidateValueContentsCalledWithConfiguredDimensionTypeAliasReturnsWi Assert.IsNotNull(result); Assert.HasCount(1, result); Assert.AreEqual(ValidationFeedbackRule.InvalidValueFound, result.First().Key.Rule); - Assert.AreEqual(ValidationFeedbackLevel.Warning, result.First().Key.Level); } [TestMethod] diff --git a/Px.Utils.UnitTests/Validation/Fixtures/ContentValidationFixtures.cs b/Px.Utils.UnitTests/Validation/Fixtures/ContentValidationFixtures.cs index ae4ee466..d69aec8b 100644 --- a/Px.Utils.UnitTests/Validation/Fixtures/ContentValidationFixtures.cs +++ b/Px.Utils.UnitTests/Validation/Fixtures/ContentValidationFixtures.cs @@ -755,12 +755,6 @@ internal static class ContentValidationFixtures Utils.Validation.ValueType.StringValue), ]; - internal static ValidationStructuredEntry[] STRUCTURED_ENTRY_ARRAY_WITH_KNOWN_DIMENSIONTYPE_ALIASES => - [ - new ValidationStructuredEntry(filename, dimensionTypeEntryKey, "Contents", 0, [], 0, Utils.Validation.ValueType.StringValue), - new ValidationStructuredEntry(filename, dimensionTypeEntryKey, "Region", 1, [], 0, Utils.Validation.ValueType.StringValue), - ]; - internal static ValidationStructuredEntry StructuredEntryWithUnmatchingAmountOfElements => new(filename, codesBarEntry.Key, diff --git a/Px.Utils/ModelBuilders/ValueParserUtilities.cs b/Px.Utils/ModelBuilders/ValueParserUtilities.cs index 1a9792a5..86c9604a 100644 --- a/Px.Utils/ModelBuilders/ValueParserUtilities.cs +++ b/Px.Utils/ModelBuilders/ValueParserUtilities.cs @@ -102,7 +102,7 @@ public static string GetTimeValValueRangeString(string input, PxFileConfiguratio /// /// Parses a string into a enumeration value. /// This method maps the input string to a enumeration value based on the provided or default PxFileConfiguration configuration. - /// The first configured token for a dimension type is treated as the primary value and any additional configured tokens are treated as aliases. + /// Mapping is done using the configured variable type mappings. /// If the input string does not map to a known , the method returns . /// /// The string to parse into a enumeration value. @@ -111,40 +111,14 @@ public static string GetTimeValValueRangeString(string input, PxFileConfiguratio public static DimensionType StringToDimensionType(string input, PxFileConfiguration? conf = null) { conf ??= PxFileConfiguration.Default; - Dictionary map = GetDimensionTypeTokenMap(conf); string cleanString = input.CleanStringDelimeters(conf.Symbols.Value.StringDelimeter); - if (map.TryGetValue(cleanString, out DimensionType value)) + if (conf.Tokens.VariableTypes.Mappings.TryGetValue(cleanString, out DimensionType value)) { return value; } return DimensionType.Unknown; } - - private static Dictionary GetDimensionTypeTokenMap(PxFileConfiguration conf) - { - Dictionary map = []; - Dictionary tokensByType = new() - { - { DimensionType.Content, conf.Tokens.VariableTypes.Content }, - { DimensionType.Time, conf.Tokens.VariableTypes.Time }, - { DimensionType.Ordinal, conf.Tokens.VariableTypes.Ordinal }, - { DimensionType.Nominal, conf.Tokens.VariableTypes.Nominal }, - { DimensionType.Geographical, conf.Tokens.VariableTypes.Geographical }, - { DimensionType.Other, conf.Tokens.VariableTypes.Other }, - { DimensionType.Unknown, conf.Tokens.VariableTypes.Unknown } - }; - - foreach (KeyValuePair tokenSet in tokensByType) - { - foreach (string token in tokenSet.Value) - { - map[token] = tokenSet.Key; - } - } - - return map; - } } } diff --git a/Px.Utils/PxFile/PxFileConfiguration.cs b/Px.Utils/PxFile/PxFileConfiguration.cs index da2ba25c..7e596b6a 100644 --- a/Px.Utils/PxFile/PxFileConfiguration.cs +++ b/Px.Utils/PxFile/PxFileConfiguration.cs @@ -108,33 +108,28 @@ private TimeValue() { } } /// - /// Defines accepted tokens for parsing and validating dimension types. - /// The first token in each array is treated as the primary value, while additional tokens are treated as aliases. + /// Defines accepted tokens for parsing and validating dimension types using an expandable dictionary string to . /// public class VariableTypeTokens { - private const string CONTENT = "Content"; - private const string CONTENTS_ALIAS = "Contents"; - private const string TIME = "Time"; - private const string ORDINAL = "Ordinal"; - private const string NOMINAL = "Nominal"; - private const string GEOGRAPHICAL = "Geographical"; - private const string REGION_ALIAS = "Region"; - private const string OTHER = "Other"; - private const string UNKNOWN = "Unknown"; - private const string CLASSIFICATORY = "Classificatory"; - - public string[] Content { get; set; } = [CONTENT, CONTENTS_ALIAS]; - public string[] Time { get; set; } = [TIME]; - public string[] Ordinal { get; set; } = [ORDINAL]; - public string[] Nominal { get; set; } = [NOMINAL]; - public string[] Geographical { get; set; } = [GEOGRAPHICAL, REGION_ALIAS]; - public string[] Other { get; set; } = [OTHER]; - public string[] Unknown { get; set; } = [UNKNOWN]; - public string[] Classificatory { get; set; } = [CLASSIFICATORY]; + /// + /// Gets or sets the expandable dictionary mapping string tokens to dimension types. + /// + public Dictionary Mappings { get; set; } = new(StringComparer.Ordinal) + { + { "Time", DimensionType.Time }, + { "Contents", DimensionType.Content }, + { "Geographical", DimensionType.Geographical }, + { "Ordinal", DimensionType.Ordinal }, + { "Nominal", DimensionType.Nominal }, + { "Other", DimensionType.Other }, + }; private VariableTypeTokens() { } + /// + /// Gets the default variable type tokens configuration. + /// public static VariableTypeTokens DefaultVariableTypeTokens => new(); } diff --git a/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs b/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs index 77c17aee..c1a3018a 100644 --- a/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs +++ b/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs @@ -1,5 +1,5 @@ -using Px.Utils.PxFile; using Px.Utils.Validation.SyntaxValidation; +using Px.Utils.Models.Metadata.Enums; using System.Globalization; namespace Px.Utils.Validation.ContentValidation @@ -338,7 +338,17 @@ public sealed partial class ContentValidator } else if (keyword == validator.Conf.Tokens.KeyWords.DimensionType) { - (recommendedValueContents, knownAliases) = BuildAllowedDimensionTypes(validator.Conf); + if (Enum.TryParse(value, out DimensionType enumType) && value == enumType.ToString()) + { + return null; + } + + if (validator.Conf.Tokens.VariableTypes.Mappings.TryGetValue(value, out DimensionType mappedType)) + { + return null; + } + + return CreateInvalidValueFeedback(entry, validator, ValidationFeedbackLevel.Error); } else if (keyword == validator.Conf.Tokens.KeyWords.ContentVariableIdentifier) { @@ -472,42 +482,5 @@ validator._dimensionValueNames is null || } return null; } - - private static (string[] RecommendedValueContents, string[] KnownAliases) BuildAllowedDimensionTypes(PxFileConfiguration conf) - { - List primaryDimensionTypes = []; - List aliasDimensionTypes = []; - string[][] dimensionTypeTokenSets = - [ - conf.Tokens.VariableTypes.Content, - conf.Tokens.VariableTypes.Time, - conf.Tokens.VariableTypes.Ordinal, - conf.Tokens.VariableTypes.Nominal, - conf.Tokens.VariableTypes.Geographical, - conf.Tokens.VariableTypes.Other, - conf.Tokens.VariableTypes.Unknown, - conf.Tokens.VariableTypes.Classificatory - ]; - - foreach (string[] tokenSet in dimensionTypeTokenSets) - { - if (tokenSet.Length == 0 || string.IsNullOrWhiteSpace(tokenSet[0])) - { - continue; - } - - primaryDimensionTypes.Add(tokenSet[0]); - - for (int i = 1; i < tokenSet.Length; i++) - { - if (!string.IsNullOrWhiteSpace(tokenSet[i])) - { - aliasDimensionTypes.Add(tokenSet[i]); - } - } - } - - return ([.. primaryDimensionTypes], [.. aliasDimensionTypes]); - } } } From 44267235ed06f8bf0bc98ef09026f9a92de3a6fc Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Mon, 1 Jun 2026 12:04:18 +0300 Subject: [PATCH 15/21] Fixes validation and metadata building mismatch and tests --- .../StringToDimensionTypeTests.cs | 11 +-------- .../ContentValidationTests.cs | 24 ------------------- .../DatabaseValidatorTests.cs | 2 +- .../Fixtures/ContentValidationFixtures.cs | 5 +--- .../Validation/Fixtures/PxFileFixtures.cs | 10 ++++---- ...ntentValidator.ValidationEntryFunctions.cs | 5 ---- 6 files changed, 8 insertions(+), 49 deletions(-) diff --git a/Px.Utils.UnitTests/ModelBuilderTests/ValueParserUtilitiesTests/StringToDimensionTypeTests.cs b/Px.Utils.UnitTests/ModelBuilderTests/ValueParserUtilitiesTests/StringToDimensionTypeTests.cs index a48fa7df..42629104 100644 --- a/Px.Utils.UnitTests/ModelBuilderTests/ValueParserUtilitiesTests/StringToDimensionTypeTests.cs +++ b/Px.Utils.UnitTests/ModelBuilderTests/ValueParserUtilitiesTests/StringToDimensionTypeTests.cs @@ -17,16 +17,7 @@ public void TimeTest() } [TestMethod] - public void ContentTest() - { - string input = "Content"; - DimensionType expected = DimensionType.Content; - DimensionType actual = ValueParserUtilities.StringToDimensionType(input); - Assert.AreEqual(expected, actual); - } - - [TestMethod] - public void ContentAliasTest() + public void ContentsTest() { string input = "Contents"; DimensionType expected = DimensionType.Content; diff --git a/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs b/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs index 9b13e776..519dbca2 100644 --- a/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs +++ b/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs @@ -538,30 +538,6 @@ public void ValidateValueTypesCalledWithStructuredEntryArrayWithWrongValuesRetur } } - [TestMethod] - public void ValidateValueContentsCalledWithDimensionTypeEnumValueReturnsWithoutFeedback() - { - // Arrange - PxFileConfiguration conf = PxFileConfiguration.Default; - - ValidationStructuredEntry entry = new( - filename, - new ValidationStructuredEntryKey("VARIABLE-TYPE", "fi", "foo"), - "Content", - 0, - [], - 0, - Utils.Validation.ValueType.StringValue); - - ContentValidator validator = new(filename, encoding, [entry], conf: conf); - - // Act - ValidationFeedback? result = ContentValidator.ValidateValueContents(entry, validator); - - // Assert - Assert.IsNull(result); - } - [TestMethod] public void ValidateValueContentsCalledWithCustomDimensionTypeValueReturnsWithoutFeedback() { diff --git a/Px.Utils.UnitTests/Validation/DatabaseValidation/DatabaseValidatorTests.cs b/Px.Utils.UnitTests/Validation/DatabaseValidation/DatabaseValidatorTests.cs index f9657cf5..4b6e2921 100644 --- a/Px.Utils.UnitTests/Validation/DatabaseValidation/DatabaseValidatorTests.cs +++ b/Px.Utils.UnitTests/Validation/DatabaseValidation/DatabaseValidatorTests.cs @@ -161,7 +161,7 @@ public async Task ValidateDatabasAsynceWithCustomFunctionsReturnsFeedback() IDatabaseValidator[] customValidators = [ new MockCustomDatabaseValidator() - ]; + ]; DatabaseValidator validator = new( "database_invalid", customPxFileValidators: customValidators, diff --git a/Px.Utils.UnitTests/Validation/Fixtures/ContentValidationFixtures.cs b/Px.Utils.UnitTests/Validation/Fixtures/ContentValidationFixtures.cs index d69aec8b..eada5c99 100644 --- a/Px.Utils.UnitTests/Validation/Fixtures/ContentValidationFixtures.cs +++ b/Px.Utils.UnitTests/Validation/Fixtures/ContentValidationFixtures.cs @@ -306,7 +306,7 @@ internal static class ContentValidationFixtures private static readonly ValidationStructuredEntry variableTypeBarEntry = new(filename, new ValidationStructuredEntryKey("VARIABLE-TYPE", null, "bar"), - "Content", + "Contents", 23, [], 13, @@ -429,9 +429,6 @@ internal static class ContentValidationFixtures 17, Utils.Validation.ValueType.StringValue); - private static readonly ValidationStructuredEntryKey dimensionTypeEntryKey = - new ("VARIABLE-TYPE", "fi", "foo"); - internal static ValidationStructuredEntry[] MINIMAL_STRUCTURED_ENTRY_ARRAY => [ charsetEntry, diff --git a/Px.Utils.UnitTests/Validation/Fixtures/PxFileFixtures.cs b/Px.Utils.UnitTests/Validation/Fixtures/PxFileFixtures.cs index 542a4491..d28536eb 100644 --- a/Px.Utils.UnitTests/Validation/Fixtures/PxFileFixtures.cs +++ b/Px.Utils.UnitTests/Validation/Fixtures/PxFileFixtures.cs @@ -1,4 +1,4 @@ -namespace Px.Utils.UnitTests.Validation.Fixtures +namespace Px.Utils.UnitTests.Validation.Fixtures { internal static class PxFileFixtures { @@ -22,7 +22,7 @@ internal static class PxFileFixtures "\r\nCODES(\"Vuosi\")=\"2015\",\"2016\",\"2017\",\"2018\",\"2019\",\"2020\",\"2021\",\"2022\",\"2023\",\"2024\";" + "\r\nCODES(\"Tiedot\")=\"code-foo-val-a\",\"code-foo-val-b\";" + "\r\nVARIABLE-TYPE(\"Vuosi\")=\"Time\";" + - "\r\nVARIABLE-TYPE(\"Tiedot\")=\"Content\";" + + "\r\nVARIABLE-TYPE(\"Tiedot\")=\"Contents\";" + "\r\nPRECISION(\"Tiedot\",\"foo-val-a\")=1;" + "\r\nPRECISION(\"Tiedot\",\"foo-val-b\")=1;" + "\r\nLAST-UPDATED(\"Tiedot\",\"foo-val-a\")=\"20231101 08:00\";" + @@ -64,7 +64,7 @@ internal static class PxFileFixtures "\r\nCODES(\"Vuosi\")=\"2015\",\"2016\",\"2017\",\"2018\",\"2019\",\"2020\",\"2021\",\"2022\",\"2023\",\"2024\";" + "\r\nCODES(\"Tiedot\")=\"code-foo-val-a\",\"code-foo-val-b\";" + "\r\nVARIABLE-TYPE(\"Vuosi\")=\"Time\";" + - "\r\nVARIABLE-TYPE(\"Tiedot\")=\"Content\";" + + "\r\nVARIABLE-TYPE(\"Tiedot\")=\"Contents\";" + "\r\nPRECISION(\"Tiedot\",\"foo-val-a\")=1;" + "\r\nPRECISION(\"Tiedot\",\"foo-val-b\")=1;" + "\r\nLAST-UPDATED(\"Tiedot\",\"foo-val-a\")=\"20231101 08:00\";" + @@ -94,7 +94,7 @@ internal static class PxFileFixtures "\r\nCODES(\"Vuosi\")=\"2015\",\"2016\",\"2017\",\"2018\",\"2019\",\"2020\",\"2021\",\"2022\",\"2023\",\"2024\";" + "\r\nCODES(\"Tiedot\")=\"code-foo-val-a\",\"code-foo-val-b\";" + "\r\nVARIABLE-TYPE(\"Vuosi\")=\"Time\";" + - "\r\nVARIABLE-TYPE(\"Tiedot\")=\"Content\";" + + "\r\nVARIABLE-TYPE(\"Tiedot\")=\"Contents\";" + "\r\nPRECISION(\"Tiedot\",\"foo-val-a\")=1;" + "\r\nPRECISION(\"Tiedot\",\"foo-val-b\")=1;" + "\r\nLAST-UPDATED(\"Tiedot\",\"foo-val-a\")=\"20231101 08:00\";" + @@ -135,7 +135,7 @@ internal static class PxFileFixtures "\r\nCODES(\"Vuosi\")=\"2015\",\"2016\",\"2017\",\"2018\",\"2019\",\"2020\",\"2021\",\"2022\",\"2023\",\"2024\";" + "\r\nCODES(\"Tiedot\")=\"code-foo-val-a\",\"code-foo-val-b\";" + "\r\nVARIABLE-TYPE(\"Vuosi\")=\"Time\";" + - "\r\nVARIABLE-TYPE(\"Tiedot\")=\"Content\";" + + "\r\nVARIABLE-TYPE(\"Tiedot\")=\"Contents\";" + "\r\nPRECISION(\"Tiedot\",\"foo-val-a\")=1;" + "\r\nLAST-UPDATED(\"Tiedot\",\"foo-val-a\")=\"20231101 08:00\";" + "\r\nUNITS(\"Tiedot\",\"foo-val-a\")=\"kpl\";" + diff --git a/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs b/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs index c1a3018a..e1099a93 100644 --- a/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs +++ b/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs @@ -338,11 +338,6 @@ public sealed partial class ContentValidator } else if (keyword == validator.Conf.Tokens.KeyWords.DimensionType) { - if (Enum.TryParse(value, out DimensionType enumType) && value == enumType.ToString()) - { - return null; - } - if (validator.Conf.Tokens.VariableTypes.Mappings.TryGetValue(value, out DimensionType mappedType)) { return null; From bf37c8f53946508ecd7e5506f813c71e321955b4 Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Mon, 1 Jun 2026 12:07:43 +0300 Subject: [PATCH 16/21] Raise minor version number due to scale of changes --- Px.Utils/Px.Utils.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Px.Utils/Px.Utils.csproj b/Px.Utils/Px.Utils.csproj index bd2ca8d5..2e855475 100644 --- a/Px.Utils/Px.Utils.csproj +++ b/Px.Utils/Px.Utils.csproj @@ -2,7 +2,7 @@ Px.Utils - 1.4.1 + 1.5.0 net10.0 enable enable From 6744b0243be0cc7f3951834c1c4ef1f9ab86522d Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Tue, 2 Jun 2026 15:37:15 +0300 Subject: [PATCH 17/21] Update documentation and make dimension type mapping case insensitive --- .../StringToDimensionTypeTests.cs | 10 ++++++---- .../ContentValidationTests.cs | 10 +++++++--- Px.Utils.sln | 17 +++++++++++++++-- Px.Utils/ModelBuilders/ValueParserUtilities.cs | 1 + Px.Utils/PxFile/PxFileConfiguration.cs | 2 +- docs/PXFILE_SPECIFICATION.md | 3 ++- docs/architecture.models.md | 2 +- docs/architecture.overview.md | 1 + 8 files changed, 34 insertions(+), 12 deletions(-) diff --git a/Px.Utils.UnitTests/ModelBuilderTests/ValueParserUtilitiesTests/StringToDimensionTypeTests.cs b/Px.Utils.UnitTests/ModelBuilderTests/ValueParserUtilitiesTests/StringToDimensionTypeTests.cs index 42629104..85b97de8 100644 --- a/Px.Utils.UnitTests/ModelBuilderTests/ValueParserUtilitiesTests/StringToDimensionTypeTests.cs +++ b/Px.Utils.UnitTests/ModelBuilderTests/ValueParserUtilitiesTests/StringToDimensionTypeTests.cs @@ -1,6 +1,7 @@ using Px.Utils.ModelBuilders; using Px.Utils.Models.Metadata.Enums; using Px.Utils.PxFile; +using System.Globalization; namespace Px.Utils.UnitTests.ModelBuilderTests.ValueParserUtilitiesTests { @@ -26,13 +27,14 @@ public void ContentsTest() } [TestMethod] - public void CustomAliasTest() + [DataRow("RANKING", DimensionType.Ordinal)] + [DataRow("region", DimensionType.Geographical)] + public void CustomAliasTest(string alias, DimensionType expected) { - string input = "Ranking"; + string input = alias.ToUpper(CultureInfo.InvariantCulture); // Testing case-insensitivity PxFileConfiguration conf = PxFileConfiguration.Default; - conf.Tokens.VariableTypes.Mappings["Ranking"] = DimensionType.Ordinal; + conf.Tokens.VariableTypes.Mappings[alias] = expected; - DimensionType expected = DimensionType.Ordinal; DimensionType actual = ValueParserUtilities.StringToDimensionType(input, conf); Assert.AreEqual(expected, actual); } diff --git a/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs b/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs index 519dbca2..fa12843b 100644 --- a/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs +++ b/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs @@ -6,6 +6,7 @@ using Px.Utils.Models.Metadata.Enums; using System.Text; using System.Reflection; +using System.Globalization; namespace Px.Utils.UnitTests.Validation.ContentValidationTests { @@ -539,21 +540,24 @@ public void ValidateValueTypesCalledWithStructuredEntryArrayWithWrongValuesRetur } [TestMethod] - public void ValidateValueContentsCalledWithCustomDimensionTypeValueReturnsWithoutFeedback() + [DataRow("ranking", DimensionType.Ordinal)] + [DataRow("REGION", DimensionType.Geographical)] + public void ValidateValueContentsCalledWithCustomDimensionTypesValueReturnsWithoutFeedback(string alias, DimensionType dimensionType) { // Arrange PxFileConfiguration conf = PxFileConfiguration.Default; - conf.Tokens.VariableTypes.Mappings["Ranking"] = DimensionType.Ordinal; + conf.Tokens.VariableTypes.Mappings[alias] = dimensionType; ValidationStructuredEntry entry = new( filename, new ValidationStructuredEntryKey("VARIABLE-TYPE", "fi", "foo"), - "Ranking", + alias.ToUpper(CultureInfo.InvariantCulture), // The value is converted to upper case to verify case insensitivity of the dimension type matching 0, [], 0, Utils.Validation.ValueType.StringValue); + ContentValidator validator = new(filename, encoding, [entry], conf: conf); // Act diff --git a/Px.Utils.sln b/Px.Utils.sln index 4fe9d1ae..a111bbda 100644 --- a/Px.Utils.sln +++ b/Px.Utils.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 17 -VisualStudioVersion = 17.8.34408.163 +# Visual Studio Version 18 +VisualStudioVersion = 18.5.11709.299 stable MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Px.Utils", "Px.Utils\Px.Utils.csproj", "{D94934AF-6B41-4C66-818A-C4C33230E9DD}" EndProject @@ -36,6 +36,18 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Px.Utils.TestingApp", "Px.U {D94934AF-6B41-4C66-818A-C4C33230E9DD} = {D94934AF-6B41-4C66-818A-C4C33230E9DD} EndProjectSection EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "architecture", "architecture", "{BFA8E05B-EBD8-4085-B765-EAED69DD0874}" + ProjectSection(SolutionItems) = preProject + docs\architecture.md = docs\architecture.md + docs\architecture.models.md = docs\architecture.models.md + docs\architecture.operations.md = docs\architecture.operations.md + docs\architecture.overview.md = docs\architecture.overview.md + docs\architecture.readers.md = docs\architecture.readers.md + docs\architecture.serializers.md = docs\architecture.serializers.md + docs\architecture.testing.md = docs\architecture.testing.md + docs\architecture.validation.md = docs\architecture.validation.md + EndProjectSection +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -60,6 +72,7 @@ Global EndGlobalSection GlobalSection(NestedProjects) = preSolution {EB4119B8-8292-418A-9CD3-2905DA67B889} = {BC1C9772-9B8F-49A7-A8F3-09E3A94789ED} + {BFA8E05B-EBD8-4085-B765-EAED69DD0874} = {42DFE3AF-7507-415A-806B-99B74519BF97} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {B1F81A3B-9FF5-4557-8A5A-A9519C42C2B2} diff --git a/Px.Utils/ModelBuilders/ValueParserUtilities.cs b/Px.Utils/ModelBuilders/ValueParserUtilities.cs index 86c9604a..8ff707e2 100644 --- a/Px.Utils/ModelBuilders/ValueParserUtilities.cs +++ b/Px.Utils/ModelBuilders/ValueParserUtilities.cs @@ -1,6 +1,7 @@ using Px.Utils.Models.Metadata.Enums; using Px.Utils.Models.Metadata.ExtensionMethods; using Px.Utils.PxFile; +using System.Globalization; namespace Px.Utils.ModelBuilders { diff --git a/Px.Utils/PxFile/PxFileConfiguration.cs b/Px.Utils/PxFile/PxFileConfiguration.cs index 7e596b6a..9b4f2f61 100644 --- a/Px.Utils/PxFile/PxFileConfiguration.cs +++ b/Px.Utils/PxFile/PxFileConfiguration.cs @@ -115,7 +115,7 @@ public class VariableTypeTokens /// /// Gets or sets the expandable dictionary mapping string tokens to dimension types. /// - public Dictionary Mappings { get; set; } = new(StringComparer.Ordinal) + public Dictionary Mappings { get; set; } = new(StringComparer.OrdinalIgnoreCase) { { "Time", DimensionType.Time }, { "Contents", DimensionType.Content }, diff --git a/docs/PXFILE_SPECIFICATION.md b/docs/PXFILE_SPECIFICATION.md index ac5ecb0e..8c3d0a65 100644 --- a/docs/PXFILE_SPECIFICATION.md +++ b/docs/PXFILE_SPECIFICATION.md @@ -96,7 +96,7 @@ Whitespace characters are not significant in the value outside of ```"``` separa - Decimal separator is period ```.```. - Thousands separator is not allowed. - Whitespace characters are not allowed between the characters of the number. -- Number must be in range of ±7.9228 x 10^28. +- Number must be in range of ±7.9228 x 10^28. #### List of strings - List items are separated by a comma ```,```. @@ -281,6 +281,7 @@ More spesific encoding information. The value must be a string that matches the - Recommended that the variable type is defined for each variable defined by the STUB or HEADING. - Has a set of allowed values: ```Content```, ```Time```, ```Geographical```, ```Ordinal```, ```Nominal```, ```Other```, ```Classificatory```., ```Unknown```. - Can be defined for each language, but this is not recommended. +- Value is not case sensitive, but uppercase characters are recommended. #### TIMEVAL - See the TIMEVAL entry for the syntax and content requirements. \ No newline at end of file diff --git a/docs/architecture.models.md b/docs/architecture.models.md index 2ebfaa53..d83ff085 100644 --- a/docs/architecture.models.md +++ b/docs/architecture.models.md @@ -62,7 +62,7 @@ Task BuildAsync(IAsyncEnumerable> m Helpers: `MetadataEntryKeyBuilder`, `ValueParserUtilities`, `MetadataEntryKey`. -`PxFileConfiguration.TokenDefinitions.VariableTypeTokens` exposes configurable string arrays for each dimension type. The first value is the primary recommended token for validation and parsing, and any additional values are treated as aliases during parsing. +`PxFileConfiguration.TokenDefinitions.VariableTypeTokens` exposes a configurable `Dictionary` (`Mappings`) used by both parsing and validation for `VARIABLE-TYPE` values. The dictionary uses `StringComparer.OrdinalIgnoreCase`, so lookups are case-insensitive by default. Custom aliases can be added by inserting additional keys into `Mappings`. ## Extension Methods diff --git a/docs/architecture.overview.md b/docs/architecture.overview.md index 3940ea1c..152f4c25 100644 --- a/docs/architecture.overview.md +++ b/docs/architecture.overview.md @@ -27,6 +27,7 @@ No DI container — all types instantiated directly via constructors. Configurat | `Symbols.KeywordSeparator` | Keyword = value separator | `=` | | `Symbols.EntrySeparator` | Entry separator | `;` | | `Tokens.KeyWords.Data` | Data section keyword | `DATA` | +| `Tokens.VariableTypes.Mappings` | `VARIABLE-TYPE` token to `DimensionType` mapping | Case-insensitive dictionary (`StringComparer.OrdinalIgnoreCase`) | ## Key Patterns From 6a3977c4b7c88ed9d5cd3de8d5e3335578fdf7ff Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Tue, 2 Jun 2026 15:40:30 +0300 Subject: [PATCH 18/21] Cleanup --- Px.Utils.sln | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Px.Utils.sln b/Px.Utils.sln index a111bbda..9196bd04 100644 --- a/Px.Utils.sln +++ b/Px.Utils.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 18 -VisualStudioVersion = 18.5.11709.299 stable +# Visual Studio Version 17 +VisualStudioVersion = 17.8.34408.163 MinimumVisualStudioVersion = 10.0.40219.1 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Px.Utils", "Px.Utils\Px.Utils.csproj", "{D94934AF-6B41-4C66-818A-C4C33230E9DD}" EndProject From 3ee5b63462cfa486e2f7b8b421689aba5ea3c10a Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Tue, 2 Jun 2026 15:44:49 +0300 Subject: [PATCH 19/21] Invalid test name --- .../ContentValidationTests/ContentValidationTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs b/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs index fa12843b..b0db7c28 100644 --- a/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs +++ b/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs @@ -568,7 +568,7 @@ public void ValidateValueContentsCalledWithCustomDimensionTypesValueReturnsWitho } [TestMethod] - public void ValidateValueContentsCalledWithUnknownDimensionTypeValueReturnsWithoutFeedback() + public void ValidateValueContentsCalledWithUnknownDimensionTypeValueReturnsWithError() { // Arrange PxFileConfiguration conf = PxFileConfiguration.Default; @@ -591,6 +591,7 @@ public void ValidateValueContentsCalledWithUnknownDimensionTypeValueReturnsWitho Assert.IsNotNull(result); Assert.HasCount(1, result); Assert.AreEqual(ValidationFeedbackRule.InvalidValueFound, result.First().Key.Rule); + Assert.AreEqual(ValidationFeedbackLevel.Error, result.First().Key.Level); } [TestMethod] From 7ff57aeb07167b1301b75007ae62152a81a56b27 Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Tue, 2 Jun 2026 15:57:41 +0300 Subject: [PATCH 20/21] Remove references to removed alias functionality in validation --- ...ntentValidator.ValidationEntryFunctions.cs | 30 ++++++++----------- 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs b/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs index e1099a93..5f0ab0ad 100644 --- a/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs +++ b/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs @@ -318,32 +318,31 @@ public sealed partial class ContentValidator /// /// Entry in the Px file metadata. Represented by a object /// object that stores information that is gathered during the validation process - /// Key value pair containing information about the rule violation is returned if an unexpected or unrecommended value is detected from the entry + /// Key value pair containing information about the rule violation is returned if an unexpected value is detected from the entry public static ValidationFeedback? ValidateValueContents(ValidationStructuredEntry entry, ContentValidator validator) { - string[] recommendedValueContents = []; - string[] knownAliases = []; + string[] allowedValueContents = []; StringComparer comparer = StringComparer.Ordinal; string keyword = entry.Key.Keyword; string value = SyntaxValidationUtilityMethods.CleanString(entry.Value, validator.Conf); if (keyword == validator.Conf.Tokens.KeyWords.Charset) { - recommendedValueContents = ["ANSI", "Unicode"]; + allowedValueContents = ["ANSI", "Unicode"]; } else if (keyword == validator.Conf.Tokens.KeyWords.CodePage) { - recommendedValueContents = [validator._encoding.BodyName]; + allowedValueContents = [validator._encoding.BodyName]; comparer = StringComparer.OrdinalIgnoreCase; } else if (keyword == validator.Conf.Tokens.KeyWords.DimensionType) { - if (validator.Conf.Tokens.VariableTypes.Mappings.TryGetValue(value, out DimensionType mappedType)) + if (validator.Conf.Tokens.VariableTypes.Mappings.TryGetValue(value, out DimensionType _)) { return null; } - return CreateInvalidValueFeedback(entry, validator, ValidationFeedbackLevel.Error); + return CreateInvalidValueFeedback(entry, validator); } else if (keyword == validator.Conf.Tokens.KeyWords.ContentVariableIdentifier) { @@ -361,25 +360,20 @@ public sealed partial class ContentValidator dimensionNames.AddRange(headingValues); } - recommendedValueContents = [.. dimensionNames]; + allowedValueContents = [.. dimensionNames]; } - if (recommendedValueContents.Length > 0 && !recommendedValueContents.Contains(value, comparer)) + if (allowedValueContents.Length == 0 || allowedValueContents.Contains(value, comparer)) { - ValidationFeedbackLevel level = knownAliases.Contains(value, comparer) - ? ValidationFeedbackLevel.Warning - : ValidationFeedbackLevel.Error; - - return CreateInvalidValueFeedback(entry, validator, level); + return null; } - return null; + return CreateInvalidValueFeedback(entry, validator); } private static ValidationFeedback CreateInvalidValueFeedback( ValidationStructuredEntry entry, - ContentValidator validator, - ValidationFeedbackLevel level) + ContentValidator validator) { KeyValuePair feedbackIndexes = SyntaxValidationUtilityMethods.GetLineAndCharacterIndex( entry.KeyStartLineIndex, @@ -387,7 +381,7 @@ private static ValidationFeedback CreateInvalidValueFeedback( entry.LineChangeIndexes); KeyValuePair feedback = new( - new(level, + new(ValidationFeedbackLevel.Error, ValidationFeedbackRule.InvalidValueFound), new(validator._filename, feedbackIndexes.Key, From 47ac546cc5daad17faa2c255adfbb8a860dd9385 Mon Sep 17 00:00:00 2001 From: Sakari Malkki Date: Tue, 2 Jun 2026 15:58:51 +0300 Subject: [PATCH 21/21] Remove excess whitespace --- .../Validation/ContentValidationTests/ContentValidationTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs b/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs index b0db7c28..3fcc6928 100644 --- a/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs +++ b/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs @@ -557,7 +557,6 @@ public void ValidateValueContentsCalledWithCustomDimensionTypesValueReturnsWitho 0, Utils.Validation.ValueType.StringValue); - ContentValidator validator = new(filename, encoding, [entry], conf: conf); // Act