diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index d86b596a..72fae3e1 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -24,5 +24,15 @@ Unit testing: - Use Moq for mocking dependencies in unit tests. - Use PascalCase for naming test methods, following the pattern: MethodNameStateUnderTestExpectedBehavior. +Architecture: +- See `docs/architecture.md` for the architecture index. It links to focused sub-files — read only the one relevant to your task: + - `docs/architecture.overview.md` — solution structure, config, patterns, file map + - `docs/architecture.validation.md` — validation pipeline, interfaces, extensibility + - `docs/architecture.models.md` — models, metadata, dimensions, builders + - `docs/architecture.readers.md` — metadata/data readers, binary codecs + - `docs/architecture.operations.md` — matrix operations + - `docs/architecture.serializers.md` — JSON converters + - `docs/architecture.testing.md` — unit tests, mocks, benchmarks + Other: - When refactoring code, update the documentation comments to reflect the changes made. \ No newline at end of file diff --git a/Px.Utils.UnitTests/ModelBuilderTests/ValueParserUtilitiesTests/StringToDimensionTypeTests.cs b/Px.Utils.UnitTests/ModelBuilderTests/ValueParserUtilitiesTests/StringToDimensionTypeTests.cs index c059b6d9..85b97de8 100644 --- a/Px.Utils.UnitTests/ModelBuilderTests/ValueParserUtilitiesTests/StringToDimensionTypeTests.cs +++ b/Px.Utils.UnitTests/ModelBuilderTests/ValueParserUtilitiesTests/StringToDimensionTypeTests.cs @@ -1,7 +1,9 @@ -using Px.Utils.ModelBuilders; +using Px.Utils.ModelBuilders; using Px.Utils.Models.Metadata.Enums; +using Px.Utils.PxFile; +using System.Globalization; -namespace ModelBuilderTests.ValueParserUtilitiesTests +namespace Px.Utils.UnitTests.ModelBuilderTests.ValueParserUtilitiesTests { [TestClass] public class StringToDimensionTypeTests @@ -16,14 +18,27 @@ public void TimeTest() } [TestMethod] - public void ContentTest() + public void ContentsTest() { - string input = "Content"; + string input = "Contents"; DimensionType expected = DimensionType.Content; DimensionType actual = ValueParserUtilities.StringToDimensionType(input); Assert.AreEqual(expected, actual); } + [TestMethod] + [DataRow("RANKING", DimensionType.Ordinal)] + [DataRow("region", DimensionType.Geographical)] + public void CustomAliasTest(string alias, DimensionType expected) + { + string input = alias.ToUpper(CultureInfo.InvariantCulture); // Testing case-insensitivity + PxFileConfiguration conf = PxFileConfiguration.Default; + conf.Tokens.VariableTypes.Mappings[alias] = expected; + + DimensionType actual = ValueParserUtilities.StringToDimensionType(input, conf); + Assert.AreEqual(expected, actual); + } + [TestMethod] public void DefaultValueTest() { diff --git a/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs b/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs index c0d34ebb..3fcc6928 100644 --- a/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs +++ b/Px.Utils.UnitTests/Validation/ContentValidationTests/ContentValidationTests.cs @@ -2,8 +2,11 @@ using Px.Utils.Validation; using Px.Utils.Validation.ContentValidation; using Px.Utils.Validation.SyntaxValidation; +using Px.Utils.PxFile; +using Px.Utils.Models.Metadata.Enums; using System.Text; using System.Reflection; +using System.Globalization; namespace Px.Utils.UnitTests.Validation.ContentValidationTests { @@ -536,6 +539,60 @@ public void ValidateValueTypesCalledWithStructuredEntryArrayWithWrongValuesRetur } } + [TestMethod] + [DataRow("ranking", DimensionType.Ordinal)] + [DataRow("REGION", DimensionType.Geographical)] + public void ValidateValueContentsCalledWithCustomDimensionTypesValueReturnsWithoutFeedback(string alias, DimensionType dimensionType) + { + // Arrange + PxFileConfiguration conf = PxFileConfiguration.Default; + conf.Tokens.VariableTypes.Mappings[alias] = dimensionType; + + ValidationStructuredEntry entry = new( + filename, + new ValidationStructuredEntryKey("VARIABLE-TYPE", "fi", "foo"), + alias.ToUpper(CultureInfo.InvariantCulture), // The value is converted to upper case to verify case insensitivity of the dimension type matching + 0, + [], + 0, + Utils.Validation.ValueType.StringValue); + + ContentValidator validator = new(filename, encoding, [entry], conf: conf); + + // Act + ValidationFeedback? result = ContentValidator.ValidateValueContents(entry, validator); + + // Assert + Assert.IsNull(result); + } + + [TestMethod] + public void ValidateValueContentsCalledWithUnknownDimensionTypeValueReturnsWithError() + { + // Arrange + PxFileConfiguration conf = PxFileConfiguration.Default; + + ValidationStructuredEntry entry = new( + filename, + new ValidationStructuredEntryKey("VARIABLE-TYPE", "fi", "foo"), + "Ranking", + 0, + [], + 0, + Utils.Validation.ValueType.StringValue); + + ContentValidator validator = new(filename, encoding, [entry], conf: conf); + + // Act + ValidationFeedback? result = ContentValidator.ValidateValueContents(entry, validator); + + // Assert + Assert.IsNotNull(result); + Assert.HasCount(1, result); + Assert.AreEqual(ValidationFeedbackRule.InvalidValueFound, result.First().Key.Rule); + Assert.AreEqual(ValidationFeedbackLevel.Error, result.First().Key.Level); + } + [TestMethod] public void ValidateValueAmountsCalledWithUnmatchingAmountOfElementsReturnsWithError() { @@ -559,6 +616,27 @@ public void ValidateValueAmountsCalledWithUnmatchingAmountOfElementsReturnsWithE Assert.AreEqual(ValidationFeedbackRule.UnmatchingValueAmount, result.First().Key.Rule); } + [TestMethod] + public void ValidateValueAmountsCalledWithMatchingAmountOfElementsWithListDelimetersReturnsNoError() + { + // Arrange + ValidationStructuredEntry[] entries = [ContentValidationFixtures.StructuredEntryWithMatchingAmountOfElementsWithListDelimeters]; + ContentValidator validator = new(filename, encoding, entries); + SetValidatorField(validator, "_defaultLanguage", defaultLanguage); + SetValidatorField(validator, "_availableLanguages", availableLanguages); + SetValidatorField(validator, "_stubDimensionNames", stubDimensionNames); + SetValidatorField(validator, "_dimensionValueNames", dimensionValueNames); + + // Act + ValidationFeedback? result = ContentValidator.ValidateValueAmounts( + entries[0], + validator + ); + + // Assert + Assert.IsNull(result); + } + [TestMethod] public void ValidateValueUppercaseRecommendationsCalledWithLowerCaseEntryReturnsWithWarning() { diff --git a/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs b/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs index b83652ce..66846a97 100644 --- a/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs +++ b/Px.Utils.UnitTests/Validation/DataValidationTests/DataValidationTest.cs @@ -1,5 +1,5 @@ +using System.Linq; using System.Text; -using Microsoft.VisualStudio.TestTools.UnitTesting.Logging; using Px.Utils.UnitTests.Validation.Fixtures; using Px.Utils.Validation; using Px.Utils.Validation.DataValidation; @@ -11,215 +11,71 @@ public class DataValidationTest { public TestContext TestContext { get; set; } - [TestMethod] - public void TestValidateWithoutErrors() - { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.SIMPLE_VALID_DATA)); - stream.Seek(6, 0); - DataValidator validator = new(5, 4, 1); - - ValidationFeedback validationFeedbacks = validator.Validate(stream, "foo", Encoding.UTF8).FeedbackItems; - - foreach (KeyValuePair> validationFeedback in validationFeedbacks) - { - foreach (ValidationFeedbackValue instance in validationFeedback.Value) - { - Logger.LogMessage($"Line {instance.Line}, Char {instance.Character}: " - + $"{validationFeedback.Key.Rule} {instance.AdditionalInfo}"); - } - } - Assert.HasCount(0, validationFeedbacks); - } - - [TestMethod] - public void TestValidateWithoutMissingCodeDelimetersReturnsWithoutErrors() - { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.SIMPLE_VALID_DATA_WITHOUT_MISISNG_CODE_DELIMETERS)); - stream.Seek(6, 0); - DataValidator validator = new(5, 4, 1); - - ValidationFeedback validationFeedbacks = validator.Validate(stream, "foo", Encoding.UTF8).FeedbackItems; - - foreach (KeyValuePair> validationFeedback in validationFeedbacks) - { - foreach (ValidationFeedbackValue instance in validationFeedback.Value) - { - Logger.LogMessage($"Line {instance.Line}, Char {instance.Character}: " - + $"{validationFeedback.Key.Rule} {instance.AdditionalInfo}"); - } - } - Assert.HasCount(0, validationFeedbacks); - } - - [TestMethod] - public async Task TestValidateAsyncWithoutErrors() - { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.SIMPLE_VALID_DATA)); - stream.Seek(6, 0); - DataValidator validator = new(5, 4, 1); - - ValidationResult result = await validator.ValidateAsync(stream, "foo", Encoding.UTF8, cancellationToken: TestContext.CancellationToken); - ValidationFeedback validationFeedbacks = result.FeedbackItems; - - foreach (KeyValuePair> validationFeedback in validationFeedbacks) - { - foreach (ValidationFeedbackValue instance in validationFeedback.Value) - { - Logger.LogMessage($"Line {instance.Line}, Char {instance.Character}: " - + $"{validationFeedback.Key.Rule} {instance.AdditionalInfo}"); - } - } - Assert.HasCount(0, validationFeedbacks); - } + private const int DataStartOffset = 6; [TestMethod] - public async Task TestValidateAsyncWithoutMissingCodeDelimetersReturnsWithoutErrors() + [DataRow(DataStreamContents.SIMPLE_VALID_DATA, 0, 0)] + [DataRow(DataStreamContents.SIMPLE_VALID_DATA_WITH_INCONSISTENT_LINEBREAKS, 0, 0)] + [DataRow(DataStreamContents.SIMPLE_VALID_DATA_WITHOUT_MISISNG_CODE_DELIMETERS, 0, 0)] + [DataRow(DataStreamContents.SIMPLE_INVALID_DATA, 7, 12)] + [DataRow(DataStreamContents.NO_DATA, 2, 6, false)] + [DataRow(DataStreamContents.DATA_ON_SINGLE_ROW, 2, 6, false)] + [DataRow(DataStreamContents.DATA_STARTING_WITH_ENCLOSED_MISSING_VALUE, 0, 0)] + [DataRow(DataStreamContents.DATA_STARTING_WITH_UNENCLOSED_MISSING_VALUE, 0, 0)] + [DataRow(DataStreamContents.DATA_STARTING_WITH_NIL_VALUE, 0, 0)] + [DataRow(DataStreamContents.DATA_STARTING_WITH_UNENCLOSED_NIL_VALUE, 0, 0)] + public void ValidateDataReturnsExpectedErrorCount( + string dataContents, + int expectedUniqueErrorCount, + int expectedTotalErrorCount, + bool shouldSeekDataStart = true) { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.SIMPLE_VALID_DATA_WITHOUT_MISISNG_CODE_DELIMETERS)); - stream.Seek(6, 0); - DataValidator validator = new(5, 4, 1); - - ValidationResult result = await validator.ValidateAsync(stream, "foo", Encoding.UTF8, cancellationToken: TestContext.CancellationToken); - ValidationFeedback validationFeedbacks = result.FeedbackItems; - - foreach (KeyValuePair> validationFeedback in validationFeedbacks) + using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(dataContents)); + if (shouldSeekDataStart) { - foreach (ValidationFeedbackValue instance in validationFeedback.Value) - { - Logger.LogMessage($"Line {instance.Line}, Char {instance.Character}: " - + $"{validationFeedback.Key.Rule} {instance.AdditionalInfo}"); - } + stream.Seek(DataStartOffset, SeekOrigin.Begin); } - Assert.HasCount(0, validationFeedbacks); - } - [TestMethod] - public void TestValidateWithErrors() - { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.SIMPLE_INVALID_DATA)); - stream.Seek(6, 0); DataValidator validator = new(5, 4, 1); ValidationFeedback validationFeedbacks = validator.Validate(stream, "foo", Encoding.UTF8).FeedbackItems; + int actualErrorCount = validationFeedbacks.Values.SelectMany(f => f).Count(); - foreach (KeyValuePair> validationFeedback in validationFeedbacks) - { - foreach (ValidationFeedbackValue instance in validationFeedback.Value) - { - Logger.LogMessage($"Line {instance.Line}, Char {instance.Character}: " - + $"{validationFeedback.Key.Rule} {instance.AdditionalInfo}"); - } - } - - Assert.HasCount(7, validationFeedbacks); // Unique feedbacks - Assert.HasCount(13, validationFeedbacks.Values.SelectMany(f => f)); // Total feedbacks including duplicates + Assert.HasCount(expectedUniqueErrorCount, validationFeedbacks); + Assert.AreEqual(expectedTotalErrorCount, actualErrorCount); } [TestMethod] - public async Task TestValidateAsyncWithErrors() + [DataRow(DataStreamContents.SIMPLE_VALID_DATA, 0, 0)] + [DataRow(DataStreamContents.SIMPLE_VALID_DATA_WITH_INCONSISTENT_LINEBREAKS, 0, 0)] + [DataRow(DataStreamContents.SIMPLE_VALID_DATA_WITHOUT_MISISNG_CODE_DELIMETERS, 0, 0)] + [DataRow(DataStreamContents.SIMPLE_INVALID_DATA, 7, 12)] + [DataRow(DataStreamContents.NO_DATA, 2, 6, false)] + [DataRow(DataStreamContents.DATA_ON_SINGLE_ROW, 2, 6, false)] + [DataRow(DataStreamContents.DATA_STARTING_WITH_ENCLOSED_MISSING_VALUE, 0, 0)] + [DataRow(DataStreamContents.DATA_STARTING_WITH_UNENCLOSED_MISSING_VALUE, 0, 0)] + [DataRow(DataStreamContents.DATA_STARTING_WITH_NIL_VALUE, 0, 0)] + [DataRow(DataStreamContents.DATA_STARTING_WITH_UNENCLOSED_NIL_VALUE, 0, 0)] + public async Task ValidateAsyncDataReturnsExpectedErrorCount( + string dataContents, + int expectedUniqueErrorCount, + int expectedTotalErrorCount, + bool shouldSeekToDataStart = true) { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.SIMPLE_INVALID_DATA)); - stream.Seek(6, 0); - DataValidator validator = new(5, 4, 1); - - ValidationResult result = await validator.ValidateAsync(stream, "foo", Encoding.UTF8, cancellationToken: TestContext.CancellationToken); - ValidationFeedback validationFeedbacks = result.FeedbackItems; - - foreach (KeyValuePair> validationFeedback in validationFeedbacks) - { - foreach (ValidationFeedbackValue instance in validationFeedback.Value) - { - Logger.LogMessage($"Line {instance.Line}, Char {instance.Character}: " - + $"{validationFeedback.Key.Rule} {instance.AdditionalInfo}"); - } - } - - Assert.HasCount(7, validationFeedbacks); // Unique feedbacks - Assert.HasCount(13, validationFeedbacks.Values.SelectMany(f => f)); // Total feedbacks including duplicates - } - - [TestMethod] - public void ValidateWithoutDataReturnsErrors() - { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.NO_DATA)); - DataValidator validator = new(5, 4, 1); - - ValidationFeedback validationFeedbacks = validator.Validate(stream, "foo", Encoding.UTF8).FeedbackItems; - - foreach (KeyValuePair> validationFeedback in validationFeedbacks) + using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(dataContents)); + if (shouldSeekToDataStart) { - foreach (ValidationFeedbackValue instance in validationFeedback.Value) - { - Logger.LogMessage($"Line {instance.Line}, Char {instance.Character}: " - + $"{validationFeedback.Key.Rule} {instance.AdditionalInfo}"); - } + stream.Seek(DataStartOffset, SeekOrigin.Begin); } - Assert.HasCount(2, validationFeedbacks); - } - - [TestMethod] - public async Task ValidateAsyncWithoutDataReturnsErrors() - { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.NO_DATA)); DataValidator validator = new(5, 4, 1); ValidationResult result = await validator.ValidateAsync(stream, "foo", Encoding.UTF8, cancellationToken: TestContext.CancellationToken); ValidationFeedback validationFeedbacks = result.FeedbackItems; + int actualErrorCount = validationFeedbacks.Values.SelectMany(f => f).Count(); - foreach (KeyValuePair> validationFeedback in validationFeedbacks) - { - foreach (ValidationFeedbackValue instance in validationFeedback.Value) - { - Logger.LogMessage($"Line {instance.Line}, Char {instance.Character}: " - + $"{validationFeedback.Key.Rule} {instance.AdditionalInfo}"); - } - } - - Assert.HasCount(2, validationFeedbacks); - } - - [TestMethod] - public void ValidateWithDataOnOnSingleRowReturnsErrors() - { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.DATA_ON_SINGLE_ROW)); - DataValidator validator = new(5, 4, 1); - - ValidationFeedback validationFeedbacks = validator.Validate(stream, "foo", Encoding.UTF8).FeedbackItems; - - foreach (KeyValuePair> validationFeedback in validationFeedbacks) - { - foreach (ValidationFeedbackValue instance in validationFeedback.Value) - { - Logger.LogMessage($"Line {instance.Line}, Char {instance.Character}: " - + $"{validationFeedback.Key.Rule} {instance.AdditionalInfo}"); - } - } - - Assert.HasCount(2, validationFeedbacks); // Unique feedbacks - Assert.HasCount(6, validationFeedbacks.Values.SelectMany(f => f)); // Total feedbacks including duplicates - } - - [TestMethod] - public async Task ValidateAsyncWithDataOnSingleRowReturnsErrors() - { - using Stream stream = new MemoryStream(Encoding.UTF8.GetBytes(DataStreamContents.DATA_ON_SINGLE_ROW)); - DataValidator validator = new(5, 4, 1); - - ValidationResult result = await validator.ValidateAsync(stream, "foo", Encoding.UTF8, cancellationToken: TestContext.CancellationToken); - ValidationFeedback validationFeedbacks = result.FeedbackItems; - - foreach (KeyValuePair> validationFeedback in validationFeedbacks) - { - foreach (ValidationFeedbackValue instance in validationFeedback.Value) - { - Logger.LogMessage($"Line {instance.Line}, Char {instance.Character}: " - + $"{validationFeedback.Key.Rule} {instance.AdditionalInfo}"); - } - } - - Assert.HasCount(2, validationFeedbacks);// Unique feedbacks - Assert.HasCount(6, validationFeedbacks.Values.SelectMany(f => f)); // Total feedbacks including duplicates + Assert.HasCount(expectedUniqueErrorCount, validationFeedbacks); + Assert.AreEqual(expectedTotalErrorCount, actualErrorCount); } } } \ No newline at end of file diff --git a/Px.Utils.UnitTests/Validation/DatabaseValidation/DatabaseValidatorTests.cs b/Px.Utils.UnitTests/Validation/DatabaseValidation/DatabaseValidatorTests.cs index f9657cf5..4b6e2921 100644 --- a/Px.Utils.UnitTests/Validation/DatabaseValidation/DatabaseValidatorTests.cs +++ b/Px.Utils.UnitTests/Validation/DatabaseValidation/DatabaseValidatorTests.cs @@ -161,7 +161,7 @@ public async Task ValidateDatabasAsynceWithCustomFunctionsReturnsFeedback() IDatabaseValidator[] customValidators = [ new MockCustomDatabaseValidator() - ]; + ]; DatabaseValidator validator = new( "database_invalid", customPxFileValidators: customValidators, diff --git a/Px.Utils.UnitTests/Validation/Fixtures/ContentValidationFixtures.cs b/Px.Utils.UnitTests/Validation/Fixtures/ContentValidationFixtures.cs index b220a5e3..eada5c99 100644 --- a/Px.Utils.UnitTests/Validation/Fixtures/ContentValidationFixtures.cs +++ b/Px.Utils.UnitTests/Validation/Fixtures/ContentValidationFixtures.cs @@ -306,7 +306,7 @@ internal static class ContentValidationFixtures private static readonly ValidationStructuredEntry variableTypeBarEntry = new(filename, new ValidationStructuredEntryKey("VARIABLE-TYPE", null, "bar"), - "Content", + "Contents", 23, [], 13, @@ -761,6 +761,15 @@ internal static class ContentValidationFixtures codesBarEntry.ValueStartIndex, Utils.Validation.ValueType.ListOfStrings); + internal static ValidationStructuredEntry StructuredEntryWithMatchingAmountOfElementsWithListDelimeters => + new(filename, + codesBarEntry.Key, + "\"foo,bar\"", // foo,bar should be treated as one element due to the quotes + codesBarEntry.ValueStartIndex, + codesBarEntry.LineChangeIndexes, + codesBarEntry.ValueStartIndex, + Utils.Validation.ValueType.ListOfStrings); + internal static ValidationStructuredEntry StructuredEntryWithLowerCaseValue => new(filename, codepageEntry.Key, diff --git a/Px.Utils.UnitTests/Validation/Fixtures/DataStreamContents.cs b/Px.Utils.UnitTests/Validation/Fixtures/DataStreamContents.cs index 65c6d8b8..f0ccdfda 100644 --- a/Px.Utils.UnitTests/Validation/Fixtures/DataStreamContents.cs +++ b/Px.Utils.UnitTests/Validation/Fixtures/DataStreamContents.cs @@ -1,22 +1,29 @@ -namespace Px.Utils.UnitTests.Validation.Fixtures +namespace Px.Utils.UnitTests.Validation.Fixtures { internal static class DataStreamContents { - internal static string SIMPLE_VALID_DATA => + internal const string SIMPLE_VALID_DATA = "DATA=\n"+ "1 2 3 4 5 \r\n" + "6 7 8 9 10 \n\r" + "\".\" \"..\" \"...\" \"....\" \".....\" \r"+ "\"......\" \"-\" -1 1.2 -1.3; \r\n"; - internal static string SIMPLE_VALID_DATA_WITHOUT_MISISNG_CODE_DELIMETERS => + internal const string SIMPLE_VALID_DATA_WITH_INCONSISTENT_LINEBREAKS = + "DATA=\r\n" + + "1 2 3 4 5 \n" + + "6 7 8 9 10 \r\n" + + "\".\" \"..\" \"...\" \"....\" \".....\" \n" + + "\"......\" \"-\" -1 1.2 -1.3; \r\n"; + + internal const string SIMPLE_VALID_DATA_WITHOUT_MISISNG_CODE_DELIMETERS = "DATA=\n" + "1 2 3 4 5 \r\n" + "6 7 8 9 10 \n\r" + ". .. ... .... ..... \r" + "...... - -1 1.2 -1.3; \r\n"; - internal static string SIMPLE_INVALID_DATA => + internal const string SIMPLE_INVALID_DATA = "DATA=a\n" + "\"b\" 1 2. 3 4 5 \r\n" + "6 7 +8\t9 10 \n\r" + @@ -24,10 +31,38 @@ internal static class DataStreamContents "\"dots\" \"-\" \"..123\" -1 1.2 -1.3 \r\n" + "1 2 \0 4 5 \r\n;"; - internal static string NO_DATA => + internal const string NO_DATA = "DATA=\n"; - internal static string DATA_ON_SINGLE_ROW => + internal const string DATA_ON_SINGLE_ROW = "DATA=1 2 3 4 5 6 7 8 9 10;"; + + internal const string DATA_STARTING_WITH_ENCLOSED_MISSING_VALUE = + "DATA=\n" + + "\".\" 2 3 4 5 \r\n" + + "6 7 8 9 10 \n\r" + + "\".\" \"..\" \"...\" \"....\" \".....\" \r" + + "\"......\" \"-\" -1 1.2 -1.3; \r\n"; + + internal const string DATA_STARTING_WITH_UNENCLOSED_MISSING_VALUE = + "DATA=\n" + + ". 2 3 4 5 \r\n" + + "6 7 8 9 10 \n\r" + + ". .. ... .... ..... \r" + + "...... - -1 1.2 -1.3; \r\n"; + + internal const string DATA_STARTING_WITH_NIL_VALUE = + "DATA=\n" + + "\"-\" 2 3 4 5 \r\n" + + "6 7 8 9 10 \n\r" + + "\".\" \"..\" \"...\" \"....\" \".....\" \r" + + "\"......\" \"-\" -1 1.2 -1.3; \r\n"; + + internal const string DATA_STARTING_WITH_UNENCLOSED_NIL_VALUE = + "DATA=\n" + + "- 2 3 4 5 \r\n" + + "6 7 8 9 10 \n\r" + + ". .. ... .... ..... \r" + + "...... - -1 1.2 -1.3; \r\n"; } } \ No newline at end of file diff --git a/Px.Utils.UnitTests/Validation/Fixtures/PxFileFixtures.cs b/Px.Utils.UnitTests/Validation/Fixtures/PxFileFixtures.cs index 542a4491..d28536eb 100644 --- a/Px.Utils.UnitTests/Validation/Fixtures/PxFileFixtures.cs +++ b/Px.Utils.UnitTests/Validation/Fixtures/PxFileFixtures.cs @@ -1,4 +1,4 @@ -namespace Px.Utils.UnitTests.Validation.Fixtures +namespace Px.Utils.UnitTests.Validation.Fixtures { internal static class PxFileFixtures { @@ -22,7 +22,7 @@ internal static class PxFileFixtures "\r\nCODES(\"Vuosi\")=\"2015\",\"2016\",\"2017\",\"2018\",\"2019\",\"2020\",\"2021\",\"2022\",\"2023\",\"2024\";" + "\r\nCODES(\"Tiedot\")=\"code-foo-val-a\",\"code-foo-val-b\";" + "\r\nVARIABLE-TYPE(\"Vuosi\")=\"Time\";" + - "\r\nVARIABLE-TYPE(\"Tiedot\")=\"Content\";" + + "\r\nVARIABLE-TYPE(\"Tiedot\")=\"Contents\";" + "\r\nPRECISION(\"Tiedot\",\"foo-val-a\")=1;" + "\r\nPRECISION(\"Tiedot\",\"foo-val-b\")=1;" + "\r\nLAST-UPDATED(\"Tiedot\",\"foo-val-a\")=\"20231101 08:00\";" + @@ -64,7 +64,7 @@ internal static class PxFileFixtures "\r\nCODES(\"Vuosi\")=\"2015\",\"2016\",\"2017\",\"2018\",\"2019\",\"2020\",\"2021\",\"2022\",\"2023\",\"2024\";" + "\r\nCODES(\"Tiedot\")=\"code-foo-val-a\",\"code-foo-val-b\";" + "\r\nVARIABLE-TYPE(\"Vuosi\")=\"Time\";" + - "\r\nVARIABLE-TYPE(\"Tiedot\")=\"Content\";" + + "\r\nVARIABLE-TYPE(\"Tiedot\")=\"Contents\";" + "\r\nPRECISION(\"Tiedot\",\"foo-val-a\")=1;" + "\r\nPRECISION(\"Tiedot\",\"foo-val-b\")=1;" + "\r\nLAST-UPDATED(\"Tiedot\",\"foo-val-a\")=\"20231101 08:00\";" + @@ -94,7 +94,7 @@ internal static class PxFileFixtures "\r\nCODES(\"Vuosi\")=\"2015\",\"2016\",\"2017\",\"2018\",\"2019\",\"2020\",\"2021\",\"2022\",\"2023\",\"2024\";" + "\r\nCODES(\"Tiedot\")=\"code-foo-val-a\",\"code-foo-val-b\";" + "\r\nVARIABLE-TYPE(\"Vuosi\")=\"Time\";" + - "\r\nVARIABLE-TYPE(\"Tiedot\")=\"Content\";" + + "\r\nVARIABLE-TYPE(\"Tiedot\")=\"Contents\";" + "\r\nPRECISION(\"Tiedot\",\"foo-val-a\")=1;" + "\r\nPRECISION(\"Tiedot\",\"foo-val-b\")=1;" + "\r\nLAST-UPDATED(\"Tiedot\",\"foo-val-a\")=\"20231101 08:00\";" + @@ -135,7 +135,7 @@ internal static class PxFileFixtures "\r\nCODES(\"Vuosi\")=\"2015\",\"2016\",\"2017\",\"2018\",\"2019\",\"2020\",\"2021\",\"2022\",\"2023\",\"2024\";" + "\r\nCODES(\"Tiedot\")=\"code-foo-val-a\",\"code-foo-val-b\";" + "\r\nVARIABLE-TYPE(\"Vuosi\")=\"Time\";" + - "\r\nVARIABLE-TYPE(\"Tiedot\")=\"Content\";" + + "\r\nVARIABLE-TYPE(\"Tiedot\")=\"Contents\";" + "\r\nPRECISION(\"Tiedot\",\"foo-val-a\")=1;" + "\r\nLAST-UPDATED(\"Tiedot\",\"foo-val-a\")=\"20231101 08:00\";" + "\r\nUNITS(\"Tiedot\",\"foo-val-a\")=\"kpl\";" + diff --git a/Px.Utils.UnitTests/Validation/SyntaxValidationTests/StreamSyntaxValidationTests.cs b/Px.Utils.UnitTests/Validation/SyntaxValidationTests/StreamSyntaxValidationTests.cs index aa6e7945..15f16d87 100644 --- a/Px.Utils.UnitTests/Validation/SyntaxValidationTests/StreamSyntaxValidationTests.cs +++ b/Px.Utils.UnitTests/Validation/SyntaxValidationTests/StreamSyntaxValidationTests.cs @@ -419,6 +419,7 @@ public void GetValueTypeFromStringCalledWithValidListsValuesCorrectValueType(str [TestMethod] [DataRow("TLIST(A1),\"2000\",\"2001\",\"2003\"", Utils.Validation.ValueType.TimeValSeries)] + [DataRow("TLIST(A1),\"2000\"", Utils.Validation.ValueType.TimeValSeries)] // Only one value - not a list [DataRow("TLIST(H1),\"20001\", \"20002\", \"20011\", \"20012\"", Utils.Validation.ValueType.TimeValSeries)] // Has spaces between values [DataRow("TLIST(T1),\"20001\",\"20002\",\"20003\"", Utils.Validation.ValueType.TimeValSeries)] [DataRow("TLIST(Q1),\"20001\",\"20002\",\"20003\",\"20004\"", Utils.Validation.ValueType.TimeValSeries)] diff --git a/Px.Utils.sln b/Px.Utils.sln index 4fe9d1ae..9196bd04 100644 --- a/Px.Utils.sln +++ b/Px.Utils.sln @@ -36,6 +36,18 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Px.Utils.TestingApp", "Px.U {D94934AF-6B41-4C66-818A-C4C33230E9DD} = {D94934AF-6B41-4C66-818A-C4C33230E9DD} EndProjectSection EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "architecture", "architecture", "{BFA8E05B-EBD8-4085-B765-EAED69DD0874}" + ProjectSection(SolutionItems) = preProject + docs\architecture.md = docs\architecture.md + docs\architecture.models.md = docs\architecture.models.md + docs\architecture.operations.md = docs\architecture.operations.md + docs\architecture.overview.md = docs\architecture.overview.md + docs\architecture.readers.md = docs\architecture.readers.md + docs\architecture.serializers.md = docs\architecture.serializers.md + docs\architecture.testing.md = docs\architecture.testing.md + docs\architecture.validation.md = docs\architecture.validation.md + EndProjectSection +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -60,6 +72,7 @@ Global EndGlobalSection GlobalSection(NestedProjects) = preSolution {EB4119B8-8292-418A-9CD3-2905DA67B889} = {BC1C9772-9B8F-49A7-A8F3-09E3A94789ED} + {BFA8E05B-EBD8-4085-B765-EAED69DD0874} = {42DFE3AF-7507-415A-806B-99B74519BF97} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {B1F81A3B-9FF5-4557-8A5A-A9519C42C2B2} diff --git a/Px.Utils/ModelBuilders/ValueParserUtilities.cs b/Px.Utils/ModelBuilders/ValueParserUtilities.cs index 33cbe847..8ff707e2 100644 --- a/Px.Utils/ModelBuilders/ValueParserUtilities.cs +++ b/Px.Utils/ModelBuilders/ValueParserUtilities.cs @@ -1,6 +1,7 @@ -using Px.Utils.Models.Metadata.Enums; +using Px.Utils.Models.Metadata.Enums; using Px.Utils.Models.Metadata.ExtensionMethods; using Px.Utils.PxFile; +using System.Globalization; namespace Px.Utils.ModelBuilders { @@ -102,6 +103,7 @@ public static string GetTimeValValueRangeString(string input, PxFileConfiguratio /// /// Parses a string into a enumeration value. /// This method maps the input string to a enumeration value based on the provided or default PxFileConfiguration configuration. + /// Mapping is done using the configured variable type mappings. /// If the input string does not map to a known , the method returns . /// /// The string to parse into a enumeration value. @@ -110,20 +112,14 @@ public static string GetTimeValValueRangeString(string input, PxFileConfiguratio public static DimensionType StringToDimensionType(string input, PxFileConfiguration? conf = null) { conf ??= PxFileConfiguration.Default; - Dictionary map = new() - { - {conf.Tokens.VariableTypes.Content, DimensionType.Content}, - {conf.Tokens.VariableTypes.Time, DimensionType.Time}, - {conf.Tokens.VariableTypes.Ordinal, DimensionType.Ordinal}, - {conf.Tokens.VariableTypes.Nominal, DimensionType.Nominal}, - {conf.Tokens.VariableTypes.Geographical, DimensionType.Geographical}, - {conf.Tokens.VariableTypes.Other, DimensionType.Other}, - {conf.Tokens.VariableTypes.Unknown, DimensionType.Unknown} - }; string cleanString = input.CleanStringDelimeters(conf.Symbols.Value.StringDelimeter); - if (map.TryGetValue(cleanString, out DimensionType value)) return value; - else return DimensionType.Unknown; + if (conf.Tokens.VariableTypes.Mappings.TryGetValue(cleanString, out DimensionType value)) + { + return value; + } + + return DimensionType.Unknown; } } } diff --git a/Px.Utils/Px.Utils.csproj b/Px.Utils/Px.Utils.csproj index dfac49e2..2e855475 100644 --- a/Px.Utils/Px.Utils.csproj +++ b/Px.Utils/Px.Utils.csproj @@ -2,7 +2,7 @@ Px.Utils - 1.4.0 + 1.5.0 net10.0 enable enable @@ -19,6 +19,7 @@ true snupkg LICENSE.md + true diff --git a/Px.Utils/PxFile/PxFileConfiguration.cs b/Px.Utils/PxFile/PxFileConfiguration.cs index f1cbf897..9b4f2f61 100644 --- a/Px.Utils/PxFile/PxFileConfiguration.cs +++ b/Px.Utils/PxFile/PxFileConfiguration.cs @@ -1,4 +1,4 @@ -using Px.Utils.Models.Metadata.Enums; +using Px.Utils.Models.Metadata.Enums; namespace Px.Utils.PxFile { @@ -107,28 +107,29 @@ private TimeValue() { } public static TimeValue DefaultTimeValue => new(); } + /// + /// Defines accepted tokens for parsing and validating dimension types using an expandable dictionary string to . + /// public class VariableTypeTokens { - private const string CONTENT = "Content"; - private const string TIME = "Time"; - private const string ORDINAL = "Ordinal"; - private const string NOMINAL = "Nominal"; - private const string GEOGRAPHICAL = "Geographical"; - private const string OTHER = "Other"; - private const string UNKNOWN = "Unknown"; - private const string CLASSIFICATORY = "Classificatory"; - - public string Content { get; set; } = CONTENT; - public string Time { get; set; } = TIME; - public string Ordinal { get; set; } = ORDINAL; - public string Nominal { get; set; } = NOMINAL; - public string Geographical { get; set; } = GEOGRAPHICAL; - public string Other { get; set; } = OTHER; - public string Unknown { get; set; } = UNKNOWN; - public string Classificatory { get; set; } = CLASSIFICATORY; + /// + /// Gets or sets the expandable dictionary mapping string tokens to dimension types. + /// + public Dictionary Mappings { get; set; } = new(StringComparer.OrdinalIgnoreCase) + { + { "Time", DimensionType.Time }, + { "Contents", DimensionType.Content }, + { "Geographical", DimensionType.Geographical }, + { "Ordinal", DimensionType.Ordinal }, + { "Nominal", DimensionType.Nominal }, + { "Other", DimensionType.Other }, + }; private VariableTypeTokens() { } + /// + /// Gets the default variable type tokens configuration. + /// public static VariableTypeTokens DefaultVariableTypeTokens => new(); } diff --git a/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs b/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs index d78a8682..5f0ab0ad 100644 --- a/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs +++ b/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationEntryFunctions.cs @@ -1,4 +1,5 @@ -using Px.Utils.Validation.SyntaxValidation; +using Px.Utils.Validation.SyntaxValidation; +using Px.Utils.Models.Metadata.Enums; using System.Globalization; namespace Px.Utils.Validation.ContentValidation @@ -317,71 +318,78 @@ public sealed partial class ContentValidator /// /// Entry in the Px file metadata. Represented by a object /// object that stores information that is gathered during the validation process - /// Key value pair containing information about the rule violation is returned if an unexpected value is detected + /// Key value pair containing information about the rule violation is returned if an unexpected value is detected from the entry public static ValidationFeedback? ValidateValueContents(ValidationStructuredEntry entry, ContentValidator validator) { - string[] allowedCharsets = ["ANSI", "Unicode"]; - - string[] dimensionTypes = [ - validator.Conf.Tokens.VariableTypes.Content, - validator.Conf.Tokens.VariableTypes.Time, - validator.Conf.Tokens.VariableTypes.Geographical, - validator.Conf.Tokens.VariableTypes.Ordinal, - validator.Conf.Tokens.VariableTypes.Nominal, - validator.Conf.Tokens.VariableTypes.Other, - validator.Conf.Tokens.VariableTypes.Unknown, - validator.Conf.Tokens.VariableTypes.Classificatory - ]; - + string[] allowedValueContents = []; + StringComparer comparer = StringComparer.Ordinal; + string keyword = entry.Key.Keyword; string value = SyntaxValidationUtilityMethods.CleanString(entry.Value, validator.Conf); - if ((entry.Key.Keyword == validator.Conf.Tokens.KeyWords.Charset && !allowedCharsets.Contains(value)) || - (entry.Key.Keyword == validator.Conf.Tokens.KeyWords.CodePage && !value.Equals(validator._encoding.BodyName, StringComparison.OrdinalIgnoreCase)) || - (entry.Key.Keyword == validator.Conf.Tokens.KeyWords.DimensionType && !dimensionTypes.Contains(value))) - { - KeyValuePair feedbackIndexes = SyntaxValidationUtilityMethods.GetLineAndCharacterIndex( - entry.KeyStartLineIndex, - entry.ValueStartIndex, - entry.LineChangeIndexes); - KeyValuePair feedback = new( - new(ValidationFeedbackLevel.Error, - ValidationFeedbackRule.InvalidValueFound), - new(validator._filename, - feedbackIndexes.Key, - feedbackIndexes.Value, - $"{entry.Key.Keyword}: {entry.Value}") - ); + if (keyword == validator.Conf.Tokens.KeyWords.Charset) + { + allowedValueContents = ["ANSI", "Unicode"]; + } + else if (keyword == validator.Conf.Tokens.KeyWords.CodePage) + { + allowedValueContents = [validator._encoding.BodyName]; + comparer = StringComparer.OrdinalIgnoreCase; + } + else if (keyword == validator.Conf.Tokens.KeyWords.DimensionType) + { + if (validator.Conf.Tokens.VariableTypes.Mappings.TryGetValue(value, out DimensionType _)) + { + return null; + } - return new(feedback); + return CreateInvalidValueFeedback(entry, validator); } - else if (entry.Key.Keyword == validator.Conf.Tokens.KeyWords.ContentVariableIdentifier) + else if (keyword == validator.Conf.Tokens.KeyWords.ContentVariableIdentifier) { string defaultLanguage = validator._defaultLanguage ?? string.Empty; string lang = entry.Key.Language ?? defaultLanguage; - if (validator._stubDimensionNames is not null && validator._stubDimensionNames.TryGetValue(lang, out string[]? stubValues) && - !Array.Exists(stubValues, d => d == value) && - (validator._headingDimensionNames is not null && validator._headingDimensionNames.TryGetValue(lang, out string[]? headingValues) && - !Array.Exists(headingValues, d => d == value))) - { - KeyValuePair feedbackIndexes = SyntaxValidationUtilityMethods.GetLineAndCharacterIndex( - entry.KeyStartLineIndex, - entry.ValueStartIndex, - entry.LineChangeIndexes); + List dimensionNames = []; - KeyValuePair feedback = new( - new(ValidationFeedbackLevel.Error, - ValidationFeedbackRule.InvalidValueFound), - new(validator._filename, - feedbackIndexes.Key, - feedbackIndexes.Value, - $"{entry.Key.Keyword}: {entry.Value}") - ); + if (validator._stubDimensionNames is not null && validator._stubDimensionNames.TryGetValue(lang, out string[]? stubValues)) + { + dimensionNames.AddRange(stubValues); + } - return new(feedback); + if (validator._headingDimensionNames is not null && validator._headingDimensionNames.TryGetValue(lang, out string[]? headingValues)) + { + dimensionNames.AddRange(headingValues); } + + allowedValueContents = [.. dimensionNames]; } - return null; + if (allowedValueContents.Length == 0 || allowedValueContents.Contains(value, comparer)) + { + return null; + } + + return CreateInvalidValueFeedback(entry, validator); + } + + private static ValidationFeedback CreateInvalidValueFeedback( + ValidationStructuredEntry entry, + ContentValidator validator) + { + KeyValuePair feedbackIndexes = SyntaxValidationUtilityMethods.GetLineAndCharacterIndex( + entry.KeyStartLineIndex, + entry.ValueStartIndex, + entry.LineChangeIndexes); + + KeyValuePair feedback = new( + new(ValidationFeedbackLevel.Error, + ValidationFeedbackRule.InvalidValueFound), + new(validator._filename, + feedbackIndexes.Key, + feedbackIndexes.Value, + $"{entry.Key.Keyword}: {entry.Value}") + ); + + return new(feedback); } /// @@ -399,7 +407,7 @@ validator._dimensionValueNames is null || return null; } - string[] codes = entry.Value.Split(validator.Conf.Symbols.Value.ListSeparator); + string[] codes = [.. SyntaxValidationUtilityMethods.GetListItemsFromString(entry.Value, validator.Conf.Symbols.Value.ListSeparator, validator.Conf.Symbols.Value.StringDelimeter)]; string defaultLanguage = validator._defaultLanguage ?? string.Empty; string lang = entry.Key.Language ?? defaultLanguage; if (codes.Length != validator._dimensionValueNames[new(lang, entry.Key.FirstSpecifier)].Length) diff --git a/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationFindKeywordFunctions.cs b/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationFindKeywordFunctions.cs index 5dbbae0f..f48839d8 100644 --- a/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationFindKeywordFunctions.cs +++ b/Px.Utils/Validation/ContentValidation/ContentValidator.ValidationFindKeywordFunctions.cs @@ -1,4 +1,4 @@ -using Px.Utils.Validation.SyntaxValidation; +using Px.Utils.Validation.SyntaxValidation; using System; using System.Collections.Generic; using System.Linq; @@ -106,7 +106,10 @@ public sealed partial class ContentValidator if (availableLanguageEntries.Length == 1) { - List languages = availableLanguageEntries[0].Value.Split(validator.Conf.Symbols.Value.ListSeparator).ToList(); + List languages = SyntaxValidationUtilityMethods.GetListItemsFromString( + availableLanguageEntries[0].Value, + validator.Conf.Symbols.Value.ListSeparator, + validator.Conf.Symbols.Value.StringDelimeter); validator._availableLanguages = [.. languages.Select(lang => SyntaxValidationUtilityMethods.CleanString(lang, validator.Conf))]; return null; } diff --git a/Px.Utils/Validation/DataValidation/DataValidator.cs b/Px.Utils/Validation/DataValidation/DataValidator.cs index 6025418e..f726cec2 100644 --- a/Px.Utils/Validation/DataValidation/DataValidator.cs +++ b/Px.Utils/Validation/DataValidation/DataValidator.cs @@ -1,4 +1,4 @@ -using System.Runtime.CompilerServices; +using System.Runtime.CompilerServices; using System.Text; using Px.Utils.PxFile; using Px.Utils.Validation.DatabaseValidation; @@ -15,6 +15,7 @@ namespace Px.Utils.Validation.DataValidation public class DataValidator(int rowLen, int numOfRows, int startRow, PxFileConfiguration? conf = null) : IPxFileStreamValidator, IPxFileStreamValidatorAsync { private const int _streamBufferSize = 4096; + private static ReadOnlySpan MissingValueStartBytes => [CharacterConstants.QUOTATIONMARK, (byte)'-', (byte)'.']; private readonly PxFileConfiguration _conf = conf ?? PxFileConfiguration.Default; @@ -267,11 +268,13 @@ private int GetStreamIndexOfFirstDataValue(Stream stream, ref ValidationFeedback bytesRead = stream.Read(buffer, 0, buffer.Length); for (int i = 0; i < bytesRead; i++) { - if (buffer[i] >= CharacterConstants.Zero && buffer[i] <= CharacterConstants.Nine) + byte currentByte = buffer[i]; + char currentChar = (char)currentByte; + if (IsDataValueStartByte(currentByte)) { return (int)stream.Position - bytesRead + i; } - else if (!CharacterConstants.WhitespaceCharacters.Contains((char)buffer[i])) + else if (!CharacterConstants.WhitespaceCharacters.Contains(currentChar)) { feedbacks.Add(new( new(ValidationFeedbackLevel.Error, @@ -284,6 +287,12 @@ private int GetStreamIndexOfFirstDataValue(Stream stream, ref ValidationFeedback return -1; } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool IsDataValueStartByte(byte currentByte) + { + return currentByte >= CharacterConstants.Zero && currentByte <= CharacterConstants.Nine || MissingValueStartBytes.Contains(currentByte); + } } /// diff --git a/Px.Utils/Validation/DatabaseValidation/DatabaseValidator.cs b/Px.Utils/Validation/DatabaseValidation/DatabaseValidator.cs index 9cd9ac70..3a164955 100644 --- a/Px.Utils/Validation/DatabaseValidation/DatabaseValidator.cs +++ b/Px.Utils/Validation/DatabaseValidation/DatabaseValidator.cs @@ -1,4 +1,4 @@ -using Px.Utils.Exceptions; +using Px.Utils.Exceptions; using Px.Utils.PxFile; using Px.Utils.PxFile.Metadata; using Px.Utils.Validation.SyntaxValidation; @@ -373,7 +373,10 @@ private void ProcessEntry(string entry, ref string defaultLanguage, ref string[] } else if (splitEntry[0] == _conf.Tokens.KeyWords.AvailableLanguages) { - languages = splitEntry[1].Split(_conf.Symbols.Value.ListSeparator); + languages = [.. SyntaxValidationUtilityMethods.GetListItemsFromString( + splitEntry[1], + _conf.Symbols.Value.ListSeparator, + _conf.Symbols.Value.StringDelimeter)]; for (int j = 0; j < languages.Length; j++) { languages[j] = languages[j].Trim(_conf.Symbols.Key.StringDelimeter); diff --git a/Px.Utils/Validation/SyntaxValidation/SyntaxValidationUtilityMethods.cs b/Px.Utils/Validation/SyntaxValidation/SyntaxValidationUtilityMethods.cs index 86156b2c..035b050e 100644 --- a/Px.Utils/Validation/SyntaxValidation/SyntaxValidationUtilityMethods.cs +++ b/Px.Utils/Validation/SyntaxValidation/SyntaxValidationUtilityMethods.cs @@ -458,8 +458,8 @@ private static bool GetTimeValueFormat(string input, out ValueType? valueFormat, } // Remove preceding list separator from input - input = input.Remove(0, 1); - if (!IsStringListFormat(input, listSeparator, conf.Symbols.Value.StringDelimeter)) + input = input[1..]; + if (!IsStringListFormat(input, listSeparator, conf.Symbols.Value.StringDelimeter) && !IsStringFormat(input, conf.Symbols.Value.StringDelimeter)) { return null; } diff --git a/docs/PXFILE_SPECIFICATION.md b/docs/PXFILE_SPECIFICATION.md index ac5ecb0e..8c3d0a65 100644 --- a/docs/PXFILE_SPECIFICATION.md +++ b/docs/PXFILE_SPECIFICATION.md @@ -96,7 +96,7 @@ Whitespace characters are not significant in the value outside of ```"``` separa - Decimal separator is period ```.```. - Thousands separator is not allowed. - Whitespace characters are not allowed between the characters of the number. -- Number must be in range of ±7.9228 x 10^28. +- Number must be in range of ±7.9228 x 10^28. #### List of strings - List items are separated by a comma ```,```. @@ -281,6 +281,7 @@ More spesific encoding information. The value must be a string that matches the - Recommended that the variable type is defined for each variable defined by the STUB or HEADING. - Has a set of allowed values: ```Content```, ```Time```, ```Geographical```, ```Ordinal```, ```Nominal```, ```Other```, ```Classificatory```., ```Unknown```. - Can be defined for each language, but this is not recommended. +- Value is not case sensitive, but uppercase characters are recommended. #### TIMEVAL - See the TIMEVAL entry for the syntax and content requirements. \ No newline at end of file diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 00000000..63cdeabd --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,15 @@ +# Architecture Reference — Px.Utils + +> Machine-readable architecture index for AI coding agents. Read only the sub-file relevant to your task. + +## Sub-files + +| File | Scope | When to read | +|---|---|---| +| [architecture.overview.md](architecture.overview.md) | Solution structure, configuration, patterns, dependencies, file map | Onboarding, cross-cutting changes, understanding project layout | +| [architecture.validation.md](architecture.validation.md) | Validation pipeline, interfaces, result types, extensibility | Working on any validator, feedback types, or custom validation | +| [architecture.models.md](architecture.models.md) | Matrix, metadata, dimensions, properties, data values, model builders | Working on data models, metadata, or model building | +| [architecture.readers.md](architecture.readers.md) | Metadata reader, data reader, binary data codecs | Working on file reading, stream processing, or binary I/O | +| [architecture.operations.md](architecture.operations.md) | Matrix aggregation and arithmetic extensions | Working on sum/multiply/divide or adding new operations | +| [architecture.serializers.md](architecture.serializers.md) | JSON converters for model types | Working on JSON serialization/deserialization | +| [architecture.testing.md](architecture.testing.md) | Unit tests, mocks, benchmarking CLI | Writing tests, finding test coverage, or running benchmarks | diff --git a/docs/architecture.models.md b/docs/architecture.models.md new file mode 100644 index 00000000..d83ff085 --- /dev/null +++ b/docs/architecture.models.md @@ -0,0 +1,88 @@ +# Architecture — Models + +> Matrix, metadata, dimensions, properties, data values, and model builders. + +## Core Types + +| Type | Purpose | +|---|---| +| `Matrix` | Container for metadata + data array; supports `GetTransform(IMatrixMap)` | +| `IReadOnlyMatrixMetadata` | Read-only metadata: languages, dimensions, additional properties | +| `IMatrixMap` / `IDimensionMap` | Structural map of matrix dimensions and value codes | +| `MatrixMetadata` | Mutable metadata implementation of `IReadOnlyMatrixMetadata` | +| `MatrixMap` | Implementation of `IMatrixMap` | +| `MultilanguageString` | Dictionary-backed multilingual string with JSON serialization support | + +## Dimensions + +| Type | Purpose | +|---|---| +| `Dimension` | Base dimension with code, name, and `ValueList` | +| `ContentDimension` | Dimension with `ContentDimensionValue` entries (units, precision) | +| `TimeDimension` | Dimension with time interval information | +| `DimensionValue` | Single value in a dimension | +| `ContentDimensionValue` | Value with unit and decimal precision | +| `ValueList` | Ordered list of dimension values | + +Enums: `DimensionType`, `TimeDimensionInterval` + +## Meta Properties + +Abstract base: `MetaProperty` (`MetaPropertyType` enum). + +| Subtype | Value type | +|---|---| +| `StringProperty` | `string` | +| `NumericProperty` | `decimal` | +| `BooleanProperty` | `bool` | +| `StringListProperty` | `IReadOnlyList` | +| `MultilanguageStringProperty` | `MultilanguageString` | +| `MultilanguageStringListProperty` | `IReadOnlyList` | + +## Data Values + +| Type | Purpose | +|---|---| +| `IDataValue` | Marker interface | +| `DoubleDataValue` | `double` value + `DataValueType` | +| `DecimalDataValue` | `decimal` value + `DataValueType` | +| `DataValueType` | Enum classifying value type | + +## Model Building + +| Interface | Implementation | +|---|---| +| `IMatrixMetadataBuilder` | `MatrixMetadataBuilder` | + +``` +MatrixMetadata Build(IEnumerable> metadataInput) +MatrixMetadata Build(IReadOnlyDictionary metadataInput) +Task BuildAsync(IAsyncEnumerable> metadataInput) +``` + +Helpers: `MetadataEntryKeyBuilder`, `ValueParserUtilities`, `MetadataEntryKey`. + +`PxFileConfiguration.TokenDefinitions.VariableTypeTokens` exposes a configurable `Dictionary` (`Mappings`) used by both parsing and validation for `VARIABLE-TYPE` values. The dictionary uses `StringComparer.OrdinalIgnoreCase`, so lookups are case-insensitive by default. Custom aliases can be added by inserting additional keys into `Mappings`. + +## Extension Methods + +| File | Purpose | +|---|---| +| `MatrixMapExtensions` | Size calculations, dimension collapsing | +| `MatrixMetadataExtensions` | Metadata transforms and queries | +| `DimensionMapExtensions` | Dimension map utilities | +| `PropertyUtilities` | Property parsing and cleaning | + +## Dependency Graph + +``` +Matrix +├── IReadOnlyMatrixMetadata → MatrixMetadata +│ ├── Dimension / ContentDimension / TimeDimension +│ │ └── ValueList → DimensionValue / ContentDimensionValue +│ └── MetaProperty subtypes +└── DataIndexer (for GetTransform) + +MatrixMetadataBuilder : IMatrixMetadataBuilder +└── PxFileConfiguration +``` diff --git a/docs/architecture.operations.md b/docs/architecture.operations.md new file mode 100644 index 00000000..0925290f --- /dev/null +++ b/docs/architecture.operations.md @@ -0,0 +1,20 @@ +# Architecture — Operations + +> Matrix aggregation and arithmetic extension methods. + +## Extension Methods on `Matrix` + +| Class | File | Purpose | +|---|---|---| +| `MatrixFunctionExtensions` | `Operations/MatrixFunctionExtensions.cs` | `ApplyOverDimension` — abstract aggregation over a dimension | +| `SumMatrixFunctionExtensions` | `Operations/SumMatrixFunctionExtensions.cs` | Sum operations | +| `MultiplicationMatrixFunction` | `Operations/MultiplicationMatrixFunction.cs` | Multiplication operations | +| `DivisionMatrixFunctionExtensions` | `Operations/DivisionMatrixFunctionExtensions.cs` | Division operations | + +## Pattern + +All operations are extension methods on `Matrix`. They use `ApplyOverDimension` which: + +1. Takes a source dimension map, an aggregation function, and a function identity value +2. Adds a new `DimensionValue` to the target dimension holding the result +3. Returns a new `Matrix` with the aggregated data diff --git a/docs/architecture.overview.md b/docs/architecture.overview.md new file mode 100644 index 00000000..152f4c25 --- /dev/null +++ b/docs/architecture.overview.md @@ -0,0 +1,71 @@ +# Architecture — Overview + +> Solution structure, configuration, patterns, dependencies, and file map. + +## Solution + +Library for reading, processing, and validating PX (PC-Axis) statistical file data and metadata. + +| Project | Type | Target | Description | +|---|---|---|---| +| `Px.Utils` | Class library (NuGet) | `net10.0` | Core library — validation, models, readers, serializers, operations | +| `Px.Utils.TestingApp` | Console app | `net10.0` | Benchmarking CLI for performance testing | +| `Px.Utils.UnitTests` | Test project (MSTest) | `net10.0` | Unit tests for `Px.Utils` | + +No DI container — all types instantiated directly via constructors. Configuration via `PxFileConfiguration`. + +## Configuration + +`PxFileConfiguration` — configurable symbols and tokens for PX file format. Access default via `PxFileConfiguration.Default`. + +| Property path | Purpose | Default | +|---|---|---| +| `Symbols.Key.ListSeparator` | Separator in keys | `,` | +| `Symbols.Key.StringDelimeter` | String delimiter in keys | `"` | +| `Symbols.Value.StringDelimeter` | String delimiter in values | `"` | +| `Symbols.Value.ListSeparator` | Separator in values | `,` | +| `Symbols.KeywordSeparator` | Keyword = value separator | `=` | +| `Symbols.EntrySeparator` | Entry separator | `;` | +| `Tokens.KeyWords.Data` | Data section keyword | `DATA` | +| `Tokens.VariableTypes.Mappings` | `VARIABLE-TYPE` token to `DimensionType` mapping | Case-insensitive dictionary (`StringComparer.OrdinalIgnoreCase`) | + +## Key Patterns + +- **No DI container** — manual wiring via constructors; primary constructors used extensively. +- **Configuration object** — `PxFileConfiguration` passed through constructors; `.Default` as fallback. +- **Sync + async pairs** — most validators and readers expose both sync and async APIs. +- **Partial classes** — `ContentValidator` and `SyntaxValidationFunctions` split across multiple files by concern. +- **Primary constructors** — used for validators, readers, builders, and model types. +- **Stream-oriented** — PX file processing operates on `Stream` inputs throughout. +- **`IFileSystem` abstraction** — enables testing without real file I/O. +- **InternalsVisibleTo** — `Px.Utils` exposes internals to `Px.Utils.UnitTests`. + +## External Dependencies (Px.Utils only) + +| Dependency | Purpose | +|---|---| +| `System.Text.Json` | JSON serialization of metadata models | + +No third-party NuGet dependencies in the main library. + +## File Map + +``` +Px.Utils/ +├── BinaryData/ -- Binary data reading and codec implementations +├── Exceptions/ -- Custom exceptions +├── Language/ -- MultilanguageString and extensions +├── ModelBuilders/ -- IMatrixMetadataBuilder, MetadataEntryKeyBuilder, ValueParserUtilities +├── Models/ +│ ├── Matrix.cs -- Matrix container +│ ├── Data/ -- IDataValue, DataValueType, DoubleDataValue, DecimalDataValue +│ └── Metadata/ -- Metadata interfaces, implementations, dimensions, enums, properties, extensions +├── Operations/ -- Matrix aggregation/sum/multiply/divide extensions +├── PxFile/ +│ ├── PxFileConfiguration.cs -- Format configuration +│ ├── CharacterConstants.cs -- Byte constants +│ ├── Data/ -- IPxFileStreamDataReader, PxFileStreamDataReader, DataIndexer, parsers +│ └── Metadata/ -- IPxFileMetadataReader, PxFileMetadataReader +├── Serializers/Json/ -- JSON converters for models +└── Validation/ -- Validation pipeline (syntax → content → data → database) +``` diff --git a/docs/architecture.readers.md b/docs/architecture.readers.md new file mode 100644 index 00000000..635b2e81 --- /dev/null +++ b/docs/architecture.readers.md @@ -0,0 +1,58 @@ +# Architecture — Readers & Binary Data + +> Metadata reading, data reading, and binary data codecs. + +## Metadata Reading + +| Interface | Implementation | File | +|---|---|---| +| `IPxFileMetadataReader` | `PxFileMetadataReader` | `PxFile/Metadata/PxFileMetadataReader.cs` | + +``` +IEnumerable> ReadMetadata(Stream, Encoding, int bufferSize = 4096) +Encoding GetEncoding(Stream) +Task GetEncodingAsync(Stream, CancellationToken) +IAsyncEnumerable> ReadMetadataAsync(Stream, Encoding, int bufferSize = 4096) +``` + +Depends on: `PxFileConfiguration`. + +## Data Reading + +| Interface | Implementation | File | +|---|---|---| +| `IPxFileStreamDataReader` | `PxFileStreamDataReader` | `PxFile/Data/PxFileStreamDataReader.cs` | + +``` +void ReadUnsafeDoubles(double[] buffer, int offset, IMatrixMap target, IMatrixMap complete, double[] missingValueEncodings) +void ReadDoubleDataValues(DoubleDataValue[] buffer, int offset, IMatrixMap target, IMatrixMap complete) +void ReadDecimalDataValues(DecimalDataValue[] buffer, int offset, IMatrixMap target, IMatrixMap complete) +``` + +Async variants available. Implements `IDisposable`. +Depends on: `PxFileConfiguration`. + +### Helpers + +| File | Purpose | +|---|---| +| `DataIndexer.cs` | Index mapping between source and target `IMatrixMap` | +| `DataValueParsers.cs` | Parse raw data values from byte spans | +| `StreamUtilities.cs` | Stream helper methods | + +## Binary Data + +| Type | File | Purpose | +|---|---|---| +| `BinaryDataReader` | `BinaryData/BinaryDataReader.cs` | Generic windowed reader for binary-encoded matrix data | +| `BinaryDataReaderStatic` | `BinaryData/BinaryDataReaderStatic.cs` | Static helper methods | +| `IBinaryValueCodec` | `BinaryData/ValueConverters/IBinaryValueCodec.cs` | Static interface for encode/decode | +| `BinaryValueCodecSelector` | `BinaryData/ValueConverters/BinaryValueCodecSelector.cs` | Codec factory by `BinaryValueCodecType` | + +### Codecs + +All implement `IBinaryValueCodec` (static abstract `ByteCount`): + +`DoubleCodec`, `FloatCodec`, `Int16Codec`, `Int24Codec`, `Int32Codec`, `UInt16Codec`, `UInt24Codec`, `UInt32Codec` + +Pattern: static interface members (`static abstract`) for zero-allocation encode/decode on spans. diff --git a/docs/architecture.serializers.md b/docs/architecture.serializers.md new file mode 100644 index 00000000..7a17251b --- /dev/null +++ b/docs/architecture.serializers.md @@ -0,0 +1,17 @@ +# Architecture — JSON Serializers + +> Custom `System.Text.Json` converters for model types. + +## Converters + +| Type | File | Serializes | +|---|---|---| +| `MultilanguageStringConverter` | `Serializers/Json/MultilanguageStringConverter.cs` | `MultilanguageString` | +| `DimensionConverter` | `Serializers/Json/DimensionConverter.cs` | `Dimension` and subtypes | +| `MetaPropertyConverter` | `Serializers/Json/MetaPropertyConverter.cs` | `MetaProperty` subtypes | +| `ContentValueListConverter` | `Serializers/Json/ContentValueListConverter.cs` | Content dimension value lists | +| `ValueListConverter` | `Serializers/Json/ValueListConverter.cs` | Generic value lists | + +### Helpers + +`ExtensionMethods.cs` — `JsonElement.GetProperty` with case-insensitive support. diff --git a/docs/architecture.testing.md b/docs/architecture.testing.md new file mode 100644 index 00000000..b3305d6b --- /dev/null +++ b/docs/architecture.testing.md @@ -0,0 +1,104 @@ +# Architecture — Testing + +> Unit tests, test mocks, and benchmarking CLI. + +## Unit Tests (`Px.Utils.UnitTests`) + +Framework: MSTest + Moq. Naming: `MethodNameStateUnderTestExpectedBehavior`. + +### Validation Tests + +| Test class | File | Covers | +|---|---|---| +| `StreamSyntaxValidationTests` | `Validation/SyntaxValidationTests/StreamSyntaxValidationTests.cs` | `SyntaxValidator` sync | +| `StreamSyntaxValidationAsyncTests` | `Validation/SyntaxValidationTests/StreamSyntaxValidationAsyncTests.cs` | `SyntaxValidator` async | +| `ContentValidationTests` | `Validation/ContentValidationTests/ContentValidationTests.cs` | `ContentValidator` | +| `DataValidationTest` | `Validation/DataValidationTests/DataValidationTest.cs` | `DataValidator` integration | +| `DataNumberValueValidatorTest` | `Validation/DataValidationTests/DataNumberValueValidatorTest.cs` | Numeric data validation | +| `DataStringValueValidatorTests` | `Validation/DataValidationTests/DataStringValueValidatorTests.cs` | String data validation | +| `DataSeparatorValidatorTest` | `Validation/DataValidationTests/DataSeparatorValidatorTest.cs` | Separator validation | +| `DataStructureValidationTests` | `Validation/DataValidationTests/DataStructureValidationTests.cs` | Data structure validation | +| `DatabaseValidatorTests` | `Validation/DatabaseValidation/DatabaseValidatorTests.cs` | `DatabaseValidator` | +| `DatabaseValidatorFunctionTests` | `Validation/DatabaseValidation/DatabaseValidatorFunctionTests.cs` | Database validator functions | +| `PxFileValidationTests` | `Validation/PxFileValidationTests/PxFileValidationTests.cs` | `PxFileValidator` | + +### Reader Tests + +| Test class | File | Covers | +|---|---|---| +| `ReadMetadataTests` | `PxFileTests/PxFileMetadataReaderTests/ReadMetadataTests.cs` | `PxFileMetadataReader` | +| `ReadMetadataAsyncTests` | `PxFileTests/PxFileMetadataReaderTests/ReadMetadataAsyncTests.cs` | `PxFileMetadataReader` async | +| `GetEncodingTests` | `PxFileTests/PxFileMetadataReaderTests/GetEncodingTests.cs` | Encoding detection | +| `DataReaderTests` | `PxFileTests/DataTests/PxFileStreamDataReaderTests/DataReaderTests.cs` | `PxFileStreamDataReader` | +| `AsyncDataReaderTests` | `PxFileTests/DataTests/PxFileStreamDataReaderTests/AsyncDataReaderTests.cs` | Async data reading | +| `MultiPartReadingTests` | `PxFileTests/DataTests/PxFileStreamDataReaderTests/MultiPartReadingTests.cs` | Multi-part reads | +| `DataIndexerTests` | `PxFileTests/DataTests/DataIndexerTests.cs` | `DataIndexer` | +| `DataValueParserTests` | `PxFileTests/DataTests/DataValueParserTests.cs` | `DataValueParsers` | + +### Model & Builder Tests + +| Test class | File | Covers | +|---|---|---| +| `MatrixMetadataBuilderTests` | `ModelBuilderTests/MatrixMetadataBuilderTests.cs` | `MatrixMetadataBuilder` | +| `MetadataEntryKeyBuilderTests` | `ModelBuilderTests/MetadataEntryKeyBuilderTests.cs` | `MetadataEntryKeyBuilder` | +| `MultilanguageStringTests` | `LanguageTests/MultilanguageStringTests.cs` | `MultilanguageString` | +| `MatrixTransformationsTests` | `ModelTests/MatrixTransformationsTests.cs` | `Matrix.GetTransform` | +| `ValueListTests` | `ModelTests/ValueListTests.cs` | `ValueList` | +| `PropertyTests` | `ModelTests/PropertyTests.cs` | `MetaProperty` subtypes | +| `DataValueTests` | `ModelTests/DataValueTests.cs` | Data value types | + +### Operations Tests + +| Test class | File | Covers | +|---|---|---| +| `SumMatrixFunctionExtensionTests` | `OperationsTests/SumMatrixFunctionExtensionTests.cs` | Sum | +| `MultiplicationMatrixFunctionExtensionTests` | `OperationsTests/MultiplicationMatrixFunctionExtensionTests.cs` | Multiplication | +| `DivisionMatrixFunctionExtensionTests` | `OperationsTests/DivisionMatrixFunctionExtensionTests.cs` | Division | + +### Binary Data Tests + +| Test class | File | Covers | +|---|---|---| +| `BinaryDataReaderCreateTests` | `BinaryData/BinaryDataReaderCreateTests.cs` | Reader creation | +| `BinaryDataReaderStreamTests` | `BinaryData/BinaryDataReaderStreamTests.cs` | Stream reading | +| `BinaryDataReaderChunkTests` | `BinaryData/BinaryDataReaderChunkTests.cs` | Chunk reading | +| `*CodecTests` | `BinaryData/ValueConverters/*CodecTests.cs` | Individual codecs | + +### Serializer Tests + +| Test class | File | Covers | +|---|---|---| +| `MatrixMetadataConverterSerializeTests` | `SerializerTests/MatrixMetadataConverterSerializeTests.cs` | JSON round-trip | +| `DimensionConverterTests` | `SerializerTests/DimensionConverterTests.cs` | Dimension JSON | +| `MetaPropertyConverterTests` | `SerializerTests/MetaPropertyConverterTests.cs` | MetaProperty JSON | +| `MultilanguageStringConverterTests` | `SerializerTests/MultilanguageStringConverterTests.cs` | MultilanguageString JSON | +| `ValueListConverterTests` | `SerializerTests/ValueListConverterTests.cs` | ValueList JSON | + +### Mocks + +`MockFileSystem`, `MockCustomValidatorFunctions`, `MockCustomSyntaxValidationFunctions`, `MockCustomContentValidationFunctions`, `MockDatabaseFileStreams`. + +--- + +## Benchmarking CLI (`Px.Utils.TestingApp`) + +Entry point: `TestingApp.Main(args)` → no args starts `InteractiveFlow`; batch mode not yet implemented. + +All commands inherit `Command`. Benchmarks inherit `Benchmark`. + +| Command | Purpose | +|---|---| +| `PxFileValidationBenchmark` | Full PX file validation | +| `DatabaseValidationBenchmark` | Database validation | +| `MetadataSyntaxValidationBenchmark` | Syntax validation | +| `MetadataContentValidationBenchmark` | Content validation | +| `DataValidationBenchmark` | Data validation | +| `MetadataReaderBenchmark` | Metadata reading | +| `MetadataBuilderBenchmark` | Metadata building | +| `DataReadBenchmark` | Data reading | +| `FileBenchmark` | File I/O | +| `ComputationBenchmark` | Matrix operations | +| `BinaryReadBenchmark` | Binary read | +| `BinaryWriteBenchmark` | Binary write | + +Supporting: `BenchmarkRunner.cs`, `TestAppConsole.cs`, `InteractiveFlow.cs`, `TestDataGenerator/`. diff --git a/docs/architecture.validation.md b/docs/architecture.validation.md new file mode 100644 index 00000000..84736d06 --- /dev/null +++ b/docs/architecture.validation.md @@ -0,0 +1,119 @@ +# Architecture — Validation + +> Validation pipeline, interfaces, result types, and extensibility. + +## Interfaces + +| Interface | Implementations | Purpose | +|---|---|---| +| `IValidator` | `ContentValidator`, `DatabaseValidator` | Sync validation → `ValidationResult` | +| `IValidatorAsync` | `DatabaseValidator` | Async validation → `Task` | +| `IPxFileStreamValidator` | `SyntaxValidator`, `DataValidator`, `PxFileValidator` | Sync stream-based PX file validation | +| `IPxFileStreamValidatorAsync` | `SyntaxValidator`, `DataValidator`, `PxFileValidator` | Async stream-based PX file validation | +| `IDatabaseValidator` | — (consumer-provided) | Extension point for database-level custom validators | +| `IFileSystem` | `LocalFileSystem` | File system abstraction for database validation | + +## Pipeline + +`PxFileValidator` orchestrates: `SyntaxValidator` → `ContentValidator` → `DataValidator` → custom validators. + +``` +Validate(stream, filename, encoding?, fileSystem?) +ValidateAsync(stream, filename, encoding?, fileSystem?, cancellationToken) +``` + +### SyntaxValidator + +Validates PX file metadata syntax (key-value structure, encoding, characters). +File: `Validation/SyntaxValidation/SyntaxValidator.cs` +Partial helpers: `SyntaxValidationFunctions.StringValidationFunctions.cs`, `KeyValueValidationFunctions.cs`, `StructuredValidationFunctions.cs` + +### ContentValidator + +Validates metadata content (required keys, language definitions, dimension consistency). +File: `Validation/ContentValidation/ContentValidator.cs` +Partial files: `ValidationEntryFunctions.cs`, `ValidationFindKeywordFunctions.cs`, `UtilityMethods.cs` + +Dimension type validation uses `PxFileConfiguration.TokenDefinitions.VariableTypeTokens`. The first configured token for each dimension type is treated as the recommended primary value, while additional configured tokens are accepted as aliases and reported as warnings. + +### DataValidator + +Validates data section (row counts, row lengths, value types, separators). +File: `Validation/DataValidation/DataValidator.cs` + +### DatabaseValidator + +Validates entire PX database directory (all `.px` files, alias files, directory structure). +File: `Validation/DatabaseValidation/DatabaseValidator.cs` + +## Extensibility + +- `CustomSyntaxValidationFunctions` — inject custom string/key-value/structured validation functions. +- `CustomContentValidationFunctions` — inject custom content validation functions. +- `PxFileValidator.SetCustomValidators(...)` — inject `IPxFileStreamValidator[]`, `IPxFileStreamValidatorAsync[]`, `IValidator[]`, `IValidatorAsync[]`. +- `DatabaseValidator` accepts `IDatabaseValidator[]` arrays for px files, alias files, and directories. + +## Result Types + +- `ValidationResult` — wraps `ValidationFeedback` +- `ValidationFeedback` — `ConcurrentDictionary>` +- `ValidationFeedbackKey` — (`ValidationFeedbackLevel`, `ValidationFeedbackRule`) +- `ValidationFeedbackLevel` — `Warning` | `Error` +- `ValidationFeedbackRule` — enum with 59 rule codes (see `Validation/Enums.cs`) + +## Dependency Graph + +``` +PxFileValidator +├── SyntaxValidator : IPxFileStreamValidator +│ ├── PxFileConfiguration +│ ├── CustomSyntaxValidationFunctions (optional) +│ └── IFileSystem → LocalFileSystem +├── ContentValidator : IValidator +│ ├── PxFileConfiguration +│ ├── ValidationStructuredEntry[] (from SyntaxValidator) +│ └── CustomContentValidationFunctions (optional) +├── DataValidator : IPxFileStreamValidator +│ └── PxFileConfiguration +└── Custom validators (optional) + +DatabaseValidator : IValidator, IValidatorAsync +├── IFileSystem → LocalFileSystem +├── PxFileConfiguration +├── PxFileMetadataReader +├── SyntaxValidator +└── IDatabaseValidator[] (optional custom) +``` + +## File Map + +``` +Validation/ +├── IValidator.cs -- IValidator, IValidatorAsync +├── IPxFileStreamValidator.cs -- IPxFileStreamValidator, IPxFileStreamValidatorAsync +├── IValidationResult.cs -- ValidationResult +├── ValidationFeedback.cs -- ValidationFeedbackKey, ValidationFeedbackValue, ValidationFeedback +├── ValidationObject.cs -- Validation context +├── Enums.cs -- ValidationFeedbackLevel, ValidationFeedbackRule, ValueType +├── PxFileValidator.cs -- Orchestrator +├── ContentValidation/ +│ ├── ContentValidator.cs -- + partial files +│ ├── ContentValidationResult.cs +│ └── CustomContentValidationFunctions.cs +├── DataValidation/ +│ ├── DataValidator.cs +│ └── DataValidatorFunctions.cs +├── DatabaseValidation/ +│ ├── DatabaseValidator.cs +│ ├── DatabaseValidatorFunctions.cs +│ ├── IFileSystem.cs +│ └── LocalFileSystem.cs +└── SyntaxValidation/ + ├── SyntaxValidator.cs + ├── SyntaxValidationFunctions.*.cs + ├── SyntaxValidationResult.cs + ├── SyntaxValidationUtilityMethods.cs + ├── CustomSyntaxValidationFunctions.cs + ├── ValidationEntry.cs / ValidationStructuredEntry.cs / ValidationKeyValuePair.cs + └── Bcp47Codes.cs +```