From 82f74d29f55ebdc37605a340ab80da7b8debb11f Mon Sep 17 00:00:00 2001 From: "Rossi, Davide" Date: Thu, 10 Feb 2022 17:29:14 +0100 Subject: [PATCH 1/3] Make neptune-export cross-platform by removing hardcoded line and path separators Signed-off-by: Rossi, Davide --- .../neptune/propertygraph/schema/DataType.java | 2 +- .../services/neptune/util/S3ObjectInfo.java | 12 ++++-------- .../services/neptune/io/DirectoriesTest.java | 2 +- .../io/JsonPropertyGraphPrinterTest.java | 8 ++++---- .../io/VariableRowCsvPropertyGraphPrinterTest.java | 10 +++++----- .../neptune/propertygraph/schema/DataTypeTest.java | 9 ++++++--- .../services/neptune/util/S3ObjectInfoTest.java | 10 ++++++++-- 7 files changed, 29 insertions(+), 24 deletions(-) diff --git a/neptune-export/src/main/java/com/amazonaws/services/neptune/propertygraph/schema/DataType.java b/neptune-export/src/main/java/com/amazonaws/services/neptune/propertygraph/schema/DataType.java index 2ab4ae78..6652d913 100644 --- a/neptune-export/src/main/java/com/amazonaws/services/neptune/propertygraph/schema/DataType.java +++ b/neptune-export/src/main/java/com/amazonaws/services/neptune/propertygraph/schema/DataType.java @@ -257,7 +257,7 @@ public String format(Object value, boolean escapeNewline) { } private String escapeNewlineChar(String value) { - return value.replace("\n", "\\n"); + return value.replace("\r", "\\r").replace("\n", "\\n"); } diff --git a/neptune-export/src/main/java/com/amazonaws/services/neptune/util/S3ObjectInfo.java b/neptune-export/src/main/java/com/amazonaws/services/neptune/util/S3ObjectInfo.java index b6acaa9a..40759436 100644 --- a/neptune-export/src/main/java/com/amazonaws/services/neptune/util/S3ObjectInfo.java +++ b/neptune-export/src/main/java/com/amazonaws/services/neptune/util/S3ObjectInfo.java @@ -44,17 +44,13 @@ public File createDownloadFile(String parent) { } public S3ObjectInfo withNewKeySuffix(String suffix) { - File file = StringUtils.isNotEmpty(key) ? new File(key, suffix) : new File(suffix); - return new S3ObjectInfo( String.format("s3://%s/%s", bucket, file.getPath())); + String newKey = StringUtils.isNotEmpty(key) ? key.replaceFirst("([^/])$","$1/") + suffix : suffix; + return new S3ObjectInfo( String.format("s3://%s/%s", bucket, newKey)); } public S3ObjectInfo replaceOrAppendKey(String placeholder, String ifPresent, String ifAbsent) { - - File file = key.contains(placeholder) ? - new File(key.replace(placeholder, ifPresent)) : - new File(key, ifAbsent); - - return new S3ObjectInfo( String.format("s3://%s/%s", bucket, file.getPath())); + String finalKey = key.contains(placeholder) ? key.replace(placeholder, ifPresent) : key.replaceFirst("([^/])$|^$","$1/") + ifAbsent; + return new S3ObjectInfo( String.format("s3://%s/%s", bucket, finalKey)); } public S3ObjectInfo replaceOrAppendKey(String placeholder, String ifPresent) { diff --git a/neptune-export/src/test/java/com/amazonaws/services/neptune/io/DirectoriesTest.java b/neptune-export/src/test/java/com/amazonaws/services/neptune/io/DirectoriesTest.java index 28084867..403e8361 100644 --- a/neptune-export/src/test/java/com/amazonaws/services/neptune/io/DirectoriesTest.java +++ b/neptune-export/src/test/java/com/amazonaws/services/neptune/io/DirectoriesTest.java @@ -41,7 +41,7 @@ public void createsDigestFilePathsForVeryLongFilenames() throws IOException { Directories directories = Directories.createFor(DirectoryStructure.PropertyGraph, new File("home"), "export-id", "", ""); Path filePath = directories.createFilePath(path, longName, PropertyGraphExportFormat.csv); - assertEquals("/export/8044f12c352773b7ff400ef524da6e90db419e4a.csv", filePath.toString()); + assertEquals(File.separator + "export" + File.separator + "8044f12c352773b7ff400ef524da6e90db419e4a.csv", filePath.toString()); } diff --git a/neptune-export/src/test/java/com/amazonaws/services/neptune/propertygraph/io/JsonPropertyGraphPrinterTest.java b/neptune-export/src/test/java/com/amazonaws/services/neptune/propertygraph/io/JsonPropertyGraphPrinterTest.java index 17e9492a..12f6d103 100644 --- a/neptune-export/src/test/java/com/amazonaws/services/neptune/propertygraph/io/JsonPropertyGraphPrinterTest.java +++ b/neptune-export/src/test/java/com/amazonaws/services/neptune/propertygraph/io/JsonPropertyGraphPrinterTest.java @@ -197,10 +197,10 @@ public void appendsPreviouslyUnseenValuesToObjectWhenInferringSchema() throws IO map(entry("fname", "fname5"), entry("lname", "lname5"), entry("age", 50)) ); - String expectedOutput = "{\"fname\":\"fname1\"}\n" + - "{\"fname\":\"fname2\",\"lname\":\"lname2\"}\n" + - "{\"fname\":\"fname3\",\"age\":30}\n" + - "{\"lname\":\"lname4\",\"age\":40}\n" + + String expectedOutput = "{\"fname\":\"fname1\"}" + System.lineSeparator() + + "{\"fname\":\"fname2\",\"lname\":\"lname2\"}" + System.lineSeparator() + + "{\"fname\":\"fname3\",\"age\":30}" + System.lineSeparator() + + "{\"lname\":\"lname4\",\"age\":40}" + System.lineSeparator() + "{\"fname\":\"fname5\",\"lname\":\"lname5\",\"age\":50}"; assertEquals(expectedOutput, stringWriter.toString()); diff --git a/neptune-export/src/test/java/com/amazonaws/services/neptune/propertygraph/io/VariableRowCsvPropertyGraphPrinterTest.java b/neptune-export/src/test/java/com/amazonaws/services/neptune/propertygraph/io/VariableRowCsvPropertyGraphPrinterTest.java index fddb73bc..0e0fa1ee 100644 --- a/neptune-export/src/test/java/com/amazonaws/services/neptune/propertygraph/io/VariableRowCsvPropertyGraphPrinterTest.java +++ b/neptune-export/src/test/java/com/amazonaws/services/neptune/propertygraph/io/VariableRowCsvPropertyGraphPrinterTest.java @@ -50,11 +50,11 @@ public void appendsPreviouslyUnseenColumnsToEndOfRow() throws IOException { map(entry("fname", "fname5"), entry("lname", "lname5"), entry("age", 50)) ); - String expectedOutput = "\"fname1\"\n" + - "\"fname2\",\"lname2\"\n" + - "\"fname3\",,30\n" + - ",\"lname4\",40\n" + - "\"fname5\",\"lname5\",50\n"; + String expectedOutput = "\"fname1\"" + System.lineSeparator() + + "\"fname2\",\"lname2\"" + System.lineSeparator() + + "\"fname3\",,30" + System.lineSeparator() + + ",\"lname4\",40" + System.lineSeparator() + + "\"fname5\",\"lname5\",50" + System.lineSeparator(); assertEquals(expectedOutput, stringWriter.toString()); diff --git a/neptune-export/src/test/java/com/amazonaws/services/neptune/propertygraph/schema/DataTypeTest.java b/neptune-export/src/test/java/com/amazonaws/services/neptune/propertygraph/schema/DataTypeTest.java index 416b80ba..5501e81a 100644 --- a/neptune-export/src/test/java/com/amazonaws/services/neptune/propertygraph/schema/DataTypeTest.java +++ b/neptune-export/src/test/java/com/amazonaws/services/neptune/propertygraph/schema/DataTypeTest.java @@ -71,7 +71,7 @@ public void shouldNotEscapeNewlineChar(){ @Test public void shouldNotEscapeNewline(){ String result = DataType.String.format("A" + System.lineSeparator() + "B"); - assertEquals("\"A\nB\"", result); + assertEquals("\"A" + System.lineSeparator() + "B\"", result); } @Test @@ -82,8 +82,11 @@ public void shouldEscapeNewlineCharIfEscapeNewlineSetToTrue(){ @Test public void shouldEscapeNewlineIfEscapeNewlineSetToTrue(){ - String result = DataType.String.format("A" + System.lineSeparator() + "B", true); - assertEquals("\"A\\nB\"", result); + String result1 = DataType.String.format("A\r\nB", true); + assertEquals("\"A\\r\\nB\"", result1); + + String result2 = DataType.String.format("A\nB", true); + assertEquals("\"A\\nB\"", result2); } @Test diff --git a/neptune-export/src/test/java/com/amazonaws/services/neptune/util/S3ObjectInfoTest.java b/neptune-export/src/test/java/com/amazonaws/services/neptune/util/S3ObjectInfoTest.java index fe33edc9..caac605f 100644 --- a/neptune-export/src/test/java/com/amazonaws/services/neptune/util/S3ObjectInfoTest.java +++ b/neptune-export/src/test/java/com/amazonaws/services/neptune/util/S3ObjectInfoTest.java @@ -14,6 +14,8 @@ import org.junit.Test; +import java.io.File; + import static org.junit.Assert.assertEquals; public class S3ObjectInfoTest { @@ -51,7 +53,9 @@ public void canCreateDownloadFileForKeyWithoutTrailingSlash(){ S3ObjectInfo s3ObjectInfo = new S3ObjectInfo(s3Uri); - assertEquals("/temp/c.txt", s3ObjectInfo.createDownloadFile("/temp").getAbsolutePath()); + String parent = new File(System.getProperty("java.io.tmpdir")).getAbsolutePath(); + String absolutePath = s3ObjectInfo.createDownloadFile(parent).getAbsolutePath(); + assertEquals(parent + File.separator + "c.txt", absolutePath); } @Test @@ -60,7 +64,9 @@ public void canCreateDownloadFileForKeyWithTrailingSlash(){ S3ObjectInfo s3ObjectInfo = new S3ObjectInfo(s3Uri); - assertEquals("/temp/c", s3ObjectInfo.createDownloadFile("/temp").getAbsolutePath()); + String parent = new File(System.getProperty("java.io.tmpdir")).getAbsolutePath(); + String absolutePath = s3ObjectInfo.createDownloadFile(parent).getAbsolutePath(); + assertEquals(parent + File.separator + "c", absolutePath); } @Test From 728aed9d5341727a41d7a0266f167617a9019c4f Mon Sep 17 00:00:00 2001 From: Hamza Ait El Fatmi Date: Wed, 2 Mar 2022 19:02:57 +0100 Subject: [PATCH 2/3] print nested properties map as nested json object instead of string --- .../io/JsonPropertyGraphPrinter.java | 27 ++++++++++++++++--- .../io/JsonPropertyGraphPrinterTest.java | 24 +++++++++++++++++ 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/neptune-export/src/main/java/com/amazonaws/services/neptune/propertygraph/io/JsonPropertyGraphPrinter.java b/neptune-export/src/main/java/com/amazonaws/services/neptune/propertygraph/io/JsonPropertyGraphPrinter.java index afdda192..ff9cb1ce 100644 --- a/neptune-export/src/main/java/com/amazonaws/services/neptune/propertygraph/io/JsonPropertyGraphPrinter.java +++ b/neptune-export/src/main/java/com/amazonaws/services/neptune/propertygraph/io/JsonPropertyGraphPrinter.java @@ -113,12 +113,27 @@ public void printProperties(Map properties) throws IOException { } private void printProperty(Object value, PropertySchema propertySchema) throws IOException { - - DataType dataType = propertySchema.dataType(); String formattedKey = propertySchema.nameWithoutDataType(); - boolean isMultiValue = propertySchema.isMultiValue(); - printProperty(value, dataType, formattedKey, isMultiValue); + if (isMap(value)) { + generator.writeFieldName(formattedKey); + printStartRow(); + printNestedProperties((Map) value); + printEndRow(); + } else { + DataType dataType = propertySchema.dataType(); + boolean isMultiValue = propertySchema.isMultiValue(); + + printProperty(value, dataType, formattedKey, isMultiValue); + } + } + + private void printNestedProperties(Map value) throws IOException { + for (Map.Entry property : value.entrySet()) { + PropertySchema propertySchema = new PropertySchema(property.getKey()); + propertySchema.accept(property.getValue(), true); + printProperty(property.getValue(), propertySchema); + } } private void printProperty(Object value, DataType dataType, String formattedKey, boolean forceMultiValue) throws IOException { @@ -210,4 +225,8 @@ public void close() throws Exception { private boolean isList(Object value) { return value instanceof List; } + + private boolean isMap(Object value) { + return value instanceof Map; + } } diff --git a/neptune-export/src/test/java/com/amazonaws/services/neptune/propertygraph/io/JsonPropertyGraphPrinterTest.java b/neptune-export/src/test/java/com/amazonaws/services/neptune/propertygraph/io/JsonPropertyGraphPrinterTest.java index 12f6d103..e5a48c42 100644 --- a/neptune-export/src/test/java/com/amazonaws/services/neptune/propertygraph/io/JsonPropertyGraphPrinterTest.java +++ b/neptune-export/src/test/java/com/amazonaws/services/neptune/propertygraph/io/JsonPropertyGraphPrinterTest.java @@ -177,6 +177,30 @@ public void shouldPrintMultiValueListAsArrayIrrespectiveOfWhetherMultiValueIsTru stringWriter.toString()); } + @Test + public void shouldPrintNestedPropertiesMapAsJsonObject() throws Exception { + StringWriter stringWriter = new StringWriter(); + + PropertySchema propertySchema1 = new PropertySchema("property", false, DataType.String, true); + + LabelSchema labelSchema = new LabelSchema(new Label("Entity")); + labelSchema.put("property", propertySchema1); + + Map props = map(entry("property", map( + entry("nestedProperty1", "value1"), + entry("nestedProperty2", "value2")))); + + try (PropertyGraphPrinter propertyGraphPrinter = PropertyGraphExportFormat.json.createPrinter(new PrintOutputWriter("outputId", stringWriter), labelSchema, PrinterOptions.NULL_OPTIONS)) { + propertyGraphPrinter.printStartRow(); + propertyGraphPrinter.printProperties(props); + propertyGraphPrinter.printEndRow(); + } + + assertEquals( + "{\"property\":{\"nestedProperty1\":\"value1\",\"nestedProperty2\":\"value2\"}}", + stringWriter.toString()); + } + @Test public void appendsPreviouslyUnseenValuesToObjectWhenInferringSchema() throws IOException { From acbbe1ad5f170ab289c68709a33a8016f2504991 Mon Sep 17 00:00:00 2001 From: Hamza Ait El Fatmi Date: Thu, 3 Mar 2022 08:55:51 +0100 Subject: [PATCH 3/3] make strictCardinality configurable at command level --- .../neptune/ExportPropertyGraphFromGremlinQueries.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/neptune-export/src/main/java/com/amazonaws/services/neptune/ExportPropertyGraphFromGremlinQueries.java b/neptune-export/src/main/java/com/amazonaws/services/neptune/ExportPropertyGraphFromGremlinQueries.java index 90e0a4cf..4dde9eb4 100644 --- a/neptune-export/src/main/java/com/amazonaws/services/neptune/ExportPropertyGraphFromGremlinQueries.java +++ b/neptune-export/src/main/java/com/amazonaws/services/neptune/ExportPropertyGraphFromGremlinQueries.java @@ -77,6 +77,10 @@ public class ExportPropertyGraphFromGremlinQueries extends NeptuneExportCommand @Once private boolean includeTypeDefinitions = false; + @Option(name = {"--strict-cardinality"}, description = "Use strict cardinality (optional, default 'false').") + @Once + private boolean strictCardinality = false; + @Option(name = {"--timeout-millis"}, description = "Query timeout in milliseconds (optional).") @Once private Long timeoutMillis = null; @@ -94,7 +98,7 @@ public void run() { directories.queriesResource(); CsvPrinterOptions csvPrinterOptions = CsvPrinterOptions.builder().setIncludeTypeDefinitions(includeTypeDefinitions).build(); - JsonPrinterOptions jsonPrinterOptions = JsonPrinterOptions.builder().setStrictCardinality(true).build(); + JsonPrinterOptions jsonPrinterOptions = JsonPrinterOptions.builder().setStrictCardinality(strictCardinality).build(); PropertyGraphTargetConfig targetConfig = target.config(directories, new PrinterOptions(csvPrinterOptions, jsonPrinterOptions)); NamedQueriesCollection namedQueries = getNamedQueriesCollection(queries, queriesFile, queriesResource);