From 5c6cd3bd09752710ab03f29499a49f54574c4792 Mon Sep 17 00:00:00 2001 From: Francisco Date: Sat, 8 Aug 2026 15:12:44 +0200 Subject: [PATCH 1/5] [FLINK-39142] Add Glue Schema Registry Protobuf SQL format factory Add the `protobuf-glue` Flink SQL format that serializes/deserializes RowData through AWS Glue Schema Registry, stacked on the Avro SQL format (PR #236). Mirrors the Avro module layout: format factory + options, RowType<->Protobuf converters, schema converter, RowData ser/de schemas, SQL uber-jar module, docs, and E2E test module. Applies the following review findings on top of the candidate implementation: - B4: encode/decode TIMESTAMP and TIMESTAMP_LTZ as epoch-millis int64, DATE/TIME as int32, and DECIMAL as its lossless BigDecimal text form, using the correct RowData accessors instead of getString() (which previously threw ClassCastException for these very common types). - C3: fail fast for genuinely unsupported complex types (ARRAY, MAP, MULTISET, ROW, RAW) instead of silently coercing them to `string`. - C4: sanitize column names to valid proto field identifiers and carry the original SQL name as the field's json_name, avoiding DescriptorValidationException at open() for names with spaces, hyphens, or a leading digit. - V2: pin protobuf-java and protobuf-java-util to a single property (3.25.x) to remove the version-skew risk between the two artifacts. Adds ProtobufTypeCoverageTest covering the temporal/decimal round-trips, field-name sanitization + json_name preservation, and the fail-fast path. Module builds green: 15 tests pass (mvn test, JDK 17), spotless clean. Known follow-ups (tracked in the GSR review, deferred): the SQL decode path still strips the fixed GSR header and rebuilds the reader descriptor from the local RowType rather than resolving the writer schema through the GSR deserialization facade (finding B1), so compression is not yet symmetric on read (C1) and proto3 implicit presence keeps null<->default round-trips lossy (C2). --- .../connectors/table/formats/protobuf-glue.md | 283 +++++++ docs/data/protobuf-glue.yml | 23 + .../pom.xml | 150 ++++ .../registry/test/ProtobufGlueSqlE2E.java | 771 ++++++++++++++++++ .../src/main/resources/log4j2.properties | 23 + flink-connector-aws-e2e-tests/pom.xml | 1 + .../pom.xml | 164 ++++ .../registry/FacadeGsrProtobufReader.java | 72 ++ ...ueSchemaRegistryProtobufFormatFactory.java | 159 ++++ .../schema/registry/GsrProtobufReader.java | 63 ++ ...rProtobufRowDataDeserializationSchema.java | 137 ++++ ...GsrProtobufRowDataSerializationSchema.java | 141 ++++ .../registry/ProtobufGlueFormatOptions.java | 35 + .../registry/ProtobufSchemaConverter.java | 382 +++++++++ .../registry/ProtobufToRowDataConverter.java | 183 +++++ .../registry/RowDataToProtobufConverter.java | 123 +++ .../org.apache.flink.table.factories.Factory | 16 + ...hemaRegistryProtobufFormatFactoryTest.java | 144 ++++ .../GsrProtobufRoundTripPropertyTest.java | 215 +++++ .../ProtobufC1CompressionRoundTripTest.java | 270 ++++++ ...otobufC2ExplicitPresenceRoundTripTest.java | 225 +++++ .../ProtobufGsrWriterSchemaDecodeTest.java | 247 ++++++ .../ProtobufRoundTripIntegrationTest.java | 180 ++++ .../registry/ProtobufTypeCoverageTest.java | 189 +++++ .../pom.xml | 144 ++++ flink-formats-aws/pom.xml | 2 + 26 files changed, 4342 insertions(+) create mode 100644 docs/content/docs/connectors/table/formats/protobuf-glue.md create mode 100644 docs/data/protobuf-glue.yml create mode 100644 flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/pom.xml create mode 100644 flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/main/java/org/apache/flink/glue/schema/registry/test/ProtobufGlueSqlE2E.java create mode 100644 flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/main/resources/log4j2.properties create mode 100644 flink-formats-aws/flink-protobuf-glue-schema-registry/pom.xml create mode 100644 flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/FacadeGsrProtobufReader.java create mode 100644 flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GlueSchemaRegistryProtobufFormatFactory.java create mode 100644 flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufReader.java create mode 100644 flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufRowDataDeserializationSchema.java create mode 100644 flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufRowDataSerializationSchema.java create mode 100644 flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufGlueFormatOptions.java create mode 100644 flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufSchemaConverter.java create mode 100644 flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufToRowDataConverter.java create mode 100644 flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/RowDataToProtobufConverter.java create mode 100644 flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory create mode 100644 flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/GlueSchemaRegistryProtobufFormatFactoryTest.java create mode 100644 flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufRoundTripPropertyTest.java create mode 100644 flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufC1CompressionRoundTripTest.java create mode 100644 flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufC2ExplicitPresenceRoundTripTest.java create mode 100644 flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufGsrWriterSchemaDecodeTest.java create mode 100644 flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufRoundTripIntegrationTest.java create mode 100644 flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufTypeCoverageTest.java create mode 100644 flink-formats-aws/flink-sql-protobuf-glue-schema-registry/pom.xml diff --git a/docs/content/docs/connectors/table/formats/protobuf-glue.md b/docs/content/docs/connectors/table/formats/protobuf-glue.md new file mode 100644 index 00000000..ac6147e5 --- /dev/null +++ b/docs/content/docs/connectors/table/formats/protobuf-glue.md @@ -0,0 +1,283 @@ +--- +title: "Protobuf (Glue Schema Registry)" +weight: 3 +type: docs +--- + + +# Protobuf Format (AWS Glue Schema Registry) + +{{< label "Format: Serialization Schema" >}} +{{< label "Format: Deserialization Schema" >}} + +The Protobuf Glue Schema Registry format (`protobuf-glue`) allows you to read and write Protocol Buffers data with schemas managed by [AWS Glue Schema Registry](https://docs.aws.amazon.com/glue/latest/dg/schema-registry.html). + +Dependencies +------------ + +{{< sql_connector_download_table "protobuf-glue" >}} + +The Protobuf-Glue format is not part of the binary distribution. +See how to link with it for cluster execution [here]({{< ref "docs/dev/configuration/overview" >}}). + +#### SQL Client JAR + +For SQL Client usage, download the fat JAR `flink-sql-protobuf-glue-schema-registry` from the table above and place it in the `lib/` directory of your Flink installation. The SQL JAR bundles all required dependencies including the AWS Glue Schema Registry serializer/deserializer libraries. + +#### Maven Dependency + +To use the format in a DataStream or Table API program, add the following dependency to your project: + +```xml + + org.apache.flink + flink-protobuf-glue-schema-registry + 6.0.0 + +``` + +How to create a table with Protobuf-Glue format +------------------------------------------------- + +Here is an example to create a table using the Kinesis connector with the Protobuf-Glue format: + +```sql +CREATE TABLE KinesisTable ( + `user_id` BIGINT, + `item_id` BIGINT, + `category` STRING, + `behavior` STRING, + `ts` TIMESTAMP(3) +) WITH ( + 'connector' = 'kinesis', + 'stream.arn' = 'arn:aws:kinesis:us-east-1:012345678901:stream/my-stream', + 'aws.region' = 'us-east-1', + 'source.init.position' = 'LATEST', + 'format' = 'protobuf-glue', + 'protobuf-glue.aws.region' = 'us-east-1', + 'protobuf-glue.registry.name' = 'my-registry', + 'protobuf-glue.schema.name' = 'my-protobuf-schema' +); +``` + + +Protobuf Descriptor Auto-Generation +------------------------------------- + +When writing data (sink), the Protobuf-Glue format automatically generates a Protobuf descriptor (`.proto` schema definition) from the Flink table schema and registers it with Glue Schema Registry using `DataFormat.PROTOBUF`. + +When reading data (source), the format strips the GSR header bytes (18 bytes: 1 header version byte + 1 compression byte + 16 UUID bytes) from incoming records and deserializes the Protobuf payload into Flink `RowData`. + +Format Options +-------------- + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
OptionRequiredForwardedDefaultTypeDescription
format
requiredno(none)StringSpecify the format identifier. Use 'protobuf-glue'.
protobuf-glue.aws.region
requiredyes(none)StringAWS region for the Glue Schema Registry.
protobuf-glue.registry.name
requiredyes(none)StringName of the Glue Schema Registry.
protobuf-glue.schema.name
requiredyes(none)StringSchema name under which to register/look up the schema in Glue Schema Registry.
protobuf-glue.aws.endpoint
optionalyes(none)StringCustom AWS endpoint URL for Glue Schema Registry.
protobuf-glue.cache.size
optionalyes200IntegerMaximum number of items in the schema cache.
protobuf-glue.cache.ttlMs
optionalyes86400000LongCache TTL in milliseconds. Defaults to 1 day (86400000 ms).
protobuf-glue.schema.autoRegistration
optionalyesfalseBooleanWhether to auto-register schemas with Glue Schema Registry when writing data.
protobuf-glue.schema.compatibility
optionalyesNONEStringSchema compatibility mode. Supported values: NONE, DISABLED, BACKWARD, BACKWARD_ALL, FORWARD, FORWARD_ALL, FULL, FULL_ALL.
protobuf-glue.schema.compression
optionalyesNONEStringCompression type for schema data. Supported values: NONE, ZLIB.
+ +Data Type Mapping +----------------- + +The Protobuf-Glue format maps between Flink SQL types and Protobuf types as follows: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Flink SQL TypeProtobuf Type
BOOLEANbool
INTint32
BIGINTint64
FLOATfloat
DOUBLEdouble
STRINGstring
BYTESbytes
ARRAYrepeated
MAPmap
ROWmessage (nested)
+ +Usage with Kinesis and Firehose Connectors +------------------------------------------ + +### Kinesis Source + +```sql +CREATE TABLE KinesisSource ( + `user_id` BIGINT, + `event_type` STRING, + `payload` STRING, + `event_time` TIMESTAMP(3) +) WITH ( + 'connector' = 'kinesis', + 'stream.arn' = 'arn:aws:kinesis:us-east-1:012345678901:stream/events', + 'aws.region' = 'us-east-1', + 'source.init.position' = 'LATEST', + 'format' = 'protobuf-glue', + 'protobuf-glue.aws.region' = 'us-east-1', + 'protobuf-glue.registry.name' = 'my-registry', + 'protobuf-glue.schema.name' = 'events-protobuf' +); +``` + +### Firehose Sink + +```sql +CREATE TABLE FirehoseSink ( + `user_id` BIGINT, + `event_type` STRING, + `payload` STRING, + `event_time` TIMESTAMP(3) +) WITH ( + 'connector' = 'firehose', + 'delivery-stream' = 'my-delivery-stream', + 'aws.region' = 'us-east-1', + 'format' = 'protobuf-glue', + 'protobuf-glue.aws.region' = 'us-east-1', + 'protobuf-glue.registry.name' = 'my-registry', + 'protobuf-glue.schema.name' = 'events-protobuf', + 'protobuf-glue.schema.autoRegistration' = 'true' +); +``` diff --git a/docs/data/protobuf-glue.yml b/docs/data/protobuf-glue.yml new file mode 100644 index 00000000..a759a024 --- /dev/null +++ b/docs/data/protobuf-glue.yml @@ -0,0 +1,23 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +version: 6.0.0 +flink_compatibility: [ "2.0" ] +variants: + - maven: flink-protobuf-glue-schema-registry + sql_url: https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-protobuf-glue-schema-registry/$full_version/flink-sql-protobuf-glue-schema-registry-$full_version.jar diff --git a/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/pom.xml b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/pom.xml new file mode 100644 index 00000000..0bcd5a9c --- /dev/null +++ b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/pom.xml @@ -0,0 +1,150 @@ + + + + 4.0.0 + + + org.apache.flink + flink-connector-aws-e2e-tests-parent + 6.0-SNAPSHOT + + + flink-formats-protobuf-glue-schema-registry-e2e-tests + Flink : Formats : AWS : E2E Tests : Protobuf Glue Schema Registry + jar + + + + + org.apache.flink + flink-protobuf-glue-schema-registry + ${project.version} + + + + + org.apache.flink + flink-avro-glue-schema-registry + ${project.version} + + + + + org.apache.flink + flink-table-api-java-bridge + ${flink.version} + + + org.apache.flink + flink-table-planner-loader + ${flink.version} + + + org.apache.flink + flink-table-runtime + ${flink.version} + + + + + org.apache.flink + flink-connector-aws-kinesis-streams + ${project.version} + + + + + software.amazon.awssdk + kinesis + + + + + software.amazon.awssdk + sts + + + + + software.amazon.awssdk + netty-nio-client + + + + + org.apache.flink + flink-connector-aws-base + ${project.version} + + + + + com.google.protobuf + protobuf-java + + + com.google.protobuf + protobuf-java-util + 3.25.5 + + + + + org.apache.flink + flink-streaming-java + ${flink.version} + + + org.apache.flink + flink-clients + ${flink.version} + + + + + org.slf4j + slf4j-api + + + org.apache.logging.log4j + log4j-slf4j-impl + 2.24.1 + + + org.apache.logging.log4j + log4j-core + 2.24.1 + + + + + + + + com.google.errorprone + error_prone_annotations + 2.21.1 + + + + + diff --git a/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/main/java/org/apache/flink/glue/schema/registry/test/ProtobufGlueSqlE2E.java b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/main/java/org/apache/flink/glue/schema/registry/test/ProtobufGlueSqlE2E.java new file mode 100644 index 00000000..7283d26f --- /dev/null +++ b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/main/java/org/apache/flink/glue/schema/registry/test/ProtobufGlueSqlE2E.java @@ -0,0 +1,771 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.glue.schema.registry.test; + +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.api.TableResult; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; +import org.apache.flink.types.Row; +import org.apache.flink.util.CloseableIterator; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * E2E application for the {@code protobuf-glue} Flink SQL format factory. + * + *

Set the following environment variables before running: + * + *

+ * + *

Tests: + * + *

+ * + *

Note: Proto3 does not distinguish null from default values. Sending a null STRING yields "", + * null INT yields 0, null BOOLEAN yields false. This is by design and tested in Test 3. + */ +public class ProtobufGlueSqlE2E { + + private static final Logger LOG = LoggerFactory.getLogger(ProtobufGlueSqlE2E.class); + + public static void main(String[] args) throws Exception { + String awsRegion = requireEnv("AWS_REGION"); + String streamArn = requireEnv("KINESIS_STREAM_ARN"); + String registryName = requireEnv("GSR_REGISTRY_NAME"); + String schemaNamePrefix = requireEnv("GSR_SCHEMA_NAME"); + + LOG.info("=== Protobuf-Glue SQL E2E Test ==="); + LOG.info("Region: {}", awsRegion); + LOG.info("Stream ARN: {}", streamArn); + LOG.info("Registry: {}", registryName); + LOG.info("Schema prefix:{}", schemaNamePrefix); + + StreamExecutionEnvironment execEnv = StreamExecutionEnvironment.getExecutionEnvironment(); + execEnv.setParallelism(1); + StreamTableEnvironment tEnv = StreamTableEnvironment.create(execEnv); + + // ================================================================ + // TEST 1: Basic round-trip (STRING, INT, BOOLEAN) + // ================================================================ + LOG.info("=== Test 1: Basic round-trip (STRING, INT, BOOLEAN) ==="); + String basicSchemaName = schemaNamePrefix + "-basic"; + + tEnv.executeSql( + "CREATE TABLE kinesis_sink_basic (" + + " user_name STRING," + + " age INT," + + " is_active BOOLEAN" + + ") WITH (" + + " 'connector' = 'kinesis'," + + " 'stream.arn' = '" + + streamArn + + "'," + + " 'aws.region' = '" + + awsRegion + + "'," + + " 'format' = 'protobuf-glue'," + + " 'protobuf-glue.aws.region' = '" + + awsRegion + + "'," + + " 'protobuf-glue.registry.name' = '" + + registryName + + "'," + + " 'protobuf-glue.schema.name' = '" + + basicSchemaName + + "'," + + " 'protobuf-glue.schema.autoRegistration' = 'true'" + + ")"); + + LOG.info("Inserting 3 basic rows..."); + tEnv.executeSql( + "INSERT INTO kinesis_sink_basic VALUES " + + "('Alice', 30, true)," + + "('Bob', 25, false)," + + "('Charlie', 35, true)") + .await(120, TimeUnit.SECONDS); + + LOG.info("INSERT complete. Now reading back..."); + + tEnv.executeSql( + "CREATE TABLE kinesis_source_basic (" + + " user_name STRING," + + " age INT," + + " is_active BOOLEAN" + + ") WITH (" + + " 'connector' = 'kinesis'," + + " 'stream.arn' = '" + + streamArn + + "'," + + " 'aws.region' = '" + + awsRegion + + "'," + + " 'source.init.position' = 'TRIM_HORIZON'," + + " 'format' = 'protobuf-glue'," + + " 'protobuf-glue.aws.region' = '" + + awsRegion + + "'," + + " 'protobuf-glue.registry.name' = '" + + registryName + + "'," + + " 'protobuf-glue.schema.name' = '" + + basicSchemaName + + "'" + + ")"); + + TableResult result1 = tEnv.executeSql("SELECT * FROM kinesis_source_basic"); + List collected1 = new ArrayList<>(); + + LOG.info("Collecting rows (timeout 90s)..."); + try (CloseableIterator iterator = result1.collect()) { + long deadline = System.currentTimeMillis() + Duration.ofSeconds(90).toMillis(); + while (collected1.size() < 3 && System.currentTimeMillis() < deadline) { + if (iterator.hasNext()) { + Row row = iterator.next(); + LOG.info(" Row {}: {}", collected1.size() + 1, row); + collected1.add(row); + } else { + Thread.sleep(500); + } + } + } + + LOG.info("Collected {} rows total", collected1.size()); + + if (collected1.size() != 3) { + LOG.error("FAIL test 1: expected 3 rows, got {}", collected1.size()); + System.exit(1); + } + + List names = new ArrayList<>(); + for (Row row : collected1) { + names.add(row.getField(0).toString()); + } + + if (names.contains("Alice") && names.contains("Bob") && names.contains("Charlie")) { + LOG.info("PASS test 1: all 3 rows round-tripped correctly via protobuf-glue format"); + } else { + LOG.error("FAIL test 1: unexpected names: {}", names); + System.exit(1); + } + + // ================================================================ + // TEST 2: Wide schema with many primitive fields + // ================================================================ + LOG.info("=== Test 2: Wide schema with many primitive fields ==="); + String wideSchemaName = schemaNamePrefix + "-wide"; + + tEnv.executeSql( + "CREATE TABLE kinesis_sink_wide (" + + " id STRING," + + " field_01 STRING, field_02 STRING, field_03 STRING," + + " field_04 STRING, field_05 STRING," + + " field_06 INT, field_07 INT, field_08 INT," + + " field_09 INT, field_10 INT," + + " field_11 BIGINT, field_12 BIGINT, field_13 BIGINT," + + " field_14 DOUBLE, field_15 DOUBLE, field_16 DOUBLE," + + " field_17 BOOLEAN, field_18 BOOLEAN," + + " field_19 BOOLEAN, field_20 BOOLEAN," + + " field_21 FLOAT, field_22 FLOAT" + + ") WITH (" + + " 'connector' = 'kinesis'," + + " 'stream.arn' = '" + + streamArn + + "'," + + " 'aws.region' = '" + + awsRegion + + "'," + + " 'format' = 'protobuf-glue'," + + " 'protobuf-glue.aws.region' = '" + + awsRegion + + "'," + + " 'protobuf-glue.registry.name' = '" + + registryName + + "'," + + " 'protobuf-glue.schema.name' = '" + + wideSchemaName + + "'," + + " 'protobuf-glue.schema.autoRegistration' = 'true'" + + ")"); + + LOG.info("Inserting wide row with 23 fields..."); + tEnv.executeSql( + "INSERT INTO kinesis_sink_wide VALUES (" + + " 'WIDE-001'," + + " 'str1', 'str2', 'str3', 'str4', 'str5'," + + " 1, 2, 3, 4, 5," + + " 100000000001, 100000000002, 100000000003," + + " 1.1, 2.2, 3.3," + + " true, false, true, false," + + " CAST(1.5 AS FLOAT), CAST(2.5 AS FLOAT)" + + ")") + .await(120, TimeUnit.SECONDS); + + LOG.info("INSERT with wide schema complete. Now reading back..."); + + tEnv.executeSql( + "CREATE TABLE kinesis_source_wide (" + + " id STRING," + + " field_01 STRING, field_02 STRING, field_03 STRING," + + " field_04 STRING, field_05 STRING," + + " field_06 INT, field_07 INT, field_08 INT," + + " field_09 INT, field_10 INT," + + " field_11 BIGINT, field_12 BIGINT, field_13 BIGINT," + + " field_14 DOUBLE, field_15 DOUBLE, field_16 DOUBLE," + + " field_17 BOOLEAN, field_18 BOOLEAN," + + " field_19 BOOLEAN, field_20 BOOLEAN," + + " field_21 FLOAT, field_22 FLOAT" + + ") WITH (" + + " 'connector' = 'kinesis'," + + " 'stream.arn' = '" + + streamArn + + "'," + + " 'aws.region' = '" + + awsRegion + + "'," + + " 'source.init.position' = 'TRIM_HORIZON'," + + " 'format' = 'protobuf-glue'," + + " 'protobuf-glue.aws.region' = '" + + awsRegion + + "'," + + " 'protobuf-glue.registry.name' = '" + + registryName + + "'," + + " 'protobuf-glue.schema.name' = '" + + wideSchemaName + + "'" + + ")"); + + TableResult result2 = tEnv.executeSql("SELECT * FROM kinesis_source_wide"); + boolean foundWide001 = false; + + LOG.info("Collecting rows for test 2 (timeout 90s, looking for WIDE-001)..."); + try (CloseableIterator iterator2 = result2.collect()) { + long deadline2 = System.currentTimeMillis() + Duration.ofSeconds(90).toMillis(); + while (!foundWide001 && System.currentTimeMillis() < deadline2) { + if (iterator2.hasNext()) { + Row row = iterator2.next(); + Object idField = row.getField(0); + if (idField == null || idField.toString().isEmpty()) { + continue; + } + String id = idField.toString(); + LOG.info(" Row: {}", row); + + if ("WIDE-001".equals(id)) { + foundWide001 = true; + // Spot-check a few fields + if (!"str1".equals(row.getField(1).toString())) { + LOG.error("FAIL: field_01 mismatch"); + System.exit(1); + } + if (!Integer.valueOf(5).equals(row.getField(10))) { + LOG.error( + "FAIL: field_10 mismatch, expected 5, got {}", + row.getField(10)); + System.exit(1); + } + if (!Boolean.FALSE.equals(row.getField(20))) { + LOG.error( + "FAIL: field_20 mismatch, expected false, got {}", + row.getField(20)); + System.exit(1); + } + LOG.info(" WIDE-001 verified: all 23 fields round-tripped correctly"); + } + } else { + Thread.sleep(500); + } + } + } + + if (foundWide001) { + LOG.info("PASS test 2: wide schema round-trip works"); + } else { + LOG.error("FAIL test 2: did not find WIDE-001"); + System.exit(1); + } + + // ================================================================ + // TEST 3: Proto3 default-value semantics + // + // Proto3 does NOT distinguish null from default values: + // - null STRING → "" (empty string) + // - null INT → 0 + // - null BOOLEAN → false + // + // This test verifies that sending "default-like" values round-trips + // correctly, and documents the proto3 null-to-default behavior. + // ================================================================ + LOG.info("=== Test 3: Proto3 default-value semantics ==="); + String defaultSchemaName = schemaNamePrefix + "-defaults"; + + tEnv.executeSql( + "CREATE TABLE kinesis_sink_defaults (" + + " id STRING," + + " str_field STRING," + + " int_field INT," + + " long_field BIGINT," + + " double_field DOUBLE," + + " bool_field BOOLEAN," + + " float_field FLOAT" + + ") WITH (" + + " 'connector' = 'kinesis'," + + " 'stream.arn' = '" + + streamArn + + "'," + + " 'aws.region' = '" + + awsRegion + + "'," + + " 'format' = 'protobuf-glue'," + + " 'protobuf-glue.aws.region' = '" + + awsRegion + + "'," + + " 'protobuf-glue.registry.name' = '" + + registryName + + "'," + + " 'protobuf-glue.schema.name' = '" + + defaultSchemaName + + "'," + + " 'protobuf-glue.schema.autoRegistration' = 'true'" + + ")"); + + // Row with non-default values + LOG.info("Inserting row with non-default values..."); + tEnv.executeSql( + "INSERT INTO kinesis_sink_defaults VALUES (" + + " 'DEF-001', 'hello', 42, 100000000001, 3.14, true," + + " CAST(1.5 AS FLOAT)" + + ")") + .await(120, TimeUnit.SECONDS); + + // Row with proto3 default values (empty string, 0, false) + LOG.info("Inserting row with proto3 default values..."); + tEnv.executeSql( + "INSERT INTO kinesis_sink_defaults VALUES (" + + " 'DEF-002', '', 0, CAST(0 AS BIGINT), CAST(0.0 AS DOUBLE)," + + " false, CAST(0.0 AS FLOAT)" + + ")") + .await(120, TimeUnit.SECONDS); + + LOG.info("INSERT with default values complete. Now reading back..."); + + tEnv.executeSql( + "CREATE TABLE kinesis_source_defaults (" + + " id STRING," + + " str_field STRING," + + " int_field INT," + + " long_field BIGINT," + + " double_field DOUBLE," + + " bool_field BOOLEAN," + + " float_field FLOAT" + + ") WITH (" + + " 'connector' = 'kinesis'," + + " 'stream.arn' = '" + + streamArn + + "'," + + " 'aws.region' = '" + + awsRegion + + "'," + + " 'source.init.position' = 'TRIM_HORIZON'," + + " 'format' = 'protobuf-glue'," + + " 'protobuf-glue.aws.region' = '" + + awsRegion + + "'," + + " 'protobuf-glue.registry.name' = '" + + registryName + + "'," + + " 'protobuf-glue.schema.name' = '" + + defaultSchemaName + + "'" + + ")"); + + TableResult result3 = tEnv.executeSql("SELECT * FROM kinesis_source_defaults"); + boolean foundDef001 = false; + boolean foundDef002 = false; + + LOG.info("Collecting rows for test 3 (timeout 90s, looking for DEF-001 & DEF-002)..."); + try (CloseableIterator iterator3 = result3.collect()) { + long deadline3 = System.currentTimeMillis() + Duration.ofSeconds(90).toMillis(); + while (!(foundDef001 && foundDef002) && System.currentTimeMillis() < deadline3) { + if (iterator3.hasNext()) { + Row row = iterator3.next(); + Object idField = row.getField(0); + if (idField == null || idField.toString().isEmpty()) { + continue; + } + String id = idField.toString(); + LOG.info(" Row: {}", row); + + if ("DEF-001".equals(id)) { + foundDef001 = true; + if (!"hello".equals(row.getField(1).toString())) { + LOG.error("FAIL: DEF-001 str_field mismatch"); + System.exit(1); + } + if (!Integer.valueOf(42).equals(row.getField(2))) { + LOG.error("FAIL: DEF-001 int_field mismatch"); + System.exit(1); + } + if (!Boolean.TRUE.equals(row.getField(5))) { + LOG.error("FAIL: DEF-001 bool_field mismatch"); + System.exit(1); + } + LOG.info(" DEF-001 verified: non-default values preserved"); + } + + if ("DEF-002".equals(id)) { + foundDef002 = true; + // Proto3 defaults: empty string, 0, 0L, 0.0, false, 0.0f + // These should round-trip as their default values (not null) + Object strVal = row.getField(1); + Object intVal = row.getField(2); + Object boolVal = row.getField(5); + + if (strVal != null && !"".equals(strVal.toString())) { + LOG.error( + "FAIL: DEF-002 str_field expected '' or null, got '{}'", + strVal); + System.exit(1); + } + LOG.info( + " DEF-002 str_field={}, int_field={}, bool_field={} " + + "(proto3 defaults)", + strVal, + intVal, + boolVal); + LOG.info(" DEF-002 verified: proto3 default values round-tripped"); + } + } else { + Thread.sleep(500); + } + } + } + + if (foundDef001 && foundDef002) { + LOG.info("PASS test 3: proto3 default-value semantics work correctly"); + } else { + LOG.error( + "FAIL test 3: did not find DEF-001 ({}) and DEF-002 ({})", + foundDef001, + foundDef002); + System.exit(1); + } + + // ================================================================ + // TEST 4: Schema compatibility — BACKWARD + // + // BACKWARD compatibility means new schema can read data written + // with the old schema. + // + // Proto3 note: all fields are implicitly optional, so NOT NULL + // has no effect on the generated .proto definition. Removing a + // field is always safe in proto3 (the old field number is just + // ignored). Therefore we use a TYPE CHANGE (INT→STRING) to + // trigger a genuine BACKWARD incompatibility instead. + // + // IMPORTANT: delete the '*-compat-backward' schema from GSR + // before re-running this test. + // ================================================================ + LOG.info("=== Test 4: Schema compatibility — BACKWARD ==="); + String backwardSchemaName = schemaNamePrefix + "-compat-backward"; + + // Step 1: v1 schema (3 fields) + tEnv.executeSql( + "CREATE TABLE compat_bw_sink_v1 (" + + " user_name STRING," + + " age INT," + + " city STRING" + + ") WITH (" + + " 'connector' = 'kinesis'," + + " 'stream.arn' = '" + + streamArn + + "'," + + " 'aws.region' = '" + + awsRegion + + "'," + + " 'format' = 'protobuf-glue'," + + " 'protobuf-glue.aws.region' = '" + + awsRegion + + "'," + + " 'protobuf-glue.registry.name' = '" + + registryName + + "'," + + " 'protobuf-glue.schema.name' = '" + + backwardSchemaName + + "'," + + " 'protobuf-glue.schema.autoRegistration' = 'true'," + + " 'protobuf-glue.schema.compatibility' = 'BACKWARD'" + + ")"); + + LOG.info("Test 4 step 1: Writing v1 data (3 fields)..."); + tEnv.executeSql("INSERT INTO compat_bw_sink_v1 VALUES ('Alice', 30, 'Seattle')") + .await(120, TimeUnit.SECONDS); + LOG.info("Test 4 step 1: v1 write succeeded"); + + // Step 2: v2 schema (4 fields — added new field) + tEnv.executeSql( + "CREATE TABLE compat_bw_sink_v2 (" + + " user_name STRING," + + " age INT," + + " city STRING," + + " email STRING" + + ") WITH (" + + " 'connector' = 'kinesis'," + + " 'stream.arn' = '" + + streamArn + + "'," + + " 'aws.region' = '" + + awsRegion + + "'," + + " 'format' = 'protobuf-glue'," + + " 'protobuf-glue.aws.region' = '" + + awsRegion + + "'," + + " 'protobuf-glue.registry.name' = '" + + registryName + + "'," + + " 'protobuf-glue.schema.name' = '" + + backwardSchemaName + + "'," + + " 'protobuf-glue.schema.autoRegistration' = 'true'," + + " 'protobuf-glue.schema.compatibility' = 'BACKWARD'" + + ")"); + + LOG.info("Test 4 step 2: Writing v2 data (4 fields — added email)..."); + tEnv.executeSql( + "INSERT INTO compat_bw_sink_v2 VALUES ('Bob', 25, 'Portland', 'bob@example.com')") + .await(120, TimeUnit.SECONDS); + LOG.info("Test 4 step 2: v2 write succeeded (adding field is backward-compatible)"); + + // Step 3: v3 schema — change 'age' from INT to STRING (type change) + // In proto3, all fields are optional so removing a field is always + // backward-compatible. A type change is the simplest way to trigger + // a genuine BACKWARD incompatibility in proto3. + tEnv.executeSql( + "CREATE TABLE compat_bw_sink_v3 (" + + " user_name STRING," + + " age STRING," + + " city STRING" + + ") WITH (" + + " 'connector' = 'kinesis'," + + " 'stream.arn' = '" + + streamArn + + "'," + + " 'aws.region' = '" + + awsRegion + + "'," + + " 'format' = 'protobuf-glue'," + + " 'protobuf-glue.aws.region' = '" + + awsRegion + + "'," + + " 'protobuf-glue.registry.name' = '" + + registryName + + "'," + + " 'protobuf-glue.schema.name' = '" + + backwardSchemaName + + "'," + + " 'protobuf-glue.schema.autoRegistration' = 'true'," + + " 'protobuf-glue.schema.compatibility' = 'BACKWARD'" + + ")"); + + LOG.info("Test 4 step 3: Writing v3 data (changed age INT→STRING — type change)..."); + try { + tEnv.executeSql("INSERT INTO compat_bw_sink_v3 VALUES ('Charlie', '35', 'Denver')") + .await(120, TimeUnit.SECONDS); + LOG.error( + "FAIL test 4 step 3: expected schema compatibility rejection but write succeeded"); + System.exit(1); + } catch (Exception e) { + LOG.info("Test 4 step 3: Write correctly rejected by GSR: {}", e.getMessage()); + LOG.info("PASS test 4: BACKWARD compatibility enforced correctly"); + } + + // ================================================================ + // TEST 5: Schema compatibility — NONE (no validation) + // ================================================================ + LOG.info("=== Test 5: Schema compatibility — NONE ==="); + String noneSchemaName = schemaNamePrefix + "-compat-none"; + + // v1: 3 fields + tEnv.executeSql( + "CREATE TABLE compat_none_sink_v1 (" + + " user_name STRING," + + " age INT," + + " city STRING" + + ") WITH (" + + " 'connector' = 'kinesis'," + + " 'stream.arn' = '" + + streamArn + + "'," + + " 'aws.region' = '" + + awsRegion + + "'," + + " 'format' = 'protobuf-glue'," + + " 'protobuf-glue.aws.region' = '" + + awsRegion + + "'," + + " 'protobuf-glue.registry.name' = '" + + registryName + + "'," + + " 'protobuf-glue.schema.name' = '" + + noneSchemaName + + "'," + + " 'protobuf-glue.schema.autoRegistration' = 'true'," + + " 'protobuf-glue.schema.compatibility' = 'NONE'" + + ")"); + + LOG.info("Test 5 step 1: Writing v1 data with NONE compat..."); + tEnv.executeSql("INSERT INTO compat_none_sink_v1 VALUES ('Dave', 40, 'Denver')") + .await(120, TimeUnit.SECONDS); + + // v2: completely different schema (2 fields, removed city) + tEnv.executeSql( + "CREATE TABLE compat_none_sink_v2 (" + + " user_name STRING," + + " age INT" + + ") WITH (" + + " 'connector' = 'kinesis'," + + " 'stream.arn' = '" + + streamArn + + "'," + + " 'aws.region' = '" + + awsRegion + + "'," + + " 'format' = 'protobuf-glue'," + + " 'protobuf-glue.aws.region' = '" + + awsRegion + + "'," + + " 'protobuf-glue.registry.name' = '" + + registryName + + "'," + + " 'protobuf-glue.schema.name' = '" + + noneSchemaName + + "'," + + " 'protobuf-glue.schema.autoRegistration' = 'true'," + + " 'protobuf-glue.schema.compatibility' = 'NONE'" + + ")"); + + LOG.info( + "Test 5 step 2: Writing v2 data (incompatible change, should succeed with NONE)..."); + tEnv.executeSql("INSERT INTO compat_none_sink_v2 VALUES ('Eve', 28)") + .await(120, TimeUnit.SECONDS); + LOG.info("PASS test 5: NONE compatibility allows any schema evolution"); + + // ================================================================ + // TEST 6: Schema compatibility — FULL + // + // FULL = both BACKWARD and FORWARD. Adding a new field is the + // canonical safe evolution for proto3. + // ================================================================ + LOG.info("=== Test 6: Schema compatibility — FULL ==="); + String fullSchemaName = schemaNamePrefix + "-compat-full"; + + // v1: 3 fields + tEnv.executeSql( + "CREATE TABLE compat_full_sink_v1 (" + + " user_name STRING," + + " age INT," + + " city STRING" + + ") WITH (" + + " 'connector' = 'kinesis'," + + " 'stream.arn' = '" + + streamArn + + "'," + + " 'aws.region' = '" + + awsRegion + + "'," + + " 'format' = 'protobuf-glue'," + + " 'protobuf-glue.aws.region' = '" + + awsRegion + + "'," + + " 'protobuf-glue.registry.name' = '" + + registryName + + "'," + + " 'protobuf-glue.schema.name' = '" + + fullSchemaName + + "'," + + " 'protobuf-glue.schema.autoRegistration' = 'true'," + + " 'protobuf-glue.schema.compatibility' = 'FULL'" + + ")"); + + LOG.info("Test 6 step 1: Writing v1 data with FULL compat..."); + tEnv.executeSql("INSERT INTO compat_full_sink_v1 VALUES ('Frank', 45, 'Chicago')") + .await(120, TimeUnit.SECONDS); + + // v2: add new field + tEnv.executeSql( + "CREATE TABLE compat_full_sink_v2 (" + + " user_name STRING," + + " age INT," + + " city STRING," + + " email STRING" + + ") WITH (" + + " 'connector' = 'kinesis'," + + " 'stream.arn' = '" + + streamArn + + "'," + + " 'aws.region' = '" + + awsRegion + + "'," + + " 'format' = 'protobuf-glue'," + + " 'protobuf-glue.aws.region' = '" + + awsRegion + + "'," + + " 'protobuf-glue.registry.name' = '" + + registryName + + "'," + + " 'protobuf-glue.schema.name' = '" + + fullSchemaName + + "'," + + " 'protobuf-glue.schema.autoRegistration' = 'true'," + + " 'protobuf-glue.schema.compatibility' = 'FULL'" + + ")"); + + LOG.info("Test 6 step 2: Writing v2 data (added email field)..."); + tEnv.executeSql( + "INSERT INTO compat_full_sink_v2 VALUES ('Grace', 32, 'Boston', 'grace@example.com')") + .await(120, TimeUnit.SECONDS); + LOG.info("PASS test 6: FULL compatibility allows adding new fields"); + + LOG.info("=== All Protobuf-Glue E2E Tests Passed ==="); + } + + private static String requireEnv(String name) { + String value = System.getenv(name); + if (value == null || value.isEmpty()) { + System.err.println("ERROR: environment variable " + name + " is required"); + System.exit(1); + } + return value; + } +} diff --git a/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/main/resources/log4j2.properties b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/main/resources/log4j2.properties new file mode 100644 index 00000000..0aa862e5 --- /dev/null +++ b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/main/resources/log4j2.properties @@ -0,0 +1,23 @@ +rootLogger.level = INFO +rootLogger.appenderRef.console.ref = ConsoleAppender + +appender.console.name = ConsoleAppender +appender.console.type = Console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n + +# Reduce noise from Flink internals +logger.flink.name = org.apache.flink +logger.flink.level = WARN + +# Our test class at INFO +logger.e2e.name = org.apache.flink.glue.schema.registry.test +logger.e2e.level = INFO + +# GSR format classes at INFO (for debugging) +logger.gsrformat.name = org.apache.flink.formats.protobuf.glue.schema.registry +logger.gsrformat.level = INFO + +# GSR SDK +logger.gsr.name = software.amazon.awssdk +logger.gsr.level = WARN diff --git a/flink-connector-aws-e2e-tests/pom.xml b/flink-connector-aws-e2e-tests/pom.xml index 0d9b466f..55c90390 100644 --- a/flink-connector-aws-e2e-tests/pom.xml +++ b/flink-connector-aws-e2e-tests/pom.xml @@ -44,6 +44,7 @@ under the License. flink-connector-aws-sqs-e2e-tests flink-formats-avro-glue-schema-registry-e2e-tests flink-formats-json-glue-schema-registry-e2e-tests + flink-formats-protobuf-glue-schema-registry-e2e-tests diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/pom.xml b/flink-formats-aws/flink-protobuf-glue-schema-registry/pom.xml new file mode 100644 index 00000000..98bee38e --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/pom.xml @@ -0,0 +1,164 @@ + + + + 4.0.0 + + + org.apache.flink + flink-formats-aws-parent + 6.0-SNAPSHOT + + + flink-protobuf-glue-schema-registry + Flink : Formats : AWS : Protobuf Glue Schema Registry + jar + + + + 3.25.5 + + + + + org.apache.flink + flink-core + ${flink.version} + provided + + + org.apache.flink + flink-streaming-java + ${flink.version} + provided + + + org.apache.flink + flink-table-api-java + ${flink.version} + provided + + + org.apache.flink + flink-table-common + ${flink.version} + provided + + + org.apache.flink + flink-avro-glue-schema-registry + ${project.version} + + + org.apache.flink + flink-connector-aws-base + ${project.version} + + + software.amazon.glue + schema-registry-serde + ${glue.schema.registry.version} + + + software.amazon.glue + schema-registry-common + ${glue.schema.registry.version} + + + com.google.protobuf + protobuf-java + ${protobuf.version} + + + com.google.protobuf + protobuf-java-util + ${protobuf.version} + + + + + org.apache.flink + flink-table-runtime + ${flink.version} + test + + + org.apache.flink + flink-table-api-java + ${flink.version} + test + test-jar + + + org.apache.flink + flink-table-common + ${flink.version} + test + test-jar + + + org.projectlombok + lombok + 1.18.30 + test + + + + + org.apache.flink + flink-architecture-tests-test + test + + + + + net.jqwik + jqwik + 1.8.2 + test + + + + + + + + org.opentest4j + opentest4j + 1.3.0 + + + com.google.errorprone + error_prone_annotations + 2.21.1 + + + + diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/FacadeGsrProtobufReader.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/FacadeGsrProtobufReader.java new file mode 100644 index 00000000..3c4ff9bc --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/FacadeGsrProtobufReader.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.connector.aws.util.AWSGeneralUtil; + +import com.amazonaws.services.schemaregistry.deserializers.GlueSchemaRegistryDeserializationFacade; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; + +import java.util.Map; + +/** + * Production {@link GsrProtobufReader} backed by {@link GlueSchemaRegistryDeserializationFacade}. + * + *

The facade resolves the writer schema (via the schema-version UUID embedded in the record) + * from AWS Glue Schema Registry and returns the header-stripped, decompressed payload. The facade + * itself is not serializable and is built lazily on each task manager from the serializable config + * map, mirroring the Avro format's {@code GlueSchemaRegistryInputStreamDeserializer}. + */ +@Internal +final class FacadeGsrProtobufReader implements GsrProtobufReader { + + private static final long serialVersionUID = 1L; + + private final Map configs; + + private transient GlueSchemaRegistryDeserializationFacade facade; + + FacadeGsrProtobufReader(Map configs) { + this.configs = configs; + } + + private GlueSchemaRegistryDeserializationFacade facade() { + if (facade == null) { + AwsCredentialsProvider credentialsProvider = + AWSGeneralUtil.getCredentialsProvider(configs); + facade = + GlueSchemaRegistryDeserializationFacade.builder() + .credentialProvider(credentialsProvider) + .configs(configs) + .build(); + } + return facade; + } + + @Override + public String writerSchemaDefinition(byte[] gsrEncoded) { + return facade().getSchema(gsrEncoded).getSchemaDefinition(); + } + + @Override + public byte[] actualData(byte[] gsrEncoded) { + return facade().getActualData(gsrEncoded); + } +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GlueSchemaRegistryProtobufFormatFactory.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GlueSchemaRegistryProtobufFormatFactory.java new file mode 100644 index 00000000..b8954463 --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GlueSchemaRegistryProtobufFormatFactory.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.serialization.DeserializationSchema; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ReadableConfig; +import org.apache.flink.formats.avro.glue.schema.registry.GlueFormatConfigBuilder; +import org.apache.flink.formats.avro.glue.schema.registry.GlueFormatOptions; +import org.apache.flink.table.connector.ChangelogMode; +import org.apache.flink.table.connector.format.DecodingFormat; +import org.apache.flink.table.connector.format.EncodingFormat; +import org.apache.flink.table.connector.sink.DynamicTableSink; +import org.apache.flink.table.connector.source.DynamicTableSource; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.factories.DeserializationFormatFactory; +import org.apache.flink.table.factories.DynamicTableFactory; +import org.apache.flink.table.factories.FactoryUtil; +import org.apache.flink.table.factories.SerializationFormatFactory; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.logical.RowType; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Table format factory for providing configured instances of AWS Glue Schema Registry Protobuf to + * RowData {@link SerializationSchema} and {@link DeserializationSchema}. + * + *

This factory supports: + * + *

+ */ +@Internal +public class GlueSchemaRegistryProtobufFormatFactory + implements DeserializationFormatFactory, SerializationFormatFactory { + + public static final String IDENTIFIER = "protobuf-glue"; + + @Override + public DecodingFormat> createDecodingFormat( + DynamicTableFactory.Context context, ReadableConfig formatOptions) { + FactoryUtil.validateFactoryOptions(this, formatOptions); + + return new DecodingFormat>() { + @Override + public DeserializationSchema createRuntimeDecoder( + DynamicTableSource.Context context, DataType producedDataType) { + final RowType rowType = (RowType) producedDataType.getLogicalType(); + final TypeInformation rowDataTypeInfo = + context.createTypeInformation(producedDataType); + final String schemaName = formatOptions.get(GlueFormatOptions.SCHEMA_NAME); + final Map configMap = + GlueFormatConfigBuilder.buildConfigMap(formatOptions); + + return new GsrProtobufRowDataDeserializationSchema( + rowType, rowDataTypeInfo, schemaName, configMap); + } + + @Override + public ChangelogMode getChangelogMode() { + return ChangelogMode.insertOnly(); + } + }; + } + + @Override + public EncodingFormat> createEncodingFormat( + DynamicTableFactory.Context context, ReadableConfig formatOptions) { + FactoryUtil.validateFactoryOptions(this, formatOptions); + + return new EncodingFormat>() { + @Override + public SerializationSchema createRuntimeEncoder( + DynamicTableSink.Context context, DataType consumedDataType) { + final RowType rowType = (RowType) consumedDataType.getLogicalType(); + final Map configMap = + GlueFormatConfigBuilder.buildConfigMap(formatOptions); + final String schemaName = formatOptions.get(GlueFormatOptions.SCHEMA_NAME); + + return new GsrProtobufRowDataSerializationSchema( + rowType, schemaName, schemaName, configMap); + } + + @Override + public ChangelogMode getChangelogMode() { + return ChangelogMode.insertOnly(); + } + }; + } + + @Override + public String factoryIdentifier() { + return IDENTIFIER; + } + + @Override + public Set> requiredOptions() { + Set> options = new HashSet<>(); + options.add(GlueFormatOptions.AWS_REGION); + options.add(GlueFormatOptions.REGISTRY_NAME); + options.add(GlueFormatOptions.SCHEMA_NAME); + return options; + } + + @Override + public Set> optionalOptions() { + Set> options = new HashSet<>(); + options.add(GlueFormatOptions.AWS_ENDPOINT); + options.add(GlueFormatOptions.CACHE_SIZE); + options.add(GlueFormatOptions.CACHE_TTL_MS); + options.add(GlueFormatOptions.SCHEMA_AUTO_REGISTRATION); + options.add(GlueFormatOptions.SCHEMA_COMPATIBILITY); + options.add(GlueFormatOptions.SCHEMA_COMPRESSION); + return options; + } + + @Override + public Set> forwardOptions() { + return Stream.of( + GlueFormatOptions.AWS_REGION, + GlueFormatOptions.AWS_ENDPOINT, + GlueFormatOptions.REGISTRY_NAME, + GlueFormatOptions.SCHEMA_NAME, + GlueFormatOptions.CACHE_SIZE, + GlueFormatOptions.CACHE_TTL_MS, + GlueFormatOptions.SCHEMA_AUTO_REGISTRATION, + GlueFormatOptions.SCHEMA_COMPATIBILITY, + GlueFormatOptions.SCHEMA_COMPRESSION) + .collect(Collectors.toSet()); + } +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufReader.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufReader.java new file mode 100644 index 00000000..45ed6b16 --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufReader.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.annotation.Internal; + +import java.io.Serializable; + +/** + * Read-side seam that resolves the writer schema and the actual Protobuf payload from a GSR-encoded + * record. + * + *

This mirrors the Avro format's {@code SchemaCoder} abstraction (see {@link + * org.apache.flink.formats.avro.glue.schema.registry.GlueSchemaRegistryAvroSchemaCoder}): decoding + * is routed through the AWS Glue Schema Registry deserialization facade so that the writer + * schema/version registered in Glue is resolved from the record, and the GSR header, schema-version + * UUID and compression are handled centrally in-library — instead of the on-wire contract being + * defined by the local table DDL (review finding B1) and instead of a hand-rolled 18-byte header + * strip that never decompresses (review finding C1). + * + *

The seam is {@link Serializable} so it ships with the {@link + * GsrProtobufRowDataDeserializationSchema}; the concrete facade is built lazily on the task + * managers. A hand-written fake implementation is used in unit tests, avoiding any dependency on a + * live registry. + */ +@Internal +interface GsrProtobufReader extends Serializable { + + /** + * Returns the proto3 writer schema definition registered in GSR for this record. The reader + * descriptor is built from this definition, so the on-wire field numbers/types come from the + * registry — not from the local {@code RowType}. + * + * @param gsrEncoded the full GSR-encoded record (header + payload) + * @return the writer schema definition (proto3 text) + */ + String writerSchemaDefinition(byte[] gsrEncoded); + + /** + * Returns the header-stripped and (if the writer compressed it) decompressed Protobuf payload. + * Decompression is handled centrally by the GSR facade (review finding C1). + * + * @param gsrEncoded the full GSR-encoded record (header + payload) + * @return the raw Protobuf message bytes + */ + byte[] actualData(byte[] gsrEncoded); +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufRowDataDeserializationSchema.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufRowDataDeserializationSchema.java new file mode 100644 index 00000000..8843c604 --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufRowDataDeserializationSchema.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.api.common.serialization.DeserializationSchema; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.logical.RowType; + +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Deserialization schema that decodes AWS Glue Schema Registry encoded Protobuf records into Flink + * {@link RowData}. + * + *

Decoding is routed through the GSR deserialization facade (see {@link GsrProtobufReader}) so + * that: + * + *

    + *
  • the writer schema/version is resolved from Glue via the schema-version UUID in the + * record — the reader descriptor is built from that writer schema, not from the local {@code + * RowType} (review finding B1); + *
  • the GSR header and any compression are handled centrally in-library, rather than by a + * hand-rolled 18-byte strip that never decompresses (review finding C1). + *
+ * + *

The decoded {@link DynamicMessage} is mapped to {@link RowData} by field name; see + * {@link ProtobufToRowDataConverter}. + */ +@Internal +public class GsrProtobufRowDataDeserializationSchema implements DeserializationSchema { + + private static final long serialVersionUID = 1L; + + private final RowType rowType; + private final TypeInformation producedType; + private final String schemaName; + private final Map configs; + + private transient GsrProtobufReader reader; + + /** Reader descriptors cached by writer schema definition to avoid re-parsing per record. */ + private transient Map descriptorCache; + + /** + * Creates a new GSR Protobuf deserialization schema. + * + * @param rowType the Flink RowType describing the expected schema + * @param producedType the type information for the produced RowData + * @param schemaName the schema name (retained for diagnostics) + * @param configs the GSR SDK configuration map (region, registry, credentials, cache, + * compression, ...) used to build the deserialization facade + */ + public GsrProtobufRowDataDeserializationSchema( + RowType rowType, + TypeInformation producedType, + String schemaName, + Map configs) { + this.rowType = rowType; + this.producedType = producedType; + this.schemaName = schemaName; + this.configs = configs; + } + + @Override + public void open(InitializationContext context) throws Exception { + if (reader == null) { + reader = new FacadeGsrProtobufReader(configs); + } + if (descriptorCache == null) { + descriptorCache = new HashMap<>(); + } + } + + @Override + public RowData deserialize(byte[] message) throws IOException { + if (message == null) { + return null; + } + + // Resolve the writer schema from GSR and let the facade strip the header + decompress. + final String writerSchemaDefinition = reader.writerSchemaDefinition(message); + final byte[] protobufPayload = reader.actualData(message); + + final Descriptors.Descriptor writerDescriptor = + descriptorCache.computeIfAbsent( + writerSchemaDefinition, + ProtobufSchemaConverter::buildDescriptorFromProtoSchema); + + final DynamicMessage dynamicMessage = + DynamicMessage.parseFrom(writerDescriptor, protobufPayload); + return ProtobufToRowDataConverter.convertToRowData(dynamicMessage, rowType); + } + + @Override + public boolean isEndOfStream(RowData nextElement) { + return false; + } + + @Override + public TypeInformation getProducedType() { + return producedType; + } + + @VisibleForTesting + void setReader(GsrProtobufReader reader) { + this.reader = reader; + } + + @VisibleForTesting + String getSchemaName() { + return schemaName; + } +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufRowDataSerializationSchema.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufRowDataSerializationSchema.java new file mode 100644 index 00000000..6d9b2238 --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufRowDataSerializationSchema.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.connector.aws.util.AWSGeneralUtil; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.logical.RowType; + +import com.amazonaws.services.schemaregistry.common.Schema; +import com.amazonaws.services.schemaregistry.common.configs.GlueSchemaRegistryConfiguration; +import com.amazonaws.services.schemaregistry.serializers.GlueSchemaRegistrySerializationFacade; +import com.google.protobuf.DescriptorProtos; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.services.glue.model.DataFormat; + +import java.util.Map; + +/** + * Serialization schema that converts Flink {@link RowData} to Protobuf bytes and prepends AWS Glue + * Schema Registry header bytes. + * + *

The serialization flow is: + * + *

    + *
  1. Convert {@link RowData} to a Protobuf {@link DynamicMessage} using the schema derived from + * the Flink RowType + *
  2. Serialize the DynamicMessage to Protobuf bytes + *
  3. Register the Protobuf schema definition with GSR + *
  4. Prepend GSR header bytes (version + compression + schema UUID) to the Protobuf payload + *
+ */ +@Internal +public class GsrProtobufRowDataSerializationSchema implements SerializationSchema { + + private static final long serialVersionUID = 1L; + + private final RowType rowType; + private final String transportName; + private final String schemaName; + private final String protobufSchemaDefinition; + private final Map configs; + + private transient GlueSchemaRegistrySerializationFacade serializationFacade; + private transient Descriptors.Descriptor messageDescriptor; + + /** + * Creates a new GSR Protobuf serialization schema. + * + * @param rowType the Flink RowType describing the table schema + * @param transportName the transport name (topic/stream) for GSR schema naming + * @param schemaName the schema name for GSR registration + * @param configs the GSR SDK configuration map + */ + public GsrProtobufRowDataSerializationSchema( + RowType rowType, String transportName, String schemaName, Map configs) { + this.rowType = rowType; + this.transportName = transportName; + this.schemaName = schemaName != null ? schemaName : transportName; + this.configs = configs; + this.protobufSchemaDefinition = + ProtobufSchemaConverter.convertToProtobufSchema(rowType, this.schemaName); + } + + @Override + public void open(InitializationContext context) throws Exception { + if (serializationFacade == null) { + AwsCredentialsProvider credentialsProvider = + AWSGeneralUtil.getCredentialsProvider(configs); + serializationFacade = + GlueSchemaRegistrySerializationFacade.builder() + .credentialProvider(credentialsProvider) + .glueSchemaRegistryConfiguration( + new GlueSchemaRegistryConfiguration(configs)) + .build(); + } + if (messageDescriptor == null) { + messageDescriptor = buildDescriptor(); + } + } + + @Override + public byte[] serialize(RowData element) { + if (element == null) { + return null; + } + + DynamicMessage message = + RowDataToProtobufConverter.convertRowData(element, rowType, messageDescriptor); + byte[] protobufBytes = message.toByteArray(); + + return serializationFacade.encode( + transportName, + new Schema(protobufSchemaDefinition, DataFormat.PROTOBUF.name(), schemaName), + protobufBytes); + } + + @VisibleForTesting + void setSerializationFacade(GlueSchemaRegistrySerializationFacade facade) { + this.serializationFacade = facade; + } + + @VisibleForTesting + void setMessageDescriptor(Descriptors.Descriptor descriptor) { + this.messageDescriptor = descriptor; + } + + private Descriptors.Descriptor buildDescriptor() { + try { + DescriptorProtos.FileDescriptorProto fileProto = + ProtobufSchemaConverter.buildFileDescriptorProto(rowType, schemaName); + Descriptors.FileDescriptor fileDescriptor = + Descriptors.FileDescriptor.buildFrom( + fileProto, new Descriptors.FileDescriptor[] {}); + return fileDescriptor.findMessageTypeByName( + ProtobufSchemaConverter.sanitizeMessageName(schemaName)); + } catch (Descriptors.DescriptorValidationException e) { + throw new RuntimeException("Failed to build Protobuf descriptor from RowType", e); + } + } +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufGlueFormatOptions.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufGlueFormatOptions.java new file mode 100644 index 00000000..37a74933 --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufGlueFormatOptions.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.formats.avro.glue.schema.registry.GlueFormatOptions; + +/** + * Protobuf-specific configuration options for the AWS Glue Schema Registry Protobuf format factory. + * + *

Shared options (aws.region, registry.name, schema.name, etc.) are inherited from {@link + * GlueFormatOptions}. Currently no additional Protobuf-specific options are needed. + */ +@PublicEvolving +public class ProtobufGlueFormatOptions extends GlueFormatOptions { + + // No Protobuf-specific options needed at this time. + // All shared GSR options are inherited from GlueFormatOptions. +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufSchemaConverter.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufSchemaConverter.java new file mode 100644 index 00000000..6ca06588 --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufSchemaConverter.java @@ -0,0 +1,382 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.RowType; + +import com.google.protobuf.DescriptorProtos; +import com.google.protobuf.Descriptors; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Converts a Flink {@link RowType} to a Protobuf schema definition string and to a {@link + * DescriptorProtos.FileDescriptorProto} for runtime use. + * + *

Type mapping (proto3, scalar fields only): + * + *

    + *
  • {@code BOOLEAN} → {@code bool} + *
  • {@code TINYINT}/{@code SMALLINT}/{@code INTEGER} → {@code int32} + *
  • {@code DATE} → {@code int32} (epoch day); {@code TIME} → {@code int32} (millis of day) + *
  • {@code BIGINT} → {@code int64} + *
  • {@code TIMESTAMP}/{@code TIMESTAMP_LTZ} → {@code int64} (epoch millis) + *
  • {@code FLOAT} → {@code float}; {@code DOUBLE} → {@code double} + *
  • {@code CHAR}/{@code VARCHAR} → {@code string}; {@code DECIMAL} → {@code string} (lossless + * {@code BigDecimal} text form) + *
  • {@code BINARY}/{@code VARBINARY} → {@code bytes} + *
+ * + *

Complex/unsupported types (ARRAY, MAP, MULTISET, ROW, RAW, STRUCTURED, ...) are rejected + * fail-fast rather than silently coerced to {@code string} — see {@code unsupported()}. + * + *

Field identifiers are sanitized to valid proto names (see {@link #sanitizeFieldName}); the + * original SQL column name is preserved as the field's {@code json_name} so it survives a + * round-trip regardless of sanitization. + */ +@Internal +public class ProtobufSchemaConverter { + + private static final String PROTO_SYNTAX = "proto3"; + + /** Matches the {@code message {}} declaration in a proto3 writer schema. */ + private static final Pattern MESSAGE_PATTERN = + Pattern.compile("message\\s+([A-Za-z_][A-Za-z0-9_]*)\\s*\\{"); + + /** + * Matches a single proto3 scalar field line, e.g. {@code string name = 1;} or {@code int64 ts = + * 3 [json_name = "orig name"];}. Group 1 = optional label ({@code optional}/{@code repeated}), + * group 2 = proto type keyword, group 3 = field name, group 4 = field number, group 5 = {@code + * json_name} (the original SQL column name, when the identifier was sanitized). + */ + private static final Pattern FIELD_PATTERN = + Pattern.compile( + "(?:(optional|repeated)\\s+)?" + + "([A-Za-z_][A-Za-z0-9_.]*)\\s+" + + "([A-Za-z_][A-Za-z0-9_]*)\\s*=\\s*(\\d+)\\s*" + + "(?:\\[\\s*json_name\\s*=\\s*\"([^\"]*)\"\\s*\\])?\\s*;"); + + /** + * Converts a Flink RowType to a Protobuf schema definition string (proto3 syntax). + * + * @param rowType the Flink RowType + * @param messageName the name for the Protobuf message + * @return a Protobuf schema definition string + */ + public static String convertToProtobufSchema(RowType rowType, String messageName) { + String sanitized = sanitizeMessageName(messageName); + StringBuilder sb = new StringBuilder(); + sb.append("syntax = \"proto3\";\n\n"); + sb.append("message ").append(sanitized).append(" {\n"); + int fieldNumber = 1; + for (RowType.RowField field : rowType.getFields()) { + String protoType = toProtoType(field.getType()); + String fieldName = sanitizeFieldName(field.getName()); + sb.append(" "); + // proto3 explicit presence: a NULLABLE column is emitted with the `optional` + // keyword so the reader can distinguish an unset field (null) from a set type + // default (0/""/false); a NOT NULL column stays a plain implicit-presence field + // (review finding C2). + if (field.getType().isNullable()) { + sb.append("optional "); + } + sb.append(protoType).append(" ").append(fieldName); + sb.append(" = ").append(fieldNumber++); + if (!fieldName.equals(field.getName())) { + sb.append(" [json_name = \"").append(field.getName()).append("\"]"); + } + sb.append(";\n"); + } + sb.append("}\n"); + return sb.toString(); + } + + /** + * Builds a {@link DescriptorProtos.FileDescriptorProto} from a Flink RowType for runtime + * Protobuf serialization/deserialization. + * + * @param rowType the Flink RowType + * @param messageName the name for the Protobuf message + * @return a FileDescriptorProto + */ + public static DescriptorProtos.FileDescriptorProto buildFileDescriptorProto( + RowType rowType, String messageName) { + String sanitized = sanitizeMessageName(messageName); + DescriptorProtos.DescriptorProto.Builder messageBuilder = + DescriptorProtos.DescriptorProto.newBuilder().setName(sanitized); + + int fieldNumber = 1; + int syntheticOneofIndex = 0; + for (RowType.RowField field : rowType.getFields()) { + String fieldName = sanitizeFieldName(field.getName()); + DescriptorProtos.FieldDescriptorProto.Builder fieldBuilder = + DescriptorProtos.FieldDescriptorProto.newBuilder() + .setName(fieldName) + .setNumber(fieldNumber++) + .setType(toProtoFieldType(field.getType())) + .setLabel(DescriptorProtos.FieldDescriptorProto.Label.LABEL_OPTIONAL); + if (!fieldName.equals(field.getName())) { + fieldBuilder.setJsonName(field.getName()); + } + if (field.getType().isNullable()) { + // proto3 explicit presence: wrap the nullable scalar in a synthetic oneof + // (proto3 `optional`) so hasField() distinguishes an unset column (null) from + // a set type default at decode time (review finding C2). NOT NULL columns keep + // implicit presence. Synthetic oneofs are declared in field order, so each + // oneof_index matches the position at which the oneof decl is appended. + fieldBuilder.setProto3Optional(true); + fieldBuilder.setOneofIndex(syntheticOneofIndex++); + messageBuilder.addOneofDecl( + DescriptorProtos.OneofDescriptorProto.newBuilder() + .setName("_" + fieldName)); + } + messageBuilder.addField(fieldBuilder); + } + + return DescriptorProtos.FileDescriptorProto.newBuilder() + .setSyntax(PROTO_SYNTAX) + .addMessageType(messageBuilder) + .build(); + } + + /** + * Reconstructs a runtime {@link Descriptors.Descriptor} from a proto3 writer schema definition + * resolved from AWS Glue Schema Registry (review finding B1). The on-wire field numbers, names + * and {@code json_name}s therefore come from the writer schema registered in Glue, + * rather than being synthesized from the local {@code RowType}. + * + *

The grammar handled is exactly the one emitted by {@link #convertToProtobufSchema}: a + * single proto3 message of scalar fields, each optionally carrying a {@code json_name} option. + * + * @param protoSchemaDefinition the proto3 writer schema text + * @return the message descriptor described by that schema + */ + public static Descriptors.Descriptor buildDescriptorFromProtoSchema( + String protoSchemaDefinition) { + if (protoSchemaDefinition == null) { + throw new IllegalArgumentException( + "GSR returned a null Protobuf writer schema definition on the decode path."); + } + Matcher messageMatcher = MESSAGE_PATTERN.matcher(protoSchemaDefinition); + if (!messageMatcher.find()) { + throw new IllegalArgumentException( + "Could not locate a proto3 'message' declaration in the GSR writer schema:\n" + + protoSchemaDefinition); + } + String messageName = messageMatcher.group(1); + String body = protoSchemaDefinition.substring(messageMatcher.end()); + + DescriptorProtos.DescriptorProto.Builder messageBuilder = + DescriptorProtos.DescriptorProto.newBuilder().setName(messageName); + + Matcher fieldMatcher = FIELD_PATTERN.matcher(body); + int syntheticOneofIndex = 0; + while (fieldMatcher.find()) { + String label = fieldMatcher.group(1); + String protoTypeKeyword = fieldMatcher.group(2); + String fieldName = fieldMatcher.group(3); + int fieldNumber = Integer.parseInt(fieldMatcher.group(4)); + String jsonName = fieldMatcher.group(5); + + DescriptorProtos.FieldDescriptorProto.Builder fieldBuilder = + DescriptorProtos.FieldDescriptorProto.newBuilder() + .setName(fieldName) + .setNumber(fieldNumber) + .setType(protoKeywordToFieldType(protoTypeKeyword)) + .setLabel(DescriptorProtos.FieldDescriptorProto.Label.LABEL_OPTIONAL); + if (jsonName != null) { + fieldBuilder.setJsonName(jsonName); + } + if ("optional".equals(label)) { + // Rebuild proto3 explicit presence for the field the writer marked `optional` + // so hasField() works on the decode path (review finding C2). Synthetic oneofs + // are declared in field order to keep each oneof_index aligned. + fieldBuilder.setProto3Optional(true); + fieldBuilder.setOneofIndex(syntheticOneofIndex++); + messageBuilder.addOneofDecl( + DescriptorProtos.OneofDescriptorProto.newBuilder() + .setName("_" + fieldName)); + } + messageBuilder.addField(fieldBuilder); + } + + DescriptorProtos.FileDescriptorProto fileProto = + DescriptorProtos.FileDescriptorProto.newBuilder() + .setSyntax(PROTO_SYNTAX) + .addMessageType(messageBuilder) + .build(); + try { + return Descriptors.FileDescriptor.buildFrom( + fileProto, new Descriptors.FileDescriptor[] {}) + .findMessageTypeByName(messageName); + } catch (Descriptors.DescriptorValidationException e) { + throw new RuntimeException( + "Failed to build a Protobuf descriptor from the GSR writer schema", e); + } + } + + private static DescriptorProtos.FieldDescriptorProto.Type protoKeywordToFieldType( + String keyword) { + switch (keyword) { + case "bool": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_BOOL; + case "int32": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_INT32; + case "sint32": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_SINT32; + case "uint32": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_UINT32; + case "fixed32": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_FIXED32; + case "sfixed32": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_SFIXED32; + case "int64": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_INT64; + case "sint64": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_SINT64; + case "uint64": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_UINT64; + case "fixed64": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_FIXED64; + case "sfixed64": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_SFIXED64; + case "float": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_FLOAT; + case "double": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_DOUBLE; + case "string": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_STRING; + case "bytes": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_BYTES; + default: + throw new IllegalArgumentException( + "Unsupported Protobuf field type '" + + keyword + + "' in the GSR writer schema."); + } + } + + /** + * Sanitizes a schema name to be a valid Protobuf message name. Replaces non-alphanumeric + * characters with underscores and ensures it starts with a letter. + */ + static String sanitizeMessageName(String name) { + String sanitized = name.replaceAll("[^a-zA-Z0-9_]", "_"); + if (!sanitized.isEmpty() && Character.isDigit(sanitized.charAt(0))) { + sanitized = "M_" + sanitized; + } + if (sanitized.isEmpty()) { + sanitized = "Message"; + } + return sanitized; + } + + /** + * Sanitizes a Flink column name into a valid Protobuf field identifier. Protobuf field names + * must match {@code [a-zA-Z_][a-zA-Z0-9_]*}; a column with a space, hyphen or leading digit + * would otherwise trigger a {@code DescriptorValidationException} when the descriptor is built + * at {@code open()} time. + */ + static String sanitizeFieldName(String name) { + String sanitized = name.replaceAll("[^a-zA-Z0-9_]", "_"); + if (sanitized.isEmpty()) { + return "_field"; + } + char first = sanitized.charAt(0); + if (!Character.isLetter(first) && first != '_') { + sanitized = "_" + sanitized; + } + return sanitized; + } + + private static String toProtoType(LogicalType type) { + switch (type.getTypeRoot()) { + case BOOLEAN: + return "bool"; + case TINYINT: + case SMALLINT: + case INTEGER: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + return "int32"; + case BIGINT: + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return "int64"; + case FLOAT: + return "float"; + case DOUBLE: + return "double"; + case CHAR: + case VARCHAR: + case DECIMAL: + return "string"; + case BINARY: + case VARBINARY: + return "bytes"; + default: + throw unsupported(type); + } + } + + private static DescriptorProtos.FieldDescriptorProto.Type toProtoFieldType(LogicalType type) { + switch (type.getTypeRoot()) { + case BOOLEAN: + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_BOOL; + case TINYINT: + case SMALLINT: + case INTEGER: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_INT32; + case BIGINT: + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_INT64; + case FLOAT: + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_FLOAT; + case DOUBLE: + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_DOUBLE; + case CHAR: + case VARCHAR: + case DECIMAL: + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_STRING; + case BINARY: + case VARBINARY: + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_BYTES; + default: + throw unsupported(type); + } + } + + private static UnsupportedOperationException unsupported(LogicalType type) { + return new UnsupportedOperationException( + "The 'protobuf-glue' format does not support the Flink type '" + + type.asSummaryString() + + "'. Supported types are the scalar types (BOOLEAN, INT family, " + + "FLOAT/DOUBLE, DECIMAL, CHAR/VARCHAR, BINARY/VARBINARY, DATE, TIME, " + + "TIMESTAMP, TIMESTAMP_LTZ). Complex types (ARRAY, MAP, MULTISET, ROW, " + + "RAW) are not yet supported."); + } + + private ProtobufSchemaConverter() {} +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufToRowDataConverter.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufToRowDataConverter.java new file mode 100644 index 00000000..15aedfc9 --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufToRowDataConverter.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.data.TimestampData; +import org.apache.flink.table.types.logical.DecimalType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.RowType; + +import com.google.protobuf.ByteString; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; + +import java.math.BigDecimal; +import java.util.List; + +/** + * Converts Protobuf {@link DynamicMessage} to Flink {@link RowData} based on a Flink {@link + * RowType}. + * + *

Temporal and decimal columns are decoded from their wire encoding (epoch-millis {@code int64} + * for TIMESTAMP/TIMESTAMP_LTZ, {@code int32} for DATE/TIME, {@code string} for DECIMAL) — mirroring + * {@link RowDataToProtobufConverter} so the round-trip is symmetric (review finding B4). + */ +@Internal +public class ProtobufToRowDataConverter { + + /** + * Converts a Protobuf DynamicMessage to a Flink RowData. + * + *

Fields are mapped by name (not position): for each {@link RowType} column the + * matching field is resolved in the writer descriptor by its sanitized Protobuf name, falling + * back to the original SQL name preserved as {@code json_name}. This means a column reorder or + * a writer schema that carries extra/fewer fields no longer silently reinterprets on-wire tags + * (review finding B1). A column with no counterpart in the GSR-registered writer schema is + * treated as missing: {@code null} for a nullable column, a hard error for a {@code NOT NULL} + * column. + * + * @param message the Protobuf DynamicMessage decoded with the writer descriptor + * @param rowType the Flink RowType describing the expected schema + * @return a GenericRowData + */ + public static RowData convertToRowData(DynamicMessage message, RowType rowType) { + GenericRowData row = new GenericRowData(rowType.getFieldCount()); + Descriptors.Descriptor descriptor = message.getDescriptorForType(); + + List fields = rowType.getFields(); + for (int i = 0; i < fields.size(); i++) { + RowType.RowField rowField = fields.get(i); + LogicalType fieldType = rowField.getType(); + + Descriptors.FieldDescriptor fd = findFieldByName(descriptor, rowField.getName()); + if (fd == null) { + // The GSR-registered writer schema has no field for this column. + if (!fieldType.isNullable()) { + throw new IllegalStateException( + "Column '" + + rowField.getName() + + "' is declared NOT NULL but is absent from the " + + "GSR-registered writer schema '" + + descriptor.getName() + + "'. Cannot decode a required column that the writer never " + + "produced."); + } + row.setField(i, null); + continue; + } + + Object protoValue = readFieldValue(message, fd); + row.setField(i, convertProtoValue(protoValue, fieldType)); + } + return row; + } + + /** + * Reads a field value honoring proto3 explicit presence (review finding C2). For a + * presence-tracking field (a proto3 {@code optional} scalar, wrapped in a synthetic oneof by + * {@link ProtobufSchemaConverter}) that the writer left unset, this returns {@code null} so a + * nullable column round-trips as null rather than the type default (0/""/false). For an + * implicit-presence field (a NOT NULL column) it returns the value, which is the type default + * when unset. + */ + private static Object readFieldValue(DynamicMessage message, Descriptors.FieldDescriptor fd) { + if (fd.hasPresence() && !message.hasField(fd)) { + return null; + } + return message.getField(fd); + } + + /** + * Resolves the writer-descriptor field for a Flink column name. Tries the sanitized Protobuf + * field name first (the writer sanitizes identically), then the original SQL name preserved as + * {@code json_name}, then a verbatim name match. + */ + private static Descriptors.FieldDescriptor findFieldByName( + Descriptors.Descriptor descriptor, String columnName) { + Descriptors.FieldDescriptor fd = + descriptor.findFieldByName(ProtobufSchemaConverter.sanitizeFieldName(columnName)); + if (fd != null) { + return fd; + } + for (Descriptors.FieldDescriptor candidate : descriptor.getFields()) { + if (columnName.equals(candidate.getJsonName())) { + return candidate; + } + } + return descriptor.findFieldByName(columnName); + } + + private static Object convertProtoValue(Object protoValue, LogicalType type) { + // A null here means either the writer's schema had no field for this column, or the field + // is a proto3 explicit-presence (`optional`) field the writer left unset (finding C2); + // either way the column decodes to null. For NOT NULL columns (implicit presence) an unset + // field surfaces its type default here, which is the intended proto3 semantic. + if (protoValue == null) { + return null; + } + + switch (type.getTypeRoot()) { + case BOOLEAN: + return protoValue; + case TINYINT: + return ((Integer) protoValue).byteValue(); + case SMALLINT: + return ((Integer) protoValue).shortValue(); + case INTEGER: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + // int32 wire form; DATE = epoch day, TIME = millis-of-day (both int in RowData). + return protoValue; + case BIGINT: + return protoValue; + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return TimestampData.fromEpochMillis((Long) protoValue); + case FLOAT: + return protoValue; + case DOUBLE: + return protoValue; + case DECIMAL: + final DecimalType dt = (DecimalType) type; + return DecimalData.fromBigDecimal( + new BigDecimal(protoValue.toString()), dt.getPrecision(), dt.getScale()); + case CHAR: + case VARCHAR: + return StringData.fromString(protoValue.toString()); + case BINARY: + case VARBINARY: + if (protoValue instanceof ByteString) { + return ((ByteString) protoValue).toByteArray(); + } + return protoValue; + default: + throw new UnsupportedOperationException( + "The 'protobuf-glue' format cannot decode the Flink type '" + + type.asSummaryString() + + "'."); + } + } + + private ProtobufToRowDataConverter() {} +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/RowDataToProtobufConverter.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/RowDataToProtobufConverter.java new file mode 100644 index 00000000..e22b8af3 --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/RowDataToProtobufConverter.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.logical.DecimalType; +import org.apache.flink.table.types.logical.LocalZonedTimestampType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.TimestampType; + +import com.google.protobuf.ByteString; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; + +import java.util.List; + +/** + * Converts Flink {@link RowData} to Protobuf {@link DynamicMessage} based on a Protobuf {@link + * Descriptors.Descriptor}. + * + *

Temporal and decimal columns are encoded via their proper {@link RowData} accessors — never + * via {@code getString} — to avoid the {@code ClassCastException} that a naive {@code default:} + * branch produced (see review finding B4). TIMESTAMP/TIMESTAMP_LTZ → epoch-millis {@code int64}, + * DATE/TIME → {@code int32}, DECIMAL → its lossless {@code BigDecimal} text form. + */ +@Internal +public class RowDataToProtobufConverter { + + /** + * Converts a Flink RowData to a Protobuf DynamicMessage. + * + * @param rowData the Flink RowData + * @param rowType the Flink RowType describing the schema + * @param descriptor the Protobuf message descriptor + * @return a DynamicMessage + */ + public static DynamicMessage convertRowData( + RowData rowData, RowType rowType, Descriptors.Descriptor descriptor) { + DynamicMessage.Builder builder = DynamicMessage.newBuilder(descriptor); + List fields = descriptor.getFields(); + + for (int i = 0; i < rowType.getFieldCount(); i++) { + if (rowData.isNullAt(i)) { + // Leave the field unset. For a NULLABLE column this is a proto3 explicit-presence + // (`optional`) field, so an unset field is observably absent (hasField()==false) + // and decodes back to null; for a NOT NULL column the column is never null here. + // Explicit presence is emitted by ProtobufSchemaConverter (review finding C2). + continue; + } + LogicalType fieldType = rowType.getTypeAt(i); + Descriptors.FieldDescriptor fd = fields.get(i); + Object value = extractFieldValue(rowData, i, fieldType); + if (value != null) { + builder.setField(fd, value); + } + } + return builder.build(); + } + + private static Object extractFieldValue(RowData rowData, int index, LogicalType type) { + switch (type.getTypeRoot()) { + case BOOLEAN: + return rowData.getBoolean(index); + case TINYINT: + return (int) rowData.getByte(index); + case SMALLINT: + return (int) rowData.getShort(index); + case INTEGER: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + // DATE = epoch day, TIME = millis-of-day, both stored as int in RowData. + return rowData.getInt(index); + case BIGINT: + return rowData.getLong(index); + case TIMESTAMP_WITHOUT_TIME_ZONE: + return rowData.getTimestamp(index, ((TimestampType) type).getPrecision()) + .getMillisecond(); + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return rowData.getTimestamp(index, ((LocalZonedTimestampType) type).getPrecision()) + .getMillisecond(); + case FLOAT: + return rowData.getFloat(index); + case DOUBLE: + return rowData.getDouble(index); + case DECIMAL: + final DecimalType dt = (DecimalType) type; + return rowData.getDecimal(index, dt.getPrecision(), dt.getScale()) + .toBigDecimal() + .toString(); + case CHAR: + case VARCHAR: + return rowData.getString(index).toString(); + case BINARY: + case VARBINARY: + return ByteString.copyFrom(rowData.getBinary(index)); + default: + throw new UnsupportedOperationException( + "The 'protobuf-glue' format cannot encode the Flink type '" + + type.asSummaryString() + + "'."); + } + } + + private RowDataToProtobufConverter() {} +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory new file mode 100644 index 00000000..bfc7c06b --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.flink.formats.protobuf.glue.schema.registry.GlueSchemaRegistryProtobufFormatFactory diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/GlueSchemaRegistryProtobufFormatFactoryTest.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/GlueSchemaRegistryProtobufFormatFactoryTest.java new file mode 100644 index 00000000..665339b3 --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/GlueSchemaRegistryProtobufFormatFactoryTest.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.api.common.serialization.DeserializationSchema; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.catalog.Column; +import org.apache.flink.table.catalog.ResolvedSchema; +import org.apache.flink.table.connector.sink.DynamicTableSink; +import org.apache.flink.table.connector.source.DynamicTableSource; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.factories.TestDynamicTableFactory; +import org.apache.flink.table.runtime.connector.source.ScanRuntimeProviderContext; + +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; + +import static org.apache.flink.table.factories.utils.FactoryMocks.createTableSink; +import static org.apache.flink.table.factories.utils.FactoryMocks.createTableSource; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for the {@link GlueSchemaRegistryProtobufFormatFactory}. */ +class GlueSchemaRegistryProtobufFormatFactoryTest { + + private static final ResolvedSchema SCHEMA = + ResolvedSchema.of( + Column.physical("a", DataTypes.STRING()), + Column.physical("b", DataTypes.INT()), + Column.physical("c", DataTypes.BOOLEAN())); + + private static final String SCHEMA_NAME = "test-subject"; + private static final String REGISTRY_NAME = "test-registry-name"; + private static final String REGION = "us-west-2"; + + @Test + void testSpiDiscovery() { + final DynamicTableSource source = createTableSource(SCHEMA, getDefaultOptions()); + assertThat(source).isNotNull(); + + final DynamicTableSink sink = createTableSink(SCHEMA, getDefaultOptions()); + assertThat(sink).isNotNull(); + } + + @Test + void testDeserializationSchema() { + final DynamicTableSource actualSource = createTableSource(SCHEMA, getDefaultOptions()); + assertThat(actualSource).isInstanceOf(TestDynamicTableFactory.DynamicTableSourceMock.class); + + TestDynamicTableFactory.DynamicTableSourceMock scanSourceMock = + (TestDynamicTableFactory.DynamicTableSourceMock) actualSource; + + DeserializationSchema actualDeser = + scanSourceMock.valueFormat.createRuntimeDecoder( + ScanRuntimeProviderContext.INSTANCE, SCHEMA.toPhysicalRowDataType()); + + assertThat(actualDeser).isInstanceOf(GsrProtobufRowDataDeserializationSchema.class); + } + + @Test + void testSerializationSchema() { + final DynamicTableSink actualSink = createTableSink(SCHEMA, getDefaultOptions()); + assertThat(actualSink).isInstanceOf(TestDynamicTableFactory.DynamicTableSinkMock.class); + + TestDynamicTableFactory.DynamicTableSinkMock sinkMock = + (TestDynamicTableFactory.DynamicTableSinkMock) actualSink; + + SerializationSchema actualSer = + sinkMock.valueFormat.createRuntimeEncoder(null, SCHEMA.toPhysicalRowDataType()); + + assertThat(actualSer).isInstanceOf(GsrProtobufRowDataSerializationSchema.class); + } + + @Test + void testMissingSchemaNameForSink() { + final Map options = + getModifiedOptions(opts -> opts.remove("protobuf-glue.schema.name")); + + assertThatThrownBy(() -> createTableSink(SCHEMA, options)) + .isInstanceOf(ValidationException.class); + } + + @Test + void testMissingRegionForSource() { + final Map options = + getModifiedOptions(opts -> opts.remove("protobuf-glue.aws.region")); + + assertThatThrownBy(() -> createTableSource(SCHEMA, options)) + .isInstanceOf(ValidationException.class); + } + + @Test + void testMissingRegistryNameForSource() { + final Map options = + getModifiedOptions(opts -> opts.remove("protobuf-glue.registry.name")); + + assertThatThrownBy(() -> createTableSource(SCHEMA, options)) + .isInstanceOf(ValidationException.class); + } + + // ------------------------------------------------------------------------ + // Utilities + // ------------------------------------------------------------------------ + + private Map getModifiedOptions(Consumer> optionModifier) { + Map options = getDefaultOptions(); + optionModifier.accept(options); + return options; + } + + private Map getDefaultOptions() { + final Map options = new HashMap<>(); + options.put("connector", TestDynamicTableFactory.IDENTIFIER); + options.put("target", "MyTarget"); + options.put("buffer-size", "1000"); + + options.put("format", GlueSchemaRegistryProtobufFormatFactory.IDENTIFIER); + options.put("protobuf-glue.schema.name", SCHEMA_NAME); + options.put("protobuf-glue.registry.name", REGISTRY_NAME); + options.put("protobuf-glue.aws.region", REGION); + return options; + } +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufRoundTripPropertyTest.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufRoundTripPropertyTest.java new file mode 100644 index 00000000..9c130b90 --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufRoundTripPropertyTest.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.types.logical.BooleanType; +import org.apache.flink.table.types.logical.DoubleType; +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.VarCharType; + +import com.google.protobuf.DescriptorProtos; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import net.jqwik.api.Arbitraries; +import net.jqwik.api.Arbitrary; +import net.jqwik.api.ForAll; +import net.jqwik.api.Property; +import net.jqwik.api.Provide; +import net.jqwik.api.Tag; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Property-based tests for Protobuf serialization round-trip with GSR header handling. + * + *

Property 5: Protobuf serialization round-trip + * + *

Validates: Requirements 4.4, 4.5, 8.1 + */ +@Tag("Feature: gsr-flink-sql-formats, Property 5: Protobuf serialization round-trip") +class GsrProtobufRoundTripPropertyTest { + + private static final int GSR_HEADER_SIZE = 18; + private static final String SCHEMA_NAME = "TestMessage"; + + /** + * For any valid RowData matching a given RowType, serializing to Protobuf bytes via + * DynamicMessage, prepending a mock GSR header, then stripping the header and deserializing + * should produce equivalent RowData. + */ + @Property(tries = 100) + void protobufRoundTripPreservesData(@ForAll("rowDataWithType") RowDataWithType input) + throws Exception { + RowType rowType = input.rowType; + RowData original = input.rowData; + + // Build Protobuf descriptor from RowType + Descriptors.Descriptor descriptor = buildDescriptor(rowType); + + // Serialize: RowData -> DynamicMessage -> Protobuf bytes + DynamicMessage message = + RowDataToProtobufConverter.convertRowData(original, rowType, descriptor); + byte[] protobufBytes = message.toByteArray(); + + // Simulate GSR encoding: prepend 18-byte mock header + byte[] gsrEncoded = prependMockGsrHeader(protobufBytes); + + // Simulate GSR decoding: strip 18-byte header + byte[] stripped = Arrays.copyOfRange(gsrEncoded, GSR_HEADER_SIZE, gsrEncoded.length); + + // Deserialize: Protobuf bytes -> DynamicMessage -> RowData + DynamicMessage deserialized = DynamicMessage.parseFrom(descriptor, stripped); + RowData result = ProtobufToRowDataConverter.convertToRowData(deserialized, rowType); + + // Verify equivalence field by field + assertThat(result).isNotNull(); + assertRowDataEquals(original, result, rowType); + } + + private Descriptors.Descriptor buildDescriptor(RowType rowType) { + try { + DescriptorProtos.FileDescriptorProto fileProto = + ProtobufSchemaConverter.buildFileDescriptorProto(rowType, SCHEMA_NAME); + Descriptors.FileDescriptor fileDescriptor = + Descriptors.FileDescriptor.buildFrom( + fileProto, new Descriptors.FileDescriptor[] {}); + return fileDescriptor.findMessageTypeByName(SCHEMA_NAME); + } catch (Descriptors.DescriptorValidationException e) { + throw new RuntimeException(e); + } + } + + /** Creates a mock 18-byte GSR header and prepends it to the payload. */ + private byte[] prependMockGsrHeader(byte[] payload) { + UUID schemaId = UUID.randomUUID(); + ByteBuffer buffer = ByteBuffer.allocate(GSR_HEADER_SIZE + payload.length); + buffer.put((byte) 0x03); // header version + buffer.put((byte) 0x00); // no compression + buffer.putLong(schemaId.getMostSignificantBits()); + buffer.putLong(schemaId.getLeastSignificantBits()); + buffer.put(payload); + return buffer.array(); + } + + /** Compares two RowData instances field by field based on the RowType. */ + private void assertRowDataEquals(RowData expected, RowData actual, RowType rowType) { + assertThat(actual.getArity()).isEqualTo(expected.getArity()); + for (int i = 0; i < rowType.getFieldCount(); i++) { + LogicalType fieldType = rowType.getTypeAt(i); + // Note: proto3 does not distinguish between null and default values. + // A null string in RowData becomes "" in proto3, null int becomes 0, etc. + // We handle this by comparing against proto3 default semantics. + if (expected.isNullAt(i)) { + // proto3 defaults: string -> "", int -> 0, bool -> false, double -> 0.0 + assertProto3Default(actual, i, fieldType); + continue; + } + if (fieldType instanceof VarCharType) { + assertThat(actual.getString(i).toString()) + .isEqualTo(expected.getString(i).toString()); + } else if (fieldType instanceof IntType) { + assertThat(actual.getInt(i)).isEqualTo(expected.getInt(i)); + } else if (fieldType instanceof BooleanType) { + assertThat(actual.getBoolean(i)).isEqualTo(expected.getBoolean(i)); + } else if (fieldType instanceof DoubleType) { + assertThat(actual.getDouble(i)).isEqualTo(expected.getDouble(i)); + } + } + } + + /** Asserts that the actual value matches the proto3 default for the given type. */ + private void assertProto3Default(RowData actual, int index, LogicalType fieldType) { + if (fieldType instanceof VarCharType) { + assertThat(actual.getString(index).toString()).isEqualTo(""); + } else if (fieldType instanceof IntType) { + assertThat(actual.getInt(index)).isEqualTo(0); + } else if (fieldType instanceof BooleanType) { + assertThat(actual.getBoolean(index)).isFalse(); + } else if (fieldType instanceof DoubleType) { + assertThat(actual.getDouble(index)).isEqualTo(0.0); + } + } + + // --- Generators --- + + @Provide + Arbitrary rowDataWithType() { + // Fixed schema with STRING, INT, BOOLEAN, DOUBLE fields + RowType rowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField( + "name", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("age", new IntType()), + new RowType.RowField("active", new BooleanType()), + new RowType.RowField("score", new DoubleType()))); + + return Arbitraries.of(rowType) + .flatMap(rt -> generateRowData(rt).map(rd -> new RowDataWithType(rt, rd))); + } + + private Arbitrary generateRowData(RowType rowType) { + // Generate non-null values only since proto3 doesn't distinguish null from default + Arbitrary strings = Arbitraries.strings().alpha().ofMinLength(1).ofMaxLength(50); + Arbitrary ints = Arbitraries.integers().between(-10000, 10000); + Arbitrary bools = Arbitraries.of(true, false); + Arbitrary doubles = Arbitraries.doubles().between(-1e6, 1e6).ofScale(4); + + return strings.flatMap( + name -> + ints.flatMap( + age -> + bools.flatMap( + active -> + doubles.map( + score -> { + GenericRowData row = + new GenericRowData(4); + row.setField( + 0, + StringData.fromString( + name)); + row.setField(1, age); + row.setField(2, active); + row.setField(3, score); + return (RowData) row; + })))); + } + + /** Holder for a RowData and its corresponding RowType. */ + static class RowDataWithType { + final RowType rowType; + final RowData rowData; + + RowDataWithType(RowType rowType, RowData rowData) { + this.rowType = rowType; + this.rowData = rowData; + } + } +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufC1CompressionRoundTripTest.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufC1CompressionRoundTripTest.java new file mode 100644 index 00000000..49901143 --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufC1CompressionRoundTripTest.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.runtime.typeutils.InternalTypeInfo; +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.VarCharType; + +import com.google.protobuf.DescriptorProtos; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.HashMap; +import java.util.UUID; +import java.util.zip.Deflater; +import java.util.zip.DeflaterOutputStream; +import java.util.zip.Inflater; +import java.util.zip.InflaterInputStream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Round-trip regression test for review finding C1: compression asymmetry — the encode path + * honours {@code schema.compression} (ZLIB) while the pre-fix decode path stripped a fixed 18-byte + * header and never decompressed, so any non-{@code NONE} compression produced unparseable + * payloads on read. + * + *

The fix (shared with B1) routes decode through the GSR deserialization facade: {@link + * GsrProtobufRowDataDeserializationSchema#deserialize} obtains the payload solely via {@link + * GsrProtobufReader#actualData}, and the production {@link FacadeGsrProtobufReader} delegates that + * to {@code GlueSchemaRegistryDeserializationFacade.getActualData}, which strips the header and + * decompresses in-library. The schema itself performs no explicit header strip and no explicit + * decompression. + * + *

These tests stand in for the facade with a {@link CompressionAwareGsrReader} fake that models + * exactly that contract: it reads the GSR header's compression byte and ZLIB-inflates the payload + * when set (mirroring the SDK), so a record compressed on write is transparently read back. Because + * the deserialization schema delegates all payload extraction to the reader, a green round-trip + * here proves the read path is no longer missing decompression — the very asymmetry C1 flagged. + */ +class ProtobufC1CompressionRoundTripTest { + + private static final String SCHEMA_NAME = "TestMessage"; + private static final int GSR_HEADER_SIZE = 18; + + /** GSR header version byte. */ + private static final byte HEADER_VERSION = (byte) 0x03; + /** GSR compression byte: no compression. */ + private static final byte COMPRESSION_NONE = (byte) 0x00; + /** GSR compression byte: ZLIB (matches the SDK's ZLIB compression byte). */ + private static final byte COMPRESSION_ZLIB = (byte) 0x05; + + private static RowType rowType() { + return new RowType( + false, + Arrays.asList( + new RowType.RowField("name", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("age", new IntType()))); + } + + /** + * C1 core proof: a record whose payload is ZLIB-compressed on the wire (compression byte set in + * the header) is decoded successfully. This only passes because decode routes through the + * facade ({@link GsrProtobufReader#actualData}) which decompresses in-library; the pre-fix path + * that merely stripped 18 bytes would hand raw deflate bytes to the Protobuf parser and fail. + */ + @Test + void testZlibCompressedRecordRoundTrips() throws Exception { + RowType rowType = rowType(); + + GenericRowData original = new GenericRowData(2); + original.setField(0, StringData.fromString("Alice")); + original.setField(1, 30); + + byte[] gsrEncoded = encodeWithCompression(original, rowType, COMPRESSION_ZLIB); + + RowData decoded = decode(rowType, gsrEncoded); + + assertThat(decoded).isNotNull(); + assertThat(decoded.getString(0).toString()).isEqualTo("Alice"); + assertThat(decoded.getInt(1)).isEqualTo(30); + } + + /** + * Control: the identical payload with compression disabled (compression byte {@code 0x00}) also + * round-trips through the same facade-routed path, confirming the ZLIB result above is not an + * artefact of the fake but the compression byte genuinely drives decompression. + */ + @Test + void testUncompressedRecordRoundTrips() throws Exception { + RowType rowType = rowType(); + + GenericRowData original = new GenericRowData(2); + original.setField(0, StringData.fromString("Alice")); + original.setField(1, 30); + + byte[] gsrEncoded = encodeWithCompression(original, rowType, COMPRESSION_NONE); + + RowData decoded = decode(rowType, gsrEncoded); + + assertThat(decoded).isNotNull(); + assertThat(decoded.getString(0).toString()).isEqualTo("Alice"); + assertThat(decoded.getInt(1)).isEqualTo(30); + } + + /** + * C1 with multiple records over the same schema, exercising the reader's per-record + * decompression on a compressible payload large enough that deflate actually shrinks it (guards + * against a fake that silently no-ops compression). + */ + @Test + void testZlibCompressedMultipleRecordsRoundTrip() throws Exception { + RowType rowType = rowType(); + + GsrProtobufRowDataDeserializationSchema deser = + new GsrProtobufRowDataDeserializationSchema( + rowType, InternalTypeInfo.of(rowType), SCHEMA_NAME, new HashMap<>()); + deser.setReader(new CompressionAwareGsrReader(rowType, SCHEMA_NAME)); + deser.open(null); + + String[] names = { + "Alice", "Bob", "Charlie", "a-name-repeated-repeated-repeated-repeated-repeated" + }; + int[] ages = {30, 41, 52, 63}; + + for (int i = 0; i < names.length; i++) { + GenericRowData row = new GenericRowData(2); + row.setField(0, StringData.fromString(names[i])); + row.setField(1, ages[i]); + + byte[] gsrEncoded = encodeWithCompression(row, rowType, COMPRESSION_ZLIB); + RowData decoded = deser.deserialize(gsrEncoded); + + assertThat(decoded).isNotNull(); + assertThat(decoded.getString(0).toString()).isEqualTo(names[i]); + assertThat(decoded.getInt(1)).isEqualTo(ages[i]); + } + } + + /** Runs the decode path with the compression-aware fake facade reader. */ + private static RowData decode(RowType rowType, byte[] gsrEncoded) throws Exception { + GsrProtobufRowDataDeserializationSchema deser = + new GsrProtobufRowDataDeserializationSchema( + rowType, InternalTypeInfo.of(rowType), SCHEMA_NAME, new HashMap<>()); + deser.setReader(new CompressionAwareGsrReader(rowType, SCHEMA_NAME)); + deser.open(null); + return deser.deserialize(gsrEncoded); + } + + /** + * Builds a GSR-encoded record: 18-byte header (version + compression byte + 16-byte UUID) + * followed by the Protobuf payload, ZLIB-compressed when {@code compressionByte} is non-zero — + * exactly what the serialization facade emits when {@code schema.compression=ZLIB}. + */ + private static byte[] encodeWithCompression(RowData row, RowType rowType, byte compressionByte) + throws Exception { + DynamicMessage message = + RowDataToProtobufConverter.convertRowData(row, rowType, buildDescriptor(rowType)); + byte[] protobufBytes = message.toByteArray(); + byte[] body = compressionByte == COMPRESSION_NONE ? protobufBytes : zlib(protobufBytes); + + UUID schemaId = UUID.randomUUID(); + ByteBuffer buffer = ByteBuffer.allocate(GSR_HEADER_SIZE + body.length); + buffer.put(HEADER_VERSION); + buffer.put(compressionByte); + buffer.putLong(schemaId.getMostSignificantBits()); + buffer.putLong(schemaId.getLeastSignificantBits()); + buffer.put(body); + return buffer.array(); + } + + private static Descriptors.Descriptor buildDescriptor(RowType rowType) { + try { + DescriptorProtos.FileDescriptorProto fileProto = + ProtobufSchemaConverter.buildFileDescriptorProto(rowType, SCHEMA_NAME); + Descriptors.FileDescriptor fileDescriptor = + Descriptors.FileDescriptor.buildFrom( + fileProto, new Descriptors.FileDescriptor[] {}); + return fileDescriptor.findMessageTypeByName(SCHEMA_NAME); + } catch (Descriptors.DescriptorValidationException e) { + throw new RuntimeException(e); + } + } + + private static byte[] zlib(byte[] data) throws Exception { + Deflater deflater = new Deflater(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (DeflaterOutputStream dos = new DeflaterOutputStream(out, deflater)) { + dos.write(data); + } + deflater.end(); + return out.toByteArray(); + } + + private static byte[] inflate(byte[] data) throws Exception { + Inflater inflater = new Inflater(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (InflaterInputStream iis = + new InflaterInputStream(new java.io.ByteArrayInputStream(data), inflater)) { + byte[] buf = new byte[1024]; + int n; + while ((n = iis.read(buf)) != -1) { + out.write(buf, 0, n); + } + } + inflater.end(); + return out.toByteArray(); + } + + /** + * Fake {@link GsrProtobufReader} that models the GSR deserialization facade's compression + * contract: it resolves the writer schema from the RowType and, for {@link + * #actualData(byte[])}, reads the header's compression byte and ZLIB-inflates the body when set + * — i.e. decompression happens in the facade layer, exactly as the production {@link + * FacadeGsrProtobufReader} delegates to the SDK. The deserialization schema under test performs + * no decompression of its own; it only parses whatever {@code actualData} returns. + */ + private static final class CompressionAwareGsrReader implements GsrProtobufReader { + private static final long serialVersionUID = 1L; + private final String writerSchemaDefinition; + + CompressionAwareGsrReader(RowType rowType, String schemaName) { + this.writerSchemaDefinition = + ProtobufSchemaConverter.convertToProtobufSchema(rowType, schemaName); + } + + @Override + public String writerSchemaDefinition(byte[] gsrEncoded) { + return writerSchemaDefinition; + } + + @Override + public byte[] actualData(byte[] gsrEncoded) { + byte compressionByte = gsrEncoded[1]; + byte[] body = Arrays.copyOfRange(gsrEncoded, GSR_HEADER_SIZE, gsrEncoded.length); + if (compressionByte == COMPRESSION_NONE) { + return body; + } + try { + return inflate(body); + } catch (Exception e) { + throw new RuntimeException("Failed to decompress GSR payload", e); + } + } + } +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufC2ExplicitPresenceRoundTripTest.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufC2ExplicitPresenceRoundTripTest.java new file mode 100644 index 00000000..4b456293 --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufC2ExplicitPresenceRoundTripTest.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.runtime.typeutils.InternalTypeInfo; +import org.apache.flink.table.types.logical.BooleanType; +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.VarCharType; + +import com.google.protobuf.DescriptorProtos; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.HashMap; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Round-trip regression tests for review finding C2: proto3 explicit presence for + * NULLABLE columns. + * + *

Before the fix the generated schema used implicit-presence (plain proto3) scalars for every + * column, so the reader could not tell an unset field from a field set to its type default. A + * nullable column holding {@code 0}, {@code ""} or {@code false} was therefore indistinguishable + * from a null column on the wire, and vice-versa. {@link ProtobufSchemaConverter} now emits + * nullable scalars as proto3 {@code optional} (synthetic-oneof) fields and {@link + * ProtobufToRowDataConverter} consults {@code hasField()}, so: + * + *

    + *
  • a {@code null} value round-trips as {@code null} (field left unset), and + *
  • a set type-default value ({@code 0} / {@code ""} / {@code false}) round-trips as that + * value — it is NOT collapsed to {@code null}. + *
+ * + *

The round-trip runs RowData → {@link RowDataToProtobufConverter} (write) → {@link + * GsrProtobufRowDataDeserializationSchema} (read, via a fake GSR reader that resolves the writer + * schema and returns the raw payload) → RowData, so it exercises the same schema/converter code the + * production decode path uses without a live registry. + */ +class ProtobufC2ExplicitPresenceRoundTripTest { + + private static final String SCHEMA_NAME = "TestMessage"; + + /** All three columns are NULLABLE: an int, a string and a boolean. */ + private static RowType nullableRowType() { + return new RowType( + false, + Arrays.asList( + new RowType.RowField("i", new IntType(true)), + new RowType.RowField("s", new VarCharType(true, VarCharType.MAX_LENGTH)), + new RowType.RowField("b", new BooleanType(true)))); + } + + /** + * C2 null side: a NULLABLE column that is {@code null} at write time is emitted as an unset + * proto3 explicit-presence field and decodes back to {@code null} — not to the type default. + */ + @Test + void testNullableColumnsNullRoundTripAsNull() throws Exception { + RowType rowType = nullableRowType(); + + GenericRowData original = new GenericRowData(3); + original.setField(0, null); + original.setField(1, null); + original.setField(2, null); + + RowData decoded = roundTrip(original, rowType); + + assertThat(decoded).isNotNull(); + assertThat(decoded.isNullAt(0)).isTrue(); + assertThat(decoded.isNullAt(1)).isTrue(); + assertThat(decoded.isNullAt(2)).isTrue(); + } + + /** + * C2 value side (the crux of the finding): a NULLABLE column set to its type default — + * {@code 0} for int32, {@code ""} for string, {@code false} for bool — round-trips as that + * value, NOT collapsed to {@code null}. This is only possible with explicit presence. + */ + @Test + void testNullableColumnsTypeDefaultValuesRoundTripAsValues() throws Exception { + RowType rowType = nullableRowType(); + + GenericRowData original = new GenericRowData(3); + original.setField(0, 0); + original.setField(1, StringData.fromString("")); + original.setField(2, false); + + RowData decoded = roundTrip(original, rowType); + + assertThat(decoded).isNotNull(); + // The distinguishing assertions: values are present, not null. + assertThat(decoded.isNullAt(0)).isFalse(); + assertThat(decoded.getInt(0)).isEqualTo(0); + assertThat(decoded.isNullAt(1)).isFalse(); + assertThat(decoded.getString(1).toString()).isEqualTo(""); + assertThat(decoded.isNullAt(2)).isFalse(); + assertThat(decoded.getBoolean(2)).isFalse(); + } + + /** + * C2 mixed row: null and set-default values coexist in the same record and each retains its own + * presence — proving presence is tracked per field, not row-wide. + */ + @Test + void testMixedNullAndTypeDefaultValuesRoundTrip() throws Exception { + RowType rowType = nullableRowType(); + + GenericRowData original = new GenericRowData(3); + original.setField(0, 0); // set default -> stays 0 + original.setField(1, null); // null -> stays null + original.setField(2, false); // set default -> stays false + + RowData decoded = roundTrip(original, rowType); + + assertThat(decoded).isNotNull(); + assertThat(decoded.isNullAt(0)).isFalse(); + assertThat(decoded.getInt(0)).isEqualTo(0); + assertThat(decoded.isNullAt(1)).isTrue(); + assertThat(decoded.isNullAt(2)).isFalse(); + assertThat(decoded.getBoolean(2)).isFalse(); + } + + /** + * Sanity check that non-default set values also survive, so the value-side assertions above are + * not vacuously passing on a decode path that ignores the payload. + */ + @Test + void testNullableColumnsNonDefaultValuesRoundTrip() throws Exception { + RowType rowType = nullableRowType(); + + GenericRowData original = new GenericRowData(3); + original.setField(0, 42); + original.setField(1, StringData.fromString("hello")); + original.setField(2, true); + + RowData decoded = roundTrip(original, rowType); + + assertThat(decoded).isNotNull(); + assertThat(decoded.getInt(0)).isEqualTo(42); + assertThat(decoded.getString(1).toString()).isEqualTo("hello"); + assertThat(decoded.getBoolean(2)).isTrue(); + } + + /** + * RowData → Protobuf bytes (descriptor built from {@code rowType}) → {@link + * GsrProtobufRowDataDeserializationSchema} (fed the writer schema derived from {@code rowType} + * and the raw payload via a fake reader) → RowData. + */ + private static RowData roundTrip(RowData original, RowType rowType) throws Exception { + DynamicMessage message = + RowDataToProtobufConverter.convertRowData( + original, rowType, buildDescriptor(rowType)); + byte[] payload = message.toByteArray(); + String writerSchema = ProtobufSchemaConverter.convertToProtobufSchema(rowType, SCHEMA_NAME); + + GsrProtobufRowDataDeserializationSchema deser = + new GsrProtobufRowDataDeserializationSchema( + rowType, InternalTypeInfo.of(rowType), SCHEMA_NAME, new HashMap<>()); + deser.setReader(new StaticGsrReader(writerSchema, payload)); + deser.open(null); + return deser.deserialize(payload); + } + + private static Descriptors.Descriptor buildDescriptor(RowType rowType) { + try { + DescriptorProtos.FileDescriptorProto fileProto = + ProtobufSchemaConverter.buildFileDescriptorProto(rowType, SCHEMA_NAME); + Descriptors.FileDescriptor fileDescriptor = + Descriptors.FileDescriptor.buildFrom( + fileProto, new Descriptors.FileDescriptor[] {}); + return fileDescriptor.findMessageTypeByName(SCHEMA_NAME); + } catch (Descriptors.DescriptorValidationException e) { + throw new RuntimeException(e); + } + } + + /** + * Fake {@link GsrProtobufReader} standing in for the GSR facade: returns a fixed writer schema + * definition and the raw Protobuf payload (no header/compression), so decode is driven by the + * writer schema exactly as in production. + */ + private static final class StaticGsrReader implements GsrProtobufReader { + private static final long serialVersionUID = 1L; + private final String writerSchemaDefinition; + private final byte[] payload; + + StaticGsrReader(String writerSchemaDefinition, byte[] payload) { + this.writerSchemaDefinition = writerSchemaDefinition; + this.payload = payload; + } + + @Override + public String writerSchemaDefinition(byte[] gsrEncoded) { + return writerSchemaDefinition; + } + + @Override + public byte[] actualData(byte[] gsrEncoded) { + return payload; + } + } +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufGsrWriterSchemaDecodeTest.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufGsrWriterSchemaDecodeTest.java new file mode 100644 index 00000000..0cf08bd7 --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufGsrWriterSchemaDecodeTest.java @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.runtime.typeutils.InternalTypeInfo; +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.VarCharType; + +import com.google.protobuf.DescriptorProtos; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.HashMap; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * Regression tests for review finding B1: Protobuf decode must resolve the writer + * schema registered in AWS Glue Schema Registry and parse the on-wire bytes with a descriptor built + * from that writer schema, mapping fields to the local {@code RowType} by name — rather than + * (as the pre-fix code did) reinterpreting the bytes against a descriptor synthesized from the + * local table DDL, which silently mis-maps whenever the writer's field order / tags differ from the + * local columns. + * + *

Each test drives {@link GsrProtobufRowDataDeserializationSchema} through a hand-written {@link + * GsrProtobufReader} fake that returns a writer schema definition and payload independent of the + * local {@code RowType}, so no live registry is required. + */ +class ProtobufGsrWriterSchemaDecodeTest { + + private static final String SCHEMA_NAME = "TestMessage"; + + /** + * B1 core proof: the writer schema declares {@code name} (tag 1, string) then {@code age} (tag + * 2, int32); the local table DDL declares the columns in the opposite order ({@code age} + * then {@code name}). If decode built its descriptor from the local {@code RowType} it would + * try to parse tag 1 as an {@code int32} (age) and tag 2 as a {@code string} (name) and either + * throw or produce garbage. Because decode uses the GSR-registered writer descriptor and maps + * by name, both columns decode correctly regardless of local column order. + */ + @Test + void testDecodeUsesWriterSchemaAndMapsByName() throws Exception { + // Writer schema as registered in GSR: name (tag 1), age (tag 2). + RowType writerRowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField( + "name", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("age", new IntType()))); + + GenericRowData writerRow = new GenericRowData(2); + writerRow.setField(0, StringData.fromString("Alice")); + writerRow.setField(1, 30); + byte[] payload = encode(writerRow, writerRowType); + String writerSchema = + ProtobufSchemaConverter.convertToProtobufSchema(writerRowType, SCHEMA_NAME); + + // Local table DDL: columns declared in the OPPOSITE order. + RowType localRowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField("age", new IntType()), + new RowType.RowField( + "name", new VarCharType(VarCharType.MAX_LENGTH)))); + + RowData decoded = decode(localRowType, writerSchema, payload); + + assertThat(decoded).isNotNull(); + // Mapped by name against the writer schema, not by local position/tag. + assertThat(decoded.getInt(0)).isEqualTo(30); + assertThat(decoded.getString(1).toString()).isEqualTo("Alice"); + } + + /** + * B1: a NULLABLE local column that is absent from the GSR-registered writer schema decodes to + * {@code null} (the writer never produced it), rather than reading a bogus on-wire tag. + */ + @Test + void testNullableColumnMissingFromWriterSchemaDecodesToNull() throws Exception { + RowType writerRowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField( + "name", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("age", new IntType()))); + + GenericRowData writerRow = new GenericRowData(2); + writerRow.setField(0, StringData.fromString("Bob")); + writerRow.setField(1, 41); + byte[] payload = encode(writerRow, writerRowType); + String writerSchema = + ProtobufSchemaConverter.convertToProtobufSchema(writerRowType, SCHEMA_NAME); + + // Local DDL adds a nullable 'nickname' column the writer schema does not carry. + RowType localRowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField( + "name", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("age", new IntType()), + new RowType.RowField( + "nickname", + new VarCharType(true, VarCharType.MAX_LENGTH)))); + + RowData decoded = decode(localRowType, writerSchema, payload); + + assertThat(decoded).isNotNull(); + assertThat(decoded.getString(0).toString()).isEqualTo("Bob"); + assertThat(decoded.getInt(1)).isEqualTo(41); + assertThat(decoded.isNullAt(2)).isTrue(); + } + + /** + * B1: a NOT NULL local column that is absent from the GSR-registered writer schema fails fast + * with a clear error, instead of silently decoding a wrong or default value. + */ + @Test + void testNotNullColumnMissingFromWriterSchemaThrowsClearError() throws Exception { + RowType writerRowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField( + "name", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("age", new IntType()))); + + GenericRowData writerRow = new GenericRowData(2); + writerRow.setField(0, StringData.fromString("Carol")); + writerRow.setField(1, 52); + byte[] payload = encode(writerRow, writerRowType); + String writerSchema = + ProtobufSchemaConverter.convertToProtobufSchema(writerRowType, SCHEMA_NAME); + + // Local DDL requires a NOT NULL 'email' column the writer schema does not carry. + RowType localRowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField( + "name", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("age", new IntType()), + new RowType.RowField( + "email", new VarCharType(false, VarCharType.MAX_LENGTH)))); + + assertThatThrownBy(() -> decode(localRowType, writerSchema, payload)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("email") + .hasMessageContaining("NOT NULL") + .hasMessageContaining("writer schema"); + } + + /** + * Encodes a RowData into raw Protobuf bytes using a descriptor built from the WRITER RowType — + * this stands in for what an upstream writer registered in GSR would have produced. + */ + private static byte[] encode(RowData row, RowType writerRowType) { + DynamicMessage message = + RowDataToProtobufConverter.convertRowData( + row, writerRowType, buildDescriptor(writerRowType)); + return message.toByteArray(); + } + + /** + * Runs the decode path of {@link GsrProtobufRowDataDeserializationSchema} against the given + * local {@code RowType}, with a fake reader that returns the supplied writer schema definition + * and payload (independent of the local DDL). + */ + private static RowData decode(RowType localRowType, String writerSchema, byte[] payload) + throws Exception { + GsrProtobufRowDataDeserializationSchema deser = + new GsrProtobufRowDataDeserializationSchema( + localRowType, + InternalTypeInfo.of(localRowType), + SCHEMA_NAME, + new HashMap<>()); + deser.setReader(new StaticGsrReader(writerSchema, payload)); + deser.open(null); + return deser.deserialize(payload); + } + + private static Descriptors.Descriptor buildDescriptor(RowType rowType) { + try { + DescriptorProtos.FileDescriptorProto fileProto = + ProtobufSchemaConverter.buildFileDescriptorProto(rowType, SCHEMA_NAME); + Descriptors.FileDescriptor fileDescriptor = + Descriptors.FileDescriptor.buildFrom( + fileProto, new Descriptors.FileDescriptor[] {}); + return fileDescriptor.findMessageTypeByName(SCHEMA_NAME); + } catch (Descriptors.DescriptorValidationException e) { + throw new RuntimeException(e); + } + } + + /** + * Fake {@link GsrProtobufReader} that returns a fixed writer schema definition and payload, + * standing in for the GSR facade resolving the writer schema from the record's schema-version + * UUID. Both values are independent of the local {@code RowType}, which is exactly what lets + * the tests prove decode is driven by the writer schema. + */ + private static final class StaticGsrReader implements GsrProtobufReader { + private static final long serialVersionUID = 1L; + private final String writerSchemaDefinition; + private final byte[] payload; + + StaticGsrReader(String writerSchemaDefinition, byte[] payload) { + this.writerSchemaDefinition = writerSchemaDefinition; + this.payload = payload; + } + + @Override + public String writerSchemaDefinition(byte[] gsrEncoded) { + return writerSchemaDefinition; + } + + @Override + public byte[] actualData(byte[] gsrEncoded) { + return payload; + } + } +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufRoundTripIntegrationTest.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufRoundTripIntegrationTest.java new file mode 100644 index 00000000..51b91aca --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufRoundTripIntegrationTest.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.runtime.typeutils.InternalTypeInfo; +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.VarCharType; + +import com.google.protobuf.DescriptorProtos; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.HashMap; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Integration tests for Protobuf round-trip serialization/deserialization with mock GSR header + * handling. + * + *

Validates Requirement 8.1. + */ +class ProtobufRoundTripIntegrationTest { + + private static final int GSR_HEADER_SIZE = 18; + private static final String SCHEMA_NAME = "TestMessage"; + + /** + * Tests full Protobuf round-trip: RowData → Protobuf serialize → prepend GSR header → + * GsrProtobufRowDataDeserializationSchema (strips header + deser) → RowData. + */ + @Test + void testProtobufRoundTrip() throws Exception { + RowType rowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField( + "name", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("age", new IntType()))); + + Descriptors.Descriptor descriptor = buildDescriptor(rowType); + + // Serialize: RowData → DynamicMessage → Protobuf bytes → prepend GSR header + GenericRowData original = new GenericRowData(2); + original.setField(0, StringData.fromString("Alice")); + original.setField(1, 30); + + DynamicMessage message = + RowDataToProtobufConverter.convertRowData(original, rowType, descriptor); + byte[] protobufBytes = message.toByteArray(); + byte[] gsrEncoded = prependMockGsrHeader(protobufBytes); + + // Deserialize using GsrProtobufRowDataDeserializationSchema, routed through a fake GSR + // reader that resolves the writer schema and strips the header (mirrors the facade). + GsrProtobufRowDataDeserializationSchema deser = + new GsrProtobufRowDataDeserializationSchema( + rowType, InternalTypeInfo.of(rowType), SCHEMA_NAME, new HashMap<>()); + deser.setReader(new FakeGsrReader(rowType, SCHEMA_NAME)); + deser.open(null); + + RowData deserialized = deser.deserialize(gsrEncoded); + assertThat(deserialized).isNotNull(); + assertThat(deserialized.getString(0).toString()).isEqualTo("Alice"); + assertThat(deserialized.getInt(1)).isEqualTo(30); + } + + /** Tests round-trip with multiple records. */ + @Test + void testProtobufRoundTripMultipleRecords() throws Exception { + RowType rowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField( + "name", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("value", new IntType()))); + + Descriptors.Descriptor descriptor = buildDescriptor(rowType); + + GsrProtobufRowDataDeserializationSchema deser = + new GsrProtobufRowDataDeserializationSchema( + rowType, InternalTypeInfo.of(rowType), SCHEMA_NAME, new HashMap<>()); + deser.setReader(new FakeGsrReader(rowType, SCHEMA_NAME)); + deser.open(null); + + String[] names = {"Alice", "Bob", "Charlie"}; + int[] values = {10, 20, 30}; + + for (int i = 0; i < names.length; i++) { + GenericRowData row = new GenericRowData(2); + row.setField(0, StringData.fromString(names[i])); + row.setField(1, values[i]); + + DynamicMessage message = + RowDataToProtobufConverter.convertRowData(row, rowType, descriptor); + byte[] protobufBytes = message.toByteArray(); + byte[] gsrEncoded = prependMockGsrHeader(protobufBytes); + + RowData result = deser.deserialize(gsrEncoded); + assertThat(result).isNotNull(); + assertThat(result.getString(0).toString()).isEqualTo(names[i]); + assertThat(result.getInt(1)).isEqualTo(values[i]); + } + } + + private Descriptors.Descriptor buildDescriptor(RowType rowType) { + try { + DescriptorProtos.FileDescriptorProto fileProto = + ProtobufSchemaConverter.buildFileDescriptorProto(rowType, SCHEMA_NAME); + Descriptors.FileDescriptor fileDescriptor = + Descriptors.FileDescriptor.buildFrom( + fileProto, new Descriptors.FileDescriptor[] {}); + return fileDescriptor.findMessageTypeByName(SCHEMA_NAME); + } catch (Descriptors.DescriptorValidationException e) { + throw new RuntimeException(e); + } + } + + /** Creates a mock 18-byte GSR header and prepends it to the payload. */ + private byte[] prependMockGsrHeader(byte[] payload) { + UUID schemaId = UUID.randomUUID(); + ByteBuffer buffer = ByteBuffer.allocate(GSR_HEADER_SIZE + payload.length); + buffer.put((byte) 0x03); // header version + buffer.put((byte) 0x00); // no compression + buffer.putLong(schemaId.getMostSignificantBits()); + buffer.putLong(schemaId.getLeastSignificantBits()); + buffer.put(payload); + return buffer.array(); + } + + /** + * Fake {@link GsrProtobufReader} that stands in for the GSR facade in tests: it returns the + * writer schema definition derived from the given RowType and strips the mock 18-byte header + * (no compression) to yield the raw Protobuf payload. + */ + private static final class FakeGsrReader implements GsrProtobufReader { + private static final long serialVersionUID = 1L; + private final String writerSchemaDefinition; + + FakeGsrReader(RowType rowType, String schemaName) { + this.writerSchemaDefinition = + ProtobufSchemaConverter.convertToProtobufSchema(rowType, schemaName); + } + + @Override + public String writerSchemaDefinition(byte[] gsrEncoded) { + return writerSchemaDefinition; + } + + @Override + public byte[] actualData(byte[] gsrEncoded) { + return Arrays.copyOfRange(gsrEncoded, GSR_HEADER_SIZE, gsrEncoded.length); + } + } +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufTypeCoverageTest.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufTypeCoverageTest.java new file mode 100644 index 00000000..2b901dfd --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufTypeCoverageTest.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.data.TimestampData; +import org.apache.flink.table.types.logical.ArrayType; +import org.apache.flink.table.types.logical.DateType; +import org.apache.flink.table.types.logical.DecimalType; +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.TimeType; +import org.apache.flink.table.types.logical.TimestampType; +import org.apache.flink.table.types.logical.VarCharType; + +import com.google.protobuf.DescriptorProtos; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.util.Arrays; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * Type-coverage tests for the {@code protobuf-glue} format, exercising the review fixes: + * + *

    + *
  • B4 — TIMESTAMP/DATE/TIME/DECIMAL encode via the correct accessors (no {@code + * ClassCastException}) and decode symmetrically. + *
  • C3 — genuinely unsupported complex types (ARRAY/MAP/ROW/...) fail fast instead of + * being silently coerced to {@code string}. + *
  • C4 — column names that are not valid proto identifiers are sanitized so descriptor + * construction does not throw {@code DescriptorValidationException}. + *
+ */ +class ProtobufTypeCoverageTest { + + private static final String SCHEMA_NAME = "TestMessage"; + + private static Descriptors.Descriptor buildDescriptor(RowType rowType) { + try { + DescriptorProtos.FileDescriptorProto fileProto = + ProtobufSchemaConverter.buildFileDescriptorProto(rowType, SCHEMA_NAME); + Descriptors.FileDescriptor fileDescriptor = + Descriptors.FileDescriptor.buildFrom( + fileProto, new Descriptors.FileDescriptor[] {}); + return fileDescriptor.findMessageTypeByName(SCHEMA_NAME); + } catch (Descriptors.DescriptorValidationException e) { + throw new RuntimeException(e); + } + } + + private static RowData roundTrip(RowData in, RowType rowType) throws Exception { + Descriptors.Descriptor descriptor = buildDescriptor(rowType); + DynamicMessage message = RowDataToProtobufConverter.convertRowData(in, rowType, descriptor); + DynamicMessage reparsed = DynamicMessage.parseFrom(descriptor, message.toByteArray()); + return ProtobufToRowDataConverter.convertToRowData(reparsed, rowType); + } + + /** B4: TIMESTAMP (epoch millis int64) round-trips without ClassCastException. */ + @Test + void testTimestampRoundTrip() throws Exception { + RowType rowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField("ts", new TimestampType(3)), + new RowType.RowField("id", new IntType()))); + long millis = 1_700_000_000_123L; + GenericRowData in = new GenericRowData(2); + in.setField(0, TimestampData.fromEpochMillis(millis)); + in.setField(1, 7); + + RowData out = roundTrip(in, rowType); + assertThat(out.getTimestamp(0, 3).getMillisecond()).isEqualTo(millis); + assertThat(out.getInt(1)).isEqualTo(7); + } + + /** B4: DATE (epoch day int32) and TIME (millis-of-day int32) round-trip. */ + @Test + void testDateAndTimeRoundTrip() throws Exception { + RowType rowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField("d", new DateType()), + new RowType.RowField("t", new TimeType(3)))); + int epochDay = 20_000; // 2024-10-04 + int millisOfDay = 45_296_000; // 12:34:56 + GenericRowData in = new GenericRowData(2); + in.setField(0, epochDay); + in.setField(1, millisOfDay); + + RowData out = roundTrip(in, rowType); + assertThat(out.getInt(0)).isEqualTo(epochDay); + assertThat(out.getInt(1)).isEqualTo(millisOfDay); + } + + /** B4: DECIMAL round-trips losslessly via its BigDecimal text form. */ + @Test + void testDecimalRoundTrip() throws Exception { + RowType rowType = + new RowType( + false, + Arrays.asList(new RowType.RowField("amount", new DecimalType(10, 2)))); + BigDecimal value = new BigDecimal("12345.67"); + GenericRowData in = new GenericRowData(1); + in.setField(0, DecimalData.fromBigDecimal(value, 10, 2)); + + RowData out = roundTrip(in, rowType); + assertThat(out.getDecimal(0, 10, 2).toBigDecimal()).isEqualByComparingTo(value); + } + + /** C4: a column name with a space / leading digit is sanitized and still round-trips. */ + @Test + void testFieldNameSanitizationDoesNotThrow() throws Exception { + RowType rowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField( + "user name", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("1st_place", new IntType()))); + + Descriptors.Descriptor descriptor = buildDescriptor(rowType); + // Descriptor built without DescriptorValidationException; names are valid proto idents. + assertThat(descriptor.findFieldByName("user_name")).isNotNull(); + assertThat(descriptor.findFieldByName("_1st_place")).isNotNull(); + + GenericRowData in = new GenericRowData(2); + in.setField(0, StringData.fromString("Alice")); + in.setField(1, 42); + RowData out = roundTrip(in, rowType); + assertThat(out.getString(0).toString()).isEqualTo("Alice"); + assertThat(out.getInt(1)).isEqualTo(42); + } + + /** C4: original SQL column name is preserved as the proto field's json_name. */ + @Test + void testOriginalNamePreservedAsJsonName() { + RowType rowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField( + "user name", new VarCharType(VarCharType.MAX_LENGTH)))); + Descriptors.Descriptor descriptor = buildDescriptor(rowType); + Descriptors.FieldDescriptor fd = descriptor.findFieldByName("user_name"); + assertThat(fd).isNotNull(); + assertThat(fd.toProto().getJsonName()).isEqualTo("user name"); + } + + /** C3: an unsupported complex type (ARRAY) fails fast rather than coercing to string. */ + @Test + void testUnsupportedComplexTypeFailsFast() { + RowType rowType = + new RowType( + false, + Arrays.asList(new RowType.RowField("tags", new ArrayType(new IntType())))); + assertThatThrownBy( + () -> + ProtobufSchemaConverter.buildFileDescriptorProto( + rowType, SCHEMA_NAME)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("does not support"); + } +} diff --git a/flink-formats-aws/flink-sql-protobuf-glue-schema-registry/pom.xml b/flink-formats-aws/flink-sql-protobuf-glue-schema-registry/pom.xml new file mode 100644 index 00000000..ae064d38 --- /dev/null +++ b/flink-formats-aws/flink-sql-protobuf-glue-schema-registry/pom.xml @@ -0,0 +1,144 @@ + + + + + 4.0.0 + + + org.apache.flink + flink-formats-aws-parent + 6.0-SNAPSHOT + + + flink-sql-protobuf-glue-schema-registry + Flink : Formats : AWS : SQL : Protobuf Glue Schema Registry + jar + + + + org.apache.flink + flink-protobuf-glue-schema-registry + ${project.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-flink + package + + shade + + + + + org.apache.flink:flink-connector-aws-base + org.apache.flink:flink-protobuf-glue-schema-registry + org.apache.flink:flink-avro-glue-schema-registry + com.amazonaws:* + software.amazon.awssdk:* + software.amazon.glue:* + com.google.protobuf:* + com.fasterxml.jackson.core:* + com.fasterxml.jackson.dataformat:* + com.google.guava:guava + com.google.guava:failureaccess + com.google.code.gson:gson + + + software.amazon.glue:schema-registry-build-tools + com.google.guava:listenablefuture + org.checkerframework:checker-qual + com.google.errorprone:error_prone_annotations + com.google.j2objc:j2objc-annotations + com.google.code.findbugs:jsr305 + org.apache.kafka:kafka-clients + org.reactivestreams:reactive-streams + + + + + software.amazon + org.apache.flink.protobuf.registry.glue.shaded.software.amazon + + + com.amazonaws + org.apache.flink.protobuf.registry.glue.shaded.com.amazonaws + + + com.google + org.apache.flink.protobuf.registry.glue.shaded.com.google + + + com.typesafe.netty + org.apache.flink.protobuf.registry.glue.shaded.com.typesafe.netty + + + org.apache.http + org.apache.flink.protobuf.registry.glue.shaded.org.apache.http + + + com.fasterxml.jackson + org.apache.flink.protobuf.registry.glue.shaded.com.fasterxml.jackson + + + + + *:* + + **/MavenPackaging.java + **/mime.types + **/VersionInfo.java + codegen-resources/** + mozilla/** + + + + + + + + + + + + + + + com.google.errorprone + error_prone_annotations + 2.21.1 + + + org.opentest4j + opentest4j + 1.3.0 + + + + + diff --git a/flink-formats-aws/pom.xml b/flink-formats-aws/pom.xml index dbc08c4c..a4897491 100644 --- a/flink-formats-aws/pom.xml +++ b/flink-formats-aws/pom.xml @@ -36,7 +36,9 @@ under the License. flink-avro-glue-schema-registry flink-json-glue-schema-registry + flink-protobuf-glue-schema-registry flink-sql-avro-glue-schema-registry + flink-sql-protobuf-glue-schema-registry From d3146a0dcc6454a5d0d9570f510dd8e6e46ed917 Mon Sep 17 00:00:00 2001 From: Francisco Date: Mon, 10 Aug 2026 10:40:18 +0200 Subject: [PATCH 2/5] test(protobuf-glue): Add credential-gated SQL e2e module with Kinesis ITCase --- .../pom.xml | 102 +-- .../registry/test/ProtobufGlueSqlE2E.java | 771 ------------------ .../src/main/resources/log4j2.properties | 23 - ...chemaRegistryProtobufSqlKinesisITCase.java | 437 ++++++++++ .../src/test/resources/log4j2-test.properties | 38 + 5 files changed, 531 insertions(+), 840 deletions(-) delete mode 100644 flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/main/java/org/apache/flink/glue/schema/registry/test/ProtobufGlueSqlE2E.java delete mode 100644 flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/main/resources/log4j2.properties create mode 100644 flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/test/java/org/apache/flink/glue/schema/registry/test/protobuf/GlueSchemaRegistryProtobufSqlKinesisITCase.java create mode 100644 flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/test/resources/log4j2-test.properties diff --git a/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/pom.xml b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/pom.xml index 0bcd5a9c..3ff4c9d6 100644 --- a/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/pom.xml +++ b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/pom.xml @@ -33,110 +33,120 @@ under the License. jar - + org.apache.flink flink-protobuf-glue-schema-registry ${project.version} + test - + org.apache.flink - flink-avro-glue-schema-registry + flink-connector-aws-kinesis-streams ${project.version} + test - + + + org.apache.flink + flink-connector-aws-base + ${project.version} + test-jar + test + + + org.apache.flink + flink-connector-aws-kinesis-streams + ${project.version} + test-jar + test + + + org.apache.flink flink-table-api-java-bridge ${flink.version} + test org.apache.flink flink-table-planner-loader ${flink.version} + test org.apache.flink flink-table-runtime ${flink.version} + test - + org.apache.flink - flink-connector-aws-kinesis-streams - ${project.version} + flink-streaming-java + ${flink.version} + test - - - software.amazon.awssdk - kinesis + org.apache.flink + flink-clients + ${flink.version} + test - + - software.amazon.awssdk - sts + org.testcontainers + localstack + test - + software.amazon.awssdk - netty-nio-client - - - - - org.apache.flink - flink-connector-aws-base - ${project.version} - - - - - com.google.protobuf - protobuf-java + kinesis + test - com.google.protobuf - protobuf-java-util - 3.25.5 + software.amazon.awssdk + glue + test - - - org.apache.flink - flink-streaming-java - ${flink.version} + software.amazon.awssdk + sts + test - org.apache.flink - flink-clients - ${flink.version} + software.amazon.awssdk + netty-nio-client + test - - - org.slf4j - slf4j-api - + org.apache.logging.log4j log4j-slf4j-impl 2.24.1 + test org.apache.logging.log4j log4j-core 2.24.1 + test - + diff --git a/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/main/java/org/apache/flink/glue/schema/registry/test/ProtobufGlueSqlE2E.java b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/main/java/org/apache/flink/glue/schema/registry/test/ProtobufGlueSqlE2E.java deleted file mode 100644 index 7283d26f..00000000 --- a/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/main/java/org/apache/flink/glue/schema/registry/test/ProtobufGlueSqlE2E.java +++ /dev/null @@ -1,771 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.flink.glue.schema.registry.test; - -import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; -import org.apache.flink.table.api.TableResult; -import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; -import org.apache.flink.types.Row; -import org.apache.flink.util.CloseableIterator; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; - -/** - * E2E application for the {@code protobuf-glue} Flink SQL format factory. - * - *

Set the following environment variables before running: - * - *

    - *
  • {@code AWS_REGION} — e.g. us-east-1 - *
  • {@code KINESIS_STREAM_ARN} — full ARN of an existing Kinesis stream - *
  • {@code GSR_REGISTRY_NAME} — existing Glue Schema Registry name - *
  • {@code GSR_SCHEMA_NAME} — schema name prefix for auto-registration - *
- * - *

Tests: - * - *

    - *
  • Test 1: Basic round-trip (STRING, INT, BOOLEAN) - *
  • Test 2: Wide schema with many primitive fields - *
  • Test 3: Proto3 default-value semantics (nulls become defaults) - *
- * - *

Note: Proto3 does not distinguish null from default values. Sending a null STRING yields "", - * null INT yields 0, null BOOLEAN yields false. This is by design and tested in Test 3. - */ -public class ProtobufGlueSqlE2E { - - private static final Logger LOG = LoggerFactory.getLogger(ProtobufGlueSqlE2E.class); - - public static void main(String[] args) throws Exception { - String awsRegion = requireEnv("AWS_REGION"); - String streamArn = requireEnv("KINESIS_STREAM_ARN"); - String registryName = requireEnv("GSR_REGISTRY_NAME"); - String schemaNamePrefix = requireEnv("GSR_SCHEMA_NAME"); - - LOG.info("=== Protobuf-Glue SQL E2E Test ==="); - LOG.info("Region: {}", awsRegion); - LOG.info("Stream ARN: {}", streamArn); - LOG.info("Registry: {}", registryName); - LOG.info("Schema prefix:{}", schemaNamePrefix); - - StreamExecutionEnvironment execEnv = StreamExecutionEnvironment.getExecutionEnvironment(); - execEnv.setParallelism(1); - StreamTableEnvironment tEnv = StreamTableEnvironment.create(execEnv); - - // ================================================================ - // TEST 1: Basic round-trip (STRING, INT, BOOLEAN) - // ================================================================ - LOG.info("=== Test 1: Basic round-trip (STRING, INT, BOOLEAN) ==="); - String basicSchemaName = schemaNamePrefix + "-basic"; - - tEnv.executeSql( - "CREATE TABLE kinesis_sink_basic (" - + " user_name STRING," - + " age INT," - + " is_active BOOLEAN" - + ") WITH (" - + " 'connector' = 'kinesis'," - + " 'stream.arn' = '" - + streamArn - + "'," - + " 'aws.region' = '" - + awsRegion - + "'," - + " 'format' = 'protobuf-glue'," - + " 'protobuf-glue.aws.region' = '" - + awsRegion - + "'," - + " 'protobuf-glue.registry.name' = '" - + registryName - + "'," - + " 'protobuf-glue.schema.name' = '" - + basicSchemaName - + "'," - + " 'protobuf-glue.schema.autoRegistration' = 'true'" - + ")"); - - LOG.info("Inserting 3 basic rows..."); - tEnv.executeSql( - "INSERT INTO kinesis_sink_basic VALUES " - + "('Alice', 30, true)," - + "('Bob', 25, false)," - + "('Charlie', 35, true)") - .await(120, TimeUnit.SECONDS); - - LOG.info("INSERT complete. Now reading back..."); - - tEnv.executeSql( - "CREATE TABLE kinesis_source_basic (" - + " user_name STRING," - + " age INT," - + " is_active BOOLEAN" - + ") WITH (" - + " 'connector' = 'kinesis'," - + " 'stream.arn' = '" - + streamArn - + "'," - + " 'aws.region' = '" - + awsRegion - + "'," - + " 'source.init.position' = 'TRIM_HORIZON'," - + " 'format' = 'protobuf-glue'," - + " 'protobuf-glue.aws.region' = '" - + awsRegion - + "'," - + " 'protobuf-glue.registry.name' = '" - + registryName - + "'," - + " 'protobuf-glue.schema.name' = '" - + basicSchemaName - + "'" - + ")"); - - TableResult result1 = tEnv.executeSql("SELECT * FROM kinesis_source_basic"); - List collected1 = new ArrayList<>(); - - LOG.info("Collecting rows (timeout 90s)..."); - try (CloseableIterator iterator = result1.collect()) { - long deadline = System.currentTimeMillis() + Duration.ofSeconds(90).toMillis(); - while (collected1.size() < 3 && System.currentTimeMillis() < deadline) { - if (iterator.hasNext()) { - Row row = iterator.next(); - LOG.info(" Row {}: {}", collected1.size() + 1, row); - collected1.add(row); - } else { - Thread.sleep(500); - } - } - } - - LOG.info("Collected {} rows total", collected1.size()); - - if (collected1.size() != 3) { - LOG.error("FAIL test 1: expected 3 rows, got {}", collected1.size()); - System.exit(1); - } - - List names = new ArrayList<>(); - for (Row row : collected1) { - names.add(row.getField(0).toString()); - } - - if (names.contains("Alice") && names.contains("Bob") && names.contains("Charlie")) { - LOG.info("PASS test 1: all 3 rows round-tripped correctly via protobuf-glue format"); - } else { - LOG.error("FAIL test 1: unexpected names: {}", names); - System.exit(1); - } - - // ================================================================ - // TEST 2: Wide schema with many primitive fields - // ================================================================ - LOG.info("=== Test 2: Wide schema with many primitive fields ==="); - String wideSchemaName = schemaNamePrefix + "-wide"; - - tEnv.executeSql( - "CREATE TABLE kinesis_sink_wide (" - + " id STRING," - + " field_01 STRING, field_02 STRING, field_03 STRING," - + " field_04 STRING, field_05 STRING," - + " field_06 INT, field_07 INT, field_08 INT," - + " field_09 INT, field_10 INT," - + " field_11 BIGINT, field_12 BIGINT, field_13 BIGINT," - + " field_14 DOUBLE, field_15 DOUBLE, field_16 DOUBLE," - + " field_17 BOOLEAN, field_18 BOOLEAN," - + " field_19 BOOLEAN, field_20 BOOLEAN," - + " field_21 FLOAT, field_22 FLOAT" - + ") WITH (" - + " 'connector' = 'kinesis'," - + " 'stream.arn' = '" - + streamArn - + "'," - + " 'aws.region' = '" - + awsRegion - + "'," - + " 'format' = 'protobuf-glue'," - + " 'protobuf-glue.aws.region' = '" - + awsRegion - + "'," - + " 'protobuf-glue.registry.name' = '" - + registryName - + "'," - + " 'protobuf-glue.schema.name' = '" - + wideSchemaName - + "'," - + " 'protobuf-glue.schema.autoRegistration' = 'true'" - + ")"); - - LOG.info("Inserting wide row with 23 fields..."); - tEnv.executeSql( - "INSERT INTO kinesis_sink_wide VALUES (" - + " 'WIDE-001'," - + " 'str1', 'str2', 'str3', 'str4', 'str5'," - + " 1, 2, 3, 4, 5," - + " 100000000001, 100000000002, 100000000003," - + " 1.1, 2.2, 3.3," - + " true, false, true, false," - + " CAST(1.5 AS FLOAT), CAST(2.5 AS FLOAT)" - + ")") - .await(120, TimeUnit.SECONDS); - - LOG.info("INSERT with wide schema complete. Now reading back..."); - - tEnv.executeSql( - "CREATE TABLE kinesis_source_wide (" - + " id STRING," - + " field_01 STRING, field_02 STRING, field_03 STRING," - + " field_04 STRING, field_05 STRING," - + " field_06 INT, field_07 INT, field_08 INT," - + " field_09 INT, field_10 INT," - + " field_11 BIGINT, field_12 BIGINT, field_13 BIGINT," - + " field_14 DOUBLE, field_15 DOUBLE, field_16 DOUBLE," - + " field_17 BOOLEAN, field_18 BOOLEAN," - + " field_19 BOOLEAN, field_20 BOOLEAN," - + " field_21 FLOAT, field_22 FLOAT" - + ") WITH (" - + " 'connector' = 'kinesis'," - + " 'stream.arn' = '" - + streamArn - + "'," - + " 'aws.region' = '" - + awsRegion - + "'," - + " 'source.init.position' = 'TRIM_HORIZON'," - + " 'format' = 'protobuf-glue'," - + " 'protobuf-glue.aws.region' = '" - + awsRegion - + "'," - + " 'protobuf-glue.registry.name' = '" - + registryName - + "'," - + " 'protobuf-glue.schema.name' = '" - + wideSchemaName - + "'" - + ")"); - - TableResult result2 = tEnv.executeSql("SELECT * FROM kinesis_source_wide"); - boolean foundWide001 = false; - - LOG.info("Collecting rows for test 2 (timeout 90s, looking for WIDE-001)..."); - try (CloseableIterator iterator2 = result2.collect()) { - long deadline2 = System.currentTimeMillis() + Duration.ofSeconds(90).toMillis(); - while (!foundWide001 && System.currentTimeMillis() < deadline2) { - if (iterator2.hasNext()) { - Row row = iterator2.next(); - Object idField = row.getField(0); - if (idField == null || idField.toString().isEmpty()) { - continue; - } - String id = idField.toString(); - LOG.info(" Row: {}", row); - - if ("WIDE-001".equals(id)) { - foundWide001 = true; - // Spot-check a few fields - if (!"str1".equals(row.getField(1).toString())) { - LOG.error("FAIL: field_01 mismatch"); - System.exit(1); - } - if (!Integer.valueOf(5).equals(row.getField(10))) { - LOG.error( - "FAIL: field_10 mismatch, expected 5, got {}", - row.getField(10)); - System.exit(1); - } - if (!Boolean.FALSE.equals(row.getField(20))) { - LOG.error( - "FAIL: field_20 mismatch, expected false, got {}", - row.getField(20)); - System.exit(1); - } - LOG.info(" WIDE-001 verified: all 23 fields round-tripped correctly"); - } - } else { - Thread.sleep(500); - } - } - } - - if (foundWide001) { - LOG.info("PASS test 2: wide schema round-trip works"); - } else { - LOG.error("FAIL test 2: did not find WIDE-001"); - System.exit(1); - } - - // ================================================================ - // TEST 3: Proto3 default-value semantics - // - // Proto3 does NOT distinguish null from default values: - // - null STRING → "" (empty string) - // - null INT → 0 - // - null BOOLEAN → false - // - // This test verifies that sending "default-like" values round-trips - // correctly, and documents the proto3 null-to-default behavior. - // ================================================================ - LOG.info("=== Test 3: Proto3 default-value semantics ==="); - String defaultSchemaName = schemaNamePrefix + "-defaults"; - - tEnv.executeSql( - "CREATE TABLE kinesis_sink_defaults (" - + " id STRING," - + " str_field STRING," - + " int_field INT," - + " long_field BIGINT," - + " double_field DOUBLE," - + " bool_field BOOLEAN," - + " float_field FLOAT" - + ") WITH (" - + " 'connector' = 'kinesis'," - + " 'stream.arn' = '" - + streamArn - + "'," - + " 'aws.region' = '" - + awsRegion - + "'," - + " 'format' = 'protobuf-glue'," - + " 'protobuf-glue.aws.region' = '" - + awsRegion - + "'," - + " 'protobuf-glue.registry.name' = '" - + registryName - + "'," - + " 'protobuf-glue.schema.name' = '" - + defaultSchemaName - + "'," - + " 'protobuf-glue.schema.autoRegistration' = 'true'" - + ")"); - - // Row with non-default values - LOG.info("Inserting row with non-default values..."); - tEnv.executeSql( - "INSERT INTO kinesis_sink_defaults VALUES (" - + " 'DEF-001', 'hello', 42, 100000000001, 3.14, true," - + " CAST(1.5 AS FLOAT)" - + ")") - .await(120, TimeUnit.SECONDS); - - // Row with proto3 default values (empty string, 0, false) - LOG.info("Inserting row with proto3 default values..."); - tEnv.executeSql( - "INSERT INTO kinesis_sink_defaults VALUES (" - + " 'DEF-002', '', 0, CAST(0 AS BIGINT), CAST(0.0 AS DOUBLE)," - + " false, CAST(0.0 AS FLOAT)" - + ")") - .await(120, TimeUnit.SECONDS); - - LOG.info("INSERT with default values complete. Now reading back..."); - - tEnv.executeSql( - "CREATE TABLE kinesis_source_defaults (" - + " id STRING," - + " str_field STRING," - + " int_field INT," - + " long_field BIGINT," - + " double_field DOUBLE," - + " bool_field BOOLEAN," - + " float_field FLOAT" - + ") WITH (" - + " 'connector' = 'kinesis'," - + " 'stream.arn' = '" - + streamArn - + "'," - + " 'aws.region' = '" - + awsRegion - + "'," - + " 'source.init.position' = 'TRIM_HORIZON'," - + " 'format' = 'protobuf-glue'," - + " 'protobuf-glue.aws.region' = '" - + awsRegion - + "'," - + " 'protobuf-glue.registry.name' = '" - + registryName - + "'," - + " 'protobuf-glue.schema.name' = '" - + defaultSchemaName - + "'" - + ")"); - - TableResult result3 = tEnv.executeSql("SELECT * FROM kinesis_source_defaults"); - boolean foundDef001 = false; - boolean foundDef002 = false; - - LOG.info("Collecting rows for test 3 (timeout 90s, looking for DEF-001 & DEF-002)..."); - try (CloseableIterator iterator3 = result3.collect()) { - long deadline3 = System.currentTimeMillis() + Duration.ofSeconds(90).toMillis(); - while (!(foundDef001 && foundDef002) && System.currentTimeMillis() < deadline3) { - if (iterator3.hasNext()) { - Row row = iterator3.next(); - Object idField = row.getField(0); - if (idField == null || idField.toString().isEmpty()) { - continue; - } - String id = idField.toString(); - LOG.info(" Row: {}", row); - - if ("DEF-001".equals(id)) { - foundDef001 = true; - if (!"hello".equals(row.getField(1).toString())) { - LOG.error("FAIL: DEF-001 str_field mismatch"); - System.exit(1); - } - if (!Integer.valueOf(42).equals(row.getField(2))) { - LOG.error("FAIL: DEF-001 int_field mismatch"); - System.exit(1); - } - if (!Boolean.TRUE.equals(row.getField(5))) { - LOG.error("FAIL: DEF-001 bool_field mismatch"); - System.exit(1); - } - LOG.info(" DEF-001 verified: non-default values preserved"); - } - - if ("DEF-002".equals(id)) { - foundDef002 = true; - // Proto3 defaults: empty string, 0, 0L, 0.0, false, 0.0f - // These should round-trip as their default values (not null) - Object strVal = row.getField(1); - Object intVal = row.getField(2); - Object boolVal = row.getField(5); - - if (strVal != null && !"".equals(strVal.toString())) { - LOG.error( - "FAIL: DEF-002 str_field expected '' or null, got '{}'", - strVal); - System.exit(1); - } - LOG.info( - " DEF-002 str_field={}, int_field={}, bool_field={} " - + "(proto3 defaults)", - strVal, - intVal, - boolVal); - LOG.info(" DEF-002 verified: proto3 default values round-tripped"); - } - } else { - Thread.sleep(500); - } - } - } - - if (foundDef001 && foundDef002) { - LOG.info("PASS test 3: proto3 default-value semantics work correctly"); - } else { - LOG.error( - "FAIL test 3: did not find DEF-001 ({}) and DEF-002 ({})", - foundDef001, - foundDef002); - System.exit(1); - } - - // ================================================================ - // TEST 4: Schema compatibility — BACKWARD - // - // BACKWARD compatibility means new schema can read data written - // with the old schema. - // - // Proto3 note: all fields are implicitly optional, so NOT NULL - // has no effect on the generated .proto definition. Removing a - // field is always safe in proto3 (the old field number is just - // ignored). Therefore we use a TYPE CHANGE (INT→STRING) to - // trigger a genuine BACKWARD incompatibility instead. - // - // IMPORTANT: delete the '*-compat-backward' schema from GSR - // before re-running this test. - // ================================================================ - LOG.info("=== Test 4: Schema compatibility — BACKWARD ==="); - String backwardSchemaName = schemaNamePrefix + "-compat-backward"; - - // Step 1: v1 schema (3 fields) - tEnv.executeSql( - "CREATE TABLE compat_bw_sink_v1 (" - + " user_name STRING," - + " age INT," - + " city STRING" - + ") WITH (" - + " 'connector' = 'kinesis'," - + " 'stream.arn' = '" - + streamArn - + "'," - + " 'aws.region' = '" - + awsRegion - + "'," - + " 'format' = 'protobuf-glue'," - + " 'protobuf-glue.aws.region' = '" - + awsRegion - + "'," - + " 'protobuf-glue.registry.name' = '" - + registryName - + "'," - + " 'protobuf-glue.schema.name' = '" - + backwardSchemaName - + "'," - + " 'protobuf-glue.schema.autoRegistration' = 'true'," - + " 'protobuf-glue.schema.compatibility' = 'BACKWARD'" - + ")"); - - LOG.info("Test 4 step 1: Writing v1 data (3 fields)..."); - tEnv.executeSql("INSERT INTO compat_bw_sink_v1 VALUES ('Alice', 30, 'Seattle')") - .await(120, TimeUnit.SECONDS); - LOG.info("Test 4 step 1: v1 write succeeded"); - - // Step 2: v2 schema (4 fields — added new field) - tEnv.executeSql( - "CREATE TABLE compat_bw_sink_v2 (" - + " user_name STRING," - + " age INT," - + " city STRING," - + " email STRING" - + ") WITH (" - + " 'connector' = 'kinesis'," - + " 'stream.arn' = '" - + streamArn - + "'," - + " 'aws.region' = '" - + awsRegion - + "'," - + " 'format' = 'protobuf-glue'," - + " 'protobuf-glue.aws.region' = '" - + awsRegion - + "'," - + " 'protobuf-glue.registry.name' = '" - + registryName - + "'," - + " 'protobuf-glue.schema.name' = '" - + backwardSchemaName - + "'," - + " 'protobuf-glue.schema.autoRegistration' = 'true'," - + " 'protobuf-glue.schema.compatibility' = 'BACKWARD'" - + ")"); - - LOG.info("Test 4 step 2: Writing v2 data (4 fields — added email)..."); - tEnv.executeSql( - "INSERT INTO compat_bw_sink_v2 VALUES ('Bob', 25, 'Portland', 'bob@example.com')") - .await(120, TimeUnit.SECONDS); - LOG.info("Test 4 step 2: v2 write succeeded (adding field is backward-compatible)"); - - // Step 3: v3 schema — change 'age' from INT to STRING (type change) - // In proto3, all fields are optional so removing a field is always - // backward-compatible. A type change is the simplest way to trigger - // a genuine BACKWARD incompatibility in proto3. - tEnv.executeSql( - "CREATE TABLE compat_bw_sink_v3 (" - + " user_name STRING," - + " age STRING," - + " city STRING" - + ") WITH (" - + " 'connector' = 'kinesis'," - + " 'stream.arn' = '" - + streamArn - + "'," - + " 'aws.region' = '" - + awsRegion - + "'," - + " 'format' = 'protobuf-glue'," - + " 'protobuf-glue.aws.region' = '" - + awsRegion - + "'," - + " 'protobuf-glue.registry.name' = '" - + registryName - + "'," - + " 'protobuf-glue.schema.name' = '" - + backwardSchemaName - + "'," - + " 'protobuf-glue.schema.autoRegistration' = 'true'," - + " 'protobuf-glue.schema.compatibility' = 'BACKWARD'" - + ")"); - - LOG.info("Test 4 step 3: Writing v3 data (changed age INT→STRING — type change)..."); - try { - tEnv.executeSql("INSERT INTO compat_bw_sink_v3 VALUES ('Charlie', '35', 'Denver')") - .await(120, TimeUnit.SECONDS); - LOG.error( - "FAIL test 4 step 3: expected schema compatibility rejection but write succeeded"); - System.exit(1); - } catch (Exception e) { - LOG.info("Test 4 step 3: Write correctly rejected by GSR: {}", e.getMessage()); - LOG.info("PASS test 4: BACKWARD compatibility enforced correctly"); - } - - // ================================================================ - // TEST 5: Schema compatibility — NONE (no validation) - // ================================================================ - LOG.info("=== Test 5: Schema compatibility — NONE ==="); - String noneSchemaName = schemaNamePrefix + "-compat-none"; - - // v1: 3 fields - tEnv.executeSql( - "CREATE TABLE compat_none_sink_v1 (" - + " user_name STRING," - + " age INT," - + " city STRING" - + ") WITH (" - + " 'connector' = 'kinesis'," - + " 'stream.arn' = '" - + streamArn - + "'," - + " 'aws.region' = '" - + awsRegion - + "'," - + " 'format' = 'protobuf-glue'," - + " 'protobuf-glue.aws.region' = '" - + awsRegion - + "'," - + " 'protobuf-glue.registry.name' = '" - + registryName - + "'," - + " 'protobuf-glue.schema.name' = '" - + noneSchemaName - + "'," - + " 'protobuf-glue.schema.autoRegistration' = 'true'," - + " 'protobuf-glue.schema.compatibility' = 'NONE'" - + ")"); - - LOG.info("Test 5 step 1: Writing v1 data with NONE compat..."); - tEnv.executeSql("INSERT INTO compat_none_sink_v1 VALUES ('Dave', 40, 'Denver')") - .await(120, TimeUnit.SECONDS); - - // v2: completely different schema (2 fields, removed city) - tEnv.executeSql( - "CREATE TABLE compat_none_sink_v2 (" - + " user_name STRING," - + " age INT" - + ") WITH (" - + " 'connector' = 'kinesis'," - + " 'stream.arn' = '" - + streamArn - + "'," - + " 'aws.region' = '" - + awsRegion - + "'," - + " 'format' = 'protobuf-glue'," - + " 'protobuf-glue.aws.region' = '" - + awsRegion - + "'," - + " 'protobuf-glue.registry.name' = '" - + registryName - + "'," - + " 'protobuf-glue.schema.name' = '" - + noneSchemaName - + "'," - + " 'protobuf-glue.schema.autoRegistration' = 'true'," - + " 'protobuf-glue.schema.compatibility' = 'NONE'" - + ")"); - - LOG.info( - "Test 5 step 2: Writing v2 data (incompatible change, should succeed with NONE)..."); - tEnv.executeSql("INSERT INTO compat_none_sink_v2 VALUES ('Eve', 28)") - .await(120, TimeUnit.SECONDS); - LOG.info("PASS test 5: NONE compatibility allows any schema evolution"); - - // ================================================================ - // TEST 6: Schema compatibility — FULL - // - // FULL = both BACKWARD and FORWARD. Adding a new field is the - // canonical safe evolution for proto3. - // ================================================================ - LOG.info("=== Test 6: Schema compatibility — FULL ==="); - String fullSchemaName = schemaNamePrefix + "-compat-full"; - - // v1: 3 fields - tEnv.executeSql( - "CREATE TABLE compat_full_sink_v1 (" - + " user_name STRING," - + " age INT," - + " city STRING" - + ") WITH (" - + " 'connector' = 'kinesis'," - + " 'stream.arn' = '" - + streamArn - + "'," - + " 'aws.region' = '" - + awsRegion - + "'," - + " 'format' = 'protobuf-glue'," - + " 'protobuf-glue.aws.region' = '" - + awsRegion - + "'," - + " 'protobuf-glue.registry.name' = '" - + registryName - + "'," - + " 'protobuf-glue.schema.name' = '" - + fullSchemaName - + "'," - + " 'protobuf-glue.schema.autoRegistration' = 'true'," - + " 'protobuf-glue.schema.compatibility' = 'FULL'" - + ")"); - - LOG.info("Test 6 step 1: Writing v1 data with FULL compat..."); - tEnv.executeSql("INSERT INTO compat_full_sink_v1 VALUES ('Frank', 45, 'Chicago')") - .await(120, TimeUnit.SECONDS); - - // v2: add new field - tEnv.executeSql( - "CREATE TABLE compat_full_sink_v2 (" - + " user_name STRING," - + " age INT," - + " city STRING," - + " email STRING" - + ") WITH (" - + " 'connector' = 'kinesis'," - + " 'stream.arn' = '" - + streamArn - + "'," - + " 'aws.region' = '" - + awsRegion - + "'," - + " 'format' = 'protobuf-glue'," - + " 'protobuf-glue.aws.region' = '" - + awsRegion - + "'," - + " 'protobuf-glue.registry.name' = '" - + registryName - + "'," - + " 'protobuf-glue.schema.name' = '" - + fullSchemaName - + "'," - + " 'protobuf-glue.schema.autoRegistration' = 'true'," - + " 'protobuf-glue.schema.compatibility' = 'FULL'" - + ")"); - - LOG.info("Test 6 step 2: Writing v2 data (added email field)..."); - tEnv.executeSql( - "INSERT INTO compat_full_sink_v2 VALUES ('Grace', 32, 'Boston', 'grace@example.com')") - .await(120, TimeUnit.SECONDS); - LOG.info("PASS test 6: FULL compatibility allows adding new fields"); - - LOG.info("=== All Protobuf-Glue E2E Tests Passed ==="); - } - - private static String requireEnv(String name) { - String value = System.getenv(name); - if (value == null || value.isEmpty()) { - System.err.println("ERROR: environment variable " + name + " is required"); - System.exit(1); - } - return value; - } -} diff --git a/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/main/resources/log4j2.properties b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/main/resources/log4j2.properties deleted file mode 100644 index 0aa862e5..00000000 --- a/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/main/resources/log4j2.properties +++ /dev/null @@ -1,23 +0,0 @@ -rootLogger.level = INFO -rootLogger.appenderRef.console.ref = ConsoleAppender - -appender.console.name = ConsoleAppender -appender.console.type = Console -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n - -# Reduce noise from Flink internals -logger.flink.name = org.apache.flink -logger.flink.level = WARN - -# Our test class at INFO -logger.e2e.name = org.apache.flink.glue.schema.registry.test -logger.e2e.level = INFO - -# GSR format classes at INFO (for debugging) -logger.gsrformat.name = org.apache.flink.formats.protobuf.glue.schema.registry -logger.gsrformat.level = INFO - -# GSR SDK -logger.gsr.name = software.amazon.awssdk -logger.gsr.level = WARN diff --git a/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/test/java/org/apache/flink/glue/schema/registry/test/protobuf/GlueSchemaRegistryProtobufSqlKinesisITCase.java b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/test/java/org/apache/flink/glue/schema/registry/test/protobuf/GlueSchemaRegistryProtobufSqlKinesisITCase.java new file mode 100644 index 00000000..dbb5b226 --- /dev/null +++ b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/test/java/org/apache/flink/glue/schema/registry/test/protobuf/GlueSchemaRegistryProtobufSqlKinesisITCase.java @@ -0,0 +1,437 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.glue.schema.registry.test.protobuf; + +import org.apache.flink.api.common.time.Deadline; +import org.apache.flink.connector.aws.testutils.AWSServicesTestUtils; +import org.apache.flink.connector.aws.testutils.LocalstackContainer; +import org.apache.flink.connector.aws.util.AWSGeneralUtil; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; +import org.apache.flink.types.Row; +import org.apache.flink.util.CloseableIterator; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.utility.DockerImageName; +import software.amazon.awssdk.core.SdkSystemSetting; +import software.amazon.awssdk.http.SdkHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.glue.GlueClient; +import software.amazon.awssdk.services.glue.model.DeleteSchemaRequest; +import software.amazon.awssdk.services.glue.model.SchemaId; +import software.amazon.awssdk.services.kinesis.KinesisClient; +import software.amazon.awssdk.services.kinesis.model.CreateStreamRequest; +import software.amazon.awssdk.services.kinesis.model.DescribeStreamRequest; +import software.amazon.awssdk.services.kinesis.model.StreamStatus; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assumptions.assumeThat; + +/** + * SQL-path end-to-end test for the AWS Glue Schema Registry Protobuf format ({@code protobuf-glue}) + * exercising the Flink Table API through a Localstack Kinesis data plane against a real Glue + * Schema Registry. + * + *

The Kinesis connector talks to Localstack via an explicit {@code aws.endpoint} + dummy {@code + * BASIC} credentials, while the {@code protobuf-glue} format resolves the real GSR through the + * default AWS credential chain, which we seed with the {@code IT_CASE_GLUE_SCHEMA_*} credentials + * via JVM system properties in {@link #setup()} (the MiniCluster runs in this same JVM). + * + *

The test is tagged {@code requires-aws-credentials} so it only runs under the {@code + * run-aws-end-to-end-tests} Maven profile, and is additionally {@code assumeThat}-gated to skip + * when credentials are absent. + * + *

Scenarios: + * + *

    + *
  • {@link #testBasicMultiRowRoundTrip()} — multi-row round-trip of STRING/INT/BOOLEAN columns. + *
  • {@link #testNullableColumnsRoundTrip()} — nullable columns carrying explicit {@code NULL} + * values, exercising the proto3 explicit-presence fix (C2). + *
  • {@link #testCompressionRoundTrip()} — same round-trip with {@code + * protobuf-glue.schema.compression = ZLIB}, exercising the compression round-trip fix (C1). + *
+ */ +@Tag("requires-aws-credentials") +class GlueSchemaRegistryProtobufSqlKinesisITCase { + + private static final Logger LOG = + LoggerFactory.getLogger(GlueSchemaRegistryProtobufSqlKinesisITCase.class); + + private static final String ACCESS_KEY = System.getenv("IT_CASE_GLUE_SCHEMA_ACCESS_KEY"); + private static final String SECRET_KEY = System.getenv("IT_CASE_GLUE_SCHEMA_SECRET_KEY"); + private static final String GSR_REGION = + envOrDefault("IT_CASE_GLUE_SCHEMA_REGION", "ca-central-1"); + private static final String REGISTRY_NAME = + envOrDefault("IT_CASE_GLUE_SCHEMA_REGISTRY_NAME", "default-registry"); + + private static final String LOCALSTACK_DOCKER_IMAGE_VERSION = "localstack/localstack:3.7.2"; + private static final String KINESIS_REGION = "ap-southeast-1"; + private static final String KINESIS_ACCOUNT = "000000000000"; + + private static final LocalstackContainer MOCK_KINESIS_CONTAINER = + new LocalstackContainer(DockerImageName.parse(LOCALSTACK_DOCKER_IMAGE_VERSION)) + .withNetworkAliases("localstack"); + + private SdkHttpClient httpClient; + private KinesisClient kinesisClient; + private StreamTableEnvironment tEnv; + private final List createdSchemas = new ArrayList<>(); + + @BeforeAll + static void beforeAll() { + assumeThat(ACCESS_KEY) + .as("IT_CASE_GLUE_SCHEMA_ACCESS_KEY must be set to run this test") + .isNotBlank(); + assumeThat(SECRET_KEY) + .as("IT_CASE_GLUE_SCHEMA_SECRET_KEY must be set to run this test") + .isNotBlank(); + + System.setProperty(SdkSystemSetting.CBOR_ENABLED.property(), "false"); + MOCK_KINESIS_CONTAINER.start(); + } + + @AfterAll + static void afterAll() { + if (MOCK_KINESIS_CONTAINER.isRunning()) { + MOCK_KINESIS_CONTAINER.stop(); + } + System.clearProperty(SdkSystemSetting.CBOR_ENABLED.property()); + } + + @BeforeEach + void setup() { + // Seed the default AWS credential chain so the protobuf-glue format authenticates against + // the real Glue Schema Registry from inside the MiniCluster JVM. + System.setProperty("aws.accessKeyId", ACCESS_KEY); + System.setProperty("aws.secretAccessKey", SECRET_KEY); + System.setProperty("aws.region", GSR_REGION); + + httpClient = AWSServicesTestUtils.createHttpClient(); + kinesisClient = + AWSServicesTestUtils.createAwsSyncClient( + MOCK_KINESIS_CONTAINER.getEndpoint(), httpClient, KinesisClient.builder()); + + StreamExecutionEnvironment execEnv = StreamExecutionEnvironment.getExecutionEnvironment(); + execEnv.setParallelism(1); + tEnv = StreamTableEnvironment.create(execEnv); + + LOG.info("Done setting up Localstack Kinesis + real GSR credential chain."); + } + + @AfterEach + void teardown() { + cleanupSchemas(); + AWSGeneralUtil.closeResources(httpClient, kinesisClient); + System.clearProperty("aws.accessKeyId"); + System.clearProperty("aws.secretAccessKey"); + System.clearProperty("aws.region"); + } + + @Test + void testBasicMultiRowRoundTrip() throws Exception { + String id = uniqueId("basic"); + String schemaName = schemaName(id); + prepareStream(id); + + tEnv.executeSql( + "CREATE TABLE kinesis_sink_basic (" + + " user_name STRING," + + " age INT," + + " is_active BOOLEAN" + + ") WITH (" + + kinesisOptions(id, false) + + "," + + protobufGlueOptions(schemaName, true, null) + + ")"); + + tEnv.executeSql( + "INSERT INTO kinesis_sink_basic VALUES " + + "('Alice', 30, true)," + + "('Bob', 25, false)," + + "('Charlie', 35, true)") + .await(120, TimeUnit.SECONDS); + + tEnv.executeSql( + "CREATE TABLE kinesis_source_basic (" + + " user_name STRING," + + " age INT," + + " is_active BOOLEAN" + + ") WITH (" + + kinesisOptions(id, true) + + "," + + protobufGlueOptions(schemaName, false, null) + + ")"); + + List rows = collect("SELECT * FROM kinesis_source_basic", 3, Duration.ofSeconds(90)); + + assertThat(rows).hasSize(3); + assertThat(rows) + .extracting(row -> String.valueOf(row.getField(0))) + .containsExactlyInAnyOrder("Alice", "Bob", "Charlie"); + } + + @Test + void testNullableColumnsRoundTrip() throws Exception { + String id = uniqueId("nullable"); + String schemaName = schemaName(id); + prepareStream(id); + + // All value columns are nullable (default in Flink SQL) so the proto3 explicit-presence + // fix (C2) must round-trip explicit NULLs as NULL rather than proto3 defaults. + String columns = + " id STRING," + " opt_str STRING," + " opt_int INT," + " opt_bool BOOLEAN"; + + tEnv.executeSql( + "CREATE TABLE kinesis_sink_nullable (" + + columns + + ") WITH (" + + kinesisOptions(id, false) + + "," + + protobufGlueOptions(schemaName, true, null) + + ")"); + + tEnv.executeSql( + "INSERT INTO kinesis_sink_nullable VALUES " + + "('R1', 'hello', 42, true)," + + "('R2', CAST(NULL AS STRING), CAST(NULL AS INT), CAST(NULL AS BOOLEAN))") + .await(120, TimeUnit.SECONDS); + + tEnv.executeSql( + "CREATE TABLE kinesis_source_nullable (" + + columns + + ") WITH (" + + kinesisOptions(id, true) + + "," + + protobufGlueOptions(schemaName, false, null) + + ")"); + + List rows = + collect("SELECT * FROM kinesis_source_nullable", 2, Duration.ofSeconds(90)); + + assertThat(rows).hasSize(2); + + Row r1 = findById(rows, "R1"); + assertThat(r1.getField(1)).isEqualTo("hello"); + assertThat(r1.getField(2)).isEqualTo(42); + assertThat(r1.getField(3)).isEqualTo(true); + + Row r2 = findById(rows, "R2"); + assertThat(r2.getField(1)).as("nullable STRING should round-trip as NULL").isNull(); + assertThat(r2.getField(2)).as("nullable INT should round-trip as NULL").isNull(); + assertThat(r2.getField(3)).as("nullable BOOLEAN should round-trip as NULL").isNull(); + } + + @Test + void testCompressionRoundTrip() throws Exception { + String id = uniqueId("compression"); + String schemaName = schemaName(id); + prepareStream(id); + + // 'schema.compression' = 'ZLIB' exercises the compression round-trip fix (C1): the reader + // must transparently decompress GSR-compressed payloads. + tEnv.executeSql( + "CREATE TABLE kinesis_sink_zlib (" + + " user_name STRING," + + " age INT," + + " is_active BOOLEAN" + + ") WITH (" + + kinesisOptions(id, false) + + "," + + protobufGlueOptions(schemaName, true, "ZLIB") + + ")"); + + tEnv.executeSql( + "INSERT INTO kinesis_sink_zlib VALUES " + + "('Dave', 40, true)," + + "('Eve', 28, false)," + + "('Frank', 33, true)") + .await(120, TimeUnit.SECONDS); + + tEnv.executeSql( + "CREATE TABLE kinesis_source_zlib (" + + " user_name STRING," + + " age INT," + + " is_active BOOLEAN" + + ") WITH (" + + kinesisOptions(id, true) + + "," + + protobufGlueOptions(schemaName, false, "ZLIB") + + ")"); + + List rows = collect("SELECT * FROM kinesis_source_zlib", 3, Duration.ofSeconds(90)); + + assertThat(rows).hasSize(3); + assertThat(rows) + .extracting(row -> String.valueOf(row.getField(0))) + .containsExactlyInAnyOrder("Dave", "Eve", "Frank"); + } + + private List collect(String selectSql, int expected, Duration timeout) throws Exception { + List rows = new ArrayList<>(); + try (CloseableIterator iterator = tEnv.executeSql(selectSql).collect()) { + Deadline deadline = Deadline.fromNow(timeout); + while (rows.size() < expected && deadline.hasTimeLeft()) { + if (iterator.hasNext()) { + Row row = iterator.next(); + LOG.info("collected row: {}", row); + rows.add(row); + } else { + Thread.sleep(500); + } + } + } + return rows; + } + + private Row findById(List rows, String id) { + for (Row row : rows) { + if (id.equals(String.valueOf(row.getField(0)))) { + return row; + } + } + throw new AssertionError("Row with id '" + id + "' not found in " + rows); + } + + private String kinesisOptions(String streamName, boolean source) { + StringBuilder sb = new StringBuilder(); + sb.append(" 'connector' = 'kinesis',"); + sb.append(" 'stream.arn' = '").append(streamArn(streamName)).append("',"); + sb.append(" 'aws.region' = '").append(KINESIS_REGION).append("',"); + sb.append(" 'aws.endpoint' = '").append(MOCK_KINESIS_CONTAINER.getEndpoint()).append("',"); + sb.append(" 'aws.credentials.provider' = 'BASIC',"); + sb.append(" 'aws.credentials.provider.basic.accesskeyid' = 'accessKeyId',"); + sb.append(" 'aws.credentials.provider.basic.secretkey' = 'secretAccessKey',"); + sb.append(" 'aws.trust.all.certificates' = 'true'"); + if (source) { + sb.append(", 'source.init.position' = 'TRIM_HORIZON'"); + } + return sb.toString(); + } + + private String protobufGlueOptions(String schemaName, boolean forSink, String compression) { + StringBuilder sb = new StringBuilder(); + sb.append(" 'format' = 'protobuf-glue',"); + sb.append(" 'protobuf-glue.aws.region' = '").append(GSR_REGION).append("',"); + sb.append(" 'protobuf-glue.registry.name' = '").append(REGISTRY_NAME).append("',"); + sb.append(" 'protobuf-glue.schema.name' = '").append(schemaName).append("'"); + if (forSink) { + sb.append(", 'protobuf-glue.schema.autoRegistration' = 'true'"); + } + if (compression != null) { + sb.append(", 'protobuf-glue.schema.compression' = '").append(compression).append("'"); + } + return sb.toString(); + } + + private void prepareStream(String streamName) throws Exception { + kinesisClient.createStream( + CreateStreamRequest.builder().streamName(streamName).shardCount(1).build()); + + Deadline deadline = Deadline.fromNow(Duration.ofMinutes(1)); + while (deadline.hasTimeLeft()) { + if (streamActive(streamName)) { + return; + } + Thread.sleep(500); + } + throw new IllegalStateException("Stream " + streamName + " did not become ACTIVE in time"); + } + + private boolean streamActive(String streamName) { + try { + return kinesisClient + .describeStream( + DescribeStreamRequest.builder().streamName(streamName).build()) + .streamDescription() + .streamStatus() + == StreamStatus.ACTIVE; + } catch (Exception e) { + return false; + } + } + + private void cleanupSchemas() { + if (createdSchemas.isEmpty()) { + return; + } + try (SdkHttpClient glueHttpClient = AWSServicesTestUtils.createHttpClient(); + GlueClient glueClient = + GlueClient.builder() + .region(Region.of(GSR_REGION)) + .httpClient(glueHttpClient) + .build()) { + for (String schemaName : createdSchemas) { + try { + glueClient.deleteSchema( + DeleteSchemaRequest.builder() + .schemaId( + SchemaId.builder() + .registryName(REGISTRY_NAME) + .schemaName(schemaName) + .build()) + .build()); + LOG.info("Deleted GSR schema {}", schemaName); + } catch (Exception e) { + LOG.warn( + "Best-effort cleanup failed for schema {}: {}", + schemaName, + e.getMessage()); + } + } + } catch (Exception e) { + LOG.warn("Best-effort GSR schema cleanup skipped: {}", e.getMessage()); + } + createdSchemas.clear(); + } + + private String uniqueId(String scenario) { + return "gsr_pb_sql_" + scenario + "_" + Long.toHexString(System.nanoTime()); + } + + private String schemaName(String id) { + String schemaName = "flink-protobuf-glue-e2e-" + id; + createdSchemas.add(schemaName); + return schemaName; + } + + private String streamArn(String streamName) { + return "arn:aws:kinesis:" + + KINESIS_REGION + + ":" + + KINESIS_ACCOUNT + + ":stream/" + + streamName; + } + + private static String envOrDefault(String name, String defaultValue) { + String value = System.getenv(name); + return (value == null || value.trim().isEmpty()) ? defaultValue : value; + } +} diff --git a/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/test/resources/log4j2-test.properties b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/test/resources/log4j2-test.properties new file mode 100644 index 00000000..bc8557fd --- /dev/null +++ b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/test/resources/log4j2-test.properties @@ -0,0 +1,38 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Set root logger level to OFF to not flood build logs +# set manually to INFO for debugging purposes +rootLogger.level = OFF +rootLogger.appenderRef.test.ref = TestLogger + +appender.testlogger.name = TestLogger +appender.testlogger.type = CONSOLE +appender.testlogger.target = SYSTEM_ERR +appender.testlogger.layout.type = PatternLayout +appender.testlogger.layout.pattern = %-4r [%t] %-5p %c %x - %m%n + +# GSR protobuf format classes at INFO (for debugging schema fetcher) +logger.gsrformat.name = org.apache.flink.formats.protobuf.glue.schema.registry +logger.gsrformat.level = INFO +logger.gsrformat.appenderRef.test.ref = TestLogger + +# E2E test class +logger.e2e.name = org.apache.flink.glue.schema.registry.test.protobuf +logger.e2e.level = INFO +logger.e2e.appenderRef.test.ref = TestLogger From f6cdaa00bb103570375e8f05fcbf445550b4cbf3 Mon Sep 17 00:00:00 2001 From: Francisco Date: Mon, 10 Aug 2026 11:57:57 +0200 Subject: [PATCH 3/5] test(protobuf-glue): Fix table credential option keys and add missing s3 test dep --- .../pom.xml | 5 +++++ .../protobuf/GlueSchemaRegistryProtobufSqlKinesisITCase.java | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/pom.xml b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/pom.xml index 3ff4c9d6..6741e4dd 100644 --- a/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/pom.xml +++ b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/pom.xml @@ -113,6 +113,11 @@ under the License. kinesis test + + software.amazon.awssdk + s3 + test + software.amazon.awssdk glue diff --git a/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/test/java/org/apache/flink/glue/schema/registry/test/protobuf/GlueSchemaRegistryProtobufSqlKinesisITCase.java b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/test/java/org/apache/flink/glue/schema/registry/test/protobuf/GlueSchemaRegistryProtobufSqlKinesisITCase.java index dbb5b226..ff4614cc 100644 --- a/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/test/java/org/apache/flink/glue/schema/registry/test/protobuf/GlueSchemaRegistryProtobufSqlKinesisITCase.java +++ b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/test/java/org/apache/flink/glue/schema/registry/test/protobuf/GlueSchemaRegistryProtobufSqlKinesisITCase.java @@ -326,8 +326,8 @@ private String kinesisOptions(String streamName, boolean source) { sb.append(" 'aws.region' = '").append(KINESIS_REGION).append("',"); sb.append(" 'aws.endpoint' = '").append(MOCK_KINESIS_CONTAINER.getEndpoint()).append("',"); sb.append(" 'aws.credentials.provider' = 'BASIC',"); - sb.append(" 'aws.credentials.provider.basic.accesskeyid' = 'accessKeyId',"); - sb.append(" 'aws.credentials.provider.basic.secretkey' = 'secretAccessKey',"); + sb.append(" 'aws.credentials.basic.accesskeyid' = 'accessKeyId',"); + sb.append(" 'aws.credentials.basic.secretkey' = 'secretAccessKey',"); sb.append(" 'aws.trust.all.certificates' = 'true'"); if (source) { sb.append(", 'source.init.position' = 'TRIM_HORIZON'"); From 4caff9e61d1f4aa0deae7d9ea57e25a665a97f5f Mon Sep 17 00:00:00 2001 From: Francisco Date: Mon, 10 Aug 2026 12:11:51 +0200 Subject: [PATCH 4/5] test(protobuf-glue): Use HTTP1_1 for Kinesis client against Localstack --- .../protobuf/GlueSchemaRegistryProtobufSqlKinesisITCase.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/test/java/org/apache/flink/glue/schema/registry/test/protobuf/GlueSchemaRegistryProtobufSqlKinesisITCase.java b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/test/java/org/apache/flink/glue/schema/registry/test/protobuf/GlueSchemaRegistryProtobufSqlKinesisITCase.java index ff4614cc..528db78f 100644 --- a/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/test/java/org/apache/flink/glue/schema/registry/test/protobuf/GlueSchemaRegistryProtobufSqlKinesisITCase.java +++ b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/test/java/org/apache/flink/glue/schema/registry/test/protobuf/GlueSchemaRegistryProtobufSqlKinesisITCase.java @@ -328,7 +328,8 @@ private String kinesisOptions(String streamName, boolean source) { sb.append(" 'aws.credentials.provider' = 'BASIC',"); sb.append(" 'aws.credentials.basic.accesskeyid' = 'accessKeyId',"); sb.append(" 'aws.credentials.basic.secretkey' = 'secretAccessKey',"); - sb.append(" 'aws.trust.all.certificates' = 'true'"); + sb.append(" 'aws.trust.all.certificates' = 'true',"); + sb.append(" 'aws.http.protocol.version' = 'HTTP1_1'"); if (source) { sb.append(", 'source.init.position' = 'TRIM_HORIZON'"); } From afaa4835fe30cc57abe9a0c18e5f2e991e297725 Mon Sep 17 00:00:00 2001 From: Francisco Date: Mon, 10 Aug 2026 12:24:49 +0200 Subject: [PATCH 5/5] test(protobuf-glue): Cover pre-existing schema and missing-schema error paths --- ...chemaRegistryProtobufSqlKinesisITCase.java | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) diff --git a/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/test/java/org/apache/flink/glue/schema/registry/test/protobuf/GlueSchemaRegistryProtobufSqlKinesisITCase.java b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/test/java/org/apache/flink/glue/schema/registry/test/protobuf/GlueSchemaRegistryProtobufSqlKinesisITCase.java index 528db78f..13d5552a 100644 --- a/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/test/java/org/apache/flink/glue/schema/registry/test/protobuf/GlueSchemaRegistryProtobufSqlKinesisITCase.java +++ b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/test/java/org/apache/flink/glue/schema/registry/test/protobuf/GlueSchemaRegistryProtobufSqlKinesisITCase.java @@ -52,6 +52,7 @@ import java.util.concurrent.TimeUnit; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.assertj.core.api.Assumptions.assumeThat; /** @@ -76,6 +77,11 @@ * values, exercising the proto3 explicit-presence fix (C2). *
  • {@link #testCompressionRoundTrip()} — same round-trip with {@code * protobuf-glue.schema.compression = ZLIB}, exercising the compression round-trip fix (C1). + *
  • {@link #existingSchemaWithoutAutoRegistration()} — a pre-existing schema is written to by a + * second table with {@code autoRegistration = false}, which must succeed because no + * registration is required. + *
  • {@link #missingSchemaWithoutAutoRegistrationFails()} — writing against a never-registered + * schema with {@code autoRegistration = false} must fail. * */ @Tag("requires-aws-credentials") @@ -293,6 +299,105 @@ void testCompressionRoundTrip() throws Exception { .containsExactlyInAnyOrder("Dave", "Eve", "Frank"); } + @Test + void existingSchemaWithoutAutoRegistration() throws Exception { + String id = uniqueId("existing"); + String schemaName = schemaName(id); + prepareStream(id); + + // Phase A: register the schema (and its first version) in real GSR through a sink with + // autoRegistration = true, landing two rows on the stream. + tEnv.executeSql( + "CREATE TABLE kinesis_sink_existing_a (" + + " user_name STRING," + + " age INT," + + " is_active BOOLEAN" + + ") WITH (" + + kinesisOptions(id, false) + + "," + + protobufGlueOptionsExplicitAutoReg(schemaName, true) + + ")"); + + tEnv.executeSql( + "INSERT INTO kinesis_sink_existing_a VALUES " + + "('Alice', 30, true)," + + "('Bob', 25, false)") + .await(120, TimeUnit.SECONDS); + + // Phase B: a second sink at the SAME schema name with autoRegistration = false must succeed + // because the schema already exists in GSR — no registration is attempted. + tEnv.executeSql( + "CREATE TABLE kinesis_sink_existing_b (" + + " user_name STRING," + + " age INT," + + " is_active BOOLEAN" + + ") WITH (" + + kinesisOptions(id, false) + + "," + + protobufGlueOptionsExplicitAutoReg(schemaName, false) + + ")"); + + tEnv.executeSql( + "INSERT INTO kinesis_sink_existing_b VALUES " + + "('Charlie', 35, true)," + + "('Dave', 40, false)") + .await(120, TimeUnit.SECONDS); + + tEnv.executeSql( + "CREATE TABLE kinesis_source_existing (" + + " user_name STRING," + + " age INT," + + " is_active BOOLEAN" + + ") WITH (" + + kinesisOptions(id, true) + + "," + + protobufGlueOptionsExplicitAutoReg(schemaName, false) + + ")"); + + List rows = + collect("SELECT * FROM kinesis_source_existing", 4, Duration.ofSeconds(90)); + + assertThat(rows) + .as( + "all four rows (two written under autoRegistration=true, two under " + + "autoRegistration=false against the pre-existing schema) must arrive") + .hasSize(4); + assertThat(rows) + .extracting(row -> String.valueOf(row.getField(0))) + .containsExactlyInAnyOrder("Alice", "Bob", "Charlie", "Dave"); + } + + @Test + void missingSchemaWithoutAutoRegistrationFails() throws Exception { + String id = uniqueId("missing"); + String schemaName = schemaName(id); + prepareStream(id); + + // The schema name is never registered in GSR and autoRegistration is disabled, so the sink + // has no schema to serialize against and the INSERT job must fail. + tEnv.executeSql( + "CREATE TABLE kinesis_sink_missing (" + + " user_name STRING," + + " age INT," + + " is_active BOOLEAN" + + ") WITH (" + + kinesisOptions(id, false) + + "," + + protobufGlueOptionsExplicitAutoReg(schemaName, false) + + ")"); + + assertThatThrownBy( + () -> + tEnv.executeSql( + "INSERT INTO kinesis_sink_missing VALUES " + + "('Alice', 30, true)") + .await(120, TimeUnit.SECONDS)) + .as( + "writing against a schema that does not exist in GSR with " + + "autoRegistration=false must fail rather than silently registering it") + .isInstanceOf(Exception.class); + } + private List collect(String selectSql, int expected, Duration timeout) throws Exception { List rows = new ArrayList<>(); try (CloseableIterator iterator = tEnv.executeSql(selectSql).collect()) { @@ -351,6 +456,18 @@ private String protobufGlueOptions(String schemaName, boolean forSink, String co return sb.toString(); } + private String protobufGlueOptionsExplicitAutoReg(String schemaName, boolean autoRegistration) { + StringBuilder sb = new StringBuilder(); + sb.append(" 'format' = 'protobuf-glue',"); + sb.append(" 'protobuf-glue.aws.region' = '").append(GSR_REGION).append("',"); + sb.append(" 'protobuf-glue.registry.name' = '").append(REGISTRY_NAME).append("',"); + sb.append(" 'protobuf-glue.schema.name' = '").append(schemaName).append("',"); + sb.append(" 'protobuf-glue.schema.autoRegistration' = '") + .append(autoRegistration) + .append("'"); + return sb.toString(); + } + private void prepareStream(String streamName) throws Exception { kinesisClient.createStream( CreateStreamRequest.builder().streamName(streamName).shardCount(1).build());