diff --git a/docs/content.zh/docs/connectors/table/formats/_index.md b/docs/content.zh/docs/connectors/table/formats/_index.md new file mode 100644 index 000000000..531814b3e --- /dev/null +++ b/docs/content.zh/docs/connectors/table/formats/_index.md @@ -0,0 +1,33 @@ +--- +title: "Formats" +weight: 10 +type: docs +--- + + +# Formats + +This section describes the format factories available for use with the AWS connectors (Kinesis, Firehose, etc.) in Flink SQL. + +## AWS Glue Schema Registry Formats + +- [Avro (avro-glue)]({{< ref "docs/connectors/table/formats/avro-glue" >}}) +- [JSON (json-glue)]({{< ref "docs/connectors/table/formats/json-glue" >}}) +- [Protobuf (protobuf-glue)]({{< ref "docs/connectors/table/formats/protobuf-glue" >}}) diff --git a/docs/content/docs/connectors/table/formats/_index.md b/docs/content/docs/connectors/table/formats/_index.md new file mode 100644 index 000000000..f1f3e48bf --- /dev/null +++ b/docs/content/docs/connectors/table/formats/_index.md @@ -0,0 +1,35 @@ +--- +title: "Formats" +weight: 10 +type: docs +--- + + +# Formats + +This section describes the format factories available for use with the AWS connectors (Kinesis, Firehose, etc.) in Flink SQL. + +## AWS Glue Schema Registry Formats + +The following formats integrate with [AWS Glue Schema Registry](https://docs.aws.amazon.com/glue/latest/dg/schema-registry.html) for schema management: + +- [Avro (avro-glue)]({{< ref "docs/connectors/table/formats/avro-glue" >}}) +- [JSON (json-glue)]({{< ref "docs/connectors/table/formats/json-glue" >}}) +- [Protobuf (protobuf-glue)]({{< ref "docs/connectors/table/formats/protobuf-glue" >}}) diff --git a/docs/content/docs/connectors/table/formats/avro-glue.md b/docs/content/docs/connectors/table/formats/avro-glue.md new file mode 100644 index 000000000..630c78493 --- /dev/null +++ b/docs/content/docs/connectors/table/formats/avro-glue.md @@ -0,0 +1,376 @@ +--- +title: "Avro (Glue Schema Registry)" +weight: 1 +type: docs +--- + + +# Avro Format (AWS Glue Schema Registry) + +{{< label "Format: Serialization Schema" >}} +{{< label "Format: Deserialization Schema" >}} + +The Avro Glue Schema Registry format (`avro-glue`) allows you to read and write Avro data with schemas managed by [AWS Glue Schema Registry](https://docs.aws.amazon.com/glue/latest/dg/schema-registry.html). + +Dependencies +------------ + +{{< sql_connector_download_table "avro-glue" >}} + +The Avro-Glue format is not part of the binary distribution. +See how to link with it for cluster execution [here]({{< ref "docs/dev/configuration/overview" >}}). + +#### SQL Client JAR + +For SQL Client usage, download the fat JAR `flink-sql-avro-glue-schema-registry` from the table above and place it in the `lib/` directory of your Flink installation. The SQL JAR bundles all required dependencies including the AWS Glue Schema Registry serializer/deserializer libraries. + +#### Maven Dependency + +To use the format in a DataStream or Table API program, add the following dependency to your project: + +```xml + + org.apache.flink + flink-avro-glue-schema-registry + 6.0.0 + +``` + +How to create a table with Avro-Glue format +-------------------------------------------- + +Here is an example to create a table using the Kinesis connector with the Avro-Glue format: + +```sql +CREATE TABLE KinesisTable ( + `user_id` BIGINT, + `item_id` BIGINT, + `category` STRING, + `behavior` STRING, + `ts` TIMESTAMP(3) +) WITH ( + 'connector' = 'kinesis', + 'stream.arn' = 'arn:aws:kinesis:us-east-1:012345678901:stream/my-stream', + 'aws.region' = 'us-east-1', + 'source.init.position' = 'LATEST', + 'format' = 'avro-glue', + 'avro-glue.aws.region' = 'us-east-1', + 'avro-glue.registry.name' = 'my-registry', + 'avro-glue.schema.name' = 'my-avro-schema' +); +``` + + +Schema Namespace Override +------------------------- + +Flink's `AvroSchemaConverter` auto-generates an Avro schema from the SQL table definition. The generated schema uses a default namespace (`org.apache.flink.avro.generated`) and record name (`record`) that may differ from schemas already registered in Glue Schema Registry. + +If your GSR registry already contains a schema with a specific namespace or record name, you can override the auto-generated values using the `avro.namespace` and `avro.record-name` options: + +```sql +CREATE TABLE KinesisTable ( + `user_id` BIGINT, + `item_id` BIGINT, + `category` STRING +) WITH ( + 'connector' = 'kinesis', + 'stream.arn' = 'arn:aws:kinesis:us-east-1:012345678901:stream/my-stream', + 'aws.region' = 'us-east-1', + 'format' = 'avro-glue', + 'avro-glue.aws.region' = 'us-east-1', + 'avro-glue.registry.name' = 'my-registry', + 'avro-glue.schema.name' = 'my-avro-schema', + 'avro-glue.avro.namespace' = 'com.mycompany.events', + 'avro-glue.avro.record-name' = 'UserEvent' +); +``` + +Alternatively, you can set `schema.fetchFromRegistry` to `true` to fetch the schema directly from GSR at runtime instead of using the auto-generated one: + +```sql +CREATE TABLE KinesisTable ( + `user_id` BIGINT, + `item_id` BIGINT, + `category` STRING +) WITH ( + 'connector' = 'kinesis', + 'stream.arn' = 'arn:aws:kinesis:us-east-1:012345678901:stream/my-stream', + 'aws.region' = 'us-east-1', + 'format' = 'avro-glue', + 'avro-glue.aws.region' = 'us-east-1', + 'avro-glue.registry.name' = 'my-registry', + 'avro-glue.schema.name' = 'my-avro-schema', + 'avro-glue.schema.fetchFromRegistry' = 'true' +); +``` + +If the schema is not found in GSR, the format falls back to the auto-generated schema (optionally patched with `avro.namespace` and `avro.record-name` if provided). + +Format Options +-------------- + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
OptionRequiredForwardedDefaultTypeDescription
format
requiredno(none)StringSpecify the format identifier. Use 'avro-glue'.
avro-glue.aws.region
requiredyes(none)StringAWS region for the Glue Schema Registry.
avro-glue.registry.name
requiredyes(none)StringName of the Glue Schema Registry.
avro-glue.schema.name
requiredyes(none)StringSchema name under which to register/look up the schema in Glue Schema Registry.
avro-glue.aws.endpoint
optionalyes(none)StringCustom AWS endpoint URL for Glue Schema Registry.
avro-glue.schema.type
optionalyesGENERIC_RECORDStringAvro record type. Supported values: GENERIC_RECORD, SPECIFIC_RECORD.
avro-glue.avro.namespace
optionalyes(none)StringOverride the namespace in the auto-generated Avro schema. Use this to match schemas already registered in GSR with a different namespace.
avro-glue.avro.record-name
optionalyes(none)StringOverride the record name in the auto-generated Avro schema. Use this to match schemas already registered in GSR with a different record name.
avro-glue.schema.fetchFromRegistry
optionalyesfalseBooleanWhether to fetch the schema from GSR instead of using the auto-generated one. Falls back to auto-generated schema if the schema is not found.
avro-glue.cache.size
optionalyes200IntegerMaximum number of items in the schema cache.
avro-glue.cache.ttlMs
optionalyes86400000LongCache TTL in milliseconds. Defaults to 1 day (86400000 ms).
avro-glue.schema.autoRegistration
optionalyesfalseBooleanWhether to auto-register schemas with Glue Schema Registry when writing data.
avro-glue.schema.compatibility
optionalyesNONEStringSchema compatibility mode. Supported values: NONE, DISABLED, BACKWARD, BACKWARD_ALL, FORWARD, FORWARD_ALL, FULL, FULL_ALL.
avro-glue.schema.compression
optionalyesNONEStringCompression type for schema data. Supported values: NONE, ZLIB.
+ +Data Type Mapping +----------------- + +The Avro-Glue format uses Flink's built-in `AvroSchemaConverter` to map between Flink SQL types and Avro types. The mapping follows the same rules as the standard Flink Avro format: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Flink SQL TypeAvro Type
BOOLEANboolean
TINYINT / SMALLINT / INTint
BIGINTlong
FLOATfloat
DOUBLEdouble
STRINGstring
BYTESbytes
DECIMALbytes (logical type: decimal)
DATEint (logical type: date)
TIMEint (logical type: time-millis)
TIMESTAMPlong (logical type: timestamp-millis)
ARRAYarray
MAP (key must be STRING)map
ROWrecord
+ +{{< hint info >}} +Nullable Flink SQL types are mapped to Avro union types `["null", "type"]`. Flink SQL types are nullable by default, so the auto-generated Avro schema will use union types for all fields unless `NOT NULL` constraints are specified. +{{< /hint >}} + +Usage with Kinesis and Firehose Connectors +------------------------------------------ + +The Avro-Glue format can be used with any Flink SQL connector that supports custom formats. Here are examples with the Kinesis and Firehose connectors: + +### Kinesis Source + +```sql +CREATE TABLE KinesisSource ( + `user_id` BIGINT, + `event_type` STRING, + `payload` STRING, + `event_time` TIMESTAMP(3) +) WITH ( + 'connector' = 'kinesis', + 'stream.arn' = 'arn:aws:kinesis:us-east-1:012345678901:stream/events', + 'aws.region' = 'us-east-1', + 'source.init.position' = 'LATEST', + 'format' = 'avro-glue', + 'avro-glue.aws.region' = 'us-east-1', + 'avro-glue.registry.name' = 'my-registry', + 'avro-glue.schema.name' = 'events-avro' +); +``` + +### Firehose Sink + +```sql +CREATE TABLE FirehoseSink ( + `user_id` BIGINT, + `event_type` STRING, + `payload` STRING, + `event_time` TIMESTAMP(3) +) WITH ( + 'connector' = 'firehose', + 'delivery-stream' = 'my-delivery-stream', + 'aws.region' = 'us-east-1', + 'format' = 'avro-glue', + 'avro-glue.aws.region' = 'us-east-1', + 'avro-glue.registry.name' = 'my-registry', + 'avro-glue.schema.name' = 'events-avro', + 'avro-glue.schema.autoRegistration' = 'true' +); +``` diff --git a/docs/content/docs/connectors/table/formats/protobuf-glue.md b/docs/content/docs/connectors/table/formats/protobuf-glue.md new file mode 100644 index 000000000..ac6147e5a --- /dev/null +++ b/docs/content/docs/connectors/table/formats/protobuf-glue.md @@ -0,0 +1,283 @@ +--- +title: "Protobuf (Glue Schema Registry)" +weight: 3 +type: docs +--- + + +# Protobuf Format (AWS Glue Schema Registry) + +{{< label "Format: Serialization Schema" >}} +{{< label "Format: Deserialization Schema" >}} + +The Protobuf Glue Schema Registry format (`protobuf-glue`) allows you to read and write Protocol Buffers data with schemas managed by [AWS Glue Schema Registry](https://docs.aws.amazon.com/glue/latest/dg/schema-registry.html). + +Dependencies +------------ + +{{< sql_connector_download_table "protobuf-glue" >}} + +The Protobuf-Glue format is not part of the binary distribution. +See how to link with it for cluster execution [here]({{< ref "docs/dev/configuration/overview" >}}). + +#### SQL Client JAR + +For SQL Client usage, download the fat JAR `flink-sql-protobuf-glue-schema-registry` from the table above and place it in the `lib/` directory of your Flink installation. The SQL JAR bundles all required dependencies including the AWS Glue Schema Registry serializer/deserializer libraries. + +#### Maven Dependency + +To use the format in a DataStream or Table API program, add the following dependency to your project: + +```xml + + org.apache.flink + flink-protobuf-glue-schema-registry + 6.0.0 + +``` + +How to create a table with Protobuf-Glue format +------------------------------------------------- + +Here is an example to create a table using the Kinesis connector with the Protobuf-Glue format: + +```sql +CREATE TABLE KinesisTable ( + `user_id` BIGINT, + `item_id` BIGINT, + `category` STRING, + `behavior` STRING, + `ts` TIMESTAMP(3) +) WITH ( + 'connector' = 'kinesis', + 'stream.arn' = 'arn:aws:kinesis:us-east-1:012345678901:stream/my-stream', + 'aws.region' = 'us-east-1', + 'source.init.position' = 'LATEST', + 'format' = 'protobuf-glue', + 'protobuf-glue.aws.region' = 'us-east-1', + 'protobuf-glue.registry.name' = 'my-registry', + 'protobuf-glue.schema.name' = 'my-protobuf-schema' +); +``` + + +Protobuf Descriptor Auto-Generation +------------------------------------- + +When writing data (sink), the Protobuf-Glue format automatically generates a Protobuf descriptor (`.proto` schema definition) from the Flink table schema and registers it with Glue Schema Registry using `DataFormat.PROTOBUF`. + +When reading data (source), the format strips the GSR header bytes (18 bytes: 1 header version byte + 1 compression byte + 16 UUID bytes) from incoming records and deserializes the Protobuf payload into Flink `RowData`. + +Format Options +-------------- + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
OptionRequiredForwardedDefaultTypeDescription
format
requiredno(none)StringSpecify the format identifier. Use 'protobuf-glue'.
protobuf-glue.aws.region
requiredyes(none)StringAWS region for the Glue Schema Registry.
protobuf-glue.registry.name
requiredyes(none)StringName of the Glue Schema Registry.
protobuf-glue.schema.name
requiredyes(none)StringSchema name under which to register/look up the schema in Glue Schema Registry.
protobuf-glue.aws.endpoint
optionalyes(none)StringCustom AWS endpoint URL for Glue Schema Registry.
protobuf-glue.cache.size
optionalyes200IntegerMaximum number of items in the schema cache.
protobuf-glue.cache.ttlMs
optionalyes86400000LongCache TTL in milliseconds. Defaults to 1 day (86400000 ms).
protobuf-glue.schema.autoRegistration
optionalyesfalseBooleanWhether to auto-register schemas with Glue Schema Registry when writing data.
protobuf-glue.schema.compatibility
optionalyesNONEStringSchema compatibility mode. Supported values: NONE, DISABLED, BACKWARD, BACKWARD_ALL, FORWARD, FORWARD_ALL, FULL, FULL_ALL.
protobuf-glue.schema.compression
optionalyesNONEStringCompression type for schema data. Supported values: NONE, ZLIB.
+ +Data Type Mapping +----------------- + +The Protobuf-Glue format maps between Flink SQL types and Protobuf types as follows: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Flink SQL TypeProtobuf Type
BOOLEANbool
INTint32
BIGINTint64
FLOATfloat
DOUBLEdouble
STRINGstring
BYTESbytes
ARRAYrepeated
MAPmap
ROWmessage (nested)
+ +Usage with Kinesis and Firehose Connectors +------------------------------------------ + +### Kinesis Source + +```sql +CREATE TABLE KinesisSource ( + `user_id` BIGINT, + `event_type` STRING, + `payload` STRING, + `event_time` TIMESTAMP(3) +) WITH ( + 'connector' = 'kinesis', + 'stream.arn' = 'arn:aws:kinesis:us-east-1:012345678901:stream/events', + 'aws.region' = 'us-east-1', + 'source.init.position' = 'LATEST', + 'format' = 'protobuf-glue', + 'protobuf-glue.aws.region' = 'us-east-1', + 'protobuf-glue.registry.name' = 'my-registry', + 'protobuf-glue.schema.name' = 'events-protobuf' +); +``` + +### Firehose Sink + +```sql +CREATE TABLE FirehoseSink ( + `user_id` BIGINT, + `event_type` STRING, + `payload` STRING, + `event_time` TIMESTAMP(3) +) WITH ( + 'connector' = 'firehose', + 'delivery-stream' = 'my-delivery-stream', + 'aws.region' = 'us-east-1', + 'format' = 'protobuf-glue', + 'protobuf-glue.aws.region' = 'us-east-1', + 'protobuf-glue.registry.name' = 'my-registry', + 'protobuf-glue.schema.name' = 'events-protobuf', + 'protobuf-glue.schema.autoRegistration' = 'true' +); +``` diff --git a/docs/data/avro-glue.yml b/docs/data/avro-glue.yml new file mode 100644 index 000000000..a84bfcc2a --- /dev/null +++ b/docs/data/avro-glue.yml @@ -0,0 +1,23 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +version: 6.0.0 +flink_compatibility: [ "2.0" ] +variants: + - maven: flink-avro-glue-schema-registry + sql_url: https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-avro-glue-schema-registry/$full_version/flink-sql-avro-glue-schema-registry-$full_version.jar diff --git a/docs/data/protobuf-glue.yml b/docs/data/protobuf-glue.yml new file mode 100644 index 000000000..a759a0245 --- /dev/null +++ b/docs/data/protobuf-glue.yml @@ -0,0 +1,23 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +version: 6.0.0 +flink_compatibility: [ "2.0" ] +variants: + - maven: flink-protobuf-glue-schema-registry + sql_url: https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-protobuf-glue-schema-registry/$full_version/flink-sql-protobuf-glue-schema-registry-$full_version.jar diff --git a/flink-connector-aws-e2e-tests/flink-formats-avro-glue-schema-registry-e2e-tests/pom.xml b/flink-connector-aws-e2e-tests/flink-formats-avro-glue-schema-registry-e2e-tests/pom.xml index 3fedf34b9..6a1963e73 100644 --- a/flink-connector-aws-e2e-tests/flink-formats-avro-glue-schema-registry-e2e-tests/pom.xml +++ b/flink-connector-aws-e2e-tests/flink-formats-avro-glue-schema-registry-e2e-tests/pom.xml @@ -33,22 +33,92 @@ under the License. jar + org.apache.flink flink-avro-glue-schema-registry ${project.version} + + + + + org.apache.flink + flink-table-api-java-bridge + ${flink.version} + + + org.apache.flink + flink-table-planner-loader + ${flink.version} + + + org.apache.flink + flink-table-runtime + ${flink.version} + + + + + org.apache.flink + flink-connector-aws-kinesis-streams + ${project.version} + + + + + software.amazon.awssdk + kinesis + + + + + software.amazon.awssdk + sts + + + + + software.amazon.awssdk + netty-nio-client + + + + + software.amazon.awssdk + s3 test + + org.apache.flink flink-connector-aws-base ${project.version} - test-jar - test + + org.apache.flink - flink-connector-aws-kinesis-streams + flink-avro + ${flink.version} + + + + + org.apache.flink + flink-streaming-java + ${flink.version} + + + org.apache.flink + flink-clients + ${flink.version} + + + + + org.apache.flink + flink-connector-aws-base ${project.version} test-jar test @@ -57,12 +127,48 @@ under the License. org.apache.flink flink-connector-aws-kinesis-streams ${project.version} + test-jar test + + - software.amazon.awssdk - s3 + org.testcontainers + localstack test + + + + org.slf4j + slf4j-api + + + org.apache.logging.log4j + log4j-slf4j-impl + 2.24.1 + + + org.apache.logging.log4j + log4j-core + 2.24.1 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + **/GlueSchemaRegistryAvroKinesisITCase.java + **/GSRKinesisPubsubClient.java + + + + + diff --git a/flink-connector-aws-e2e-tests/flink-formats-avro-glue-schema-registry-e2e-tests/src/main/resources/log4j2.properties b/flink-connector-aws-e2e-tests/flink-formats-avro-glue-schema-registry-e2e-tests/src/main/resources/log4j2.properties new file mode 100644 index 000000000..cac90c928 --- /dev/null +++ b/flink-connector-aws-e2e-tests/flink-formats-avro-glue-schema-registry-e2e-tests/src/main/resources/log4j2.properties @@ -0,0 +1,41 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +rootLogger.level = INFO +rootLogger.appenderRef.console.ref = ConsoleAppender + +appender.console.name = ConsoleAppender +appender.console.type = Console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n + +# Reduce noise from Flink internals +logger.flink.name = org.apache.flink +logger.flink.level = WARN + +# Our test class at INFO +logger.e2e.name = org.apache.flink.glue.schema.registry.test +logger.e2e.level = INFO + +# GSR format classes at INFO (for debugging) +logger.gsrformat.name = org.apache.flink.formats.avro.glue.schema.registry +logger.gsrformat.level = INFO + +# GSR SDK +logger.gsr.name = software.amazon.awssdk +logger.gsr.level = WARN diff --git a/flink-connector-aws-e2e-tests/flink-formats-avro-glue-schema-registry-e2e-tests/src/test/java/org/apache/flink/glue/schema/registry/test/GlueSchemaRegistryAvroSqlKinesisITCase.java b/flink-connector-aws-e2e-tests/flink-formats-avro-glue-schema-registry-e2e-tests/src/test/java/org/apache/flink/glue/schema/registry/test/GlueSchemaRegistryAvroSqlKinesisITCase.java new file mode 100644 index 000000000..15376f6d7 --- /dev/null +++ b/flink-connector-aws-e2e-tests/flink-formats-avro-glue-schema-registry-e2e-tests/src/test/java/org/apache/flink/glue/schema/registry/test/GlueSchemaRegistryAvroSqlKinesisITCase.java @@ -0,0 +1,679 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.glue.schema.registry.test; + +import org.apache.flink.api.common.time.Deadline; +import org.apache.flink.connector.aws.testutils.AWSServicesTestUtils; +import org.apache.flink.connector.aws.testutils.LocalstackContainer; +import org.apache.flink.connector.aws.util.AWSGeneralUtil; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; +import org.apache.flink.test.junit5.MiniClusterExtension; +import org.apache.flink.types.Row; +import org.apache.flink.util.CloseableIterator; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.utility.DockerImageName; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.core.SdkSystemSetting; +import software.amazon.awssdk.http.SdkHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.glue.GlueClient; +import software.amazon.awssdk.services.glue.model.DeleteSchemaRequest; +import software.amazon.awssdk.services.glue.model.SchemaId; +import software.amazon.awssdk.services.kinesis.KinesisClient; +import software.amazon.awssdk.services.kinesis.model.CreateStreamRequest; +import software.amazon.awssdk.services.kinesis.model.DescribeStreamRequest; +import software.amazon.awssdk.services.kinesis.model.StreamStatus; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assumptions.assumeThat; + +/** + * End-to-end test for the {@code avro-glue} Flink SQL format factory. + * + *

This is the SQL-path counterpart of {@link GlueSchemaRegistryAvroKinesisITCase} (which + * exercises the DataStream API). Kinesis I/O runs against a Localstack container; the Glue Schema + * Registry calls go to real AWS. The class is gated on the {@code + * IT_CASE_GLUE_SCHEMA_ACCESS_KEY} / {@code IT_CASE_GLUE_SCHEMA_SECRET_KEY} environment variables + * and tagged {@code requires-aws-credentials}, so it is excluded from the credential-free {@code + * run-end-to-end-tests} profile and only runs under {@code run-aws-end-to-end-tests}. Without + * credentials it skips cleanly (the container is never started). + * + *

Each test uses its own Localstack Kinesis stream and a run-unique GSR schema name (timestamp + * suffix) so re-runs do not collide. Created GSR schemas are best-effort deleted in {@link + * #afterAll()}. + */ +@ExtendWith(MiniClusterExtension.class) +@Tag("requires-aws-credentials") +class GlueSchemaRegistryAvroSqlKinesisITCase { + + private static final Logger LOG = + LoggerFactory.getLogger(GlueSchemaRegistryAvroSqlKinesisITCase.class); + + private static final String ACCESS_KEY = System.getenv("IT_CASE_GLUE_SCHEMA_ACCESS_KEY"); + private static final String SECRET_KEY = System.getenv("IT_CASE_GLUE_SCHEMA_SECRET_KEY"); + + /** Region for the real Glue Schema Registry calls. Overridable for the test account. */ + private static final String GSR_REGION = + envOrDefault("IT_CASE_GLUE_SCHEMA_REGION", "ca-central-1"); + + /** Registry that backs the schemas; GSR's implicit default registry when unset. */ + private static final String REGISTRY_NAME = + envOrDefault("IT_CASE_GLUE_SCHEMA_REGISTRY", "default-registry"); + + /** Region used for the Localstack Kinesis endpoint (mirrors the DataStream ITCase). */ + private static final String KINESIS_REGION = "ap-southeast-1"; + + private static final String LOCALSTACK_DOCKER_IMAGE_VERSION = "localstack/localstack:3.7.2"; + + /** Unique per JVM run so parallel/repeat runs never reuse a GSR schema name. */ + private static final String RUN_ID = String.valueOf(System.currentTimeMillis()); + + /** GSR schema names created during the run, for best-effort teardown. */ + private static final Set CREATED_SCHEMAS = new LinkedHashSet<>(); + + private static final LocalstackContainer LOCALSTACK = + new LocalstackContainer(DockerImageName.parse(LOCALSTACK_DOCKER_IMAGE_VERSION)) + .withNetworkAliases("localstack"); + + private static SdkHttpClient httpClient; + private static KinesisClient kinesisClient; + + private StreamTableEnvironment tEnv; + + @BeforeAll + static void beforeAll() { + assumeThat(ACCESS_KEY) + .as("IT_CASE_GLUE_SCHEMA_ACCESS_KEY not configured, skipping test") + .isNotBlank(); + assumeThat(SECRET_KEY) + .as("IT_CASE_GLUE_SCHEMA_SECRET_KEY not configured, skipping test") + .isNotBlank(); + + System.setProperty(SdkSystemSetting.CBOR_ENABLED.property(), "false"); + + LOCALSTACK.start(); + httpClient = AWSServicesTestUtils.createHttpClient(); + kinesisClient = + AWSServicesTestUtils.createAwsSyncClient( + LOCALSTACK.getEndpoint(), httpClient, KinesisClient.builder()); + LOG.info("Localstack Kinesis endpoint ready at {}", LOCALSTACK.getEndpoint()); + } + + @AfterAll + static void afterAll() { + deleteCreatedSchemas(); + AWSGeneralUtil.closeResources(httpClient, kinesisClient); + if (LOCALSTACK.isRunning()) { + LOCALSTACK.stop(); + } + System.clearProperty(SdkSystemSetting.CBOR_ENABLED.property()); + } + + @BeforeEach + void setUp() { + // The SQL format's GSR client resolves credentials from the default chain inside the + // MiniCluster JVM; expose the real IT credentials via system properties. + System.setProperty(SdkSystemSetting.AWS_ACCESS_KEY_ID.property(), ACCESS_KEY); + System.setProperty(SdkSystemSetting.AWS_SECRET_ACCESS_KEY.property(), SECRET_KEY); + System.setProperty(SdkSystemSetting.AWS_REGION.property(), GSR_REGION); + + StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); + env.setParallelism(1); + tEnv = StreamTableEnvironment.create(env); + } + + @AfterEach + void tearDown() { + System.clearProperty(SdkSystemSetting.AWS_ACCESS_KEY_ID.property()); + System.clearProperty(SdkSystemSetting.AWS_SECRET_ACCESS_KEY.property()); + System.clearProperty(SdkSystemSetting.AWS_REGION.property()); + } + + // --------------------------------------------------------------------------------------------- + // Scenarios (ported one-to-one from the AvroGlueSqlE2E manual driver) + // --------------------------------------------------------------------------------------------- + + @Test + void basicRoundTrip() throws Exception { + String streamArn = createStream("gsr_avro_sql_basic"); + String schemaName = schemaName("basic"); + String columns = "user_name STRING, favorite_number INT, favorite_color STRING"; + List formatOpts = + autoRegOpts(schemaName, "avro-glue.schema.autoRegistration", "true"); + + createKinesisTable("basic_sink", columns, streamArn, false, formatOpts); + tEnv.executeSql( + "INSERT INTO basic_sink VALUES " + + "('Alice', 42, 'blue')," + + "('Bob', 7, 'green')," + + "('Charlie', 99, 'red')") + .await(120, TimeUnit.SECONDS); + + createKinesisTable("basic_source", columns, streamArn, true, sourceOpts(schemaName)); + List rows = collect("SELECT * FROM basic_source", 3, Duration.ofSeconds(90)); + + List names = firstFieldStrings(rows); + assertThat(names).contains("Alice", "Bob", "Charlie"); + } + + @Test + void customNamespaceAndRecordName() throws Exception { + String streamArn = createStream("gsr_avro_sql_custom_ns"); + String schemaName = schemaName("custom-ns"); + String columns = "user_name STRING, favorite_number INT, favorite_color STRING"; + List formatOpts = + autoRegOpts( + schemaName, + "avro-glue.schema.autoRegistration", + "true", + "avro-glue.avro.namespace", + "com.example.events", + "avro-glue.avro.record-name", + "UserEvent"); + + createKinesisTable("custom_ns_sink", columns, streamArn, false, formatOpts); + tEnv.executeSql( + "INSERT INTO custom_ns_sink VALUES " + + "('Dave', 13, 'yellow')," + + "('Eve', 55, 'purple')") + .await(120, TimeUnit.SECONDS); + + createKinesisTable("custom_ns_source", columns, streamArn, true, sourceOpts(schemaName)); + List rows = collect("SELECT * FROM custom_ns_source", 2, Duration.ofSeconds(90)); + + assertThat(firstFieldStrings(rows)).contains("Dave", "Eve"); + } + + @Test + void fetchFromRegistry() throws Exception { + String streamArn = createStream("gsr_avro_sql_fetch"); + String schemaName = schemaName("fetch"); + String columns = "user_name STRING, favorite_number INT, favorite_color STRING"; + + // Seed the schema in GSR (explicit namespace/record-name) via auto-registration. + createKinesisTable( + "fetch_seed_sink", + columns, + streamArn, + false, + autoRegOpts( + schemaName, + "avro-glue.schema.autoRegistration", + "true", + "avro-glue.avro.namespace", + "com.example.events", + "avro-glue.avro.record-name", + "UserEvent")); + tEnv.executeSql("INSERT INTO fetch_seed_sink VALUES ('Seed', 0, 'none')") + .await(120, TimeUnit.SECONDS); + + // Write with fetchFromRegistry=true (no explicit namespace) — schema fetched from GSR. + createKinesisTable( + "fetch_sink", + columns, + streamArn, + false, + autoRegOpts(schemaName, "avro-glue.schema.fetchFromRegistry", "true")); + tEnv.executeSql( + "INSERT INTO fetch_sink VALUES " + + "('Frank', 21, 'orange')," + + "('Grace', 33, 'pink')") + .await(120, TimeUnit.SECONDS); + + createKinesisTable("fetch_source", columns, streamArn, true, sourceOpts(schemaName)); + List rows = collect("SELECT * FROM fetch_source", 3, Duration.ofSeconds(90)); + + assertThat(firstFieldStrings(rows)).contains("Frank", "Grace"); + } + + @Test + void complexTypes() throws Exception { + String streamArn = createStream("gsr_avro_sql_complex"); + String schemaName = schemaName("complex"); + String columns = + "order_id STRING," + + "order_time TIMESTAMP(3)," + + "total_amount DECIMAL(10, 2)," + + "customer ROW," + + "items ARRAY>," + + "tags ARRAY," + + "metadata MAP," + + "notes STRING"; + + createKinesisTable( + "complex_sink", + columns, + streamArn, + false, + autoRegOpts( + schemaName, + "avro-glue.schema.autoRegistration", + "true", + "avro-glue.avro.namespace", + "com.example.orders", + "avro-glue.avro.record-name", + "OrderEvent")); + + tEnv.executeSql( + "INSERT INTO complex_sink VALUES (" + + " 'ORD-001'," + + " TIMESTAMP '2024-01-15 10:30:00'," + + " CAST(199.99 AS DECIMAL(10, 2))," + + " ROW('John Doe', 'john@example.com', 35)," + + " ARRAY[ROW('Widget', 2, CAST(49.99 AS DECIMAL(8, 2))), " + + " ROW('Gadget', 1, CAST(99.99 AS DECIMAL(8, 2)))]," + + " ARRAY['priority', 'express']," + + " MAP['source', 'web', 'campaign', 'summer-sale']," + + " 'Handle with care'" + + ")") + .await(120, TimeUnit.SECONDS); + tEnv.executeSql( + "INSERT INTO complex_sink VALUES (" + + " 'ORD-002'," + + " TIMESTAMP '2024-01-15 11:45:00'," + + " CAST(75.50 AS DECIMAL(10, 2))," + + " ROW('Jane Smith', 'jane@example.com', 28)," + + " ARRAY[ROW('Gizmo', 3, CAST(25.00 AS DECIMAL(8, 2)))]," + + " ARRAY['standard']," + + " MAP['source', 'mobile']," + + " CAST(NULL AS STRING)" + + ")") + .await(120, TimeUnit.SECONDS); + + createKinesisTable("complex_source", columns, streamArn, true, sourceOpts(schemaName)); + List rows = collect("SELECT * FROM complex_source", 2, Duration.ofSeconds(90)); + + Row ord001 = findByOrderId(rows, "ORD-001"); + Row ord002 = findByOrderId(rows, "ORD-002"); + assertThat(ord001).as("ORD-001 present").isNotNull(); + assertThat(ord002).as("ORD-002 present").isNotNull(); + + Row customer = (Row) ord001.getField(3); + assertThat(customer).isNotNull(); + assertThat(customer.getField(0)).isEqualTo("John Doe"); + assertThat(ord001.getField(7)).as("ORD-001 notes not null").isNotNull(); + assertThat(ord002.getField(7)).as("ORD-002 notes null (nullable)").isNull(); + } + + @Test + void compatibilityBackwardRejectsNewRequiredField() throws Exception { + String streamArn = createStream("gsr_avro_sql_compat_bw"); + String schemaName = schemaName("compat-backward"); + + // v1: all fields NOT NULL -> required Avro fields. + createKinesisTable( + "compat_bw_v1", + "user_name STRING NOT NULL, age INT NOT NULL, city STRING NOT NULL", + streamArn, + false, + compatOpts(schemaName, "BACKWARD")); + tEnv.executeSql("INSERT INTO compat_bw_v1 VALUES ('Alice', 30, 'Seattle')") + .await(120, TimeUnit.SECONDS); + + // v2: add a nullable field -> backward compatible. + createKinesisTable( + "compat_bw_v2", + "user_name STRING NOT NULL, age INT NOT NULL, city STRING NOT NULL, email STRING", + streamArn, + false, + compatOpts(schemaName, "BACKWARD")); + tEnv.executeSql( + "INSERT INTO compat_bw_v2 VALUES ('Bob', 25, 'Portland', 'bob@example.com')") + .await(120, TimeUnit.SECONDS); + + // v3: ADD a required (NOT NULL, no default) field -> BACKWARD violation, + // must be rejected. (Removing a field is backward-compatible in Avro: + // a new reader simply ignores the extra field in old data. The true + // violation is a new required reader field that old data cannot supply.) + createKinesisTable( + "compat_bw_v3", + "user_name STRING NOT NULL, age INT NOT NULL, city STRING NOT NULL," + + " country STRING NOT NULL", + streamArn, + false, + compatOpts(schemaName, "BACKWARD")); + assertThatThrownBy( + () -> + tEnv.executeSql( + "INSERT INTO compat_bw_v3 VALUES" + + " ('Charlie', 35, 'Boston', 'USA')") + .await(120, TimeUnit.SECONDS)) + .as("adding a required field without default must violate BACKWARD compatibility") + .isInstanceOf(Exception.class); + } + + @Test + void compatibilityNoneAllowsAnyEvolution() throws Exception { + String streamArn = createStream("gsr_avro_sql_compat_none"); + String schemaName = schemaName("compat-none"); + + createKinesisTable( + "compat_none_v1", + "user_name STRING, age INT, city STRING", + streamArn, + false, + compatOpts(schemaName, "NONE")); + tEnv.executeSql("INSERT INTO compat_none_v1 VALUES ('Dave', 40, 'Denver')") + .await(120, TimeUnit.SECONDS); + + createKinesisTable( + "compat_none_v2", + "user_name STRING, age INT", + streamArn, + false, + compatOpts(schemaName, "NONE")); + // Removing a field is normally incompatible; NONE must allow it without throwing. + tEnv.executeSql("INSERT INTO compat_none_v2 VALUES ('Eve', 28)") + .await(120, TimeUnit.SECONDS); + } + + @Test + void compatibilityFullAllowsAddingOptional() throws Exception { + String streamArn = createStream("gsr_avro_sql_compat_full"); + String schemaName = schemaName("compat-full"); + + createKinesisTable( + "compat_full_v1", + "user_name STRING, age INT, city STRING", + streamArn, + false, + compatOpts(schemaName, "FULL")); + tEnv.executeSql("INSERT INTO compat_full_v1 VALUES ('Frank', 45, 'Chicago')") + .await(120, TimeUnit.SECONDS); + + createKinesisTable( + "compat_full_v2", + "user_name STRING, age INT, city STRING, email STRING", + streamArn, + false, + compatOpts(schemaName, "FULL")); + // Adding an optional field is safe in both directions -> allowed under FULL. + tEnv.executeSql( + "INSERT INTO compat_full_v2 VALUES ('Grace', 32, 'Boston', 'grace@example.com')") + .await(120, TimeUnit.SECONDS); + } + + @Test + void existingSchemaWithoutAutoRegistration() throws Exception { + String schemaName = schemaName("existing-no-autoreg"); + String columns = "user_name STRING, favorite_number INT, favorite_color STRING"; + + // Phase A: register the schema (and a first version) by writing through a table that + // auto-registers it. This mirrors a one-time governed schema-onboarding step. + String seedStreamArn = createStream("gsr_avro_sql_existing_seed"); + createKinesisTable( + "existing_seed_sink", + columns, + seedStreamArn, + false, + autoRegOpts(schemaName, "avro-glue.schema.autoRegistration", "true")); + tEnv.executeSql( + "INSERT INTO existing_seed_sink VALUES " + + "('Heidi', 11, 'teal')," + + "('Ivan', 22, 'olive')") + .await(120, TimeUnit.SECONDS); + + // Phase B: a NEW stream for isolation, SAME schema name, but autoRegistration=false. The + // write must succeed because the schema and a compatible version already exist in GSR — + // this is the production governance pattern where apps are forbidden from registering. + String prodStreamArn = createStream("gsr_avro_sql_existing_prod"); + createKinesisTable( + "existing_prod_sink", + columns, + prodStreamArn, + false, + autoRegOpts(schemaName, "avro-glue.schema.autoRegistration", "false")); + tEnv.executeSql( + "INSERT INTO existing_prod_sink VALUES " + + "('Judy', 33, 'maroon')," + + "('Mallory', 44, 'navy')") + .await(120, TimeUnit.SECONDS); + + createKinesisTable( + "existing_prod_source", columns, prodStreamArn, true, sourceOpts(schemaName)); + List rows = collect("SELECT * FROM existing_prod_source", 2, Duration.ofSeconds(90)); + + assertThat(firstFieldStrings(rows)) + .as( + "rows written against a pre-existing schema without auto-registration must" + + " round-trip") + .contains("Judy", "Mallory"); + } + + @Test + void missingSchemaWithoutAutoRegistrationFails() throws Exception { + String streamArn = createStream("gsr_avro_sql_missing"); + String schemaName = schemaName("missing-no-autoreg"); + String columns = "user_name STRING, favorite_number INT, favorite_color STRING"; + + // The schema name has never been registered and autoRegistration is off, so the write must + // fail with a clear error rather than silently creating the schema. + createKinesisTable( + "missing_sink", + columns, + streamArn, + false, + autoRegOpts(schemaName, "avro-glue.schema.autoRegistration", "false")); + assertThatThrownBy( + () -> + tEnv.executeSql( + "INSERT INTO missing_sink VALUES ('Oscar', 1," + + " 'gray')") + .await(120, TimeUnit.SECONDS)) + .as( + "writing against a missing schema without auto-registration must fail with" + + " a clear error rather than silently creating the schema") + .isInstanceOf(Exception.class); + } + + // --------------------------------------------------------------------------------------------- + // Helpers + // --------------------------------------------------------------------------------------------- + + private void createKinesisTable( + String tableName, + String columns, + String streamArn, + boolean source, + List formatOptions) { + List with = new ArrayList<>(); + with.add("'connector' = 'kinesis'"); + with.add("'stream.arn' = '" + streamArn + "'"); + with.add("'aws.region' = '" + KINESIS_REGION + "'"); + with.add("'aws.endpoint' = '" + LOCALSTACK.getEndpoint() + "'"); + with.add("'aws.credentials.provider' = 'BASIC'"); + with.add("'aws.credentials.basic.accesskeyid' = 'accessKeyId'"); + with.add("'aws.credentials.basic.secretkey' = 'secretAccessKey'"); + with.add("'aws.trust.all.certificates' = 'true'"); + with.add("'aws.http.protocol.version' = 'HTTP1_1'"); + if (source) { + with.add("'source.init.position' = 'TRIM_HORIZON'"); + } + with.add("'format' = 'avro-glue'"); + with.add("'avro-glue.aws.region' = '" + GSR_REGION + "'"); + with.add("'avro-glue.registry.name' = '" + REGISTRY_NAME + "'"); + with.addAll(formatOptions); + + String ddl = + "CREATE TABLE " + + tableName + + " (" + + columns + + ") WITH (" + + String.join(", ", with) + + ")"; + tEnv.executeSql(ddl); + } + + /** Builds format options starting with the (required) schema name, then key/value pairs. */ + private List autoRegOpts(String schemaName, String... kvPairs) { + List opts = new ArrayList<>(); + opts.add("'avro-glue.schema.name' = '" + schemaName + "'"); + for (int i = 0; i + 1 < kvPairs.length; i += 2) { + opts.add("'" + kvPairs[i] + "' = '" + kvPairs[i + 1] + "'"); + } + return opts; + } + + private List sourceOpts(String schemaName) { + return autoRegOpts(schemaName); + } + + private List compatOpts(String schemaName, String compatibility) { + return autoRegOpts( + schemaName, + "avro-glue.schema.autoRegistration", + "true", + "avro-glue.schema.compatibility", + compatibility); + } + + private List collect(String selectSql, int expected, Duration timeout) throws Exception { + List rows = new ArrayList<>(); + try (CloseableIterator iterator = tEnv.executeSql(selectSql).collect()) { + Deadline deadline = Deadline.fromNow(timeout); + while (rows.size() < expected && deadline.hasTimeLeft()) { + if (iterator.hasNext()) { + Row row = iterator.next(); + LOG.info("collected row: {}", row); + rows.add(row); + } else { + Thread.sleep(500); + } + } + } + return rows; + } + + private static List firstFieldStrings(List rows) { + List names = new ArrayList<>(); + for (Row row : rows) { + Object first = row.getField(0); + if (first != null) { + names.add(first.toString()); + } + } + return names; + } + + private static Row findByOrderId(List rows, String orderId) { + for (Row row : rows) { + Object field = row.getField(0); + if (field != null && orderId.equals(field.toString())) { + return row; + } + } + return null; + } + + /** Creates a Localstack Kinesis stream, waits until ACTIVE, and returns its ARN. */ + private String createStream(String baseName) throws Exception { + String streamName = baseName + "_" + RUN_ID; + kinesisClient.createStream( + CreateStreamRequest.builder().streamName(streamName).shardCount(1).build()); + + Deadline deadline = Deadline.fromNow(Duration.ofMinutes(1)); + while (!streamActive(streamName)) { + if (deadline.isOverdue()) { + throw new IllegalStateException("Stream did not become ACTIVE: " + streamName); + } + Thread.sleep(500); + } + return kinesisClient + .describeStream(DescribeStreamRequest.builder().streamName(streamName).build()) + .streamDescription() + .streamARN(); + } + + private boolean streamActive(String streamName) { + try { + return kinesisClient + .describeStream( + DescribeStreamRequest.builder().streamName(streamName).build()) + .streamDescription() + .streamStatus() + == StreamStatus.ACTIVE; + } catch (Exception e) { + return false; + } + } + + private String schemaName(String base) { + String name = "flink-avro-glue-sql-e2e-" + base + "-" + RUN_ID; + CREATED_SCHEMAS.add(name); + return name; + } + + private static void deleteCreatedSchemas() { + if (CREATED_SCHEMAS.isEmpty()) { + return; + } + try (GlueClient glue = + GlueClient.builder() + .region(Region.of(GSR_REGION)) + .credentialsProvider( + StaticCredentialsProvider.create( + AwsBasicCredentials.create(ACCESS_KEY, SECRET_KEY))) + .build()) { + for (String schema : CREATED_SCHEMAS) { + try { + glue.deleteSchema( + DeleteSchemaRequest.builder() + .schemaId( + SchemaId.builder() + .registryName(REGISTRY_NAME) + .schemaName(schema) + .build()) + .build()); + LOG.info("Deleted GSR schema {}", schema); + } catch (Exception e) { + LOG.warn( + "Best-effort delete of GSR schema {} failed: {}", + schema, + e.getMessage()); + } + } + } catch (Exception e) { + LOG.warn("Could not create GlueClient for schema cleanup: {}", e.getMessage()); + } + } + + private static String envOrDefault(String name, String defaultValue) { + String value = System.getenv(name); + return (value == null || value.isEmpty()) ? defaultValue : value; + } +} diff --git a/flink-connector-aws-e2e-tests/flink-formats-avro-glue-schema-registry-e2e-tests/src/test/resources/aws-e2e-setup.sh b/flink-connector-aws-e2e-tests/flink-formats-avro-glue-schema-registry-e2e-tests/src/test/resources/aws-e2e-setup.sh new file mode 100755 index 000000000..f3ed9fa3e --- /dev/null +++ b/flink-connector-aws-e2e-tests/flink-formats-avro-glue-schema-registry-e2e-tests/src/test/resources/aws-e2e-setup.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Setup and teardown AWS resources for avro-glue E2E tests. +# +# Usage: +# ./aws-e2e-setup.sh setup # Create Kinesis stream + GSR registry +# ./aws-e2e-setup.sh teardown # Delete Kinesis stream + GSR registry +# +# Environment variables (set before running): +# AWS_REGION - AWS region (default: us-east-1) +# KINESIS_STREAM - Kinesis stream name (default: flink-avro-glue-e2e-test) +# GSR_REGISTRY_NAME - GSR registry name (default: flink-avro-glue-e2e-registry) +# GSR_SCHEMA_NAME - Schema name prefix (default: flink-avro-glue-e2e-schema) + +set -euo pipefail + +AWS_REGION="${AWS_REGION:-us-east-1}" +KINESIS_STREAM="${KINESIS_STREAM:-flink-avro-glue-e2e-test}" +GSR_REGISTRY_NAME="${GSR_REGISTRY_NAME:-flink-avro-glue-e2e-registry}" +GSR_SCHEMA_NAME="${GSR_SCHEMA_NAME:-flink-avro-glue-e2e-schema}" + +setup() { + echo "=== Creating Kinesis stream: ${KINESIS_STREAM} ===" + aws kinesis create-stream \ + --stream-name "${KINESIS_STREAM}" \ + --shard-count 1 \ + --region "${AWS_REGION}" 2>/dev/null || echo "Stream may already exist" + + echo "Waiting for stream to become ACTIVE..." + aws kinesis wait stream-exists \ + --stream-name "${KINESIS_STREAM}" \ + --region "${AWS_REGION}" + + STREAM_ARN=$(aws kinesis describe-stream-summary \ + --stream-name "${KINESIS_STREAM}" \ + --region "${AWS_REGION}" \ + --query 'StreamDescriptionSummary.StreamARN' \ + --output text) + + echo "=== Creating GSR registry: ${GSR_REGISTRY_NAME} ===" + aws glue create-registry \ + --registry-name "${GSR_REGISTRY_NAME}" \ + --region "${AWS_REGION}" 2>/dev/null || echo "Registry may already exist" + + echo "" + echo "=== Setup complete ===" + echo "Export these before running the E2E test:" + echo " export AWS_REGION=${AWS_REGION}" + echo " export KINESIS_STREAM_ARN=${STREAM_ARN}" + echo " export GSR_REGISTRY_NAME=${GSR_REGISTRY_NAME}" + echo " export GSR_SCHEMA_NAME=${GSR_SCHEMA_NAME}" +} + +teardown() { + echo "=== Deleting Kinesis stream: ${KINESIS_STREAM} ===" + aws kinesis delete-stream \ + --stream-name "${KINESIS_STREAM}" \ + --enforce-consumer-deletion \ + --region "${AWS_REGION}" 2>/dev/null || echo "Stream may not exist" + + echo "=== Deleting GSR schemas in registry: ${GSR_REGISTRY_NAME} ===" + # Delete all schemas in the registry before deleting the registry + SCHEMAS=$(aws glue list-schemas \ + --registry-id RegistryName="${GSR_REGISTRY_NAME}" \ + --region "${AWS_REGION}" \ + --query 'Schemas[].SchemaName' \ + --output text 2>/dev/null || echo "") + + for schema in ${SCHEMAS}; do + echo " Deleting schema: ${schema}" + aws glue delete-schema \ + --schema-id SchemaName="${schema}",RegistryName="${GSR_REGISTRY_NAME}" \ + --region "${AWS_REGION}" 2>/dev/null || true + done + + echo "=== Deleting GSR registry: ${GSR_REGISTRY_NAME} ===" + aws glue delete-registry \ + --registry-id RegistryName="${GSR_REGISTRY_NAME}" \ + --region "${AWS_REGION}" 2>/dev/null || echo "Registry may not exist" + + echo "=== Teardown complete ===" +} + +case "${1:-}" in + setup) + setup + ;; + teardown) + teardown + ;; + *) + echo "Usage: $0 {setup|teardown}" + exit 1 + ;; +esac diff --git a/flink-connector-aws-e2e-tests/flink-formats-avro-glue-schema-registry-e2e-tests/src/test/resources/log4j2-test.properties b/flink-connector-aws-e2e-tests/flink-formats-avro-glue-schema-registry-e2e-tests/src/test/resources/log4j2-test.properties index 835c2ec9a..e0c3649c8 100644 --- a/flink-connector-aws-e2e-tests/flink-formats-avro-glue-schema-registry-e2e-tests/src/test/resources/log4j2-test.properties +++ b/flink-connector-aws-e2e-tests/flink-formats-avro-glue-schema-registry-e2e-tests/src/test/resources/log4j2-test.properties @@ -26,3 +26,13 @@ appender.testlogger.type = CONSOLE appender.testlogger.target = SYSTEM_ERR appender.testlogger.layout.type = PatternLayout appender.testlogger.layout.pattern = %-4r [%t] %-5p %c %x - %m%n + +# GSR format classes at INFO (for debugging schema fetcher) +logger.gsrformat.name = org.apache.flink.formats.avro.glue.schema.registry +logger.gsrformat.level = INFO +logger.gsrformat.appenderRef.test.ref = TestLogger + +# E2E test class +logger.e2e.name = org.apache.flink.glue.schema.registry.test +logger.e2e.level = INFO +logger.e2e.appenderRef.test.ref = TestLogger diff --git a/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/pom.xml b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/pom.xml new file mode 100644 index 000000000..6741e4dd9 --- /dev/null +++ b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/pom.xml @@ -0,0 +1,165 @@ + + + + 4.0.0 + + + org.apache.flink + flink-connector-aws-e2e-tests-parent + 6.0-SNAPSHOT + + + flink-formats-protobuf-glue-schema-registry-e2e-tests + Flink : Formats : AWS : E2E Tests : Protobuf Glue Schema Registry + jar + + + + + org.apache.flink + flink-protobuf-glue-schema-registry + ${project.version} + test + + + + + org.apache.flink + flink-connector-aws-kinesis-streams + ${project.version} + test + + + + + org.apache.flink + flink-connector-aws-base + ${project.version} + test-jar + test + + + org.apache.flink + flink-connector-aws-kinesis-streams + ${project.version} + test-jar + test + + + + + org.apache.flink + flink-table-api-java-bridge + ${flink.version} + test + + + org.apache.flink + flink-table-planner-loader + ${flink.version} + test + + + org.apache.flink + flink-table-runtime + ${flink.version} + test + + + + + org.apache.flink + flink-streaming-java + ${flink.version} + test + + + org.apache.flink + flink-clients + ${flink.version} + test + + + + + org.testcontainers + localstack + test + + + + + software.amazon.awssdk + kinesis + test + + + software.amazon.awssdk + s3 + test + + + software.amazon.awssdk + glue + test + + + software.amazon.awssdk + sts + test + + + software.amazon.awssdk + netty-nio-client + test + + + + + org.apache.logging.log4j + log4j-slf4j-impl + 2.24.1 + test + + + org.apache.logging.log4j + log4j-core + 2.24.1 + test + + + + + + + + com.google.errorprone + error_prone_annotations + 2.21.1 + + + + + diff --git a/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/test/java/org/apache/flink/glue/schema/registry/test/protobuf/GlueSchemaRegistryProtobufSqlKinesisITCase.java b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/test/java/org/apache/flink/glue/schema/registry/test/protobuf/GlueSchemaRegistryProtobufSqlKinesisITCase.java new file mode 100644 index 000000000..13d5552ac --- /dev/null +++ b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/test/java/org/apache/flink/glue/schema/registry/test/protobuf/GlueSchemaRegistryProtobufSqlKinesisITCase.java @@ -0,0 +1,555 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.glue.schema.registry.test.protobuf; + +import org.apache.flink.api.common.time.Deadline; +import org.apache.flink.connector.aws.testutils.AWSServicesTestUtils; +import org.apache.flink.connector.aws.testutils.LocalstackContainer; +import org.apache.flink.connector.aws.util.AWSGeneralUtil; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; +import org.apache.flink.types.Row; +import org.apache.flink.util.CloseableIterator; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.utility.DockerImageName; +import software.amazon.awssdk.core.SdkSystemSetting; +import software.amazon.awssdk.http.SdkHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.glue.GlueClient; +import software.amazon.awssdk.services.glue.model.DeleteSchemaRequest; +import software.amazon.awssdk.services.glue.model.SchemaId; +import software.amazon.awssdk.services.kinesis.KinesisClient; +import software.amazon.awssdk.services.kinesis.model.CreateStreamRequest; +import software.amazon.awssdk.services.kinesis.model.DescribeStreamRequest; +import software.amazon.awssdk.services.kinesis.model.StreamStatus; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assumptions.assumeThat; + +/** + * SQL-path end-to-end test for the AWS Glue Schema Registry Protobuf format ({@code protobuf-glue}) + * exercising the Flink Table API through a Localstack Kinesis data plane against a real Glue + * Schema Registry. + * + *

The Kinesis connector talks to Localstack via an explicit {@code aws.endpoint} + dummy {@code + * BASIC} credentials, while the {@code protobuf-glue} format resolves the real GSR through the + * default AWS credential chain, which we seed with the {@code IT_CASE_GLUE_SCHEMA_*} credentials + * via JVM system properties in {@link #setup()} (the MiniCluster runs in this same JVM). + * + *

The test is tagged {@code requires-aws-credentials} so it only runs under the {@code + * run-aws-end-to-end-tests} Maven profile, and is additionally {@code assumeThat}-gated to skip + * when credentials are absent. + * + *

Scenarios: + * + *

+ */ +@Tag("requires-aws-credentials") +class GlueSchemaRegistryProtobufSqlKinesisITCase { + + private static final Logger LOG = + LoggerFactory.getLogger(GlueSchemaRegistryProtobufSqlKinesisITCase.class); + + private static final String ACCESS_KEY = System.getenv("IT_CASE_GLUE_SCHEMA_ACCESS_KEY"); + private static final String SECRET_KEY = System.getenv("IT_CASE_GLUE_SCHEMA_SECRET_KEY"); + private static final String GSR_REGION = + envOrDefault("IT_CASE_GLUE_SCHEMA_REGION", "ca-central-1"); + private static final String REGISTRY_NAME = + envOrDefault("IT_CASE_GLUE_SCHEMA_REGISTRY_NAME", "default-registry"); + + private static final String LOCALSTACK_DOCKER_IMAGE_VERSION = "localstack/localstack:3.7.2"; + private static final String KINESIS_REGION = "ap-southeast-1"; + private static final String KINESIS_ACCOUNT = "000000000000"; + + private static final LocalstackContainer MOCK_KINESIS_CONTAINER = + new LocalstackContainer(DockerImageName.parse(LOCALSTACK_DOCKER_IMAGE_VERSION)) + .withNetworkAliases("localstack"); + + private SdkHttpClient httpClient; + private KinesisClient kinesisClient; + private StreamTableEnvironment tEnv; + private final List createdSchemas = new ArrayList<>(); + + @BeforeAll + static void beforeAll() { + assumeThat(ACCESS_KEY) + .as("IT_CASE_GLUE_SCHEMA_ACCESS_KEY must be set to run this test") + .isNotBlank(); + assumeThat(SECRET_KEY) + .as("IT_CASE_GLUE_SCHEMA_SECRET_KEY must be set to run this test") + .isNotBlank(); + + System.setProperty(SdkSystemSetting.CBOR_ENABLED.property(), "false"); + MOCK_KINESIS_CONTAINER.start(); + } + + @AfterAll + static void afterAll() { + if (MOCK_KINESIS_CONTAINER.isRunning()) { + MOCK_KINESIS_CONTAINER.stop(); + } + System.clearProperty(SdkSystemSetting.CBOR_ENABLED.property()); + } + + @BeforeEach + void setup() { + // Seed the default AWS credential chain so the protobuf-glue format authenticates against + // the real Glue Schema Registry from inside the MiniCluster JVM. + System.setProperty("aws.accessKeyId", ACCESS_KEY); + System.setProperty("aws.secretAccessKey", SECRET_KEY); + System.setProperty("aws.region", GSR_REGION); + + httpClient = AWSServicesTestUtils.createHttpClient(); + kinesisClient = + AWSServicesTestUtils.createAwsSyncClient( + MOCK_KINESIS_CONTAINER.getEndpoint(), httpClient, KinesisClient.builder()); + + StreamExecutionEnvironment execEnv = StreamExecutionEnvironment.getExecutionEnvironment(); + execEnv.setParallelism(1); + tEnv = StreamTableEnvironment.create(execEnv); + + LOG.info("Done setting up Localstack Kinesis + real GSR credential chain."); + } + + @AfterEach + void teardown() { + cleanupSchemas(); + AWSGeneralUtil.closeResources(httpClient, kinesisClient); + System.clearProperty("aws.accessKeyId"); + System.clearProperty("aws.secretAccessKey"); + System.clearProperty("aws.region"); + } + + @Test + void testBasicMultiRowRoundTrip() throws Exception { + String id = uniqueId("basic"); + String schemaName = schemaName(id); + prepareStream(id); + + tEnv.executeSql( + "CREATE TABLE kinesis_sink_basic (" + + " user_name STRING," + + " age INT," + + " is_active BOOLEAN" + + ") WITH (" + + kinesisOptions(id, false) + + "," + + protobufGlueOptions(schemaName, true, null) + + ")"); + + tEnv.executeSql( + "INSERT INTO kinesis_sink_basic VALUES " + + "('Alice', 30, true)," + + "('Bob', 25, false)," + + "('Charlie', 35, true)") + .await(120, TimeUnit.SECONDS); + + tEnv.executeSql( + "CREATE TABLE kinesis_source_basic (" + + " user_name STRING," + + " age INT," + + " is_active BOOLEAN" + + ") WITH (" + + kinesisOptions(id, true) + + "," + + protobufGlueOptions(schemaName, false, null) + + ")"); + + List rows = collect("SELECT * FROM kinesis_source_basic", 3, Duration.ofSeconds(90)); + + assertThat(rows).hasSize(3); + assertThat(rows) + .extracting(row -> String.valueOf(row.getField(0))) + .containsExactlyInAnyOrder("Alice", "Bob", "Charlie"); + } + + @Test + void testNullableColumnsRoundTrip() throws Exception { + String id = uniqueId("nullable"); + String schemaName = schemaName(id); + prepareStream(id); + + // All value columns are nullable (default in Flink SQL) so the proto3 explicit-presence + // fix (C2) must round-trip explicit NULLs as NULL rather than proto3 defaults. + String columns = + " id STRING," + " opt_str STRING," + " opt_int INT," + " opt_bool BOOLEAN"; + + tEnv.executeSql( + "CREATE TABLE kinesis_sink_nullable (" + + columns + + ") WITH (" + + kinesisOptions(id, false) + + "," + + protobufGlueOptions(schemaName, true, null) + + ")"); + + tEnv.executeSql( + "INSERT INTO kinesis_sink_nullable VALUES " + + "('R1', 'hello', 42, true)," + + "('R2', CAST(NULL AS STRING), CAST(NULL AS INT), CAST(NULL AS BOOLEAN))") + .await(120, TimeUnit.SECONDS); + + tEnv.executeSql( + "CREATE TABLE kinesis_source_nullable (" + + columns + + ") WITH (" + + kinesisOptions(id, true) + + "," + + protobufGlueOptions(schemaName, false, null) + + ")"); + + List rows = + collect("SELECT * FROM kinesis_source_nullable", 2, Duration.ofSeconds(90)); + + assertThat(rows).hasSize(2); + + Row r1 = findById(rows, "R1"); + assertThat(r1.getField(1)).isEqualTo("hello"); + assertThat(r1.getField(2)).isEqualTo(42); + assertThat(r1.getField(3)).isEqualTo(true); + + Row r2 = findById(rows, "R2"); + assertThat(r2.getField(1)).as("nullable STRING should round-trip as NULL").isNull(); + assertThat(r2.getField(2)).as("nullable INT should round-trip as NULL").isNull(); + assertThat(r2.getField(3)).as("nullable BOOLEAN should round-trip as NULL").isNull(); + } + + @Test + void testCompressionRoundTrip() throws Exception { + String id = uniqueId("compression"); + String schemaName = schemaName(id); + prepareStream(id); + + // 'schema.compression' = 'ZLIB' exercises the compression round-trip fix (C1): the reader + // must transparently decompress GSR-compressed payloads. + tEnv.executeSql( + "CREATE TABLE kinesis_sink_zlib (" + + " user_name STRING," + + " age INT," + + " is_active BOOLEAN" + + ") WITH (" + + kinesisOptions(id, false) + + "," + + protobufGlueOptions(schemaName, true, "ZLIB") + + ")"); + + tEnv.executeSql( + "INSERT INTO kinesis_sink_zlib VALUES " + + "('Dave', 40, true)," + + "('Eve', 28, false)," + + "('Frank', 33, true)") + .await(120, TimeUnit.SECONDS); + + tEnv.executeSql( + "CREATE TABLE kinesis_source_zlib (" + + " user_name STRING," + + " age INT," + + " is_active BOOLEAN" + + ") WITH (" + + kinesisOptions(id, true) + + "," + + protobufGlueOptions(schemaName, false, "ZLIB") + + ")"); + + List rows = collect("SELECT * FROM kinesis_source_zlib", 3, Duration.ofSeconds(90)); + + assertThat(rows).hasSize(3); + assertThat(rows) + .extracting(row -> String.valueOf(row.getField(0))) + .containsExactlyInAnyOrder("Dave", "Eve", "Frank"); + } + + @Test + void existingSchemaWithoutAutoRegistration() throws Exception { + String id = uniqueId("existing"); + String schemaName = schemaName(id); + prepareStream(id); + + // Phase A: register the schema (and its first version) in real GSR through a sink with + // autoRegistration = true, landing two rows on the stream. + tEnv.executeSql( + "CREATE TABLE kinesis_sink_existing_a (" + + " user_name STRING," + + " age INT," + + " is_active BOOLEAN" + + ") WITH (" + + kinesisOptions(id, false) + + "," + + protobufGlueOptionsExplicitAutoReg(schemaName, true) + + ")"); + + tEnv.executeSql( + "INSERT INTO kinesis_sink_existing_a VALUES " + + "('Alice', 30, true)," + + "('Bob', 25, false)") + .await(120, TimeUnit.SECONDS); + + // Phase B: a second sink at the SAME schema name with autoRegistration = false must succeed + // because the schema already exists in GSR — no registration is attempted. + tEnv.executeSql( + "CREATE TABLE kinesis_sink_existing_b (" + + " user_name STRING," + + " age INT," + + " is_active BOOLEAN" + + ") WITH (" + + kinesisOptions(id, false) + + "," + + protobufGlueOptionsExplicitAutoReg(schemaName, false) + + ")"); + + tEnv.executeSql( + "INSERT INTO kinesis_sink_existing_b VALUES " + + "('Charlie', 35, true)," + + "('Dave', 40, false)") + .await(120, TimeUnit.SECONDS); + + tEnv.executeSql( + "CREATE TABLE kinesis_source_existing (" + + " user_name STRING," + + " age INT," + + " is_active BOOLEAN" + + ") WITH (" + + kinesisOptions(id, true) + + "," + + protobufGlueOptionsExplicitAutoReg(schemaName, false) + + ")"); + + List rows = + collect("SELECT * FROM kinesis_source_existing", 4, Duration.ofSeconds(90)); + + assertThat(rows) + .as( + "all four rows (two written under autoRegistration=true, two under " + + "autoRegistration=false against the pre-existing schema) must arrive") + .hasSize(4); + assertThat(rows) + .extracting(row -> String.valueOf(row.getField(0))) + .containsExactlyInAnyOrder("Alice", "Bob", "Charlie", "Dave"); + } + + @Test + void missingSchemaWithoutAutoRegistrationFails() throws Exception { + String id = uniqueId("missing"); + String schemaName = schemaName(id); + prepareStream(id); + + // The schema name is never registered in GSR and autoRegistration is disabled, so the sink + // has no schema to serialize against and the INSERT job must fail. + tEnv.executeSql( + "CREATE TABLE kinesis_sink_missing (" + + " user_name STRING," + + " age INT," + + " is_active BOOLEAN" + + ") WITH (" + + kinesisOptions(id, false) + + "," + + protobufGlueOptionsExplicitAutoReg(schemaName, false) + + ")"); + + assertThatThrownBy( + () -> + tEnv.executeSql( + "INSERT INTO kinesis_sink_missing VALUES " + + "('Alice', 30, true)") + .await(120, TimeUnit.SECONDS)) + .as( + "writing against a schema that does not exist in GSR with " + + "autoRegistration=false must fail rather than silently registering it") + .isInstanceOf(Exception.class); + } + + private List collect(String selectSql, int expected, Duration timeout) throws Exception { + List rows = new ArrayList<>(); + try (CloseableIterator iterator = tEnv.executeSql(selectSql).collect()) { + Deadline deadline = Deadline.fromNow(timeout); + while (rows.size() < expected && deadline.hasTimeLeft()) { + if (iterator.hasNext()) { + Row row = iterator.next(); + LOG.info("collected row: {}", row); + rows.add(row); + } else { + Thread.sleep(500); + } + } + } + return rows; + } + + private Row findById(List rows, String id) { + for (Row row : rows) { + if (id.equals(String.valueOf(row.getField(0)))) { + return row; + } + } + throw new AssertionError("Row with id '" + id + "' not found in " + rows); + } + + private String kinesisOptions(String streamName, boolean source) { + StringBuilder sb = new StringBuilder(); + sb.append(" 'connector' = 'kinesis',"); + sb.append(" 'stream.arn' = '").append(streamArn(streamName)).append("',"); + sb.append(" 'aws.region' = '").append(KINESIS_REGION).append("',"); + sb.append(" 'aws.endpoint' = '").append(MOCK_KINESIS_CONTAINER.getEndpoint()).append("',"); + sb.append(" 'aws.credentials.provider' = 'BASIC',"); + sb.append(" 'aws.credentials.basic.accesskeyid' = 'accessKeyId',"); + sb.append(" 'aws.credentials.basic.secretkey' = 'secretAccessKey',"); + sb.append(" 'aws.trust.all.certificates' = 'true',"); + sb.append(" 'aws.http.protocol.version' = 'HTTP1_1'"); + if (source) { + sb.append(", 'source.init.position' = 'TRIM_HORIZON'"); + } + return sb.toString(); + } + + private String protobufGlueOptions(String schemaName, boolean forSink, String compression) { + StringBuilder sb = new StringBuilder(); + sb.append(" 'format' = 'protobuf-glue',"); + sb.append(" 'protobuf-glue.aws.region' = '").append(GSR_REGION).append("',"); + sb.append(" 'protobuf-glue.registry.name' = '").append(REGISTRY_NAME).append("',"); + sb.append(" 'protobuf-glue.schema.name' = '").append(schemaName).append("'"); + if (forSink) { + sb.append(", 'protobuf-glue.schema.autoRegistration' = 'true'"); + } + if (compression != null) { + sb.append(", 'protobuf-glue.schema.compression' = '").append(compression).append("'"); + } + return sb.toString(); + } + + private String protobufGlueOptionsExplicitAutoReg(String schemaName, boolean autoRegistration) { + StringBuilder sb = new StringBuilder(); + sb.append(" 'format' = 'protobuf-glue',"); + sb.append(" 'protobuf-glue.aws.region' = '").append(GSR_REGION).append("',"); + sb.append(" 'protobuf-glue.registry.name' = '").append(REGISTRY_NAME).append("',"); + sb.append(" 'protobuf-glue.schema.name' = '").append(schemaName).append("',"); + sb.append(" 'protobuf-glue.schema.autoRegistration' = '") + .append(autoRegistration) + .append("'"); + return sb.toString(); + } + + private void prepareStream(String streamName) throws Exception { + kinesisClient.createStream( + CreateStreamRequest.builder().streamName(streamName).shardCount(1).build()); + + Deadline deadline = Deadline.fromNow(Duration.ofMinutes(1)); + while (deadline.hasTimeLeft()) { + if (streamActive(streamName)) { + return; + } + Thread.sleep(500); + } + throw new IllegalStateException("Stream " + streamName + " did not become ACTIVE in time"); + } + + private boolean streamActive(String streamName) { + try { + return kinesisClient + .describeStream( + DescribeStreamRequest.builder().streamName(streamName).build()) + .streamDescription() + .streamStatus() + == StreamStatus.ACTIVE; + } catch (Exception e) { + return false; + } + } + + private void cleanupSchemas() { + if (createdSchemas.isEmpty()) { + return; + } + try (SdkHttpClient glueHttpClient = AWSServicesTestUtils.createHttpClient(); + GlueClient glueClient = + GlueClient.builder() + .region(Region.of(GSR_REGION)) + .httpClient(glueHttpClient) + .build()) { + for (String schemaName : createdSchemas) { + try { + glueClient.deleteSchema( + DeleteSchemaRequest.builder() + .schemaId( + SchemaId.builder() + .registryName(REGISTRY_NAME) + .schemaName(schemaName) + .build()) + .build()); + LOG.info("Deleted GSR schema {}", schemaName); + } catch (Exception e) { + LOG.warn( + "Best-effort cleanup failed for schema {}: {}", + schemaName, + e.getMessage()); + } + } + } catch (Exception e) { + LOG.warn("Best-effort GSR schema cleanup skipped: {}", e.getMessage()); + } + createdSchemas.clear(); + } + + private String uniqueId(String scenario) { + return "gsr_pb_sql_" + scenario + "_" + Long.toHexString(System.nanoTime()); + } + + private String schemaName(String id) { + String schemaName = "flink-protobuf-glue-e2e-" + id; + createdSchemas.add(schemaName); + return schemaName; + } + + private String streamArn(String streamName) { + return "arn:aws:kinesis:" + + KINESIS_REGION + + ":" + + KINESIS_ACCOUNT + + ":stream/" + + streamName; + } + + private static String envOrDefault(String name, String defaultValue) { + String value = System.getenv(name); + return (value == null || value.trim().isEmpty()) ? defaultValue : value; + } +} diff --git a/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/test/resources/log4j2-test.properties b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/test/resources/log4j2-test.properties new file mode 100644 index 000000000..bc8557fd3 --- /dev/null +++ b/flink-connector-aws-e2e-tests/flink-formats-protobuf-glue-schema-registry-e2e-tests/src/test/resources/log4j2-test.properties @@ -0,0 +1,38 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +# Set root logger level to OFF to not flood build logs +# set manually to INFO for debugging purposes +rootLogger.level = OFF +rootLogger.appenderRef.test.ref = TestLogger + +appender.testlogger.name = TestLogger +appender.testlogger.type = CONSOLE +appender.testlogger.target = SYSTEM_ERR +appender.testlogger.layout.type = PatternLayout +appender.testlogger.layout.pattern = %-4r [%t] %-5p %c %x - %m%n + +# GSR protobuf format classes at INFO (for debugging schema fetcher) +logger.gsrformat.name = org.apache.flink.formats.protobuf.glue.schema.registry +logger.gsrformat.level = INFO +logger.gsrformat.appenderRef.test.ref = TestLogger + +# E2E test class +logger.e2e.name = org.apache.flink.glue.schema.registry.test.protobuf +logger.e2e.level = INFO +logger.e2e.appenderRef.test.ref = TestLogger diff --git a/flink-connector-aws-e2e-tests/pom.xml b/flink-connector-aws-e2e-tests/pom.xml index 0d9b466f7..55c90390c 100644 --- a/flink-connector-aws-e2e-tests/pom.xml +++ b/flink-connector-aws-e2e-tests/pom.xml @@ -44,6 +44,7 @@ under the License. flink-connector-aws-sqs-e2e-tests flink-formats-avro-glue-schema-registry-e2e-tests flink-formats-json-glue-schema-registry-e2e-tests + flink-formats-protobuf-glue-schema-registry-e2e-tests diff --git a/flink-formats-aws/flink-avro-glue-schema-registry/pom.xml b/flink-formats-aws/flink-avro-glue-schema-registry/pom.xml index 4bfe40885..da50e6a8c 100644 --- a/flink-formats-aws/flink-avro-glue-schema-registry/pom.xml +++ b/flink-formats-aws/flink-avro-glue-schema-registry/pom.xml @@ -39,6 +39,18 @@ under the License. ${flink.version} provided + + org.apache.flink + flink-table-api-java + ${flink.version} + provided + + + org.apache.flink + flink-table-common + ${flink.version} + provided + org.apache.flink flink-avro @@ -61,12 +73,60 @@ under the License. ${glue.schema.registry.version} - + + + org.apache.flink + flink-table-runtime + ${flink.version} + test + + + org.apache.flink + flink-table-api-java + ${flink.version} + test + test-jar + + + org.apache.flink + flink-table-common + ${flink.version} + test + test-jar + + + org.apache.flink + flink-avro + ${flink.version} + test + test-jar + + org.apache.flink flink-architecture-tests-test test + + + + net.jqwik + jqwik + 1.8.2 + test + + + + + + + org.opentest4j + opentest4j + 1.3.0 + + + diff --git a/flink-formats-aws/flink-avro-glue-schema-registry/src/main/java/org/apache/flink/formats/avro/glue/schema/registry/AvroGlueFormatOptions.java b/flink-formats-aws/flink-avro-glue-schema-registry/src/main/java/org/apache/flink/formats/avro/glue/schema/registry/AvroGlueFormatOptions.java new file mode 100644 index 000000000..f889f803c --- /dev/null +++ b/flink-formats-aws/flink-avro-glue-schema-registry/src/main/java/org/apache/flink/formats/avro/glue/schema/registry/AvroGlueFormatOptions.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.avro.glue.schema.registry; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; + +import com.amazonaws.services.schemaregistry.utils.AvroRecordType; + +/** + * Avro-specific configuration options for the AWS Glue Schema Registry Avro format factory. + * + *

Shared options (aws.region, registry.name, schema.name, etc.) are defined in {@link + * GlueFormatOptions}. + */ +@PublicEvolving +public class AvroGlueFormatOptions extends GlueFormatOptions { + + public static final ConfigOption SCHEMA_TYPE = + ConfigOptions.key("schema.type") + .enumType(AvroRecordType.class) + .defaultValue(AvroRecordType.GENERIC_RECORD) + .withDescription("Avro record type. Defaults to GENERIC_RECORD."); + + public static final ConfigOption AVRO_NAMESPACE = + ConfigOptions.key("avro.namespace") + .stringType() + .noDefaultValue() + .withDescription( + "Override the namespace in the auto-generated Avro schema. " + + "Use this to match schemas already registered in GSR with a different namespace."); + + public static final ConfigOption AVRO_RECORD_NAME = + ConfigOptions.key("avro.record-name") + .stringType() + .noDefaultValue() + .withDescription( + "Override the record name in the auto-generated Avro schema. " + + "Use this to match schemas already registered in GSR with a different record name."); + + public static final ConfigOption SCHEMA_FETCH_FROM_REGISTRY = + ConfigOptions.key("schema.fetchFromRegistry") + .booleanType() + .defaultValue(false) + .withDescription( + "Whether to fetch the schema from GSR instead of using the auto-generated one. " + + "Defaults to false."); + + private AvroGlueFormatOptions() {} +} diff --git a/flink-formats-aws/flink-avro-glue-schema-registry/src/main/java/org/apache/flink/formats/avro/glue/schema/registry/AvroSchemaPatcher.java b/flink-formats-aws/flink-avro-glue-schema-registry/src/main/java/org/apache/flink/formats/avro/glue/schema/registry/AvroSchemaPatcher.java new file mode 100644 index 000000000..450d93894 --- /dev/null +++ b/flink-formats-aws/flink-avro-glue-schema-registry/src/main/java/org/apache/flink/formats/avro/glue/schema/registry/AvroSchemaPatcher.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.avro.glue.schema.registry; + +import org.apache.flink.annotation.Internal; + +import org.apache.avro.Schema; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * Utility class that creates a new Avro {@link Schema} with overridden namespace and/or record name + * while preserving all field definitions from the original schema. + * + *

This addresses the Avro schema namespace bug where Flink's {@code AvroSchemaConverter} + * auto-generates a namespace (e.g. {@code org.apache.flink.avro.generated}) that may differ from + * schemas already registered in AWS Glue Schema Registry. + * + *

The patcher recursively patches all nested record types to use the same namespace, ensuring + * consistent namespace usage throughout the schema hierarchy. + */ +@Internal +public class AvroSchemaPatcher { + + /** + * Creates a new Avro Schema with overridden namespace and/or record name, preserving all field + * definitions from the original schema. Recursively patches all nested record types to use the + * same namespace. + * + * @param original the original Avro schema (can be a RECORD or UNION type) + * @param namespace the namespace override, or {@code null} to keep the original namespace + * @param recordName the record name override, or {@code null} to keep the original record name + * @return a new Schema with the overridden namespace/name and the same fields as the original + */ + public static Schema patchSchema( + Schema original, @Nullable String namespace, @Nullable String recordName) { + if (namespace == null && recordName == null) { + return original; + } + + // Track already-patched schemas to handle recursive references + Map patchedSchemas = new HashMap<>(); + + // Special handling for top-level UNION: apply recordName to the main RECORD in the union + if (original.getType() == Schema.Type.UNION) { + return patchTopLevelUnion(original, namespace, recordName, patchedSchemas); + } + + return patchSchemaRecursive(original, namespace, recordName, patchedSchemas); + } + + /** + * Patches a top-level UNION schema, applying the recordName override to the main RECORD type. + * This handles the common case where AvroSchemaConverter produces ["null", record] unions. + */ + private static Schema patchTopLevelUnion( + Schema unionSchema, + @Nullable String namespace, + @Nullable String recordName, + Map patchedSchemas) { + + List patchedTypes = new ArrayList<>(); + boolean recordNameApplied = false; + + for (Schema unionType : unionSchema.getTypes()) { + if (unionType.getType() == Schema.Type.RECORD && !recordNameApplied) { + // Apply recordName to the first RECORD in the union + patchedTypes.add( + patchSchemaRecursive(unionType, namespace, recordName, patchedSchemas)); + recordNameApplied = true; + } else { + // For other types (null, primitives, nested records), only apply namespace + patchedTypes.add(patchSchemaRecursive(unionType, namespace, null, patchedSchemas)); + } + } + return Schema.createUnion(patchedTypes); + } + + private static Schema patchSchemaRecursive( + Schema schema, + @Nullable String namespace, + @Nullable String recordName, + Map patchedSchemas) { + + switch (schema.getType()) { + case RECORD: + return patchRecordSchema(schema, namespace, recordName, patchedSchemas); + + case ARRAY: + Schema patchedElement = + patchSchemaRecursive( + schema.getElementType(), namespace, null, patchedSchemas); + return Schema.createArray(patchedElement); + + case MAP: + Schema patchedValue = + patchSchemaRecursive( + schema.getValueType(), namespace, null, patchedSchemas); + return Schema.createMap(patchedValue); + + case UNION: + List patchedTypes = new ArrayList<>(); + for (Schema unionType : schema.getTypes()) { + patchedTypes.add( + patchSchemaRecursive(unionType, namespace, null, patchedSchemas)); + } + return Schema.createUnion(patchedTypes); + + default: + // Primitive types and other types don't need patching + return schema; + } + } + + private static Schema patchRecordSchema( + Schema original, + @Nullable String namespace, + @Nullable String recordName, + Map patchedSchemas) { + + String originalFullName = original.getFullName(); + + // Check if we've already patched this schema (handles recursive references) + if (patchedSchemas.containsKey(originalFullName)) { + return patchedSchemas.get(originalFullName); + } + + String effectiveNamespace = namespace != null ? namespace : original.getNamespace(); + String effectiveName = recordName != null ? recordName : original.getName(); + + // Create the new schema first (without fields) to handle recursive references + Schema patched = + Schema.createRecord( + effectiveName, original.getDoc(), effectiveNamespace, original.isError()); + + // Register before processing fields to handle self-references + patchedSchemas.put(originalFullName, patched); + + // Now patch all fields recursively + List patchedFields = + original.getFields().stream() + .map( + f -> { + Schema patchedFieldSchema = + patchSchemaRecursive( + f.schema(), namespace, null, patchedSchemas); + return new Schema.Field( + f.name(), patchedFieldSchema, f.doc(), f.defaultVal()); + }) + .collect(Collectors.toList()); + + patched.setFields(patchedFields); + return patched; + } + + private AvroSchemaPatcher() {} +} diff --git a/flink-formats-aws/flink-avro-glue-schema-registry/src/main/java/org/apache/flink/formats/avro/glue/schema/registry/AvroSchemaResolver.java b/flink-formats-aws/flink-avro-glue-schema-registry/src/main/java/org/apache/flink/formats/avro/glue/schema/registry/AvroSchemaResolver.java new file mode 100644 index 000000000..4dbbc180c --- /dev/null +++ b/flink-formats-aws/flink-avro-glue-schema-registry/src/main/java/org/apache/flink/formats/avro/glue/schema/registry/AvroSchemaResolver.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.avro.glue.schema.registry; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.configuration.ReadableConfig; + +import org.apache.avro.Schema; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.annotation.Nullable; + +/** + * Resolves the Avro schema to use for GSR serialization/deserialization based on the configured + * options. + * + *

Resolution flow: + * + *

    + *
  1. If {@code schema.fetchFromRegistry} is true, attempt to fetch the schema from GSR + *
  2. If fetch fails or is disabled, check for {@code avro.namespace} / {@code avro.record-name} + * overrides and patch the auto-generated schema + *
  3. Otherwise, use the auto-generated schema unchanged + *
+ */ +@Internal +public class AvroSchemaResolver { + + private static final Logger LOG = LoggerFactory.getLogger(AvroSchemaResolver.class); + + /** + * Resolves the Avro schema based on the format options and the auto-generated schema. + * + * @param autoGeneratedSchema the Avro schema auto-generated from the Flink RowType via {@code + * AvroSchemaConverter.convertToSchema(RowType)} + * @param formatOptions the Flink format options from SQL DDL + * @param schemaFetcher optional callback to fetch schema from GSR; may be null if fetch is not + * supported + * @return the resolved Avro schema + */ + public static Schema resolveSchema( + Schema autoGeneratedSchema, + ReadableConfig formatOptions, + @Nullable SchemaFetcher schemaFetcher) { + + boolean fetchFromRegistry = + formatOptions.get(AvroGlueFormatOptions.SCHEMA_FETCH_FROM_REGISTRY); + + if (fetchFromRegistry && schemaFetcher != null) { + String schemaName = formatOptions.get(GlueFormatOptions.SCHEMA_NAME); + String registryName = formatOptions.get(GlueFormatOptions.REGISTRY_NAME); + try { + Schema fetched = schemaFetcher.fetchSchema(registryName, schemaName); + if (fetched != null) { + return fetched; + } + } catch (Exception e) { + LOG.warn( + "Failed to fetch schema '{}' from registry '{}', " + + "falling back to auto-generated schema.", + schemaName, + registryName, + e); + } + } + + // Apply namespace/record-name patching if configured + String namespace = + formatOptions.getOptional(AvroGlueFormatOptions.AVRO_NAMESPACE).orElse(null); + String recordName = + formatOptions.getOptional(AvroGlueFormatOptions.AVRO_RECORD_NAME).orElse(null); + + return AvroSchemaPatcher.patchSchema(autoGeneratedSchema, namespace, recordName); + } + + /** + * Functional interface for fetching a schema from GSR. This allows the format factory to inject + * the actual GSR client call without coupling this resolver to the GSR SDK directly. + */ + @FunctionalInterface + public interface SchemaFetcher { + /** + * Fetches the latest schema version from GSR. + * + * @param registryName the registry name + * @param schemaName the schema name + * @return the fetched Avro Schema, or null if not found + * @throws Exception if the fetch fails + */ + @Nullable + Schema fetchSchema(String registryName, String schemaName) throws Exception; + } + + private AvroSchemaResolver() {} +} diff --git a/flink-formats-aws/flink-avro-glue-schema-registry/src/main/java/org/apache/flink/formats/avro/glue/schema/registry/GlueFormatConfigBuilder.java b/flink-formats-aws/flink-avro-glue-schema-registry/src/main/java/org/apache/flink/formats/avro/glue/schema/registry/GlueFormatConfigBuilder.java new file mode 100644 index 000000000..6df3dbc4d --- /dev/null +++ b/flink-formats-aws/flink-avro-glue-schema-registry/src/main/java/org/apache/flink/formats/avro/glue/schema/registry/GlueFormatConfigBuilder.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.avro.glue.schema.registry; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.configuration.ReadableConfig; + +import com.amazonaws.services.schemaregistry.utils.AWSSchemaRegistryConstants; + +import java.util.HashMap; +import java.util.Map; + +/** + * Utility class that builds the {@code Map} configuration required by the AWS Glue + * Schema Registry SDK from Flink's {@link ReadableConfig} format options. + */ +@Internal +public class GlueFormatConfigBuilder { + + /** + * Builds a GSR SDK configuration map from the shared {@link GlueFormatOptions}. + * + * @param formatOptions the Flink format options from SQL DDL + * @return a map of GSR SDK configuration keys to their values + */ + public static Map buildConfigMap(ReadableConfig formatOptions) { + final Map properties = new HashMap<>(); + + formatOptions + .getOptional(GlueFormatOptions.AWS_REGION) + .ifPresent(v -> properties.put(AWSSchemaRegistryConstants.AWS_REGION, v)); + formatOptions + .getOptional(GlueFormatOptions.AWS_ENDPOINT) + .ifPresent(v -> properties.put(AWSSchemaRegistryConstants.AWS_ENDPOINT, v)); + formatOptions + .getOptional(GlueFormatOptions.REGISTRY_NAME) + .ifPresent(v -> properties.put(AWSSchemaRegistryConstants.REGISTRY_NAME, v)); + formatOptions + .getOptional(GlueFormatOptions.SCHEMA_NAME) + .ifPresent(v -> properties.put(AWSSchemaRegistryConstants.SCHEMA_NAME, v)); + formatOptions + .getOptional(GlueFormatOptions.CACHE_SIZE) + .ifPresent(v -> properties.put(AWSSchemaRegistryConstants.CACHE_SIZE, v)); + formatOptions + .getOptional(GlueFormatOptions.CACHE_TTL_MS) + .ifPresent( + v -> + properties.put( + AWSSchemaRegistryConstants.CACHE_TIME_TO_LIVE_MILLIS, v)); + formatOptions + .getOptional(GlueFormatOptions.SCHEMA_AUTO_REGISTRATION) + .ifPresent( + v -> + properties.put( + AWSSchemaRegistryConstants.SCHEMA_AUTO_REGISTRATION_SETTING, + v)); + formatOptions + .getOptional(GlueFormatOptions.SCHEMA_COMPATIBILITY) + .ifPresent( + v -> properties.put(AWSSchemaRegistryConstants.COMPATIBILITY_SETTING, v)); + formatOptions + .getOptional(GlueFormatOptions.SCHEMA_COMPRESSION) + .ifPresent( + v -> properties.put(AWSSchemaRegistryConstants.COMPRESSION_TYPE, v.name())); + + return properties; + } + + private GlueFormatConfigBuilder() {} +} diff --git a/flink-formats-aws/flink-avro-glue-schema-registry/src/main/java/org/apache/flink/formats/avro/glue/schema/registry/GlueFormatOptions.java b/flink-formats-aws/flink-avro-glue-schema-registry/src/main/java/org/apache/flink/formats/avro/glue/schema/registry/GlueFormatOptions.java new file mode 100644 index 000000000..c9a2b7438 --- /dev/null +++ b/flink-formats-aws/flink-avro-glue-schema-registry/src/main/java/org/apache/flink/formats/avro/glue/schema/registry/GlueFormatOptions.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.avro.glue.schema.registry; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ConfigOptions; + +import com.amazonaws.services.schemaregistry.utils.AWSSchemaRegistryConstants; +import software.amazon.awssdk.services.glue.model.Compatibility; + +import java.time.Duration; + +/** + * Base configuration options shared across all AWS Glue Schema Registry format factories (Avro, + * JSON, Protobuf). + */ +@PublicEvolving +public class GlueFormatOptions { + + public static final ConfigOption AWS_REGION = + ConfigOptions.key("aws.region") + .stringType() + .noDefaultValue() + .withDescription("AWS region for the Glue Schema Registry."); + + public static final ConfigOption AWS_ENDPOINT = + ConfigOptions.key("aws.endpoint") + .stringType() + .noDefaultValue() + .withDescription("Custom AWS endpoint URL."); + + public static final ConfigOption REGISTRY_NAME = + ConfigOptions.key("registry.name") + .stringType() + .noDefaultValue() + .withDescription("Name of the Glue Schema Registry."); + + public static final ConfigOption SCHEMA_NAME = + ConfigOptions.key("schema.name") + .stringType() + .noDefaultValue() + .withDescription( + "Schema name under which to register/look up the schema in Glue Schema Registry."); + + public static final ConfigOption CACHE_SIZE = + ConfigOptions.key("cache.size") + .intType() + .defaultValue(200) + .withDescription( + "Maximum number of items in the schema cache. Defaults to 200."); + + public static final ConfigOption CACHE_TTL_MS = + ConfigOptions.key("cache.ttlMs") + .longType() + .defaultValue(Duration.ofDays(1L).toMillis()) + .withDescription("Cache TTL in milliseconds. Defaults to 1 day."); + + public static final ConfigOption SCHEMA_AUTO_REGISTRATION = + ConfigOptions.key("schema.autoRegistration") + .booleanType() + .defaultValue(false) + .withDescription( + "Whether to auto-register schemas with Glue Schema Registry. Defaults to false."); + + public static final ConfigOption SCHEMA_COMPATIBILITY = + ConfigOptions.key("schema.compatibility") + .enumType(Compatibility.class) + .defaultValue(AWSSchemaRegistryConstants.DEFAULT_COMPATIBILITY_SETTING) + .withDescription("Schema compatibility mode for Glue Schema Registry."); + + public static final ConfigOption SCHEMA_COMPRESSION = + ConfigOptions.key("schema.compression") + .enumType(AWSSchemaRegistryConstants.COMPRESSION.class) + .defaultValue(AWSSchemaRegistryConstants.COMPRESSION.NONE) + .withDescription("Compression type for schema data. Defaults to NONE."); + + protected GlueFormatOptions() {} +} diff --git a/flink-formats-aws/flink-avro-glue-schema-registry/src/main/java/org/apache/flink/formats/avro/glue/schema/registry/GlueSchemaRegistryAvroFormatFactory.java b/flink-formats-aws/flink-avro-glue-schema-registry/src/main/java/org/apache/flink/formats/avro/glue/schema/registry/GlueSchemaRegistryAvroFormatFactory.java new file mode 100644 index 000000000..1a324baf2 --- /dev/null +++ b/flink-formats-aws/flink-avro-glue-schema-registry/src/main/java/org/apache/flink/formats/avro/glue/schema/registry/GlueSchemaRegistryAvroFormatFactory.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.avro.glue.schema.registry; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.serialization.DeserializationSchema; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ReadableConfig; +import org.apache.flink.formats.avro.AvroRowDataDeserializationSchema; +import org.apache.flink.formats.avro.AvroRowDataSerializationSchema; +import org.apache.flink.formats.avro.AvroToRowDataConverters; +import org.apache.flink.formats.avro.RowDataToAvroConverters; +import org.apache.flink.formats.avro.typeutils.AvroSchemaConverter; +import org.apache.flink.table.connector.ChangelogMode; +import org.apache.flink.table.connector.Projection; +import org.apache.flink.table.connector.format.DecodingFormat; +import org.apache.flink.table.connector.format.EncodingFormat; +import org.apache.flink.table.connector.format.ProjectableDecodingFormat; +import org.apache.flink.table.connector.sink.DynamicTableSink; +import org.apache.flink.table.connector.source.DynamicTableSource; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.factories.DeserializationFormatFactory; +import org.apache.flink.table.factories.DynamicTableFactory; +import org.apache.flink.table.factories.FactoryUtil; +import org.apache.flink.table.factories.SerializationFormatFactory; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.logical.RowType; + +import org.apache.avro.Schema; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Table format factory for providing configured instances of AWS Glue Schema Registry Avro to + * RowData {@link SerializationSchema} and {@link DeserializationSchema}. + * + *

This factory supports: + * + *

    + *
  • SPI discovery via identifier {@code avro-glue} + *
  • Projection pushdown via {@link ProjectableDecodingFormat} + *
  • Schema namespace/record-name patching via {@link AvroSchemaPatcher} + *
  • Schema fetching from GSR via {@link AvroSchemaResolver} + *
+ */ +@Internal +public class GlueSchemaRegistryAvroFormatFactory + implements DeserializationFormatFactory, SerializationFormatFactory { + + public static final String IDENTIFIER = "avro-glue"; + + @Override + public DecodingFormat> createDecodingFormat( + DynamicTableFactory.Context context, ReadableConfig formatOptions) { + FactoryUtil.validateFactoryOptions(this, formatOptions); + + return new ProjectableDecodingFormat>() { + @Override + public DeserializationSchema createRuntimeDecoder( + DynamicTableSource.Context context, + DataType producedDataType, + int[][] projections) { + producedDataType = Projection.of(projections).project(producedDataType); + final RowType rowType = (RowType) producedDataType.getLogicalType(); + final TypeInformation rowDataTypeInfo = + context.createTypeInformation(producedDataType); + final Schema autoGeneratedSchema = AvroSchemaConverter.convertToSchema(rowType); + final Map configMap = + GlueFormatConfigBuilder.buildConfigMap(formatOptions); + final AvroSchemaResolver.SchemaFetcher schemaFetcher = + createSchemaFetcherIfEnabled(formatOptions, configMap); + final Schema resolvedSchema = + AvroSchemaResolver.resolveSchema( + autoGeneratedSchema, formatOptions, schemaFetcher); + return new AvroRowDataDeserializationSchema( + GlueSchemaRegistryAvroDeserializationSchema.forGeneric( + resolvedSchema, configMap), + AvroToRowDataConverters.createRowConverter(rowType), + rowDataTypeInfo); + } + + @Override + public ChangelogMode getChangelogMode() { + return ChangelogMode.insertOnly(); + } + }; + } + + @Override + public EncodingFormat> createEncodingFormat( + DynamicTableFactory.Context context, ReadableConfig formatOptions) { + FactoryUtil.validateFactoryOptions(this, formatOptions); + + return new EncodingFormat>() { + @Override + public SerializationSchema createRuntimeEncoder( + DynamicTableSink.Context context, DataType consumedDataType) { + final RowType rowType = (RowType) consumedDataType.getLogicalType(); + final Schema autoGeneratedSchema = AvroSchemaConverter.convertToSchema(rowType); + final Map configMap = + GlueFormatConfigBuilder.buildConfigMap(formatOptions); + final AvroSchemaResolver.SchemaFetcher schemaFetcher = + createSchemaFetcherIfEnabled(formatOptions, configMap); + final Schema resolvedSchema = + AvroSchemaResolver.resolveSchema( + autoGeneratedSchema, formatOptions, schemaFetcher); + final String transportName = formatOptions.get(GlueFormatOptions.SCHEMA_NAME); + return new AvroRowDataSerializationSchema( + rowType, + GlueSchemaRegistryAvroSerializationSchema.forGeneric( + resolvedSchema, transportName, configMap), + RowDataToAvroConverters.createConverter(rowType)); + } + + @Override + public ChangelogMode getChangelogMode() { + return ChangelogMode.insertOnly(); + } + }; + } + + @Override + public String factoryIdentifier() { + return IDENTIFIER; + } + + @Override + public Set> requiredOptions() { + Set> options = new HashSet<>(); + options.add(GlueFormatOptions.AWS_REGION); + options.add(GlueFormatOptions.REGISTRY_NAME); + options.add(GlueFormatOptions.SCHEMA_NAME); + return options; + } + + @Override + public Set> optionalOptions() { + Set> options = new HashSet<>(); + options.add(GlueFormatOptions.AWS_ENDPOINT); + options.add(GlueFormatOptions.CACHE_SIZE); + options.add(GlueFormatOptions.CACHE_TTL_MS); + options.add(GlueFormatOptions.SCHEMA_AUTO_REGISTRATION); + options.add(GlueFormatOptions.SCHEMA_COMPATIBILITY); + options.add(GlueFormatOptions.SCHEMA_COMPRESSION); + options.add(AvroGlueFormatOptions.SCHEMA_TYPE); + options.add(AvroGlueFormatOptions.AVRO_NAMESPACE); + options.add(AvroGlueFormatOptions.AVRO_RECORD_NAME); + options.add(AvroGlueFormatOptions.SCHEMA_FETCH_FROM_REGISTRY); + return options; + } + + @Override + public Set> forwardOptions() { + return Stream.of( + GlueFormatOptions.AWS_REGION, + GlueFormatOptions.AWS_ENDPOINT, + GlueFormatOptions.REGISTRY_NAME, + GlueFormatOptions.SCHEMA_NAME, + GlueFormatOptions.CACHE_SIZE, + GlueFormatOptions.CACHE_TTL_MS, + GlueFormatOptions.SCHEMA_AUTO_REGISTRATION, + GlueFormatOptions.SCHEMA_COMPATIBILITY, + GlueFormatOptions.SCHEMA_COMPRESSION, + AvroGlueFormatOptions.SCHEMA_TYPE, + AvroGlueFormatOptions.AVRO_NAMESPACE, + AvroGlueFormatOptions.AVRO_RECORD_NAME, + AvroGlueFormatOptions.SCHEMA_FETCH_FROM_REGISTRY) + .collect(Collectors.toSet()); + } + + /** + * Creates a SchemaFetcher if fetchFromRegistry is enabled. + * + * @param formatOptions the format options + * @param configMap the GSR config map + * @return a SchemaFetcher instance, or null if fetchFromRegistry is disabled + */ + private static AvroSchemaResolver.SchemaFetcher createSchemaFetcherIfEnabled( + ReadableConfig formatOptions, Map configMap) { + boolean fetchFromRegistry = + formatOptions.get(AvroGlueFormatOptions.SCHEMA_FETCH_FROM_REGISTRY); + if (fetchFromRegistry) { + return new GlueSchemaRegistrySchemaFetcher(configMap); + } + return null; + } +} diff --git a/flink-formats-aws/flink-avro-glue-schema-registry/src/main/java/org/apache/flink/formats/avro/glue/schema/registry/GlueSchemaRegistrySchemaFetcher.java b/flink-formats-aws/flink-avro-glue-schema-registry/src/main/java/org/apache/flink/formats/avro/glue/schema/registry/GlueSchemaRegistrySchemaFetcher.java new file mode 100644 index 000000000..2dacfe38c --- /dev/null +++ b/flink-formats-aws/flink-avro-glue-schema-registry/src/main/java/org/apache/flink/formats/avro/glue/schema/registry/GlueSchemaRegistrySchemaFetcher.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.avro.glue.schema.registry; + +import org.apache.flink.annotation.Internal; + +import com.amazonaws.services.schemaregistry.utils.AWSSchemaRegistryConstants; +import org.apache.avro.Schema; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.glue.GlueClient; +import software.amazon.awssdk.services.glue.GlueClientBuilder; +import software.amazon.awssdk.services.glue.model.GetSchemaVersionRequest; +import software.amazon.awssdk.services.glue.model.GetSchemaVersionResponse; +import software.amazon.awssdk.services.glue.model.SchemaId; +import software.amazon.awssdk.services.glue.model.SchemaVersionNumber; + +import java.net.URI; +import java.util.Map; + +/** + * Fetches the latest Avro schema from AWS Glue Schema Registry using the Glue SDK. + * + *

Used when {@code schema.fetchFromRegistry=true} to resolve the actual schema from GSR instead + * of relying on the auto-generated Flink schema (which defaults to namespace {@code + * org.apache.flink.avro.generated} and record name {@code record}). + */ +@Internal +public class GlueSchemaRegistrySchemaFetcher implements AvroSchemaResolver.SchemaFetcher { + + private static final Logger LOG = + LoggerFactory.getLogger(GlueSchemaRegistrySchemaFetcher.class); + + private final GlueClient glueClient; + + public GlueSchemaRegistrySchemaFetcher(Map configMap) { + String region = (String) configMap.get(AWSSchemaRegistryConstants.AWS_REGION); + GlueClientBuilder builder = GlueClient.builder(); + if (region != null) { + builder.region(Region.of(region)); + } + Object endpoint = configMap.get(AWSSchemaRegistryConstants.AWS_ENDPOINT); + if (endpoint != null) { + builder.endpointOverride(URI.create(endpoint.toString())); + } + this.glueClient = builder.build(); + LOG.debug("GlueSchemaRegistrySchemaFetcher initialized for region: {}", region); + } + + /** Package-private constructor for testing with a pre-built GlueClient. */ + GlueSchemaRegistrySchemaFetcher(GlueClient glueClient) { + this.glueClient = glueClient; + } + + @Override + public Schema fetchSchema(String registryName, String schemaName) throws Exception { + LOG.debug( + "Fetching schema from GSR - registry: '{}', schema: '{}'", + registryName, + schemaName); + + GetSchemaVersionRequest request = + GetSchemaVersionRequest.builder() + .schemaId( + SchemaId.builder() + .registryName(registryName) + .schemaName(schemaName) + .build()) + .schemaVersionNumber( + SchemaVersionNumber.builder().latestVersion(true).build()) + .build(); + + GetSchemaVersionResponse response = glueClient.getSchemaVersion(request); + String schemaDefinition = response.schemaDefinition(); + + if (schemaDefinition == null || schemaDefinition.isEmpty()) { + LOG.warn( + "Schema definition is null or empty for registry: '{}', schema: '{}'", + registryName, + schemaName); + return null; + } + + Schema schema = new Schema.Parser().parse(schemaDefinition); + LOG.debug( + "Fetched schema - namespace: '{}', name: '{}', fields: {}", + schema.getNamespace(), + schema.getName(), + schema.getFields().size()); + + return schema; + } +} diff --git a/flink-formats-aws/flink-avro-glue-schema-registry/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-formats-aws/flink-avro-glue-schema-registry/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory new file mode 100644 index 000000000..2b14a1906 --- /dev/null +++ b/flink-formats-aws/flink-avro-glue-schema-registry/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.flink.formats.avro.glue.schema.registry.GlueSchemaRegistryAvroFormatFactory diff --git a/flink-formats-aws/flink-avro-glue-schema-registry/src/test/java/org/apache/flink/formats/avro/glue/schema/registry/AvroRoundTripIntegrationTest.java b/flink-formats-aws/flink-avro-glue-schema-registry/src/test/java/org/apache/flink/formats/avro/glue/schema/registry/AvroRoundTripIntegrationTest.java new file mode 100644 index 000000000..9257f6118 --- /dev/null +++ b/flink-formats-aws/flink-avro-glue-schema-registry/src/test/java/org/apache/flink/formats/avro/glue/schema/registry/AvroRoundTripIntegrationTest.java @@ -0,0 +1,231 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.avro.glue.schema.registry; + +import org.apache.flink.formats.avro.AvroRowDataDeserializationSchema; +import org.apache.flink.formats.avro.AvroRowDataSerializationSchema; +import org.apache.flink.formats.avro.AvroToRowDataConverters; +import org.apache.flink.formats.avro.RowDataToAvroConverters; +import org.apache.flink.formats.avro.SchemaCoder; +import org.apache.flink.formats.avro.typeutils.AvroSchemaConverter; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.runtime.typeutils.InternalTypeInfo; +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.VarCharType; + +import com.amazonaws.services.schemaregistry.utils.AWSSchemaRegistryConstants; +import org.apache.avro.Schema; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Integration tests for Avro round-trip serialization/deserialization with mock GSR facades. + * + *

Validates Requirements 8.1, 8.3. + */ +class AvroRoundTripIntegrationTest { + + private MockGlueSchemaRegistryFacades mockFacades; + private Map configs; + + @BeforeEach + void setUp() { + mockFacades = new MockGlueSchemaRegistryFacades(); + configs = new HashMap<>(); + configs.put(AWSSchemaRegistryConstants.AWS_REGION, "us-west-2"); + configs.put(AWSSchemaRegistryConstants.SCHEMA_AUTO_REGISTRATION_SETTING, true); + configs.put(AWSSchemaRegistryConstants.SCHEMA_NAME, "test-schema"); + } + + /** + * Tests basic Avro round-trip: RowData → serialize → deserialize → RowData. Requirement 8.1. + */ + @Test + void testBasicAvroRoundTrip() throws Exception { + RowType rowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField( + "name", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("age", new IntType()))); + + Schema avroSchema = AvroSchemaConverter.convertToSchema(rowType); + + // Build mock SchemaCoder using mock facades + GlueSchemaRegistryOutputStreamSerializer mockSerializer = + mockFacades.createMockOutputStreamSerializer("test-topic", configs); + GlueSchemaRegistryInputStreamDeserializer mockDeserializer = + mockFacades.createMockInputStreamDeserializer(); + + SchemaCoder serCoder = new GlueSchemaRegistryAvroSchemaCoder(mockSerializer); + SchemaCoder deserCoder = new GlueSchemaRegistryAvroSchemaCoder(mockDeserializer); + + // Create ser/deser schemas + GlueSchemaRegistryAvroSerializationSchema + gsrAvroSer = + new GlueSchemaRegistryAvroSerializationSchema<>( + org.apache.avro.generic.GenericRecord.class, avroSchema, serCoder); + + AvroRowDataSerializationSchema serSchema = + new AvroRowDataSerializationSchema( + rowType, gsrAvroSer, RowDataToAvroConverters.createConverter(rowType)); + + AvroRowDataDeserializationSchema deserSchema = + new AvroRowDataDeserializationSchema( + createDeserSchemaWithMockCoder(avroSchema, deserCoder), + AvroToRowDataConverters.createRowConverter(rowType), + InternalTypeInfo.of(rowType)); + + // Open schemas + serSchema.open(null); + deserSchema.open(null); + + // Create test RowData + GenericRowData original = new GenericRowData(2); + original.setField(0, StringData.fromString("Alice")); + original.setField(1, 30); + + // Serialize + byte[] serialized = serSchema.serialize(original); + assertThat(serialized).isNotNull(); + assertThat(serialized.length).isGreaterThan(MockGlueSchemaRegistryFacades.GSR_HEADER_SIZE); + + // Deserialize + RowData deserialized = deserSchema.deserialize(serialized); + assertThat(deserialized).isNotNull(); + assertThat(deserialized.getString(0).toString()).isEqualTo("Alice"); + assertThat(deserialized.getInt(1)).isEqualTo(30); + } + + /** + * Tests namespace bug scenario: serialize with avro.namespace override. Pre-register schema + * with custom namespace, then serialize with patched schema. Requirement 8.3. + */ + @Test + void testNamespaceBugScenario() throws Exception { + RowType rowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField( + "name", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("age", new IntType()))); + + // Auto-generated schema has namespace "org.apache.flink.avro.generated" + Schema autoGenerated = AvroSchemaConverter.convertToSchema(rowType); + assertThat(autoGenerated.getNamespace()).isEqualTo("org.apache.flink.avro.generated"); + + // Patch schema with custom namespace (simulating avro.namespace option) + String customNamespace = "com.example.myapp"; + Schema patchedSchema = AvroSchemaPatcher.patchSchema(autoGenerated, customNamespace, null); + assertThat(patchedSchema.getNamespace()).isEqualTo(customNamespace); + assertThat(patchedSchema.getFields()).hasSameSizeAs(autoGenerated.getFields()); + + // Build mock SchemaCoder with patched schema + GlueSchemaRegistryOutputStreamSerializer mockSerializer = + mockFacades.createMockOutputStreamSerializer("test-topic", configs); + GlueSchemaRegistryInputStreamDeserializer mockDeserializer = + mockFacades.createMockInputStreamDeserializer(); + + SchemaCoder serCoder = new GlueSchemaRegistryAvroSchemaCoder(mockSerializer); + SchemaCoder deserCoder = new GlueSchemaRegistryAvroSchemaCoder(mockDeserializer); + + // Create ser schema with patched schema + GlueSchemaRegistryAvroSerializationSchema + gsrAvroSer = + new GlueSchemaRegistryAvroSerializationSchema<>( + org.apache.avro.generic.GenericRecord.class, + patchedSchema, + serCoder); + + AvroRowDataSerializationSchema serSchema = + new AvroRowDataSerializationSchema( + rowType, gsrAvroSer, RowDataToAvroConverters.createConverter(rowType)); + + // Create deser schema with patched schema + AvroRowDataDeserializationSchema deserSchema = + new AvroRowDataDeserializationSchema( + createDeserSchemaWithMockCoder(patchedSchema, deserCoder), + AvroToRowDataConverters.createRowConverter(rowType), + InternalTypeInfo.of(rowType)); + + serSchema.open(null); + deserSchema.open(null); + + // Create test RowData + GenericRowData original = new GenericRowData(2); + original.setField(0, StringData.fromString("Bob")); + original.setField(1, 25); + + // Serialize with patched schema + byte[] serialized = serSchema.serialize(original); + assertThat(serialized).isNotNull(); + + // Deserialize — should succeed despite different namespace + RowData deserialized = deserSchema.deserialize(serialized); + assertThat(deserialized).isNotNull(); + assertThat(deserialized.getString(0).toString()).isEqualTo("Bob"); + assertThat(deserialized.getInt(1)).isEqualTo(25); + } + + /** + * Creates a GlueSchemaRegistryAvroDeserializationSchema with a mock SchemaCoder injected via + * reflection (the schemaCoder field is private in the parent class). + */ + private static GlueSchemaRegistryAvroDeserializationSchema< + org.apache.avro.generic.GenericRecord> + createDeserSchemaWithMockCoder(Schema schema, SchemaCoder mockCoder) { + // Create a real deser schema (configs won't be used since we override schemaCoder) + Map dummyConfigs = new HashMap<>(); + dummyConfigs.put(AWSSchemaRegistryConstants.AWS_REGION, "us-west-2"); + GlueSchemaRegistryAvroDeserializationSchema + deserSchema = + GlueSchemaRegistryAvroDeserializationSchema.forGeneric( + schema, dummyConfigs); + + // Use reflection to inject the mock SchemaCoder + try { + Class clazz = deserSchema.getClass(); + while (clazz != null) { + try { + Field field = clazz.getDeclaredField("schemaCoder"); + field.setAccessible(true); + field.set(deserSchema, mockCoder); + return deserSchema; + } catch (NoSuchFieldException e) { + clazz = clazz.getSuperclass(); + } + } + throw new RuntimeException("Could not find schemaCoder field"); + } catch (IllegalAccessException e) { + throw new RuntimeException("Failed to inject mock SchemaCoder", e); + } + } +} diff --git a/flink-formats-aws/flink-avro-glue-schema-registry/src/test/java/org/apache/flink/formats/avro/glue/schema/registry/AvroRoundTripPropertyTest.java b/flink-formats-aws/flink-avro-glue-schema-registry/src/test/java/org/apache/flink/formats/avro/glue/schema/registry/AvroRoundTripPropertyTest.java new file mode 100644 index 000000000..1e9355826 --- /dev/null +++ b/flink-formats-aws/flink-avro-glue-schema-registry/src/test/java/org/apache/flink/formats/avro/glue/schema/registry/AvroRoundTripPropertyTest.java @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.avro.glue.schema.registry; + +import org.apache.flink.formats.avro.AvroRowDataDeserializationSchema; +import org.apache.flink.formats.avro.AvroRowDataSerializationSchema; +import org.apache.flink.formats.avro.AvroToRowDataConverters; +import org.apache.flink.formats.avro.RowDataToAvroConverters; +import org.apache.flink.formats.avro.SchemaCoder; +import org.apache.flink.formats.avro.typeutils.AvroSchemaConverter; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.runtime.typeutils.InternalTypeInfo; +import org.apache.flink.table.types.logical.BooleanType; +import org.apache.flink.table.types.logical.DoubleType; +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.VarCharType; + +import com.amazonaws.services.schemaregistry.utils.AWSSchemaRegistryConstants; +import net.jqwik.api.Arbitraries; +import net.jqwik.api.Arbitrary; +import net.jqwik.api.ForAll; +import net.jqwik.api.Property; +import net.jqwik.api.Provide; +import net.jqwik.api.Tag; +import org.apache.avro.Schema; + +import java.lang.reflect.Field; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Property-based tests for Avro serialization round-trip with mock GSR facades. + * + *

Property 3: Avro serialization round-trip + * + *

Validates: Requirements 1.4, 1.5, 8.1 + */ +@Tag("Feature: gsr-flink-sql-formats, Property 3: Avro serialization round-trip") +class AvroRoundTripPropertyTest { + + /** + * For any valid RowData matching a given RowType, serializing via the Avro encoding format + * (with mock GSR facades) and then deserializing should produce equivalent RowData. + */ + @Property(tries = 100) + void avroRoundTripPreservesData(@ForAll("rowDataWithType") RowDataWithType input) + throws Exception { + RowType rowType = input.rowType; + RowData original = input.rowData; + + Schema avroSchema = AvroSchemaConverter.convertToSchema(rowType); + + // Create mock facades + MockGlueSchemaRegistryFacades mockFacades = new MockGlueSchemaRegistryFacades(); + Map configs = new HashMap<>(); + configs.put(AWSSchemaRegistryConstants.AWS_REGION, "us-west-2"); + configs.put(AWSSchemaRegistryConstants.SCHEMA_AUTO_REGISTRATION_SETTING, true); + configs.put(AWSSchemaRegistryConstants.SCHEMA_NAME, "test-schema"); + + GlueSchemaRegistryOutputStreamSerializer mockSerializer = + mockFacades.createMockOutputStreamSerializer("test-topic", configs); + GlueSchemaRegistryInputStreamDeserializer mockDeserializer = + mockFacades.createMockInputStreamDeserializer(); + + SchemaCoder serCoder = new GlueSchemaRegistryAvroSchemaCoder(mockSerializer); + SchemaCoder deserCoder = new GlueSchemaRegistryAvroSchemaCoder(mockDeserializer); + + // Create serialization schema + GlueSchemaRegistryAvroSerializationSchema + gsrAvroSer = + new GlueSchemaRegistryAvroSerializationSchema<>( + org.apache.avro.generic.GenericRecord.class, avroSchema, serCoder); + + AvroRowDataSerializationSchema serSchema = + new AvroRowDataSerializationSchema( + rowType, gsrAvroSer, RowDataToAvroConverters.createConverter(rowType)); + + // Create deserialization schema with mock coder via reflection + GlueSchemaRegistryAvroDeserializationSchema + gsrAvroDe = + GlueSchemaRegistryAvroDeserializationSchema.forGeneric(avroSchema, configs); + injectSchemaCoder(gsrAvroDe, deserCoder); + + AvroRowDataDeserializationSchema deserSchema = + new AvroRowDataDeserializationSchema( + gsrAvroDe, + AvroToRowDataConverters.createRowConverter(rowType), + InternalTypeInfo.of(rowType)); + + serSchema.open(null); + deserSchema.open(null); + + // Serialize + byte[] serialized = serSchema.serialize(original); + assertThat(serialized).isNotNull(); + + // Deserialize + RowData deserialized = deserSchema.deserialize(serialized); + assertThat(deserialized).isNotNull(); + + // Verify equivalence field by field + assertRowDataEquals(original, deserialized, rowType); + } + + /** Injects a mock SchemaCoder into a deser schema via reflection. */ + private static void injectSchemaCoder(Object target, SchemaCoder coder) { + try { + Class clazz = target.getClass(); + while (clazz != null) { + try { + Field field = clazz.getDeclaredField("schemaCoder"); + field.setAccessible(true); + field.set(target, coder); + return; + } catch (NoSuchFieldException e) { + clazz = clazz.getSuperclass(); + } + } + throw new RuntimeException("Could not find schemaCoder field"); + } catch (IllegalAccessException e) { + throw new RuntimeException("Failed to inject mock SchemaCoder", e); + } + } + + /** Compares two RowData instances field by field based on the RowType. */ + private void assertRowDataEquals(RowData expected, RowData actual, RowType rowType) { + assertThat(actual.getArity()).isEqualTo(expected.getArity()); + for (int i = 0; i < rowType.getFieldCount(); i++) { + LogicalType fieldType = rowType.getTypeAt(i); + // Avro treats null strings as null, null ints/bools/doubles as null + if (expected.isNullAt(i)) { + assertThat(actual.isNullAt(i)).isTrue(); + continue; + } + if (fieldType instanceof VarCharType) { + assertThat(actual.getString(i).toString()) + .isEqualTo(expected.getString(i).toString()); + } else if (fieldType instanceof IntType) { + assertThat(actual.getInt(i)).isEqualTo(expected.getInt(i)); + } else if (fieldType instanceof BooleanType) { + assertThat(actual.getBoolean(i)).isEqualTo(expected.getBoolean(i)); + } else if (fieldType instanceof DoubleType) { + assertThat(actual.getDouble(i)).isEqualTo(expected.getDouble(i)); + } + } + } + + // --- Generators --- + + @Provide + Arbitrary rowDataWithType() { + // Fixed schema with STRING, INT, BOOLEAN, DOUBLE fields + RowType rowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField( + "name", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("age", new IntType()), + new RowType.RowField("active", new BooleanType()), + new RowType.RowField("score", new DoubleType()))); + + return Arbitraries.of(rowType) + .flatMap(rt -> generateRowData(rt).map(rd -> new RowDataWithType(rt, rd))); + } + + private Arbitrary generateRowData(RowType rowType) { + Arbitrary strings = Arbitraries.strings().alpha().ofMinLength(0).ofMaxLength(50); + Arbitrary ints = Arbitraries.integers().between(-10000, 10000); + Arbitrary bools = Arbitraries.of(true, false); + Arbitrary doubles = Arbitraries.doubles().between(-1e6, 1e6).ofScale(4); + + return strings.flatMap( + name -> + ints.flatMap( + age -> + bools.flatMap( + active -> + doubles.map( + score -> { + GenericRowData row = + new GenericRowData(4); + row.setField( + 0, + StringData.fromString( + name)); + row.setField(1, age); + row.setField(2, active); + row.setField(3, score); + return (RowData) row; + })))); + } + + /** Holder for a RowData and its corresponding RowType. */ + static class RowDataWithType { + final RowType rowType; + final RowData rowData; + + RowDataWithType(RowType rowType, RowData rowData) { + this.rowType = rowType; + this.rowData = rowData; + } + } +} diff --git a/flink-formats-aws/flink-avro-glue-schema-registry/src/test/java/org/apache/flink/formats/avro/glue/schema/registry/AvroSchemaPatcherPropertyTest.java b/flink-formats-aws/flink-avro-glue-schema-registry/src/test/java/org/apache/flink/formats/avro/glue/schema/registry/AvroSchemaPatcherPropertyTest.java new file mode 100644 index 000000000..32e89a773 --- /dev/null +++ b/flink-formats-aws/flink-avro-glue-schema-registry/src/test/java/org/apache/flink/formats/avro/glue/schema/registry/AvroSchemaPatcherPropertyTest.java @@ -0,0 +1,414 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.avro.glue.schema.registry; + +import org.apache.flink.formats.avro.typeutils.AvroSchemaConverter; +import org.apache.flink.table.types.logical.ArrayType; +import org.apache.flink.table.types.logical.BigIntType; +import org.apache.flink.table.types.logical.BooleanType; +import org.apache.flink.table.types.logical.DoubleType; +import org.apache.flink.table.types.logical.FloatType; +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.MapType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.VarCharType; + +import net.jqwik.api.Arbitraries; +import net.jqwik.api.Arbitrary; +import net.jqwik.api.ForAll; +import net.jqwik.api.Property; +import net.jqwik.api.Provide; +import net.jqwik.api.Tag; +import net.jqwik.api.Tuple; +import org.apache.avro.Schema; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Property-based tests for {@link AvroSchemaPatcher}. + * + *

Property 2: Avro schema patching preserves fields with overridden namespace and name + * + *

Validates: Requirements 2.3, 2.4, 2.9 + */ +@Tag( + "Feature: gsr-flink-sql-formats, Property 2: Avro schema patching preserves fields with" + + " overridden namespace and name") +class AvroSchemaPatcherPropertyTest { + + /** + * For any valid Flink RowType and for any non-empty namespace and record name strings, patching + * the auto-generated Avro schema should produce a schema where the namespace and record name + * match the provided values, and all fields are preserved (with nested records also patched). + */ + @Property(tries = 100) + void patchSchemaPreservesFieldsWithOverriddenNamespaceAndName( + @ForAll("rowTypes") RowType rowType, + @ForAll("namespaces") String namespace, + @ForAll("recordNames") String recordName) { + + Schema original = AvroSchemaConverter.convertToSchema(rowType); + Schema patched = AvroSchemaPatcher.patchSchema(original, namespace, recordName); + + assertThat(patched.getNamespace()).isEqualTo(namespace); + assertThat(patched.getName()).isEqualTo(recordName); + assertThat(patched.getFields()).hasSameSizeAs(original.getFields()); + + for (int i = 0; i < original.getFields().size(); i++) { + Schema.Field originalField = original.getFields().get(i); + Schema.Field patchedField = patched.getFields().get(i); + assertThat(patchedField.name()).isEqualTo(originalField.name()); + // For nested records, the schema will be patched too, so we verify structure + assertSchemaStructureMatches(originalField.schema(), patchedField.schema(), namespace); + } + } + + /** + * Verifies that the patched schema has the same structure as the original, with nested records + * having the new namespace. + */ + private void assertSchemaStructureMatches( + Schema original, Schema patched, String expectedNamespace) { + assertThat(patched.getType()).isEqualTo(original.getType()); + + switch (original.getType()) { + case RECORD: + assertThat(patched.getNamespace()).isEqualTo(expectedNamespace); + assertThat(patched.getFields()).hasSameSizeAs(original.getFields()); + for (int i = 0; i < original.getFields().size(); i++) { + assertSchemaStructureMatches( + original.getFields().get(i).schema(), + patched.getFields().get(i).schema(), + expectedNamespace); + } + break; + case ARRAY: + assertSchemaStructureMatches( + original.getElementType(), patched.getElementType(), expectedNamespace); + break; + case MAP: + assertSchemaStructureMatches( + original.getValueType(), patched.getValueType(), expectedNamespace); + break; + case UNION: + assertThat(patched.getTypes()).hasSameSizeAs(original.getTypes()); + for (int i = 0; i < original.getTypes().size(); i++) { + assertSchemaStructureMatches( + original.getTypes().get(i), + patched.getTypes().get(i), + expectedNamespace); + } + break; + default: + // Primitive types should be equal + assertThat(patched).isEqualTo(original); + } + } + + /** + * When only namespace is provided (recordName is null), the record name should remain + * unchanged. + */ + @Property(tries = 100) + void patchSchemaWithOnlyNamespacePreservesRecordName( + @ForAll("rowTypes") RowType rowType, @ForAll("namespaces") String namespace) { + + Schema original = AvroSchemaConverter.convertToSchema(rowType); + Schema patched = AvroSchemaPatcher.patchSchema(original, namespace, null); + + assertThat(patched.getNamespace()).isEqualTo(namespace); + assertThat(patched.getName()).isEqualTo(original.getName()); + assertThat(patched.getFields()).hasSameSizeAs(original.getFields()); + } + + /** + * When only recordName is provided (namespace is null), the namespace should remain unchanged. + */ + @Property(tries = 100) + void patchSchemaWithOnlyRecordNamePreservesNamespace( + @ForAll("rowTypes") RowType rowType, @ForAll("recordNames") String recordName) { + + Schema original = AvroSchemaConverter.convertToSchema(rowType); + Schema patched = AvroSchemaPatcher.patchSchema(original, null, recordName); + + assertThat(patched.getNamespace()).isEqualTo(original.getNamespace()); + assertThat(patched.getName()).isEqualTo(recordName); + assertThat(patched.getFields()).hasSameSizeAs(original.getFields()); + } + + /** + * When both namespace and recordName are null, the original schema should be returned + * unchanged. + */ + @Property(tries = 100) + void patchSchemaWithNullOverridesReturnsOriginal(@ForAll("rowTypes") RowType rowType) { + + Schema original = AvroSchemaConverter.convertToSchema(rowType); + Schema patched = AvroSchemaPatcher.patchSchema(original, null, null); + + assertThat(patched).isSameAs(original); + } + + @Provide + Arbitrary rowTypes() { + return rowTypesWithDepth(0); + } + + /** Generates RowTypes with nested complex types up to a maximum depth. */ + private Arbitrary rowTypesWithDepth(int depth) { + Arbitrary fieldCount = Arbitraries.integers().between(1, 5); + return fieldCount.flatMap( + count -> { + Arbitrary> types = + logicalTypesWithDepth(depth).list().ofSize(count); + return types.map( + typeList -> { + List fields = + IntStream.range(0, typeList.size()) + .mapToObj( + i -> + new RowType.RowField( + "f" + i, typeList.get(i))) + .collect(Collectors.toList()); + return new RowType(false, fields); + }); + }); + } + + /** + * Generates LogicalTypes including primitives and complex types (ARRAY, MAP, ROW) with depth + * control. + */ + private Arbitrary logicalTypesWithDepth(int depth) { + // Primitive types - always available + Arbitrary primitives = + Arbitraries.of( + (LogicalType) new VarCharType(VarCharType.MAX_LENGTH), + new IntType(), + new BigIntType(), + new BooleanType(), + new FloatType(), + new DoubleType()); + + // At max depth, only return primitives + if (depth >= 2) { + return primitives; + } + + // Complex types with nested structures + Arbitrary arrayType = logicalTypesWithDepth(depth + 1).map(ArrayType::new); + + Arbitrary mapType = + logicalTypesWithDepth(depth + 1) + .map( + valueType -> + new MapType( + new VarCharType(VarCharType.MAX_LENGTH), + valueType)); + + Arbitrary nestedRowType = + rowTypesWithDepth(depth + 1).map(rt -> (LogicalType) rt); + + // Mix primitives and complex types with higher weight on primitives + return Arbitraries.frequencyOf( + Tuple.of(6, primitives), + Tuple.of(1, arrayType), + Tuple.of(1, mapType), + Tuple.of(2, nestedRowType)); + } + + @Provide + Arbitrary namespaces() { + return Arbitraries.strings() + .withCharRange('a', 'z') + .ofMinLength(1) + .ofMaxLength(20) + .flatMap( + first -> + Arbitraries.strings() + .withCharRange('a', 'z') + .ofMinLength(1) + .ofMaxLength(20) + .map(second -> first + "." + second)); + } + + @Provide + Arbitrary recordNames() { + // Avro record names must start with a letter and contain only alphanumeric + underscore + return Arbitraries.strings() + .withCharRange('A', 'Z') + .ofLength(1) + .flatMap( + first -> + Arbitraries.strings() + .withCharRange('a', 'z') + .ofMinLength(1) + .ofMaxLength(15) + .map(rest -> first + rest)); + } + + /** Test that nested record types are also patched with the same namespace. */ + @Test + void patchSchemaRecursivelyPatchesNestedRecords() { + // Create a RowType with nested ROW (record within record) + RowType nestedRowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField( + "name", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("age", new IntType()))); + + RowType outerRowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField("id", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("customer", nestedRowType))); + + Schema original = AvroSchemaConverter.convertToSchema(outerRowType); + Schema patched = + AvroSchemaPatcher.patchSchema(original, "com.example.orders", "OrderEvent"); + + // Verify root record is patched + assertThat(patched.getNamespace()).isEqualTo("com.example.orders"); + assertThat(patched.getName()).isEqualTo("OrderEvent"); + + // Verify nested record is also patched with the same namespace + Schema customerField = patched.getField("customer").schema(); + // Handle union type (nullable) + if (customerField.getType() == Schema.Type.UNION) { + customerField = + customerField.getTypes().stream() + .filter(s -> s.getType() == Schema.Type.RECORD) + .findFirst() + .orElseThrow(); + } + assertThat(customerField.getNamespace()).isEqualTo("com.example.orders"); + } + + /** Test that ARRAY of records has nested records patched. */ + @Test + void patchSchemaRecursivelyPatchesArrayOfRecords() { + // Create a RowType with ARRAY> + RowType itemRowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField( + "product_name", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("quantity", new IntType()))); + + RowType outerRowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField( + "order_id", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("items", new ArrayType(itemRowType)))); + + Schema original = AvroSchemaConverter.convertToSchema(outerRowType); + Schema patched = + AvroSchemaPatcher.patchSchema(original, "com.example.orders", "OrderEvent"); + + // Verify root record is patched + assertThat(patched.getNamespace()).isEqualTo("com.example.orders"); + assertThat(patched.getName()).isEqualTo("OrderEvent"); + + // Verify array element record is also patched + Schema itemsField = patched.getField("items").schema(); + // Handle union type (nullable) + if (itemsField.getType() == Schema.Type.UNION) { + itemsField = + itemsField.getTypes().stream() + .filter(s -> s.getType() == Schema.Type.ARRAY) + .findFirst() + .orElseThrow(); + } + Schema elementSchema = itemsField.getElementType(); + // Handle union type for element + if (elementSchema.getType() == Schema.Type.UNION) { + elementSchema = + elementSchema.getTypes().stream() + .filter(s -> s.getType() == Schema.Type.RECORD) + .findFirst() + .orElseThrow(); + } + assertThat(elementSchema.getNamespace()).isEqualTo("com.example.orders"); + } + + /** Test that MAP values with record types are patched. */ + @Test + void patchSchemaRecursivelyPatchesMapValues() { + // Create a RowType with MAP> + RowType valueRowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField( + "key", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("value", new IntType()))); + + RowType outerRowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField("id", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField( + "metadata", + new MapType( + new VarCharType(VarCharType.MAX_LENGTH), + valueRowType)))); + + Schema original = AvroSchemaConverter.convertToSchema(outerRowType); + Schema patched = AvroSchemaPatcher.patchSchema(original, "com.example.data", "DataRecord"); + + // Verify root record is patched + assertThat(patched.getNamespace()).isEqualTo("com.example.data"); + assertThat(patched.getName()).isEqualTo("DataRecord"); + + // Verify map value record is also patched + Schema metadataField = patched.getField("metadata").schema(); + // Handle union type (nullable) + if (metadataField.getType() == Schema.Type.UNION) { + metadataField = + metadataField.getTypes().stream() + .filter(s -> s.getType() == Schema.Type.MAP) + .findFirst() + .orElseThrow(); + } + Schema valueSchema = metadataField.getValueType(); + // Handle union type for value + if (valueSchema.getType() == Schema.Type.UNION) { + valueSchema = + valueSchema.getTypes().stream() + .filter(s -> s.getType() == Schema.Type.RECORD) + .findFirst() + .orElseThrow(); + } + assertThat(valueSchema.getNamespace()).isEqualTo("com.example.data"); + } +} diff --git a/flink-formats-aws/flink-avro-glue-schema-registry/src/test/java/org/apache/flink/formats/avro/glue/schema/registry/AvroSchemaResolverTest.java b/flink-formats-aws/flink-avro-glue-schema-registry/src/test/java/org/apache/flink/formats/avro/glue/schema/registry/AvroSchemaResolverTest.java new file mode 100644 index 000000000..d69791ee8 --- /dev/null +++ b/flink-formats-aws/flink-avro-glue-schema-registry/src/test/java/org/apache/flink/formats/avro/glue/schema/registry/AvroSchemaResolverTest.java @@ -0,0 +1,258 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.avro.glue.schema.registry; + +import org.apache.flink.configuration.Configuration; +import org.apache.flink.formats.avro.typeutils.AvroSchemaConverter; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.types.logical.RowType; + +import org.apache.avro.Schema; +import org.apache.avro.SchemaBuilder; +import org.junit.jupiter.api.Test; + +import javax.annotation.Nullable; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Unit tests for {@link AvroSchemaResolver}. + * + *

Tests the schema resolution flow: + * + *

    + *
  1. If fetchFromRegistry is true and fetcher succeeds, use fetched schema + *
  2. If fetchFromRegistry is true but fetcher fails, fall back to patched/auto-generated schema + *
  3. If namespace/record-name overrides are provided, patch the schema + *
  4. Otherwise, use the auto-generated schema unchanged + *
+ */ +class AvroSchemaResolverTest { + + private static final RowType TEST_ROW_TYPE = + (RowType) + DataTypes.ROW( + DataTypes.FIELD("id", DataTypes.STRING()), + DataTypes.FIELD("value", DataTypes.INT())) + .getLogicalType(); + + // Note: AvroSchemaConverter.convertToSchema returns a UNION ["null", record] for nullable rows + private static final Schema AUTO_GENERATED_SCHEMA = + AvroSchemaConverter.convertToSchema(TEST_ROW_TYPE); + + private static final String REGISTRY_NAME = "test-registry"; + private static final String SCHEMA_NAME = "test-schema"; + + /** + * Extracts the RECORD schema from a potentially UNION schema. AvroSchemaConverter wraps records + * in ["null", record] unions. + */ + private static Schema extractRecordSchema(Schema schema) { + if (schema.getType() == Schema.Type.UNION) { + return schema.getTypes().stream() + .filter(s -> s.getType() == Schema.Type.RECORD) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("No RECORD in UNION")); + } + return schema; + } + + @Test + void testResolveSchemaWithFetchFromRegistrySuccess() { + // Create a mock fetched schema with custom namespace using SchemaBuilder + Schema fetchedSchema = + SchemaBuilder.record("FetchedRecord") + .namespace("com.example.fetched") + .fields() + .optionalString("id") + .optionalInt("value") + .endRecord(); + + MockSchemaFetcher fetcher = new MockSchemaFetcher(fetchedSchema); + + Configuration config = new Configuration(); + config.set(GlueFormatOptions.REGISTRY_NAME, REGISTRY_NAME); + config.set(GlueFormatOptions.SCHEMA_NAME, SCHEMA_NAME); + config.set(AvroGlueFormatOptions.SCHEMA_FETCH_FROM_REGISTRY, true); + + Schema resolved = AvroSchemaResolver.resolveSchema(AUTO_GENERATED_SCHEMA, config, fetcher); + + assertThat(resolved.getNamespace()).isEqualTo("com.example.fetched"); + assertThat(resolved.getName()).isEqualTo("FetchedRecord"); + assertThat(fetcher.fetchCalled).isTrue(); + assertThat(fetcher.lastRegistryName).isEqualTo(REGISTRY_NAME); + assertThat(fetcher.lastSchemaName).isEqualTo(SCHEMA_NAME); + } + + @Test + void testResolveSchemaWithFetchFromRegistryFailure() { + // Fetcher that throws an exception + MockSchemaFetcher fetcher = new MockSchemaFetcher(new RuntimeException("GSR unavailable")); + + Configuration config = new Configuration(); + config.set(GlueFormatOptions.REGISTRY_NAME, REGISTRY_NAME); + config.set(GlueFormatOptions.SCHEMA_NAME, SCHEMA_NAME); + config.set(AvroGlueFormatOptions.SCHEMA_FETCH_FROM_REGISTRY, true); + config.set(AvroGlueFormatOptions.AVRO_NAMESPACE, "com.example.fallback"); + + Schema resolved = AvroSchemaResolver.resolveSchema(AUTO_GENERATED_SCHEMA, config, fetcher); + + // Should fall back to patched schema (extract RECORD from UNION) + Schema resolvedRecord = extractRecordSchema(resolved); + assertThat(resolvedRecord.getNamespace()).isEqualTo("com.example.fallback"); + assertThat(fetcher.fetchCalled).isTrue(); + } + + @Test + void testResolveSchemaWithFetchFromRegistryReturnsNull() { + // Fetcher that returns null (schema not found) + MockSchemaFetcher fetcher = new MockSchemaFetcher((Schema) null); + + Configuration config = new Configuration(); + config.set(GlueFormatOptions.REGISTRY_NAME, REGISTRY_NAME); + config.set(GlueFormatOptions.SCHEMA_NAME, SCHEMA_NAME); + config.set(AvroGlueFormatOptions.SCHEMA_FETCH_FROM_REGISTRY, true); + config.set(AvroGlueFormatOptions.AVRO_RECORD_NAME, "FallbackRecord"); + + Schema resolved = AvroSchemaResolver.resolveSchema(AUTO_GENERATED_SCHEMA, config, fetcher); + + // Should fall back to patched schema (extract RECORD from UNION) + Schema resolvedRecord = extractRecordSchema(resolved); + assertThat(resolvedRecord.getName()).isEqualTo("FallbackRecord"); + assertThat(fetcher.fetchCalled).isTrue(); + } + + @Test + void testResolveSchemaWithNamespaceOverrideOnly() { + Configuration config = new Configuration(); + config.set(GlueFormatOptions.REGISTRY_NAME, REGISTRY_NAME); + config.set(GlueFormatOptions.SCHEMA_NAME, SCHEMA_NAME); + config.set(AvroGlueFormatOptions.SCHEMA_FETCH_FROM_REGISTRY, false); + config.set(AvroGlueFormatOptions.AVRO_NAMESPACE, "com.example.custom"); + + Schema resolved = AvroSchemaResolver.resolveSchema(AUTO_GENERATED_SCHEMA, config, null); + + // Extract RECORD from UNION for assertions + Schema resolvedRecord = extractRecordSchema(resolved); + Schema originalRecord = extractRecordSchema(AUTO_GENERATED_SCHEMA); + assertThat(resolvedRecord.getNamespace()).isEqualTo("com.example.custom"); + assertThat(resolvedRecord.getName()).isEqualTo(originalRecord.getName()); + } + + @Test + void testResolveSchemaWithRecordNameOverrideOnly() { + Configuration config = new Configuration(); + config.set(GlueFormatOptions.REGISTRY_NAME, REGISTRY_NAME); + config.set(GlueFormatOptions.SCHEMA_NAME, SCHEMA_NAME); + config.set(AvroGlueFormatOptions.SCHEMA_FETCH_FROM_REGISTRY, false); + config.set(AvroGlueFormatOptions.AVRO_RECORD_NAME, "CustomRecord"); + + Schema resolved = AvroSchemaResolver.resolveSchema(AUTO_GENERATED_SCHEMA, config, null); + + // Extract RECORD from UNION for assertions + Schema resolvedRecord = extractRecordSchema(resolved); + Schema originalRecord = extractRecordSchema(AUTO_GENERATED_SCHEMA); + assertThat(resolvedRecord.getNamespace()).isEqualTo(originalRecord.getNamespace()); + assertThat(resolvedRecord.getName()).isEqualTo("CustomRecord"); + } + + @Test + void testResolveSchemaWithBothOverrides() { + Configuration config = new Configuration(); + config.set(GlueFormatOptions.REGISTRY_NAME, REGISTRY_NAME); + config.set(GlueFormatOptions.SCHEMA_NAME, SCHEMA_NAME); + config.set(AvroGlueFormatOptions.SCHEMA_FETCH_FROM_REGISTRY, false); + config.set(AvroGlueFormatOptions.AVRO_NAMESPACE, "com.example.custom"); + config.set(AvroGlueFormatOptions.AVRO_RECORD_NAME, "CustomRecord"); + + Schema resolved = AvroSchemaResolver.resolveSchema(AUTO_GENERATED_SCHEMA, config, null); + + // Extract RECORD from UNION for assertions + Schema resolvedRecord = extractRecordSchema(resolved); + assertThat(resolvedRecord.getNamespace()).isEqualTo("com.example.custom"); + assertThat(resolvedRecord.getName()).isEqualTo("CustomRecord"); + } + + @Test + void testResolveSchemaWithNoOverrides() { + Configuration config = new Configuration(); + config.set(GlueFormatOptions.REGISTRY_NAME, REGISTRY_NAME); + config.set(GlueFormatOptions.SCHEMA_NAME, SCHEMA_NAME); + config.set(AvroGlueFormatOptions.SCHEMA_FETCH_FROM_REGISTRY, false); + + Schema resolved = AvroSchemaResolver.resolveSchema(AUTO_GENERATED_SCHEMA, config, null); + + // Should return the original schema unchanged + assertThat(resolved).isSameAs(AUTO_GENERATED_SCHEMA); + } + + @Test + void testResolveSchemaWithFetchDisabledIgnoresFetcher() { + Schema fetchedSchema = + SchemaBuilder.record("FetchedRecord") + .namespace("com.example.fetched") + .fields() + .optionalString("id") + .optionalInt("value") + .endRecord(); + + MockSchemaFetcher fetcher = new MockSchemaFetcher(fetchedSchema); + + Configuration config = new Configuration(); + config.set(GlueFormatOptions.REGISTRY_NAME, REGISTRY_NAME); + config.set(GlueFormatOptions.SCHEMA_NAME, SCHEMA_NAME); + config.set(AvroGlueFormatOptions.SCHEMA_FETCH_FROM_REGISTRY, false); + + Schema resolved = AvroSchemaResolver.resolveSchema(AUTO_GENERATED_SCHEMA, config, fetcher); + + // Should NOT call fetcher when fetchFromRegistry is false + assertThat(fetcher.fetchCalled).isFalse(); + assertThat(resolved).isSameAs(AUTO_GENERATED_SCHEMA); + } + + /** Mock implementation of SchemaFetcher for testing. */ + private static class MockSchemaFetcher implements AvroSchemaResolver.SchemaFetcher { + private final Schema schemaToReturn; + private final Exception exceptionToThrow; + boolean fetchCalled = false; + String lastRegistryName; + String lastSchemaName; + + MockSchemaFetcher(@Nullable Schema schemaToReturn) { + this.schemaToReturn = schemaToReturn; + this.exceptionToThrow = null; + } + + MockSchemaFetcher(Exception exceptionToThrow) { + this.schemaToReturn = null; + this.exceptionToThrow = exceptionToThrow; + } + + @Override + public Schema fetchSchema(String registryName, String schemaName) throws Exception { + fetchCalled = true; + lastRegistryName = registryName; + lastSchemaName = schemaName; + if (exceptionToThrow != null) { + throw exceptionToThrow; + } + return schemaToReturn; + } + } +} diff --git a/flink-formats-aws/flink-avro-glue-schema-registry/src/test/java/org/apache/flink/formats/avro/glue/schema/registry/GlueFormatConfigBuilderPropertyTest.java b/flink-formats-aws/flink-avro-glue-schema-registry/src/test/java/org/apache/flink/formats/avro/glue/schema/registry/GlueFormatConfigBuilderPropertyTest.java new file mode 100644 index 000000000..ecc2e7ac0 --- /dev/null +++ b/flink-formats-aws/flink-avro-glue-schema-registry/src/test/java/org/apache/flink/formats/avro/glue/schema/registry/GlueFormatConfigBuilderPropertyTest.java @@ -0,0 +1,232 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.avro.glue.schema.registry; + +import org.apache.flink.configuration.Configuration; + +import com.amazonaws.services.schemaregistry.utils.AWSSchemaRegistryConstants; +import net.jqwik.api.Arbitraries; +import net.jqwik.api.Arbitrary; +import net.jqwik.api.Combinators; +import net.jqwik.api.ForAll; +import net.jqwik.api.Property; +import net.jqwik.api.Provide; +import net.jqwik.api.Tag; +import software.amazon.awssdk.services.glue.model.Compatibility; + +import java.util.Map; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Property-based tests for {@link GlueFormatConfigBuilder}. + * + *

Property 6: Config map builder correctly maps all provided options + * + *

Validates: Requirements 5.1, 5.2 + */ +@Tag( + "Feature: gsr-flink-sql-formats, Property 6: Config map builder correctly maps all provided" + + " options") +class GlueFormatConfigBuilderPropertyTest { + + /** + * For any valid combination of GSR config option values, building the config map via + * GlueFormatConfigBuilder.buildConfigMap() should produce a map where each provided option maps + * to the correct AWSSchemaRegistryConstants key with the same value, and absent options are not + * present in the map. + */ + @Property(tries = 100) + void configMapBuilderCorrectlyMapsAllProvidedOptions( + @ForAll("gsrConfigCombinations") GsrConfigInput input) { + + Configuration config = new Configuration(); + + input.region.ifPresent(v -> config.set(GlueFormatOptions.AWS_REGION, v)); + input.endpoint.ifPresent(v -> config.set(GlueFormatOptions.AWS_ENDPOINT, v)); + input.registryName.ifPresent(v -> config.set(GlueFormatOptions.REGISTRY_NAME, v)); + input.schemaName.ifPresent(v -> config.set(GlueFormatOptions.SCHEMA_NAME, v)); + input.cacheSize.ifPresent(v -> config.set(GlueFormatOptions.CACHE_SIZE, v)); + input.cacheTtlMs.ifPresent(v -> config.set(GlueFormatOptions.CACHE_TTL_MS, v)); + input.autoRegistration.ifPresent( + v -> config.set(GlueFormatOptions.SCHEMA_AUTO_REGISTRATION, v)); + input.compatibility.ifPresent(v -> config.set(GlueFormatOptions.SCHEMA_COMPATIBILITY, v)); + input.compression.ifPresent(v -> config.set(GlueFormatOptions.SCHEMA_COMPRESSION, v)); + + Map result = GlueFormatConfigBuilder.buildConfigMap(config); + + // Verify present options map to correct keys with correct values + assertOptionalMapping(result, input.region, AWSSchemaRegistryConstants.AWS_REGION); + assertOptionalMapping(result, input.endpoint, AWSSchemaRegistryConstants.AWS_ENDPOINT); + assertOptionalMapping(result, input.registryName, AWSSchemaRegistryConstants.REGISTRY_NAME); + assertOptionalMapping(result, input.schemaName, AWSSchemaRegistryConstants.SCHEMA_NAME); + assertOptionalMapping(result, input.cacheSize, AWSSchemaRegistryConstants.CACHE_SIZE); + assertOptionalMapping( + result, input.cacheTtlMs, AWSSchemaRegistryConstants.CACHE_TIME_TO_LIVE_MILLIS); + assertOptionalMapping( + result, + input.autoRegistration, + AWSSchemaRegistryConstants.SCHEMA_AUTO_REGISTRATION_SETTING); + assertOptionalMapping( + result, input.compatibility, AWSSchemaRegistryConstants.COMPATIBILITY_SETTING); + // Compression must land as the enum NAME string: the GSR serde casts this + // config value to String at serializer init. + assertOptionalMapping( + result, + input.compression.map(Enum::name), + AWSSchemaRegistryConstants.COMPRESSION_TYPE); + + // Verify map size equals number of provided options + long providedCount = + countPresent( + input.region, + input.endpoint, + input.registryName, + input.schemaName, + input.cacheSize, + input.cacheTtlMs, + input.autoRegistration, + input.compatibility, + input.compression); + assertThat(result).hasSize((int) providedCount); + } + + @Provide + Arbitrary gsrConfigCombinations() { + Arbitrary> optRegion = + Arbitraries.strings().alpha().ofMinLength(1).ofMaxLength(20).optional(); + Arbitrary> optEndpoint = + Arbitraries.strings() + .alpha() + .ofMinLength(1) + .ofMaxLength(30) + .map(s -> "https://" + s) + .optional(); + Arbitrary> optRegistryName = + Arbitraries.strings().alpha().ofMinLength(1).ofMaxLength(30).optional(); + Arbitrary> optSchemaName = + Arbitraries.strings().alpha().ofMinLength(1).ofMaxLength(30).optional(); + Arbitrary> optCacheSize = + Arbitraries.integers().between(1, 10000).optional(); + Arbitrary> optCacheTtlMs = + Arbitraries.longs().between(1000L, 172800000L).optional(); + Arbitrary> optAutoReg = Arbitraries.of(true, false).optional(); + Arbitrary> optCompat = + Arbitraries.of(Compatibility.knownValues().toArray(new Compatibility[0])) + .optional(); + Arbitrary> optCompress = + Arbitraries.of(AWSSchemaRegistryConstants.COMPRESSION.values()).optional(); + + // jqwik Combinators.combine supports up to 8 params, so we nest via flatMap + return Combinators.combine( + optRegion, + optEndpoint, + optRegistryName, + optSchemaName, + optCacheSize, + optCacheTtlMs, + optAutoReg, + optCompat) + .flatAs( + (region, endpoint, registry, schema, cache, ttl, autoReg, compat) -> + optCompress.map( + compress -> + new GsrConfigInput( + region, endpoint, registry, schema, cache, + ttl, autoReg, compat, compress))); + } + + private static void assertOptionalMapping( + Map result, Optional optionalValue, String expectedKey) { + if (optionalValue.isPresent()) { + assertThat(result).containsEntry(expectedKey, optionalValue.get()); + } else { + assertThat(result).doesNotContainKey(expectedKey); + } + } + + private static long countPresent(Optional... optionals) { + long count = 0; + for (Optional opt : optionals) { + if (opt.isPresent()) { + count++; + } + } + return count; + } + + /** Value object holding an arbitrary combination of GSR config options. */ + static class GsrConfigInput { + final Optional region; + final Optional endpoint; + final Optional registryName; + final Optional schemaName; + final Optional cacheSize; + final Optional cacheTtlMs; + final Optional autoRegistration; + final Optional compatibility; + final Optional compression; + + GsrConfigInput( + Optional region, + Optional endpoint, + Optional registryName, + Optional schemaName, + Optional cacheSize, + Optional cacheTtlMs, + Optional autoRegistration, + Optional compatibility, + Optional compression) { + this.region = region; + this.endpoint = endpoint; + this.registryName = registryName; + this.schemaName = schemaName; + this.cacheSize = cacheSize; + this.cacheTtlMs = cacheTtlMs; + this.autoRegistration = autoRegistration; + this.compatibility = compatibility; + this.compression = compression; + } + + @Override + public String toString() { + return "GsrConfigInput{" + + "region=" + + region + + ", endpoint=" + + endpoint + + ", registryName=" + + registryName + + ", schemaName=" + + schemaName + + ", cacheSize=" + + cacheSize + + ", cacheTtlMs=" + + cacheTtlMs + + ", autoRegistration=" + + autoRegistration + + ", compatibility=" + + compatibility + + ", compression=" + + compression + + '}'; + } + } +} diff --git a/flink-formats-aws/flink-avro-glue-schema-registry/src/test/java/org/apache/flink/formats/avro/glue/schema/registry/GlueSchemaRegistryAvroFormatFactoryPropertyTest.java b/flink-formats-aws/flink-avro-glue-schema-registry/src/test/java/org/apache/flink/formats/avro/glue/schema/registry/GlueSchemaRegistryAvroFormatFactoryPropertyTest.java new file mode 100644 index 000000000..b5875fd40 --- /dev/null +++ b/flink-formats-aws/flink-avro-glue-schema-registry/src/test/java/org/apache/flink/formats/avro/glue/schema/registry/GlueSchemaRegistryAvroFormatFactoryPropertyTest.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.avro.glue.schema.registry; + +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.catalog.Column; +import org.apache.flink.table.catalog.ResolvedSchema; +import org.apache.flink.table.factories.TestDynamicTableFactory; + +import net.jqwik.api.Arbitraries; +import net.jqwik.api.Arbitrary; +import net.jqwik.api.Combinators; +import net.jqwik.api.ForAll; +import net.jqwik.api.Property; +import net.jqwik.api.Provide; +import net.jqwik.api.Tag; + +import java.util.HashMap; +import java.util.Map; + +import static org.apache.flink.table.factories.utils.FactoryMocks.createTableSource; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * Property-based tests for required option validation in {@link + * GlueSchemaRegistryAvroFormatFactory}. + * + *

Validates: Requirements 1.2, 3.2, 4.2, 5.3 + */ +@Tag( + "Feature: gsr-flink-sql-formats, Property 1: Required option validation across all format" + + " factories") +class GlueSchemaRegistryAvroFormatFactoryPropertyTest { + + private static final ResolvedSchema SCHEMA = + ResolvedSchema.of( + Column.physical("a", DataTypes.STRING()), + Column.physical("b", DataTypes.INT()), + Column.physical("c", DataTypes.BOOLEAN())); + + /** + * For any subset of required options that is missing at least one required option, creating a + * table source should throw a ValidationException. + * + *

Required options: aws.region, registry.name, schema.name + */ + @Property(tries = 100) + void missingAnyRequiredOptionCausesValidationException( + @ForAll("incompleteRequiredOptionSets") RequiredOptionSubset subset) { + + Map options = new HashMap<>(); + options.put("connector", TestDynamicTableFactory.IDENTIFIER); + options.put("target", "MyTarget"); + options.put("buffer-size", "1000"); + options.put("format", GlueSchemaRegistryAvroFormatFactory.IDENTIFIER); + + if (subset.includeRegion) { + options.put("avro-glue.aws.region", subset.regionValue); + } + if (subset.includeRegistryName) { + options.put("avro-glue.registry.name", subset.registryNameValue); + } + if (subset.includeSchemaName) { + options.put("avro-glue.schema.name", subset.schemaNameValue); + } + + assertThatThrownBy(() -> createTableSource(SCHEMA, options)) + .isInstanceOf(ValidationException.class); + } + + @Provide + Arbitrary incompleteRequiredOptionSets() { + Arbitrary bools = Arbitraries.of(true, false); + Arbitrary regionValues = + Arbitraries.of("us-east-1", "us-west-2", "eu-west-1", "ap-southeast-1"); + Arbitrary registryValues = + Arbitraries.strings().alpha().ofMinLength(1).ofMaxLength(20); + Arbitrary schemaValues = + Arbitraries.strings().alpha().ofMinLength(1).ofMaxLength(20); + + return Combinators.combine(bools, bools, bools, regionValues, registryValues, schemaValues) + .as(RequiredOptionSubset::new) + // Filter to only keep subsets where at least one required option is missing + .filter(s -> !(s.includeRegion && s.includeRegistryName && s.includeSchemaName)); + } + + /** Value object representing a subset of required options. */ + static class RequiredOptionSubset { + final boolean includeRegion; + final boolean includeRegistryName; + final boolean includeSchemaName; + final String regionValue; + final String registryNameValue; + final String schemaNameValue; + + RequiredOptionSubset( + boolean includeRegion, + boolean includeRegistryName, + boolean includeSchemaName, + String regionValue, + String registryNameValue, + String schemaNameValue) { + this.includeRegion = includeRegion; + this.includeRegistryName = includeRegistryName; + this.includeSchemaName = includeSchemaName; + this.regionValue = regionValue; + this.registryNameValue = registryNameValue; + this.schemaNameValue = schemaNameValue; + } + + @Override + public String toString() { + return "RequiredOptionSubset{" + + "region=" + + (includeRegion ? regionValue : "") + + ", registryName=" + + (includeRegistryName ? registryNameValue : "") + + ", schemaName=" + + (includeSchemaName ? schemaNameValue : "") + + '}'; + } + } +} diff --git a/flink-formats-aws/flink-avro-glue-schema-registry/src/test/java/org/apache/flink/formats/avro/glue/schema/registry/GlueSchemaRegistryAvroFormatFactoryTest.java b/flink-formats-aws/flink-avro-glue-schema-registry/src/test/java/org/apache/flink/formats/avro/glue/schema/registry/GlueSchemaRegistryAvroFormatFactoryTest.java new file mode 100644 index 000000000..f215d400d --- /dev/null +++ b/flink-formats-aws/flink-avro-glue-schema-registry/src/test/java/org/apache/flink/formats/avro/glue/schema/registry/GlueSchemaRegistryAvroFormatFactoryTest.java @@ -0,0 +1,250 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.avro.glue.schema.registry; + +import org.apache.flink.api.common.serialization.DeserializationSchema; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.formats.avro.AvroRowDataDeserializationSchema; +import org.apache.flink.formats.avro.AvroRowDataSerializationSchema; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.catalog.Column; +import org.apache.flink.table.catalog.ResolvedSchema; +import org.apache.flink.table.connector.sink.DynamicTableSink; +import org.apache.flink.table.connector.source.DynamicTableSource; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.factories.TestDynamicTableFactory; +import org.apache.flink.table.runtime.connector.source.ScanRuntimeProviderContext; +import org.apache.flink.table.types.logical.RowType; + +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; + +import static org.apache.flink.table.factories.utils.FactoryMocks.createTableSink; +import static org.apache.flink.table.factories.utils.FactoryMocks.createTableSource; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for the {@link GlueSchemaRegistryAvroFormatFactory}. */ +class GlueSchemaRegistryAvroFormatFactoryTest { + + private static final ResolvedSchema SCHEMA = + ResolvedSchema.of( + Column.physical("a", DataTypes.STRING()), + Column.physical("b", DataTypes.INT()), + Column.physical("c", DataTypes.BOOLEAN())); + + private static final RowType ROW_TYPE = + (RowType) SCHEMA.toPhysicalRowDataType().getLogicalType(); + + private static final String SCHEMA_NAME = "test-subject"; + private static final String REGISTRY_NAME = "test-registry-name"; + private static final String REGION = "us-west-2"; + + @Test + void testDeserializationSchema() { + final DynamicTableSource actualSource = createTableSource(SCHEMA, getDefaultOptions()); + assertThat(actualSource).isInstanceOf(TestDynamicTableFactory.DynamicTableSourceMock.class); + + TestDynamicTableFactory.DynamicTableSourceMock scanSourceMock = + (TestDynamicTableFactory.DynamicTableSourceMock) actualSource; + + DeserializationSchema actualDeser = + scanSourceMock.valueFormat.createRuntimeDecoder( + ScanRuntimeProviderContext.INSTANCE, SCHEMA.toPhysicalRowDataType()); + + assertThat(actualDeser).isInstanceOf(AvroRowDataDeserializationSchema.class); + } + + @Test + void testSerializationSchema() { + final DynamicTableSink actualSink = createTableSink(SCHEMA, getDefaultOptions()); + assertThat(actualSink).isInstanceOf(TestDynamicTableFactory.DynamicTableSinkMock.class); + + TestDynamicTableFactory.DynamicTableSinkMock sinkMock = + (TestDynamicTableFactory.DynamicTableSinkMock) actualSink; + + SerializationSchema actualSer = + sinkMock.valueFormat.createRuntimeEncoder(null, SCHEMA.toPhysicalRowDataType()); + + assertThat(actualSer).isInstanceOf(AvroRowDataSerializationSchema.class); + } + + @Test + void testMissingSchemaNameForSink() { + final Map options = + getModifiedOptions(opts -> opts.remove("avro-glue.schema.name")); + + assertThatThrownBy(() -> createTableSink(SCHEMA, options)) + .isInstanceOf(ValidationException.class); + } + + @Test + void testMissingRegionForSource() { + final Map options = + getModifiedOptions(opts -> opts.remove("avro-glue.aws.region")); + + assertThatThrownBy(() -> createTableSource(SCHEMA, options)) + .isInstanceOf(ValidationException.class); + } + + @Test + void testMissingRegistryNameForSource() { + final Map options = + getModifiedOptions(opts -> opts.remove("avro-glue.registry.name")); + + assertThatThrownBy(() -> createTableSource(SCHEMA, options)) + .isInstanceOf(ValidationException.class); + } + + @Test + void testDeserializationSchemaWithNamespaceOverride() { + final String customNamespace = "com.example.custom"; + final Map options = + getModifiedOptions(opts -> opts.put("avro-glue.avro.namespace", customNamespace)); + + final DynamicTableSource actualSource = createTableSource(SCHEMA, options); + assertThat(actualSource).isInstanceOf(TestDynamicTableFactory.DynamicTableSourceMock.class); + + TestDynamicTableFactory.DynamicTableSourceMock scanSourceMock = + (TestDynamicTableFactory.DynamicTableSourceMock) actualSource; + + DeserializationSchema actualDeser = + scanSourceMock.valueFormat.createRuntimeDecoder( + ScanRuntimeProviderContext.INSTANCE, SCHEMA.toPhysicalRowDataType()); + + // Verify the schema is created successfully with namespace override + assertThat(actualDeser).isInstanceOf(AvroRowDataDeserializationSchema.class); + } + + @Test + void testSerializationSchemaWithRecordNameOverride() { + final String customRecordName = "MyCustomRecord"; + final Map options = + getModifiedOptions( + opts -> opts.put("avro-glue.avro.record-name", customRecordName)); + + final DynamicTableSink actualSink = createTableSink(SCHEMA, options); + assertThat(actualSink).isInstanceOf(TestDynamicTableFactory.DynamicTableSinkMock.class); + + TestDynamicTableFactory.DynamicTableSinkMock sinkMock = + (TestDynamicTableFactory.DynamicTableSinkMock) actualSink; + + SerializationSchema actualSer = + sinkMock.valueFormat.createRuntimeEncoder(null, SCHEMA.toPhysicalRowDataType()); + + // Verify the schema is created successfully with record name override + assertThat(actualSer).isInstanceOf(AvroRowDataSerializationSchema.class); + } + + @Test + void testSpiDiscovery() { + final DynamicTableSource source = createTableSource(SCHEMA, getDefaultOptions()); + assertThat(source).isNotNull(); + + final DynamicTableSink sink = createTableSink(SCHEMA, getDefaultOptions()); + assertThat(sink).isNotNull(); + } + + @Test + void testDeserializationSchemaWithFetchFromRegistry() { + final Map options = + getModifiedOptions(opts -> opts.put("avro-glue.schema.fetchFromRegistry", "true")); + + // Should not throw - the option should be accepted + final DynamicTableSource actualSource = createTableSource(SCHEMA, options); + assertThat(actualSource).isInstanceOf(TestDynamicTableFactory.DynamicTableSourceMock.class); + + TestDynamicTableFactory.DynamicTableSourceMock scanSourceMock = + (TestDynamicTableFactory.DynamicTableSourceMock) actualSource; + + DeserializationSchema actualDeser = + scanSourceMock.valueFormat.createRuntimeDecoder( + ScanRuntimeProviderContext.INSTANCE, SCHEMA.toPhysicalRowDataType()); + + assertThat(actualDeser).isInstanceOf(AvroRowDataDeserializationSchema.class); + } + + @Test + void testSerializationSchemaWithFetchFromRegistry() { + final Map options = + getModifiedOptions(opts -> opts.put("avro-glue.schema.fetchFromRegistry", "true")); + + // Should not throw - the option should be accepted + final DynamicTableSink actualSink = createTableSink(SCHEMA, options); + assertThat(actualSink).isInstanceOf(TestDynamicTableFactory.DynamicTableSinkMock.class); + + TestDynamicTableFactory.DynamicTableSinkMock sinkMock = + (TestDynamicTableFactory.DynamicTableSinkMock) actualSink; + + SerializationSchema actualSer = + sinkMock.valueFormat.createRuntimeEncoder(null, SCHEMA.toPhysicalRowDataType()); + + assertThat(actualSer).isInstanceOf(AvroRowDataSerializationSchema.class); + } + + @Test + void testDeserializationSchemaWithAllOverrideOptions() { + final Map options = + getModifiedOptions( + opts -> { + opts.put("avro-glue.avro.namespace", "com.example.custom"); + opts.put("avro-glue.avro.record-name", "CustomRecord"); + opts.put("avro-glue.schema.fetchFromRegistry", "false"); + }); + + final DynamicTableSource actualSource = createTableSource(SCHEMA, options); + assertThat(actualSource).isInstanceOf(TestDynamicTableFactory.DynamicTableSourceMock.class); + + TestDynamicTableFactory.DynamicTableSourceMock scanSourceMock = + (TestDynamicTableFactory.DynamicTableSourceMock) actualSource; + + DeserializationSchema actualDeser = + scanSourceMock.valueFormat.createRuntimeDecoder( + ScanRuntimeProviderContext.INSTANCE, SCHEMA.toPhysicalRowDataType()); + + assertThat(actualDeser).isInstanceOf(AvroRowDataDeserializationSchema.class); + } + + // ------------------------------------------------------------------------ + // Utilities + // ------------------------------------------------------------------------ + + private Map getModifiedOptions(Consumer> optionModifier) { + Map options = getDefaultOptions(); + optionModifier.accept(options); + return options; + } + + private Map getDefaultOptions() { + final Map options = new HashMap<>(); + options.put("connector", TestDynamicTableFactory.IDENTIFIER); + options.put("target", "MyTarget"); + options.put("buffer-size", "1000"); + + options.put("format", GlueSchemaRegistryAvroFormatFactory.IDENTIFIER); + options.put("avro-glue.schema.name", SCHEMA_NAME); + options.put("avro-glue.registry.name", REGISTRY_NAME); + options.put("avro-glue.aws.region", REGION); + return options; + } +} diff --git a/flink-formats-aws/flink-avro-glue-schema-registry/src/test/java/org/apache/flink/formats/avro/glue/schema/registry/GlueSchemaRegistrySchemaFetcherTest.java b/flink-formats-aws/flink-avro-glue-schema-registry/src/test/java/org/apache/flink/formats/avro/glue/schema/registry/GlueSchemaRegistrySchemaFetcherTest.java new file mode 100644 index 000000000..5589c9977 --- /dev/null +++ b/flink-formats-aws/flink-avro-glue-schema-registry/src/test/java/org/apache/flink/formats/avro/glue/schema/registry/GlueSchemaRegistrySchemaFetcherTest.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.avro.glue.schema.registry; + +import org.apache.avro.Schema; +import org.apache.avro.SchemaBuilder; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.glue.GlueClient; +import software.amazon.awssdk.services.glue.model.GetSchemaVersionRequest; +import software.amazon.awssdk.services.glue.model.GetSchemaVersionResponse; + +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for {@link GlueSchemaRegistrySchemaFetcher} using a mock GlueClient. */ +class GlueSchemaRegistrySchemaFetcherTest { + + private static final String REGISTRY_NAME = "test-registry"; + private static final String SCHEMA_NAME = "test-schema"; + + @Test + void testFetchSchemaReturnsValidSchema() throws Exception { + Schema expected = + SchemaBuilder.record("TestRecord") + .namespace("com.example") + .fields() + .requiredString("id") + .requiredInt("value") + .endRecord(); + + AtomicReference capturedRequest = new AtomicReference<>(); + + GlueClient mockClient = + createMockGlueClient( + request -> capturedRequest.set(request), + GetSchemaVersionResponse.builder() + .schemaDefinition(expected.toString()) + .build()); + + GlueSchemaRegistrySchemaFetcher fetcher = new GlueSchemaRegistrySchemaFetcher(mockClient); + + Schema result = fetcher.fetchSchema(REGISTRY_NAME, SCHEMA_NAME); + + assertThat(result).isNotNull(); + assertThat(result.getNamespace()).isEqualTo("com.example"); + assertThat(result.getName()).isEqualTo("TestRecord"); + assertThat(result.getFields()).hasSize(2); + + // Verify the request was constructed correctly + GetSchemaVersionRequest req = capturedRequest.get(); + assertThat(req.schemaId().registryName()).isEqualTo(REGISTRY_NAME); + assertThat(req.schemaId().schemaName()).isEqualTo(SCHEMA_NAME); + assertThat(req.schemaVersionNumber().latestVersion()).isTrue(); + } + + @Test + void testFetchSchemaReturnsNullForEmptyDefinition() throws Exception { + GlueClient mockClient = + createMockGlueClient( + request -> {}, + GetSchemaVersionResponse.builder().schemaDefinition("").build()); + + GlueSchemaRegistrySchemaFetcher fetcher = new GlueSchemaRegistrySchemaFetcher(mockClient); + + Schema result = fetcher.fetchSchema(REGISTRY_NAME, SCHEMA_NAME); + + assertThat(result).isNull(); + } + + @Test + void testFetchSchemaReturnsNullForNullDefinition() throws Exception { + GlueClient mockClient = + createMockGlueClient(request -> {}, GetSchemaVersionResponse.builder().build()); + + GlueSchemaRegistrySchemaFetcher fetcher = new GlueSchemaRegistrySchemaFetcher(mockClient); + + Schema result = fetcher.fetchSchema(REGISTRY_NAME, SCHEMA_NAME); + + assertThat(result).isNull(); + } + + @Test + void testFetchSchemaPropagatesGlueClientException() { + GlueClient mockClient = + createThrowingMockGlueClient(new RuntimeException("Service unavailable")); + + GlueSchemaRegistrySchemaFetcher fetcher = new GlueSchemaRegistrySchemaFetcher(mockClient); + + assertThatThrownBy(() -> fetcher.fetchSchema(REGISTRY_NAME, SCHEMA_NAME)) + .isInstanceOf(RuntimeException.class) + .hasMessage("Service unavailable"); + } + + /** + * Creates a mock GlueClient that captures the request and returns a fixed response. + * + *

Uses a proxy to avoid depending on Mockito. + */ + private static GlueClient createMockGlueClient( + Consumer requestCaptor, GetSchemaVersionResponse response) { + return new GlueClientStub() { + @Override + public GetSchemaVersionResponse getSchemaVersion(GetSchemaVersionRequest request) { + requestCaptor.accept(request); + return response; + } + }; + } + + /** Creates a mock GlueClient that throws on getSchemaVersion. */ + private static GlueClient createThrowingMockGlueClient(RuntimeException exception) { + return new GlueClientStub() { + @Override + public GetSchemaVersionResponse getSchemaVersion(GetSchemaVersionRequest request) { + throw exception; + } + }; + } + + /** + * Minimal stub of GlueClient. Only getSchemaVersion is used by the fetcher; other methods throw + * UnsupportedOperationException if called. + */ + private abstract static class GlueClientStub implements GlueClient { + + @Override + public String serviceName() { + return "glue"; + } + + @Override + public void close() {} + } +} diff --git a/flink-formats-aws/flink-avro-glue-schema-registry/src/test/java/org/apache/flink/formats/avro/glue/schema/registry/MockGlueSchemaRegistryFacades.java b/flink-formats-aws/flink-avro-glue-schema-registry/src/test/java/org/apache/flink/formats/avro/glue/schema/registry/MockGlueSchemaRegistryFacades.java new file mode 100644 index 000000000..a1087d538 --- /dev/null +++ b/flink-formats-aws/flink-avro-glue-schema-registry/src/test/java/org/apache/flink/formats/avro/glue/schema/registry/MockGlueSchemaRegistryFacades.java @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.avro.glue.schema.registry; + +import com.amazonaws.services.schemaregistry.utils.AWSSchemaRegistryConstants; +import org.apache.avro.Schema; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +/** + * In-memory mock implementations of GSR facade classes for integration testing without AWS + * credentials. + * + *

The mock serialization facade stores schemas in-memory and prepends a mock 18-byte GSR header. + * The mock deserialization facade strips the header and returns the schema definition from the + * in-memory store. + */ +public class MockGlueSchemaRegistryFacades { + + /** GSR header size: 1 (version) + 1 (compression) + 16 (UUID) = 18 bytes. */ + public static final int GSR_HEADER_SIZE = 18; + + /** In-memory schema store shared between serialization and deserialization mocks. */ + private final Map schemaStore = new HashMap<>(); + + /** + * Encodes payload bytes by prepending a mock GSR header and registering the schema. + * + * @param schemaDefinition the schema definition string + * @param payload the raw serialized bytes + * @return bytes with mock GSR header prepended + */ + public byte[] encode(String schemaDefinition, byte[] payload) { + UUID schemaVersionId = getOrRegister(schemaDefinition); + ByteBuffer buffer = ByteBuffer.allocate(GSR_HEADER_SIZE + payload.length); + buffer.put(AWSSchemaRegistryConstants.HEADER_VERSION_BYTE); + buffer.put((byte) 0x00); // no compression + buffer.putLong(schemaVersionId.getMostSignificantBits()); + buffer.putLong(schemaVersionId.getLeastSignificantBits()); + buffer.put(payload); + return buffer.array(); + } + + /** + * Strips the mock GSR header and returns the raw payload bytes. + * + * @param gsrEncodedBytes the GSR-encoded bytes (header + payload) + * @return the raw payload bytes + * @throws IOException if the input is too short + */ + public byte[] getActualData(byte[] gsrEncodedBytes) throws IOException { + if (gsrEncodedBytes.length < GSR_HEADER_SIZE) { + throw new IOException( + "Invalid GSR-encoded data: expected at least " + + GSR_HEADER_SIZE + + " header bytes, got " + + gsrEncodedBytes.length); + } + byte[] payload = new byte[gsrEncodedBytes.length - GSR_HEADER_SIZE]; + System.arraycopy(gsrEncodedBytes, GSR_HEADER_SIZE, payload, 0, payload.length); + return payload; + } + + /** + * Extracts the schema definition from the GSR header UUID. + * + * @param gsrEncodedBytes the GSR-encoded bytes + * @return the schema definition string + * @throws IOException if the schema is not found + */ + public String getSchemaDefinition(byte[] gsrEncodedBytes) throws IOException { + if (gsrEncodedBytes.length < GSR_HEADER_SIZE) { + throw new IOException("Invalid GSR-encoded data: too short"); + } + ByteBuffer buffer = ByteBuffer.wrap(gsrEncodedBytes, 2, 16); + UUID schemaVersionId = new UUID(buffer.getLong(), buffer.getLong()); + String definition = schemaStore.get(schemaVersionId); + if (definition == null) { + throw new IOException("Schema not found for UUID: " + schemaVersionId); + } + return definition; + } + + /** Registers a schema definition and returns its UUID. */ + private UUID getOrRegister(String schemaDefinition) { + // Check if already registered + for (Map.Entry entry : schemaStore.entrySet()) { + if (entry.getValue().equals(schemaDefinition)) { + return entry.getKey(); + } + } + UUID id = UUID.randomUUID(); + schemaStore.put(id, schemaDefinition); + return id; + } + + /** + * Creates a mock {@link GlueSchemaRegistryOutputStreamSerializer} that uses this facade's + * in-memory store. + */ + public GlueSchemaRegistryOutputStreamSerializer createMockOutputStreamSerializer( + String transportName, Map configs) { + return new MockOutputStreamSerializer(transportName, configs); + } + + /** + * Creates a mock {@link GlueSchemaRegistryInputStreamDeserializer} that uses this facade's + * in-memory store. + */ + public GlueSchemaRegistryInputStreamDeserializer createMockInputStreamDeserializer() { + return new MockInputStreamDeserializer(); + } + + // ---- Inner mock classes ---- + + /** Mock output stream serializer that stores schemas in-memory. */ + private class MockOutputStreamSerializer extends GlueSchemaRegistryOutputStreamSerializer { + + public MockOutputStreamSerializer(String transportName, Map configs) { + super(transportName, configs, null); + } + + @Override + public void registerSchemaAndSerializeStream(Schema schema, OutputStream out, byte[] data) + throws IOException { + byte[] encoded = encode(schema.toString(), data); + out.write(encoded); + } + } + + /** Mock input stream deserializer that reads from the in-memory store. */ + private class MockInputStreamDeserializer extends GlueSchemaRegistryInputStreamDeserializer { + + public MockInputStreamDeserializer() { + super( + (com.amazonaws.services.schemaregistry.deserializers + .GlueSchemaRegistryDeserializationFacade) + null); + } + + @Override + public Schema getSchemaAndDeserializedStream(InputStream in) throws IOException { + byte[] inputBytes = new byte[in.available()]; + in.read(inputBytes); + in.reset(); + + // Extract schema definition from the header + String schemaDefinition = + MockGlueSchemaRegistryFacades.this.getSchemaDefinition(inputBytes); + + // Strip header and update the stream + byte[] actualData = MockGlueSchemaRegistryFacades.this.getActualData(inputBytes); + org.apache.flink.formats.avro.utils.MutableByteArrayInputStream mutableStream = + (org.apache.flink.formats.avro.utils.MutableByteArrayInputStream) in; + mutableStream.setBuffer(actualData); + + return new Schema.Parser().parse(schemaDefinition); + } + } +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/pom.xml b/flink-formats-aws/flink-protobuf-glue-schema-registry/pom.xml new file mode 100644 index 000000000..98bee38e6 --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/pom.xml @@ -0,0 +1,164 @@ + + + + 4.0.0 + + + org.apache.flink + flink-formats-aws-parent + 6.0-SNAPSHOT + + + flink-protobuf-glue-schema-registry + Flink : Formats : AWS : Protobuf Glue Schema Registry + jar + + + + 3.25.5 + + + + + org.apache.flink + flink-core + ${flink.version} + provided + + + org.apache.flink + flink-streaming-java + ${flink.version} + provided + + + org.apache.flink + flink-table-api-java + ${flink.version} + provided + + + org.apache.flink + flink-table-common + ${flink.version} + provided + + + org.apache.flink + flink-avro-glue-schema-registry + ${project.version} + + + org.apache.flink + flink-connector-aws-base + ${project.version} + + + software.amazon.glue + schema-registry-serde + ${glue.schema.registry.version} + + + software.amazon.glue + schema-registry-common + ${glue.schema.registry.version} + + + com.google.protobuf + protobuf-java + ${protobuf.version} + + + com.google.protobuf + protobuf-java-util + ${protobuf.version} + + + + + org.apache.flink + flink-table-runtime + ${flink.version} + test + + + org.apache.flink + flink-table-api-java + ${flink.version} + test + test-jar + + + org.apache.flink + flink-table-common + ${flink.version} + test + test-jar + + + org.projectlombok + lombok + 1.18.30 + test + + + + + org.apache.flink + flink-architecture-tests-test + test + + + + + net.jqwik + jqwik + 1.8.2 + test + + + + + + + + org.opentest4j + opentest4j + 1.3.0 + + + com.google.errorprone + error_prone_annotations + 2.21.1 + + + + diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/FacadeGsrProtobufReader.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/FacadeGsrProtobufReader.java new file mode 100644 index 000000000..3c4ff9bc9 --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/FacadeGsrProtobufReader.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.connector.aws.util.AWSGeneralUtil; + +import com.amazonaws.services.schemaregistry.deserializers.GlueSchemaRegistryDeserializationFacade; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; + +import java.util.Map; + +/** + * Production {@link GsrProtobufReader} backed by {@link GlueSchemaRegistryDeserializationFacade}. + * + *

The facade resolves the writer schema (via the schema-version UUID embedded in the record) + * from AWS Glue Schema Registry and returns the header-stripped, decompressed payload. The facade + * itself is not serializable and is built lazily on each task manager from the serializable config + * map, mirroring the Avro format's {@code GlueSchemaRegistryInputStreamDeserializer}. + */ +@Internal +final class FacadeGsrProtobufReader implements GsrProtobufReader { + + private static final long serialVersionUID = 1L; + + private final Map configs; + + private transient GlueSchemaRegistryDeserializationFacade facade; + + FacadeGsrProtobufReader(Map configs) { + this.configs = configs; + } + + private GlueSchemaRegistryDeserializationFacade facade() { + if (facade == null) { + AwsCredentialsProvider credentialsProvider = + AWSGeneralUtil.getCredentialsProvider(configs); + facade = + GlueSchemaRegistryDeserializationFacade.builder() + .credentialProvider(credentialsProvider) + .configs(configs) + .build(); + } + return facade; + } + + @Override + public String writerSchemaDefinition(byte[] gsrEncoded) { + return facade().getSchema(gsrEncoded).getSchemaDefinition(); + } + + @Override + public byte[] actualData(byte[] gsrEncoded) { + return facade().getActualData(gsrEncoded); + } +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GlueSchemaRegistryProtobufFormatFactory.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GlueSchemaRegistryProtobufFormatFactory.java new file mode 100644 index 000000000..b8954463e --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GlueSchemaRegistryProtobufFormatFactory.java @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.api.common.serialization.DeserializationSchema; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.configuration.ConfigOption; +import org.apache.flink.configuration.ReadableConfig; +import org.apache.flink.formats.avro.glue.schema.registry.GlueFormatConfigBuilder; +import org.apache.flink.formats.avro.glue.schema.registry.GlueFormatOptions; +import org.apache.flink.table.connector.ChangelogMode; +import org.apache.flink.table.connector.format.DecodingFormat; +import org.apache.flink.table.connector.format.EncodingFormat; +import org.apache.flink.table.connector.sink.DynamicTableSink; +import org.apache.flink.table.connector.source.DynamicTableSource; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.factories.DeserializationFormatFactory; +import org.apache.flink.table.factories.DynamicTableFactory; +import org.apache.flink.table.factories.FactoryUtil; +import org.apache.flink.table.factories.SerializationFormatFactory; +import org.apache.flink.table.types.DataType; +import org.apache.flink.table.types.logical.RowType; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * Table format factory for providing configured instances of AWS Glue Schema Registry Protobuf to + * RowData {@link SerializationSchema} and {@link DeserializationSchema}. + * + *

This factory supports: + * + *

    + *
  • SPI discovery via identifier {@code protobuf-glue} + *
  • Protobuf serialization with GSR header prepending + *
  • Protobuf deserialization with GSR header stripping + *
  • Protobuf schema registration with GSR derived from Flink RowType + *
+ */ +@Internal +public class GlueSchemaRegistryProtobufFormatFactory + implements DeserializationFormatFactory, SerializationFormatFactory { + + public static final String IDENTIFIER = "protobuf-glue"; + + @Override + public DecodingFormat> createDecodingFormat( + DynamicTableFactory.Context context, ReadableConfig formatOptions) { + FactoryUtil.validateFactoryOptions(this, formatOptions); + + return new DecodingFormat>() { + @Override + public DeserializationSchema createRuntimeDecoder( + DynamicTableSource.Context context, DataType producedDataType) { + final RowType rowType = (RowType) producedDataType.getLogicalType(); + final TypeInformation rowDataTypeInfo = + context.createTypeInformation(producedDataType); + final String schemaName = formatOptions.get(GlueFormatOptions.SCHEMA_NAME); + final Map configMap = + GlueFormatConfigBuilder.buildConfigMap(formatOptions); + + return new GsrProtobufRowDataDeserializationSchema( + rowType, rowDataTypeInfo, schemaName, configMap); + } + + @Override + public ChangelogMode getChangelogMode() { + return ChangelogMode.insertOnly(); + } + }; + } + + @Override + public EncodingFormat> createEncodingFormat( + DynamicTableFactory.Context context, ReadableConfig formatOptions) { + FactoryUtil.validateFactoryOptions(this, formatOptions); + + return new EncodingFormat>() { + @Override + public SerializationSchema createRuntimeEncoder( + DynamicTableSink.Context context, DataType consumedDataType) { + final RowType rowType = (RowType) consumedDataType.getLogicalType(); + final Map configMap = + GlueFormatConfigBuilder.buildConfigMap(formatOptions); + final String schemaName = formatOptions.get(GlueFormatOptions.SCHEMA_NAME); + + return new GsrProtobufRowDataSerializationSchema( + rowType, schemaName, schemaName, configMap); + } + + @Override + public ChangelogMode getChangelogMode() { + return ChangelogMode.insertOnly(); + } + }; + } + + @Override + public String factoryIdentifier() { + return IDENTIFIER; + } + + @Override + public Set> requiredOptions() { + Set> options = new HashSet<>(); + options.add(GlueFormatOptions.AWS_REGION); + options.add(GlueFormatOptions.REGISTRY_NAME); + options.add(GlueFormatOptions.SCHEMA_NAME); + return options; + } + + @Override + public Set> optionalOptions() { + Set> options = new HashSet<>(); + options.add(GlueFormatOptions.AWS_ENDPOINT); + options.add(GlueFormatOptions.CACHE_SIZE); + options.add(GlueFormatOptions.CACHE_TTL_MS); + options.add(GlueFormatOptions.SCHEMA_AUTO_REGISTRATION); + options.add(GlueFormatOptions.SCHEMA_COMPATIBILITY); + options.add(GlueFormatOptions.SCHEMA_COMPRESSION); + return options; + } + + @Override + public Set> forwardOptions() { + return Stream.of( + GlueFormatOptions.AWS_REGION, + GlueFormatOptions.AWS_ENDPOINT, + GlueFormatOptions.REGISTRY_NAME, + GlueFormatOptions.SCHEMA_NAME, + GlueFormatOptions.CACHE_SIZE, + GlueFormatOptions.CACHE_TTL_MS, + GlueFormatOptions.SCHEMA_AUTO_REGISTRATION, + GlueFormatOptions.SCHEMA_COMPATIBILITY, + GlueFormatOptions.SCHEMA_COMPRESSION) + .collect(Collectors.toSet()); + } +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufReader.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufReader.java new file mode 100644 index 000000000..45ed6b163 --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufReader.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.annotation.Internal; + +import java.io.Serializable; + +/** + * Read-side seam that resolves the writer schema and the actual Protobuf payload from a GSR-encoded + * record. + * + *

This mirrors the Avro format's {@code SchemaCoder} abstraction (see {@link + * org.apache.flink.formats.avro.glue.schema.registry.GlueSchemaRegistryAvroSchemaCoder}): decoding + * is routed through the AWS Glue Schema Registry deserialization facade so that the writer + * schema/version registered in Glue is resolved from the record, and the GSR header, schema-version + * UUID and compression are handled centrally in-library — instead of the on-wire contract being + * defined by the local table DDL (review finding B1) and instead of a hand-rolled 18-byte header + * strip that never decompresses (review finding C1). + * + *

The seam is {@link Serializable} so it ships with the {@link + * GsrProtobufRowDataDeserializationSchema}; the concrete facade is built lazily on the task + * managers. A hand-written fake implementation is used in unit tests, avoiding any dependency on a + * live registry. + */ +@Internal +interface GsrProtobufReader extends Serializable { + + /** + * Returns the proto3 writer schema definition registered in GSR for this record. The reader + * descriptor is built from this definition, so the on-wire field numbers/types come from the + * registry — not from the local {@code RowType}. + * + * @param gsrEncoded the full GSR-encoded record (header + payload) + * @return the writer schema definition (proto3 text) + */ + String writerSchemaDefinition(byte[] gsrEncoded); + + /** + * Returns the header-stripped and (if the writer compressed it) decompressed Protobuf payload. + * Decompression is handled centrally by the GSR facade (review finding C1). + * + * @param gsrEncoded the full GSR-encoded record (header + payload) + * @return the raw Protobuf message bytes + */ + byte[] actualData(byte[] gsrEncoded); +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufRowDataDeserializationSchema.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufRowDataDeserializationSchema.java new file mode 100644 index 000000000..8843c604d --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufRowDataDeserializationSchema.java @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.api.common.serialization.DeserializationSchema; +import org.apache.flink.api.common.typeinfo.TypeInformation; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.logical.RowType; + +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Deserialization schema that decodes AWS Glue Schema Registry encoded Protobuf records into Flink + * {@link RowData}. + * + *

Decoding is routed through the GSR deserialization facade (see {@link GsrProtobufReader}) so + * that: + * + *

    + *
  • the writer schema/version is resolved from Glue via the schema-version UUID in the + * record — the reader descriptor is built from that writer schema, not from the local {@code + * RowType} (review finding B1); + *
  • the GSR header and any compression are handled centrally in-library, rather than by a + * hand-rolled 18-byte strip that never decompresses (review finding C1). + *
+ * + *

The decoded {@link DynamicMessage} is mapped to {@link RowData} by field name; see + * {@link ProtobufToRowDataConverter}. + */ +@Internal +public class GsrProtobufRowDataDeserializationSchema implements DeserializationSchema { + + private static final long serialVersionUID = 1L; + + private final RowType rowType; + private final TypeInformation producedType; + private final String schemaName; + private final Map configs; + + private transient GsrProtobufReader reader; + + /** Reader descriptors cached by writer schema definition to avoid re-parsing per record. */ + private transient Map descriptorCache; + + /** + * Creates a new GSR Protobuf deserialization schema. + * + * @param rowType the Flink RowType describing the expected schema + * @param producedType the type information for the produced RowData + * @param schemaName the schema name (retained for diagnostics) + * @param configs the GSR SDK configuration map (region, registry, credentials, cache, + * compression, ...) used to build the deserialization facade + */ + public GsrProtobufRowDataDeserializationSchema( + RowType rowType, + TypeInformation producedType, + String schemaName, + Map configs) { + this.rowType = rowType; + this.producedType = producedType; + this.schemaName = schemaName; + this.configs = configs; + } + + @Override + public void open(InitializationContext context) throws Exception { + if (reader == null) { + reader = new FacadeGsrProtobufReader(configs); + } + if (descriptorCache == null) { + descriptorCache = new HashMap<>(); + } + } + + @Override + public RowData deserialize(byte[] message) throws IOException { + if (message == null) { + return null; + } + + // Resolve the writer schema from GSR and let the facade strip the header + decompress. + final String writerSchemaDefinition = reader.writerSchemaDefinition(message); + final byte[] protobufPayload = reader.actualData(message); + + final Descriptors.Descriptor writerDescriptor = + descriptorCache.computeIfAbsent( + writerSchemaDefinition, + ProtobufSchemaConverter::buildDescriptorFromProtoSchema); + + final DynamicMessage dynamicMessage = + DynamicMessage.parseFrom(writerDescriptor, protobufPayload); + return ProtobufToRowDataConverter.convertToRowData(dynamicMessage, rowType); + } + + @Override + public boolean isEndOfStream(RowData nextElement) { + return false; + } + + @Override + public TypeInformation getProducedType() { + return producedType; + } + + @VisibleForTesting + void setReader(GsrProtobufReader reader) { + this.reader = reader; + } + + @VisibleForTesting + String getSchemaName() { + return schemaName; + } +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufRowDataSerializationSchema.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufRowDataSerializationSchema.java new file mode 100644 index 000000000..6d9b22384 --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufRowDataSerializationSchema.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.annotation.VisibleForTesting; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.connector.aws.util.AWSGeneralUtil; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.logical.RowType; + +import com.amazonaws.services.schemaregistry.common.Schema; +import com.amazonaws.services.schemaregistry.common.configs.GlueSchemaRegistryConfiguration; +import com.amazonaws.services.schemaregistry.serializers.GlueSchemaRegistrySerializationFacade; +import com.google.protobuf.DescriptorProtos; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.services.glue.model.DataFormat; + +import java.util.Map; + +/** + * Serialization schema that converts Flink {@link RowData} to Protobuf bytes and prepends AWS Glue + * Schema Registry header bytes. + * + *

The serialization flow is: + * + *

    + *
  1. Convert {@link RowData} to a Protobuf {@link DynamicMessage} using the schema derived from + * the Flink RowType + *
  2. Serialize the DynamicMessage to Protobuf bytes + *
  3. Register the Protobuf schema definition with GSR + *
  4. Prepend GSR header bytes (version + compression + schema UUID) to the Protobuf payload + *
+ */ +@Internal +public class GsrProtobufRowDataSerializationSchema implements SerializationSchema { + + private static final long serialVersionUID = 1L; + + private final RowType rowType; + private final String transportName; + private final String schemaName; + private final String protobufSchemaDefinition; + private final Map configs; + + private transient GlueSchemaRegistrySerializationFacade serializationFacade; + private transient Descriptors.Descriptor messageDescriptor; + + /** + * Creates a new GSR Protobuf serialization schema. + * + * @param rowType the Flink RowType describing the table schema + * @param transportName the transport name (topic/stream) for GSR schema naming + * @param schemaName the schema name for GSR registration + * @param configs the GSR SDK configuration map + */ + public GsrProtobufRowDataSerializationSchema( + RowType rowType, String transportName, String schemaName, Map configs) { + this.rowType = rowType; + this.transportName = transportName; + this.schemaName = schemaName != null ? schemaName : transportName; + this.configs = configs; + this.protobufSchemaDefinition = + ProtobufSchemaConverter.convertToProtobufSchema(rowType, this.schemaName); + } + + @Override + public void open(InitializationContext context) throws Exception { + if (serializationFacade == null) { + AwsCredentialsProvider credentialsProvider = + AWSGeneralUtil.getCredentialsProvider(configs); + serializationFacade = + GlueSchemaRegistrySerializationFacade.builder() + .credentialProvider(credentialsProvider) + .glueSchemaRegistryConfiguration( + new GlueSchemaRegistryConfiguration(configs)) + .build(); + } + if (messageDescriptor == null) { + messageDescriptor = buildDescriptor(); + } + } + + @Override + public byte[] serialize(RowData element) { + if (element == null) { + return null; + } + + DynamicMessage message = + RowDataToProtobufConverter.convertRowData(element, rowType, messageDescriptor); + byte[] protobufBytes = message.toByteArray(); + + return serializationFacade.encode( + transportName, + new Schema(protobufSchemaDefinition, DataFormat.PROTOBUF.name(), schemaName), + protobufBytes); + } + + @VisibleForTesting + void setSerializationFacade(GlueSchemaRegistrySerializationFacade facade) { + this.serializationFacade = facade; + } + + @VisibleForTesting + void setMessageDescriptor(Descriptors.Descriptor descriptor) { + this.messageDescriptor = descriptor; + } + + private Descriptors.Descriptor buildDescriptor() { + try { + DescriptorProtos.FileDescriptorProto fileProto = + ProtobufSchemaConverter.buildFileDescriptorProto(rowType, schemaName); + Descriptors.FileDescriptor fileDescriptor = + Descriptors.FileDescriptor.buildFrom( + fileProto, new Descriptors.FileDescriptor[] {}); + return fileDescriptor.findMessageTypeByName( + ProtobufSchemaConverter.sanitizeMessageName(schemaName)); + } catch (Descriptors.DescriptorValidationException e) { + throw new RuntimeException("Failed to build Protobuf descriptor from RowType", e); + } + } +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufGlueFormatOptions.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufGlueFormatOptions.java new file mode 100644 index 000000000..37a749338 --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufGlueFormatOptions.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.annotation.PublicEvolving; +import org.apache.flink.formats.avro.glue.schema.registry.GlueFormatOptions; + +/** + * Protobuf-specific configuration options for the AWS Glue Schema Registry Protobuf format factory. + * + *

Shared options (aws.region, registry.name, schema.name, etc.) are inherited from {@link + * GlueFormatOptions}. Currently no additional Protobuf-specific options are needed. + */ +@PublicEvolving +public class ProtobufGlueFormatOptions extends GlueFormatOptions { + + // No Protobuf-specific options needed at this time. + // All shared GSR options are inherited from GlueFormatOptions. +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufSchemaConverter.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufSchemaConverter.java new file mode 100644 index 000000000..6ca065882 --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufSchemaConverter.java @@ -0,0 +1,382 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.RowType; + +import com.google.protobuf.DescriptorProtos; +import com.google.protobuf.Descriptors; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Converts a Flink {@link RowType} to a Protobuf schema definition string and to a {@link + * DescriptorProtos.FileDescriptorProto} for runtime use. + * + *

Type mapping (proto3, scalar fields only): + * + *

    + *
  • {@code BOOLEAN} → {@code bool} + *
  • {@code TINYINT}/{@code SMALLINT}/{@code INTEGER} → {@code int32} + *
  • {@code DATE} → {@code int32} (epoch day); {@code TIME} → {@code int32} (millis of day) + *
  • {@code BIGINT} → {@code int64} + *
  • {@code TIMESTAMP}/{@code TIMESTAMP_LTZ} → {@code int64} (epoch millis) + *
  • {@code FLOAT} → {@code float}; {@code DOUBLE} → {@code double} + *
  • {@code CHAR}/{@code VARCHAR} → {@code string}; {@code DECIMAL} → {@code string} (lossless + * {@code BigDecimal} text form) + *
  • {@code BINARY}/{@code VARBINARY} → {@code bytes} + *
+ * + *

Complex/unsupported types (ARRAY, MAP, MULTISET, ROW, RAW, STRUCTURED, ...) are rejected + * fail-fast rather than silently coerced to {@code string} — see {@code unsupported()}. + * + *

Field identifiers are sanitized to valid proto names (see {@link #sanitizeFieldName}); the + * original SQL column name is preserved as the field's {@code json_name} so it survives a + * round-trip regardless of sanitization. + */ +@Internal +public class ProtobufSchemaConverter { + + private static final String PROTO_SYNTAX = "proto3"; + + /** Matches the {@code message {}} declaration in a proto3 writer schema. */ + private static final Pattern MESSAGE_PATTERN = + Pattern.compile("message\\s+([A-Za-z_][A-Za-z0-9_]*)\\s*\\{"); + + /** + * Matches a single proto3 scalar field line, e.g. {@code string name = 1;} or {@code int64 ts = + * 3 [json_name = "orig name"];}. Group 1 = optional label ({@code optional}/{@code repeated}), + * group 2 = proto type keyword, group 3 = field name, group 4 = field number, group 5 = {@code + * json_name} (the original SQL column name, when the identifier was sanitized). + */ + private static final Pattern FIELD_PATTERN = + Pattern.compile( + "(?:(optional|repeated)\\s+)?" + + "([A-Za-z_][A-Za-z0-9_.]*)\\s+" + + "([A-Za-z_][A-Za-z0-9_]*)\\s*=\\s*(\\d+)\\s*" + + "(?:\\[\\s*json_name\\s*=\\s*\"([^\"]*)\"\\s*\\])?\\s*;"); + + /** + * Converts a Flink RowType to a Protobuf schema definition string (proto3 syntax). + * + * @param rowType the Flink RowType + * @param messageName the name for the Protobuf message + * @return a Protobuf schema definition string + */ + public static String convertToProtobufSchema(RowType rowType, String messageName) { + String sanitized = sanitizeMessageName(messageName); + StringBuilder sb = new StringBuilder(); + sb.append("syntax = \"proto3\";\n\n"); + sb.append("message ").append(sanitized).append(" {\n"); + int fieldNumber = 1; + for (RowType.RowField field : rowType.getFields()) { + String protoType = toProtoType(field.getType()); + String fieldName = sanitizeFieldName(field.getName()); + sb.append(" "); + // proto3 explicit presence: a NULLABLE column is emitted with the `optional` + // keyword so the reader can distinguish an unset field (null) from a set type + // default (0/""/false); a NOT NULL column stays a plain implicit-presence field + // (review finding C2). + if (field.getType().isNullable()) { + sb.append("optional "); + } + sb.append(protoType).append(" ").append(fieldName); + sb.append(" = ").append(fieldNumber++); + if (!fieldName.equals(field.getName())) { + sb.append(" [json_name = \"").append(field.getName()).append("\"]"); + } + sb.append(";\n"); + } + sb.append("}\n"); + return sb.toString(); + } + + /** + * Builds a {@link DescriptorProtos.FileDescriptorProto} from a Flink RowType for runtime + * Protobuf serialization/deserialization. + * + * @param rowType the Flink RowType + * @param messageName the name for the Protobuf message + * @return a FileDescriptorProto + */ + public static DescriptorProtos.FileDescriptorProto buildFileDescriptorProto( + RowType rowType, String messageName) { + String sanitized = sanitizeMessageName(messageName); + DescriptorProtos.DescriptorProto.Builder messageBuilder = + DescriptorProtos.DescriptorProto.newBuilder().setName(sanitized); + + int fieldNumber = 1; + int syntheticOneofIndex = 0; + for (RowType.RowField field : rowType.getFields()) { + String fieldName = sanitizeFieldName(field.getName()); + DescriptorProtos.FieldDescriptorProto.Builder fieldBuilder = + DescriptorProtos.FieldDescriptorProto.newBuilder() + .setName(fieldName) + .setNumber(fieldNumber++) + .setType(toProtoFieldType(field.getType())) + .setLabel(DescriptorProtos.FieldDescriptorProto.Label.LABEL_OPTIONAL); + if (!fieldName.equals(field.getName())) { + fieldBuilder.setJsonName(field.getName()); + } + if (field.getType().isNullable()) { + // proto3 explicit presence: wrap the nullable scalar in a synthetic oneof + // (proto3 `optional`) so hasField() distinguishes an unset column (null) from + // a set type default at decode time (review finding C2). NOT NULL columns keep + // implicit presence. Synthetic oneofs are declared in field order, so each + // oneof_index matches the position at which the oneof decl is appended. + fieldBuilder.setProto3Optional(true); + fieldBuilder.setOneofIndex(syntheticOneofIndex++); + messageBuilder.addOneofDecl( + DescriptorProtos.OneofDescriptorProto.newBuilder() + .setName("_" + fieldName)); + } + messageBuilder.addField(fieldBuilder); + } + + return DescriptorProtos.FileDescriptorProto.newBuilder() + .setSyntax(PROTO_SYNTAX) + .addMessageType(messageBuilder) + .build(); + } + + /** + * Reconstructs a runtime {@link Descriptors.Descriptor} from a proto3 writer schema definition + * resolved from AWS Glue Schema Registry (review finding B1). The on-wire field numbers, names + * and {@code json_name}s therefore come from the writer schema registered in Glue, + * rather than being synthesized from the local {@code RowType}. + * + *

The grammar handled is exactly the one emitted by {@link #convertToProtobufSchema}: a + * single proto3 message of scalar fields, each optionally carrying a {@code json_name} option. + * + * @param protoSchemaDefinition the proto3 writer schema text + * @return the message descriptor described by that schema + */ + public static Descriptors.Descriptor buildDescriptorFromProtoSchema( + String protoSchemaDefinition) { + if (protoSchemaDefinition == null) { + throw new IllegalArgumentException( + "GSR returned a null Protobuf writer schema definition on the decode path."); + } + Matcher messageMatcher = MESSAGE_PATTERN.matcher(protoSchemaDefinition); + if (!messageMatcher.find()) { + throw new IllegalArgumentException( + "Could not locate a proto3 'message' declaration in the GSR writer schema:\n" + + protoSchemaDefinition); + } + String messageName = messageMatcher.group(1); + String body = protoSchemaDefinition.substring(messageMatcher.end()); + + DescriptorProtos.DescriptorProto.Builder messageBuilder = + DescriptorProtos.DescriptorProto.newBuilder().setName(messageName); + + Matcher fieldMatcher = FIELD_PATTERN.matcher(body); + int syntheticOneofIndex = 0; + while (fieldMatcher.find()) { + String label = fieldMatcher.group(1); + String protoTypeKeyword = fieldMatcher.group(2); + String fieldName = fieldMatcher.group(3); + int fieldNumber = Integer.parseInt(fieldMatcher.group(4)); + String jsonName = fieldMatcher.group(5); + + DescriptorProtos.FieldDescriptorProto.Builder fieldBuilder = + DescriptorProtos.FieldDescriptorProto.newBuilder() + .setName(fieldName) + .setNumber(fieldNumber) + .setType(protoKeywordToFieldType(protoTypeKeyword)) + .setLabel(DescriptorProtos.FieldDescriptorProto.Label.LABEL_OPTIONAL); + if (jsonName != null) { + fieldBuilder.setJsonName(jsonName); + } + if ("optional".equals(label)) { + // Rebuild proto3 explicit presence for the field the writer marked `optional` + // so hasField() works on the decode path (review finding C2). Synthetic oneofs + // are declared in field order to keep each oneof_index aligned. + fieldBuilder.setProto3Optional(true); + fieldBuilder.setOneofIndex(syntheticOneofIndex++); + messageBuilder.addOneofDecl( + DescriptorProtos.OneofDescriptorProto.newBuilder() + .setName("_" + fieldName)); + } + messageBuilder.addField(fieldBuilder); + } + + DescriptorProtos.FileDescriptorProto fileProto = + DescriptorProtos.FileDescriptorProto.newBuilder() + .setSyntax(PROTO_SYNTAX) + .addMessageType(messageBuilder) + .build(); + try { + return Descriptors.FileDescriptor.buildFrom( + fileProto, new Descriptors.FileDescriptor[] {}) + .findMessageTypeByName(messageName); + } catch (Descriptors.DescriptorValidationException e) { + throw new RuntimeException( + "Failed to build a Protobuf descriptor from the GSR writer schema", e); + } + } + + private static DescriptorProtos.FieldDescriptorProto.Type protoKeywordToFieldType( + String keyword) { + switch (keyword) { + case "bool": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_BOOL; + case "int32": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_INT32; + case "sint32": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_SINT32; + case "uint32": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_UINT32; + case "fixed32": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_FIXED32; + case "sfixed32": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_SFIXED32; + case "int64": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_INT64; + case "sint64": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_SINT64; + case "uint64": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_UINT64; + case "fixed64": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_FIXED64; + case "sfixed64": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_SFIXED64; + case "float": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_FLOAT; + case "double": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_DOUBLE; + case "string": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_STRING; + case "bytes": + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_BYTES; + default: + throw new IllegalArgumentException( + "Unsupported Protobuf field type '" + + keyword + + "' in the GSR writer schema."); + } + } + + /** + * Sanitizes a schema name to be a valid Protobuf message name. Replaces non-alphanumeric + * characters with underscores and ensures it starts with a letter. + */ + static String sanitizeMessageName(String name) { + String sanitized = name.replaceAll("[^a-zA-Z0-9_]", "_"); + if (!sanitized.isEmpty() && Character.isDigit(sanitized.charAt(0))) { + sanitized = "M_" + sanitized; + } + if (sanitized.isEmpty()) { + sanitized = "Message"; + } + return sanitized; + } + + /** + * Sanitizes a Flink column name into a valid Protobuf field identifier. Protobuf field names + * must match {@code [a-zA-Z_][a-zA-Z0-9_]*}; a column with a space, hyphen or leading digit + * would otherwise trigger a {@code DescriptorValidationException} when the descriptor is built + * at {@code open()} time. + */ + static String sanitizeFieldName(String name) { + String sanitized = name.replaceAll("[^a-zA-Z0-9_]", "_"); + if (sanitized.isEmpty()) { + return "_field"; + } + char first = sanitized.charAt(0); + if (!Character.isLetter(first) && first != '_') { + sanitized = "_" + sanitized; + } + return sanitized; + } + + private static String toProtoType(LogicalType type) { + switch (type.getTypeRoot()) { + case BOOLEAN: + return "bool"; + case TINYINT: + case SMALLINT: + case INTEGER: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + return "int32"; + case BIGINT: + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return "int64"; + case FLOAT: + return "float"; + case DOUBLE: + return "double"; + case CHAR: + case VARCHAR: + case DECIMAL: + return "string"; + case BINARY: + case VARBINARY: + return "bytes"; + default: + throw unsupported(type); + } + } + + private static DescriptorProtos.FieldDescriptorProto.Type toProtoFieldType(LogicalType type) { + switch (type.getTypeRoot()) { + case BOOLEAN: + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_BOOL; + case TINYINT: + case SMALLINT: + case INTEGER: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_INT32; + case BIGINT: + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_INT64; + case FLOAT: + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_FLOAT; + case DOUBLE: + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_DOUBLE; + case CHAR: + case VARCHAR: + case DECIMAL: + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_STRING; + case BINARY: + case VARBINARY: + return DescriptorProtos.FieldDescriptorProto.Type.TYPE_BYTES; + default: + throw unsupported(type); + } + } + + private static UnsupportedOperationException unsupported(LogicalType type) { + return new UnsupportedOperationException( + "The 'protobuf-glue' format does not support the Flink type '" + + type.asSummaryString() + + "'. Supported types are the scalar types (BOOLEAN, INT family, " + + "FLOAT/DOUBLE, DECIMAL, CHAR/VARCHAR, BINARY/VARBINARY, DATE, TIME, " + + "TIMESTAMP, TIMESTAMP_LTZ). Complex types (ARRAY, MAP, MULTISET, ROW, " + + "RAW) are not yet supported."); + } + + private ProtobufSchemaConverter() {} +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufToRowDataConverter.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufToRowDataConverter.java new file mode 100644 index 000000000..15aedfc90 --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufToRowDataConverter.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.data.TimestampData; +import org.apache.flink.table.types.logical.DecimalType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.RowType; + +import com.google.protobuf.ByteString; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; + +import java.math.BigDecimal; +import java.util.List; + +/** + * Converts Protobuf {@link DynamicMessage} to Flink {@link RowData} based on a Flink {@link + * RowType}. + * + *

Temporal and decimal columns are decoded from their wire encoding (epoch-millis {@code int64} + * for TIMESTAMP/TIMESTAMP_LTZ, {@code int32} for DATE/TIME, {@code string} for DECIMAL) — mirroring + * {@link RowDataToProtobufConverter} so the round-trip is symmetric (review finding B4). + */ +@Internal +public class ProtobufToRowDataConverter { + + /** + * Converts a Protobuf DynamicMessage to a Flink RowData. + * + *

Fields are mapped by name (not position): for each {@link RowType} column the + * matching field is resolved in the writer descriptor by its sanitized Protobuf name, falling + * back to the original SQL name preserved as {@code json_name}. This means a column reorder or + * a writer schema that carries extra/fewer fields no longer silently reinterprets on-wire tags + * (review finding B1). A column with no counterpart in the GSR-registered writer schema is + * treated as missing: {@code null} for a nullable column, a hard error for a {@code NOT NULL} + * column. + * + * @param message the Protobuf DynamicMessage decoded with the writer descriptor + * @param rowType the Flink RowType describing the expected schema + * @return a GenericRowData + */ + public static RowData convertToRowData(DynamicMessage message, RowType rowType) { + GenericRowData row = new GenericRowData(rowType.getFieldCount()); + Descriptors.Descriptor descriptor = message.getDescriptorForType(); + + List fields = rowType.getFields(); + for (int i = 0; i < fields.size(); i++) { + RowType.RowField rowField = fields.get(i); + LogicalType fieldType = rowField.getType(); + + Descriptors.FieldDescriptor fd = findFieldByName(descriptor, rowField.getName()); + if (fd == null) { + // The GSR-registered writer schema has no field for this column. + if (!fieldType.isNullable()) { + throw new IllegalStateException( + "Column '" + + rowField.getName() + + "' is declared NOT NULL but is absent from the " + + "GSR-registered writer schema '" + + descriptor.getName() + + "'. Cannot decode a required column that the writer never " + + "produced."); + } + row.setField(i, null); + continue; + } + + Object protoValue = readFieldValue(message, fd); + row.setField(i, convertProtoValue(protoValue, fieldType)); + } + return row; + } + + /** + * Reads a field value honoring proto3 explicit presence (review finding C2). For a + * presence-tracking field (a proto3 {@code optional} scalar, wrapped in a synthetic oneof by + * {@link ProtobufSchemaConverter}) that the writer left unset, this returns {@code null} so a + * nullable column round-trips as null rather than the type default (0/""/false). For an + * implicit-presence field (a NOT NULL column) it returns the value, which is the type default + * when unset. + */ + private static Object readFieldValue(DynamicMessage message, Descriptors.FieldDescriptor fd) { + if (fd.hasPresence() && !message.hasField(fd)) { + return null; + } + return message.getField(fd); + } + + /** + * Resolves the writer-descriptor field for a Flink column name. Tries the sanitized Protobuf + * field name first (the writer sanitizes identically), then the original SQL name preserved as + * {@code json_name}, then a verbatim name match. + */ + private static Descriptors.FieldDescriptor findFieldByName( + Descriptors.Descriptor descriptor, String columnName) { + Descriptors.FieldDescriptor fd = + descriptor.findFieldByName(ProtobufSchemaConverter.sanitizeFieldName(columnName)); + if (fd != null) { + return fd; + } + for (Descriptors.FieldDescriptor candidate : descriptor.getFields()) { + if (columnName.equals(candidate.getJsonName())) { + return candidate; + } + } + return descriptor.findFieldByName(columnName); + } + + private static Object convertProtoValue(Object protoValue, LogicalType type) { + // A null here means either the writer's schema had no field for this column, or the field + // is a proto3 explicit-presence (`optional`) field the writer left unset (finding C2); + // either way the column decodes to null. For NOT NULL columns (implicit presence) an unset + // field surfaces its type default here, which is the intended proto3 semantic. + if (protoValue == null) { + return null; + } + + switch (type.getTypeRoot()) { + case BOOLEAN: + return protoValue; + case TINYINT: + return ((Integer) protoValue).byteValue(); + case SMALLINT: + return ((Integer) protoValue).shortValue(); + case INTEGER: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + // int32 wire form; DATE = epoch day, TIME = millis-of-day (both int in RowData). + return protoValue; + case BIGINT: + return protoValue; + case TIMESTAMP_WITHOUT_TIME_ZONE: + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return TimestampData.fromEpochMillis((Long) protoValue); + case FLOAT: + return protoValue; + case DOUBLE: + return protoValue; + case DECIMAL: + final DecimalType dt = (DecimalType) type; + return DecimalData.fromBigDecimal( + new BigDecimal(protoValue.toString()), dt.getPrecision(), dt.getScale()); + case CHAR: + case VARCHAR: + return StringData.fromString(protoValue.toString()); + case BINARY: + case VARBINARY: + if (protoValue instanceof ByteString) { + return ((ByteString) protoValue).toByteArray(); + } + return protoValue; + default: + throw new UnsupportedOperationException( + "The 'protobuf-glue' format cannot decode the Flink type '" + + type.asSummaryString() + + "'."); + } + } + + private ProtobufToRowDataConverter() {} +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/RowDataToProtobufConverter.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/RowDataToProtobufConverter.java new file mode 100644 index 000000000..e22b8af3e --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/java/org/apache/flink/formats/protobuf/glue/schema/registry/RowDataToProtobufConverter.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.types.logical.DecimalType; +import org.apache.flink.table.types.logical.LocalZonedTimestampType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.TimestampType; + +import com.google.protobuf.ByteString; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; + +import java.util.List; + +/** + * Converts Flink {@link RowData} to Protobuf {@link DynamicMessage} based on a Protobuf {@link + * Descriptors.Descriptor}. + * + *

Temporal and decimal columns are encoded via their proper {@link RowData} accessors — never + * via {@code getString} — to avoid the {@code ClassCastException} that a naive {@code default:} + * branch produced (see review finding B4). TIMESTAMP/TIMESTAMP_LTZ → epoch-millis {@code int64}, + * DATE/TIME → {@code int32}, DECIMAL → its lossless {@code BigDecimal} text form. + */ +@Internal +public class RowDataToProtobufConverter { + + /** + * Converts a Flink RowData to a Protobuf DynamicMessage. + * + * @param rowData the Flink RowData + * @param rowType the Flink RowType describing the schema + * @param descriptor the Protobuf message descriptor + * @return a DynamicMessage + */ + public static DynamicMessage convertRowData( + RowData rowData, RowType rowType, Descriptors.Descriptor descriptor) { + DynamicMessage.Builder builder = DynamicMessage.newBuilder(descriptor); + List fields = descriptor.getFields(); + + for (int i = 0; i < rowType.getFieldCount(); i++) { + if (rowData.isNullAt(i)) { + // Leave the field unset. For a NULLABLE column this is a proto3 explicit-presence + // (`optional`) field, so an unset field is observably absent (hasField()==false) + // and decodes back to null; for a NOT NULL column the column is never null here. + // Explicit presence is emitted by ProtobufSchemaConverter (review finding C2). + continue; + } + LogicalType fieldType = rowType.getTypeAt(i); + Descriptors.FieldDescriptor fd = fields.get(i); + Object value = extractFieldValue(rowData, i, fieldType); + if (value != null) { + builder.setField(fd, value); + } + } + return builder.build(); + } + + private static Object extractFieldValue(RowData rowData, int index, LogicalType type) { + switch (type.getTypeRoot()) { + case BOOLEAN: + return rowData.getBoolean(index); + case TINYINT: + return (int) rowData.getByte(index); + case SMALLINT: + return (int) rowData.getShort(index); + case INTEGER: + case DATE: + case TIME_WITHOUT_TIME_ZONE: + // DATE = epoch day, TIME = millis-of-day, both stored as int in RowData. + return rowData.getInt(index); + case BIGINT: + return rowData.getLong(index); + case TIMESTAMP_WITHOUT_TIME_ZONE: + return rowData.getTimestamp(index, ((TimestampType) type).getPrecision()) + .getMillisecond(); + case TIMESTAMP_WITH_LOCAL_TIME_ZONE: + return rowData.getTimestamp(index, ((LocalZonedTimestampType) type).getPrecision()) + .getMillisecond(); + case FLOAT: + return rowData.getFloat(index); + case DOUBLE: + return rowData.getDouble(index); + case DECIMAL: + final DecimalType dt = (DecimalType) type; + return rowData.getDecimal(index, dt.getPrecision(), dt.getScale()) + .toBigDecimal() + .toString(); + case CHAR: + case VARCHAR: + return rowData.getString(index).toString(); + case BINARY: + case VARBINARY: + return ByteString.copyFrom(rowData.getBinary(index)); + default: + throw new UnsupportedOperationException( + "The 'protobuf-glue' format cannot encode the Flink type '" + + type.asSummaryString() + + "'."); + } + } + + private RowDataToProtobufConverter() {} +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory new file mode 100644 index 000000000..bfc7c06bc --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/main/resources/META-INF/services/org.apache.flink.table.factories.Factory @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.flink.formats.protobuf.glue.schema.registry.GlueSchemaRegistryProtobufFormatFactory diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/GlueSchemaRegistryProtobufFormatFactoryTest.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/GlueSchemaRegistryProtobufFormatFactoryTest.java new file mode 100644 index 000000000..665339b39 --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/GlueSchemaRegistryProtobufFormatFactoryTest.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.api.common.serialization.DeserializationSchema; +import org.apache.flink.api.common.serialization.SerializationSchema; +import org.apache.flink.table.api.DataTypes; +import org.apache.flink.table.api.ValidationException; +import org.apache.flink.table.catalog.Column; +import org.apache.flink.table.catalog.ResolvedSchema; +import org.apache.flink.table.connector.sink.DynamicTableSink; +import org.apache.flink.table.connector.source.DynamicTableSource; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.factories.TestDynamicTableFactory; +import org.apache.flink.table.runtime.connector.source.ScanRuntimeProviderContext; + +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; + +import static org.apache.flink.table.factories.utils.FactoryMocks.createTableSink; +import static org.apache.flink.table.factories.utils.FactoryMocks.createTableSource; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests for the {@link GlueSchemaRegistryProtobufFormatFactory}. */ +class GlueSchemaRegistryProtobufFormatFactoryTest { + + private static final ResolvedSchema SCHEMA = + ResolvedSchema.of( + Column.physical("a", DataTypes.STRING()), + Column.physical("b", DataTypes.INT()), + Column.physical("c", DataTypes.BOOLEAN())); + + private static final String SCHEMA_NAME = "test-subject"; + private static final String REGISTRY_NAME = "test-registry-name"; + private static final String REGION = "us-west-2"; + + @Test + void testSpiDiscovery() { + final DynamicTableSource source = createTableSource(SCHEMA, getDefaultOptions()); + assertThat(source).isNotNull(); + + final DynamicTableSink sink = createTableSink(SCHEMA, getDefaultOptions()); + assertThat(sink).isNotNull(); + } + + @Test + void testDeserializationSchema() { + final DynamicTableSource actualSource = createTableSource(SCHEMA, getDefaultOptions()); + assertThat(actualSource).isInstanceOf(TestDynamicTableFactory.DynamicTableSourceMock.class); + + TestDynamicTableFactory.DynamicTableSourceMock scanSourceMock = + (TestDynamicTableFactory.DynamicTableSourceMock) actualSource; + + DeserializationSchema actualDeser = + scanSourceMock.valueFormat.createRuntimeDecoder( + ScanRuntimeProviderContext.INSTANCE, SCHEMA.toPhysicalRowDataType()); + + assertThat(actualDeser).isInstanceOf(GsrProtobufRowDataDeserializationSchema.class); + } + + @Test + void testSerializationSchema() { + final DynamicTableSink actualSink = createTableSink(SCHEMA, getDefaultOptions()); + assertThat(actualSink).isInstanceOf(TestDynamicTableFactory.DynamicTableSinkMock.class); + + TestDynamicTableFactory.DynamicTableSinkMock sinkMock = + (TestDynamicTableFactory.DynamicTableSinkMock) actualSink; + + SerializationSchema actualSer = + sinkMock.valueFormat.createRuntimeEncoder(null, SCHEMA.toPhysicalRowDataType()); + + assertThat(actualSer).isInstanceOf(GsrProtobufRowDataSerializationSchema.class); + } + + @Test + void testMissingSchemaNameForSink() { + final Map options = + getModifiedOptions(opts -> opts.remove("protobuf-glue.schema.name")); + + assertThatThrownBy(() -> createTableSink(SCHEMA, options)) + .isInstanceOf(ValidationException.class); + } + + @Test + void testMissingRegionForSource() { + final Map options = + getModifiedOptions(opts -> opts.remove("protobuf-glue.aws.region")); + + assertThatThrownBy(() -> createTableSource(SCHEMA, options)) + .isInstanceOf(ValidationException.class); + } + + @Test + void testMissingRegistryNameForSource() { + final Map options = + getModifiedOptions(opts -> opts.remove("protobuf-glue.registry.name")); + + assertThatThrownBy(() -> createTableSource(SCHEMA, options)) + .isInstanceOf(ValidationException.class); + } + + // ------------------------------------------------------------------------ + // Utilities + // ------------------------------------------------------------------------ + + private Map getModifiedOptions(Consumer> optionModifier) { + Map options = getDefaultOptions(); + optionModifier.accept(options); + return options; + } + + private Map getDefaultOptions() { + final Map options = new HashMap<>(); + options.put("connector", TestDynamicTableFactory.IDENTIFIER); + options.put("target", "MyTarget"); + options.put("buffer-size", "1000"); + + options.put("format", GlueSchemaRegistryProtobufFormatFactory.IDENTIFIER); + options.put("protobuf-glue.schema.name", SCHEMA_NAME); + options.put("protobuf-glue.registry.name", REGISTRY_NAME); + options.put("protobuf-glue.aws.region", REGION); + return options; + } +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufRoundTripPropertyTest.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufRoundTripPropertyTest.java new file mode 100644 index 000000000..9c130b909 --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/GsrProtobufRoundTripPropertyTest.java @@ -0,0 +1,215 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.types.logical.BooleanType; +import org.apache.flink.table.types.logical.DoubleType; +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.LogicalType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.VarCharType; + +import com.google.protobuf.DescriptorProtos; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import net.jqwik.api.Arbitraries; +import net.jqwik.api.Arbitrary; +import net.jqwik.api.ForAll; +import net.jqwik.api.Property; +import net.jqwik.api.Provide; +import net.jqwik.api.Tag; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Property-based tests for Protobuf serialization round-trip with GSR header handling. + * + *

Property 5: Protobuf serialization round-trip + * + *

Validates: Requirements 4.4, 4.5, 8.1 + */ +@Tag("Feature: gsr-flink-sql-formats, Property 5: Protobuf serialization round-trip") +class GsrProtobufRoundTripPropertyTest { + + private static final int GSR_HEADER_SIZE = 18; + private static final String SCHEMA_NAME = "TestMessage"; + + /** + * For any valid RowData matching a given RowType, serializing to Protobuf bytes via + * DynamicMessage, prepending a mock GSR header, then stripping the header and deserializing + * should produce equivalent RowData. + */ + @Property(tries = 100) + void protobufRoundTripPreservesData(@ForAll("rowDataWithType") RowDataWithType input) + throws Exception { + RowType rowType = input.rowType; + RowData original = input.rowData; + + // Build Protobuf descriptor from RowType + Descriptors.Descriptor descriptor = buildDescriptor(rowType); + + // Serialize: RowData -> DynamicMessage -> Protobuf bytes + DynamicMessage message = + RowDataToProtobufConverter.convertRowData(original, rowType, descriptor); + byte[] protobufBytes = message.toByteArray(); + + // Simulate GSR encoding: prepend 18-byte mock header + byte[] gsrEncoded = prependMockGsrHeader(protobufBytes); + + // Simulate GSR decoding: strip 18-byte header + byte[] stripped = Arrays.copyOfRange(gsrEncoded, GSR_HEADER_SIZE, gsrEncoded.length); + + // Deserialize: Protobuf bytes -> DynamicMessage -> RowData + DynamicMessage deserialized = DynamicMessage.parseFrom(descriptor, stripped); + RowData result = ProtobufToRowDataConverter.convertToRowData(deserialized, rowType); + + // Verify equivalence field by field + assertThat(result).isNotNull(); + assertRowDataEquals(original, result, rowType); + } + + private Descriptors.Descriptor buildDescriptor(RowType rowType) { + try { + DescriptorProtos.FileDescriptorProto fileProto = + ProtobufSchemaConverter.buildFileDescriptorProto(rowType, SCHEMA_NAME); + Descriptors.FileDescriptor fileDescriptor = + Descriptors.FileDescriptor.buildFrom( + fileProto, new Descriptors.FileDescriptor[] {}); + return fileDescriptor.findMessageTypeByName(SCHEMA_NAME); + } catch (Descriptors.DescriptorValidationException e) { + throw new RuntimeException(e); + } + } + + /** Creates a mock 18-byte GSR header and prepends it to the payload. */ + private byte[] prependMockGsrHeader(byte[] payload) { + UUID schemaId = UUID.randomUUID(); + ByteBuffer buffer = ByteBuffer.allocate(GSR_HEADER_SIZE + payload.length); + buffer.put((byte) 0x03); // header version + buffer.put((byte) 0x00); // no compression + buffer.putLong(schemaId.getMostSignificantBits()); + buffer.putLong(schemaId.getLeastSignificantBits()); + buffer.put(payload); + return buffer.array(); + } + + /** Compares two RowData instances field by field based on the RowType. */ + private void assertRowDataEquals(RowData expected, RowData actual, RowType rowType) { + assertThat(actual.getArity()).isEqualTo(expected.getArity()); + for (int i = 0; i < rowType.getFieldCount(); i++) { + LogicalType fieldType = rowType.getTypeAt(i); + // Note: proto3 does not distinguish between null and default values. + // A null string in RowData becomes "" in proto3, null int becomes 0, etc. + // We handle this by comparing against proto3 default semantics. + if (expected.isNullAt(i)) { + // proto3 defaults: string -> "", int -> 0, bool -> false, double -> 0.0 + assertProto3Default(actual, i, fieldType); + continue; + } + if (fieldType instanceof VarCharType) { + assertThat(actual.getString(i).toString()) + .isEqualTo(expected.getString(i).toString()); + } else if (fieldType instanceof IntType) { + assertThat(actual.getInt(i)).isEqualTo(expected.getInt(i)); + } else if (fieldType instanceof BooleanType) { + assertThat(actual.getBoolean(i)).isEqualTo(expected.getBoolean(i)); + } else if (fieldType instanceof DoubleType) { + assertThat(actual.getDouble(i)).isEqualTo(expected.getDouble(i)); + } + } + } + + /** Asserts that the actual value matches the proto3 default for the given type. */ + private void assertProto3Default(RowData actual, int index, LogicalType fieldType) { + if (fieldType instanceof VarCharType) { + assertThat(actual.getString(index).toString()).isEqualTo(""); + } else if (fieldType instanceof IntType) { + assertThat(actual.getInt(index)).isEqualTo(0); + } else if (fieldType instanceof BooleanType) { + assertThat(actual.getBoolean(index)).isFalse(); + } else if (fieldType instanceof DoubleType) { + assertThat(actual.getDouble(index)).isEqualTo(0.0); + } + } + + // --- Generators --- + + @Provide + Arbitrary rowDataWithType() { + // Fixed schema with STRING, INT, BOOLEAN, DOUBLE fields + RowType rowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField( + "name", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("age", new IntType()), + new RowType.RowField("active", new BooleanType()), + new RowType.RowField("score", new DoubleType()))); + + return Arbitraries.of(rowType) + .flatMap(rt -> generateRowData(rt).map(rd -> new RowDataWithType(rt, rd))); + } + + private Arbitrary generateRowData(RowType rowType) { + // Generate non-null values only since proto3 doesn't distinguish null from default + Arbitrary strings = Arbitraries.strings().alpha().ofMinLength(1).ofMaxLength(50); + Arbitrary ints = Arbitraries.integers().between(-10000, 10000); + Arbitrary bools = Arbitraries.of(true, false); + Arbitrary doubles = Arbitraries.doubles().between(-1e6, 1e6).ofScale(4); + + return strings.flatMap( + name -> + ints.flatMap( + age -> + bools.flatMap( + active -> + doubles.map( + score -> { + GenericRowData row = + new GenericRowData(4); + row.setField( + 0, + StringData.fromString( + name)); + row.setField(1, age); + row.setField(2, active); + row.setField(3, score); + return (RowData) row; + })))); + } + + /** Holder for a RowData and its corresponding RowType. */ + static class RowDataWithType { + final RowType rowType; + final RowData rowData; + + RowDataWithType(RowType rowType, RowData rowData) { + this.rowType = rowType; + this.rowData = rowData; + } + } +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufC1CompressionRoundTripTest.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufC1CompressionRoundTripTest.java new file mode 100644 index 000000000..49901143c --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufC1CompressionRoundTripTest.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.runtime.typeutils.InternalTypeInfo; +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.VarCharType; + +import com.google.protobuf.DescriptorProtos; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.HashMap; +import java.util.UUID; +import java.util.zip.Deflater; +import java.util.zip.DeflaterOutputStream; +import java.util.zip.Inflater; +import java.util.zip.InflaterInputStream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Round-trip regression test for review finding C1: compression asymmetry — the encode path + * honours {@code schema.compression} (ZLIB) while the pre-fix decode path stripped a fixed 18-byte + * header and never decompressed, so any non-{@code NONE} compression produced unparseable + * payloads on read. + * + *

The fix (shared with B1) routes decode through the GSR deserialization facade: {@link + * GsrProtobufRowDataDeserializationSchema#deserialize} obtains the payload solely via {@link + * GsrProtobufReader#actualData}, and the production {@link FacadeGsrProtobufReader} delegates that + * to {@code GlueSchemaRegistryDeserializationFacade.getActualData}, which strips the header and + * decompresses in-library. The schema itself performs no explicit header strip and no explicit + * decompression. + * + *

These tests stand in for the facade with a {@link CompressionAwareGsrReader} fake that models + * exactly that contract: it reads the GSR header's compression byte and ZLIB-inflates the payload + * when set (mirroring the SDK), so a record compressed on write is transparently read back. Because + * the deserialization schema delegates all payload extraction to the reader, a green round-trip + * here proves the read path is no longer missing decompression — the very asymmetry C1 flagged. + */ +class ProtobufC1CompressionRoundTripTest { + + private static final String SCHEMA_NAME = "TestMessage"; + private static final int GSR_HEADER_SIZE = 18; + + /** GSR header version byte. */ + private static final byte HEADER_VERSION = (byte) 0x03; + /** GSR compression byte: no compression. */ + private static final byte COMPRESSION_NONE = (byte) 0x00; + /** GSR compression byte: ZLIB (matches the SDK's ZLIB compression byte). */ + private static final byte COMPRESSION_ZLIB = (byte) 0x05; + + private static RowType rowType() { + return new RowType( + false, + Arrays.asList( + new RowType.RowField("name", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("age", new IntType()))); + } + + /** + * C1 core proof: a record whose payload is ZLIB-compressed on the wire (compression byte set in + * the header) is decoded successfully. This only passes because decode routes through the + * facade ({@link GsrProtobufReader#actualData}) which decompresses in-library; the pre-fix path + * that merely stripped 18 bytes would hand raw deflate bytes to the Protobuf parser and fail. + */ + @Test + void testZlibCompressedRecordRoundTrips() throws Exception { + RowType rowType = rowType(); + + GenericRowData original = new GenericRowData(2); + original.setField(0, StringData.fromString("Alice")); + original.setField(1, 30); + + byte[] gsrEncoded = encodeWithCompression(original, rowType, COMPRESSION_ZLIB); + + RowData decoded = decode(rowType, gsrEncoded); + + assertThat(decoded).isNotNull(); + assertThat(decoded.getString(0).toString()).isEqualTo("Alice"); + assertThat(decoded.getInt(1)).isEqualTo(30); + } + + /** + * Control: the identical payload with compression disabled (compression byte {@code 0x00}) also + * round-trips through the same facade-routed path, confirming the ZLIB result above is not an + * artefact of the fake but the compression byte genuinely drives decompression. + */ + @Test + void testUncompressedRecordRoundTrips() throws Exception { + RowType rowType = rowType(); + + GenericRowData original = new GenericRowData(2); + original.setField(0, StringData.fromString("Alice")); + original.setField(1, 30); + + byte[] gsrEncoded = encodeWithCompression(original, rowType, COMPRESSION_NONE); + + RowData decoded = decode(rowType, gsrEncoded); + + assertThat(decoded).isNotNull(); + assertThat(decoded.getString(0).toString()).isEqualTo("Alice"); + assertThat(decoded.getInt(1)).isEqualTo(30); + } + + /** + * C1 with multiple records over the same schema, exercising the reader's per-record + * decompression on a compressible payload large enough that deflate actually shrinks it (guards + * against a fake that silently no-ops compression). + */ + @Test + void testZlibCompressedMultipleRecordsRoundTrip() throws Exception { + RowType rowType = rowType(); + + GsrProtobufRowDataDeserializationSchema deser = + new GsrProtobufRowDataDeserializationSchema( + rowType, InternalTypeInfo.of(rowType), SCHEMA_NAME, new HashMap<>()); + deser.setReader(new CompressionAwareGsrReader(rowType, SCHEMA_NAME)); + deser.open(null); + + String[] names = { + "Alice", "Bob", "Charlie", "a-name-repeated-repeated-repeated-repeated-repeated" + }; + int[] ages = {30, 41, 52, 63}; + + for (int i = 0; i < names.length; i++) { + GenericRowData row = new GenericRowData(2); + row.setField(0, StringData.fromString(names[i])); + row.setField(1, ages[i]); + + byte[] gsrEncoded = encodeWithCompression(row, rowType, COMPRESSION_ZLIB); + RowData decoded = deser.deserialize(gsrEncoded); + + assertThat(decoded).isNotNull(); + assertThat(decoded.getString(0).toString()).isEqualTo(names[i]); + assertThat(decoded.getInt(1)).isEqualTo(ages[i]); + } + } + + /** Runs the decode path with the compression-aware fake facade reader. */ + private static RowData decode(RowType rowType, byte[] gsrEncoded) throws Exception { + GsrProtobufRowDataDeserializationSchema deser = + new GsrProtobufRowDataDeserializationSchema( + rowType, InternalTypeInfo.of(rowType), SCHEMA_NAME, new HashMap<>()); + deser.setReader(new CompressionAwareGsrReader(rowType, SCHEMA_NAME)); + deser.open(null); + return deser.deserialize(gsrEncoded); + } + + /** + * Builds a GSR-encoded record: 18-byte header (version + compression byte + 16-byte UUID) + * followed by the Protobuf payload, ZLIB-compressed when {@code compressionByte} is non-zero — + * exactly what the serialization facade emits when {@code schema.compression=ZLIB}. + */ + private static byte[] encodeWithCompression(RowData row, RowType rowType, byte compressionByte) + throws Exception { + DynamicMessage message = + RowDataToProtobufConverter.convertRowData(row, rowType, buildDescriptor(rowType)); + byte[] protobufBytes = message.toByteArray(); + byte[] body = compressionByte == COMPRESSION_NONE ? protobufBytes : zlib(protobufBytes); + + UUID schemaId = UUID.randomUUID(); + ByteBuffer buffer = ByteBuffer.allocate(GSR_HEADER_SIZE + body.length); + buffer.put(HEADER_VERSION); + buffer.put(compressionByte); + buffer.putLong(schemaId.getMostSignificantBits()); + buffer.putLong(schemaId.getLeastSignificantBits()); + buffer.put(body); + return buffer.array(); + } + + private static Descriptors.Descriptor buildDescriptor(RowType rowType) { + try { + DescriptorProtos.FileDescriptorProto fileProto = + ProtobufSchemaConverter.buildFileDescriptorProto(rowType, SCHEMA_NAME); + Descriptors.FileDescriptor fileDescriptor = + Descriptors.FileDescriptor.buildFrom( + fileProto, new Descriptors.FileDescriptor[] {}); + return fileDescriptor.findMessageTypeByName(SCHEMA_NAME); + } catch (Descriptors.DescriptorValidationException e) { + throw new RuntimeException(e); + } + } + + private static byte[] zlib(byte[] data) throws Exception { + Deflater deflater = new Deflater(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (DeflaterOutputStream dos = new DeflaterOutputStream(out, deflater)) { + dos.write(data); + } + deflater.end(); + return out.toByteArray(); + } + + private static byte[] inflate(byte[] data) throws Exception { + Inflater inflater = new Inflater(); + ByteArrayOutputStream out = new ByteArrayOutputStream(); + try (InflaterInputStream iis = + new InflaterInputStream(new java.io.ByteArrayInputStream(data), inflater)) { + byte[] buf = new byte[1024]; + int n; + while ((n = iis.read(buf)) != -1) { + out.write(buf, 0, n); + } + } + inflater.end(); + return out.toByteArray(); + } + + /** + * Fake {@link GsrProtobufReader} that models the GSR deserialization facade's compression + * contract: it resolves the writer schema from the RowType and, for {@link + * #actualData(byte[])}, reads the header's compression byte and ZLIB-inflates the body when set + * — i.e. decompression happens in the facade layer, exactly as the production {@link + * FacadeGsrProtobufReader} delegates to the SDK. The deserialization schema under test performs + * no decompression of its own; it only parses whatever {@code actualData} returns. + */ + private static final class CompressionAwareGsrReader implements GsrProtobufReader { + private static final long serialVersionUID = 1L; + private final String writerSchemaDefinition; + + CompressionAwareGsrReader(RowType rowType, String schemaName) { + this.writerSchemaDefinition = + ProtobufSchemaConverter.convertToProtobufSchema(rowType, schemaName); + } + + @Override + public String writerSchemaDefinition(byte[] gsrEncoded) { + return writerSchemaDefinition; + } + + @Override + public byte[] actualData(byte[] gsrEncoded) { + byte compressionByte = gsrEncoded[1]; + byte[] body = Arrays.copyOfRange(gsrEncoded, GSR_HEADER_SIZE, gsrEncoded.length); + if (compressionByte == COMPRESSION_NONE) { + return body; + } + try { + return inflate(body); + } catch (Exception e) { + throw new RuntimeException("Failed to decompress GSR payload", e); + } + } + } +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufC2ExplicitPresenceRoundTripTest.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufC2ExplicitPresenceRoundTripTest.java new file mode 100644 index 000000000..4b456293d --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufC2ExplicitPresenceRoundTripTest.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.runtime.typeutils.InternalTypeInfo; +import org.apache.flink.table.types.logical.BooleanType; +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.VarCharType; + +import com.google.protobuf.DescriptorProtos; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.HashMap; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Round-trip regression tests for review finding C2: proto3 explicit presence for + * NULLABLE columns. + * + *

Before the fix the generated schema used implicit-presence (plain proto3) scalars for every + * column, so the reader could not tell an unset field from a field set to its type default. A + * nullable column holding {@code 0}, {@code ""} or {@code false} was therefore indistinguishable + * from a null column on the wire, and vice-versa. {@link ProtobufSchemaConverter} now emits + * nullable scalars as proto3 {@code optional} (synthetic-oneof) fields and {@link + * ProtobufToRowDataConverter} consults {@code hasField()}, so: + * + *

    + *
  • a {@code null} value round-trips as {@code null} (field left unset), and + *
  • a set type-default value ({@code 0} / {@code ""} / {@code false}) round-trips as that + * value — it is NOT collapsed to {@code null}. + *
+ * + *

The round-trip runs RowData → {@link RowDataToProtobufConverter} (write) → {@link + * GsrProtobufRowDataDeserializationSchema} (read, via a fake GSR reader that resolves the writer + * schema and returns the raw payload) → RowData, so it exercises the same schema/converter code the + * production decode path uses without a live registry. + */ +class ProtobufC2ExplicitPresenceRoundTripTest { + + private static final String SCHEMA_NAME = "TestMessage"; + + /** All three columns are NULLABLE: an int, a string and a boolean. */ + private static RowType nullableRowType() { + return new RowType( + false, + Arrays.asList( + new RowType.RowField("i", new IntType(true)), + new RowType.RowField("s", new VarCharType(true, VarCharType.MAX_LENGTH)), + new RowType.RowField("b", new BooleanType(true)))); + } + + /** + * C2 null side: a NULLABLE column that is {@code null} at write time is emitted as an unset + * proto3 explicit-presence field and decodes back to {@code null} — not to the type default. + */ + @Test + void testNullableColumnsNullRoundTripAsNull() throws Exception { + RowType rowType = nullableRowType(); + + GenericRowData original = new GenericRowData(3); + original.setField(0, null); + original.setField(1, null); + original.setField(2, null); + + RowData decoded = roundTrip(original, rowType); + + assertThat(decoded).isNotNull(); + assertThat(decoded.isNullAt(0)).isTrue(); + assertThat(decoded.isNullAt(1)).isTrue(); + assertThat(decoded.isNullAt(2)).isTrue(); + } + + /** + * C2 value side (the crux of the finding): a NULLABLE column set to its type default — + * {@code 0} for int32, {@code ""} for string, {@code false} for bool — round-trips as that + * value, NOT collapsed to {@code null}. This is only possible with explicit presence. + */ + @Test + void testNullableColumnsTypeDefaultValuesRoundTripAsValues() throws Exception { + RowType rowType = nullableRowType(); + + GenericRowData original = new GenericRowData(3); + original.setField(0, 0); + original.setField(1, StringData.fromString("")); + original.setField(2, false); + + RowData decoded = roundTrip(original, rowType); + + assertThat(decoded).isNotNull(); + // The distinguishing assertions: values are present, not null. + assertThat(decoded.isNullAt(0)).isFalse(); + assertThat(decoded.getInt(0)).isEqualTo(0); + assertThat(decoded.isNullAt(1)).isFalse(); + assertThat(decoded.getString(1).toString()).isEqualTo(""); + assertThat(decoded.isNullAt(2)).isFalse(); + assertThat(decoded.getBoolean(2)).isFalse(); + } + + /** + * C2 mixed row: null and set-default values coexist in the same record and each retains its own + * presence — proving presence is tracked per field, not row-wide. + */ + @Test + void testMixedNullAndTypeDefaultValuesRoundTrip() throws Exception { + RowType rowType = nullableRowType(); + + GenericRowData original = new GenericRowData(3); + original.setField(0, 0); // set default -> stays 0 + original.setField(1, null); // null -> stays null + original.setField(2, false); // set default -> stays false + + RowData decoded = roundTrip(original, rowType); + + assertThat(decoded).isNotNull(); + assertThat(decoded.isNullAt(0)).isFalse(); + assertThat(decoded.getInt(0)).isEqualTo(0); + assertThat(decoded.isNullAt(1)).isTrue(); + assertThat(decoded.isNullAt(2)).isFalse(); + assertThat(decoded.getBoolean(2)).isFalse(); + } + + /** + * Sanity check that non-default set values also survive, so the value-side assertions above are + * not vacuously passing on a decode path that ignores the payload. + */ + @Test + void testNullableColumnsNonDefaultValuesRoundTrip() throws Exception { + RowType rowType = nullableRowType(); + + GenericRowData original = new GenericRowData(3); + original.setField(0, 42); + original.setField(1, StringData.fromString("hello")); + original.setField(2, true); + + RowData decoded = roundTrip(original, rowType); + + assertThat(decoded).isNotNull(); + assertThat(decoded.getInt(0)).isEqualTo(42); + assertThat(decoded.getString(1).toString()).isEqualTo("hello"); + assertThat(decoded.getBoolean(2)).isTrue(); + } + + /** + * RowData → Protobuf bytes (descriptor built from {@code rowType}) → {@link + * GsrProtobufRowDataDeserializationSchema} (fed the writer schema derived from {@code rowType} + * and the raw payload via a fake reader) → RowData. + */ + private static RowData roundTrip(RowData original, RowType rowType) throws Exception { + DynamicMessage message = + RowDataToProtobufConverter.convertRowData( + original, rowType, buildDescriptor(rowType)); + byte[] payload = message.toByteArray(); + String writerSchema = ProtobufSchemaConverter.convertToProtobufSchema(rowType, SCHEMA_NAME); + + GsrProtobufRowDataDeserializationSchema deser = + new GsrProtobufRowDataDeserializationSchema( + rowType, InternalTypeInfo.of(rowType), SCHEMA_NAME, new HashMap<>()); + deser.setReader(new StaticGsrReader(writerSchema, payload)); + deser.open(null); + return deser.deserialize(payload); + } + + private static Descriptors.Descriptor buildDescriptor(RowType rowType) { + try { + DescriptorProtos.FileDescriptorProto fileProto = + ProtobufSchemaConverter.buildFileDescriptorProto(rowType, SCHEMA_NAME); + Descriptors.FileDescriptor fileDescriptor = + Descriptors.FileDescriptor.buildFrom( + fileProto, new Descriptors.FileDescriptor[] {}); + return fileDescriptor.findMessageTypeByName(SCHEMA_NAME); + } catch (Descriptors.DescriptorValidationException e) { + throw new RuntimeException(e); + } + } + + /** + * Fake {@link GsrProtobufReader} standing in for the GSR facade: returns a fixed writer schema + * definition and the raw Protobuf payload (no header/compression), so decode is driven by the + * writer schema exactly as in production. + */ + private static final class StaticGsrReader implements GsrProtobufReader { + private static final long serialVersionUID = 1L; + private final String writerSchemaDefinition; + private final byte[] payload; + + StaticGsrReader(String writerSchemaDefinition, byte[] payload) { + this.writerSchemaDefinition = writerSchemaDefinition; + this.payload = payload; + } + + @Override + public String writerSchemaDefinition(byte[] gsrEncoded) { + return writerSchemaDefinition; + } + + @Override + public byte[] actualData(byte[] gsrEncoded) { + return payload; + } + } +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufGsrWriterSchemaDecodeTest.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufGsrWriterSchemaDecodeTest.java new file mode 100644 index 000000000..0cf08bd74 --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufGsrWriterSchemaDecodeTest.java @@ -0,0 +1,247 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.runtime.typeutils.InternalTypeInfo; +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.VarCharType; + +import com.google.protobuf.DescriptorProtos; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.HashMap; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * Regression tests for review finding B1: Protobuf decode must resolve the writer + * schema registered in AWS Glue Schema Registry and parse the on-wire bytes with a descriptor built + * from that writer schema, mapping fields to the local {@code RowType} by name — rather than + * (as the pre-fix code did) reinterpreting the bytes against a descriptor synthesized from the + * local table DDL, which silently mis-maps whenever the writer's field order / tags differ from the + * local columns. + * + *

Each test drives {@link GsrProtobufRowDataDeserializationSchema} through a hand-written {@link + * GsrProtobufReader} fake that returns a writer schema definition and payload independent of the + * local {@code RowType}, so no live registry is required. + */ +class ProtobufGsrWriterSchemaDecodeTest { + + private static final String SCHEMA_NAME = "TestMessage"; + + /** + * B1 core proof: the writer schema declares {@code name} (tag 1, string) then {@code age} (tag + * 2, int32); the local table DDL declares the columns in the opposite order ({@code age} + * then {@code name}). If decode built its descriptor from the local {@code RowType} it would + * try to parse tag 1 as an {@code int32} (age) and tag 2 as a {@code string} (name) and either + * throw or produce garbage. Because decode uses the GSR-registered writer descriptor and maps + * by name, both columns decode correctly regardless of local column order. + */ + @Test + void testDecodeUsesWriterSchemaAndMapsByName() throws Exception { + // Writer schema as registered in GSR: name (tag 1), age (tag 2). + RowType writerRowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField( + "name", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("age", new IntType()))); + + GenericRowData writerRow = new GenericRowData(2); + writerRow.setField(0, StringData.fromString("Alice")); + writerRow.setField(1, 30); + byte[] payload = encode(writerRow, writerRowType); + String writerSchema = + ProtobufSchemaConverter.convertToProtobufSchema(writerRowType, SCHEMA_NAME); + + // Local table DDL: columns declared in the OPPOSITE order. + RowType localRowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField("age", new IntType()), + new RowType.RowField( + "name", new VarCharType(VarCharType.MAX_LENGTH)))); + + RowData decoded = decode(localRowType, writerSchema, payload); + + assertThat(decoded).isNotNull(); + // Mapped by name against the writer schema, not by local position/tag. + assertThat(decoded.getInt(0)).isEqualTo(30); + assertThat(decoded.getString(1).toString()).isEqualTo("Alice"); + } + + /** + * B1: a NULLABLE local column that is absent from the GSR-registered writer schema decodes to + * {@code null} (the writer never produced it), rather than reading a bogus on-wire tag. + */ + @Test + void testNullableColumnMissingFromWriterSchemaDecodesToNull() throws Exception { + RowType writerRowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField( + "name", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("age", new IntType()))); + + GenericRowData writerRow = new GenericRowData(2); + writerRow.setField(0, StringData.fromString("Bob")); + writerRow.setField(1, 41); + byte[] payload = encode(writerRow, writerRowType); + String writerSchema = + ProtobufSchemaConverter.convertToProtobufSchema(writerRowType, SCHEMA_NAME); + + // Local DDL adds a nullable 'nickname' column the writer schema does not carry. + RowType localRowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField( + "name", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("age", new IntType()), + new RowType.RowField( + "nickname", + new VarCharType(true, VarCharType.MAX_LENGTH)))); + + RowData decoded = decode(localRowType, writerSchema, payload); + + assertThat(decoded).isNotNull(); + assertThat(decoded.getString(0).toString()).isEqualTo("Bob"); + assertThat(decoded.getInt(1)).isEqualTo(41); + assertThat(decoded.isNullAt(2)).isTrue(); + } + + /** + * B1: a NOT NULL local column that is absent from the GSR-registered writer schema fails fast + * with a clear error, instead of silently decoding a wrong or default value. + */ + @Test + void testNotNullColumnMissingFromWriterSchemaThrowsClearError() throws Exception { + RowType writerRowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField( + "name", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("age", new IntType()))); + + GenericRowData writerRow = new GenericRowData(2); + writerRow.setField(0, StringData.fromString("Carol")); + writerRow.setField(1, 52); + byte[] payload = encode(writerRow, writerRowType); + String writerSchema = + ProtobufSchemaConverter.convertToProtobufSchema(writerRowType, SCHEMA_NAME); + + // Local DDL requires a NOT NULL 'email' column the writer schema does not carry. + RowType localRowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField( + "name", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("age", new IntType()), + new RowType.RowField( + "email", new VarCharType(false, VarCharType.MAX_LENGTH)))); + + assertThatThrownBy(() -> decode(localRowType, writerSchema, payload)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("email") + .hasMessageContaining("NOT NULL") + .hasMessageContaining("writer schema"); + } + + /** + * Encodes a RowData into raw Protobuf bytes using a descriptor built from the WRITER RowType — + * this stands in for what an upstream writer registered in GSR would have produced. + */ + private static byte[] encode(RowData row, RowType writerRowType) { + DynamicMessage message = + RowDataToProtobufConverter.convertRowData( + row, writerRowType, buildDescriptor(writerRowType)); + return message.toByteArray(); + } + + /** + * Runs the decode path of {@link GsrProtobufRowDataDeserializationSchema} against the given + * local {@code RowType}, with a fake reader that returns the supplied writer schema definition + * and payload (independent of the local DDL). + */ + private static RowData decode(RowType localRowType, String writerSchema, byte[] payload) + throws Exception { + GsrProtobufRowDataDeserializationSchema deser = + new GsrProtobufRowDataDeserializationSchema( + localRowType, + InternalTypeInfo.of(localRowType), + SCHEMA_NAME, + new HashMap<>()); + deser.setReader(new StaticGsrReader(writerSchema, payload)); + deser.open(null); + return deser.deserialize(payload); + } + + private static Descriptors.Descriptor buildDescriptor(RowType rowType) { + try { + DescriptorProtos.FileDescriptorProto fileProto = + ProtobufSchemaConverter.buildFileDescriptorProto(rowType, SCHEMA_NAME); + Descriptors.FileDescriptor fileDescriptor = + Descriptors.FileDescriptor.buildFrom( + fileProto, new Descriptors.FileDescriptor[] {}); + return fileDescriptor.findMessageTypeByName(SCHEMA_NAME); + } catch (Descriptors.DescriptorValidationException e) { + throw new RuntimeException(e); + } + } + + /** + * Fake {@link GsrProtobufReader} that returns a fixed writer schema definition and payload, + * standing in for the GSR facade resolving the writer schema from the record's schema-version + * UUID. Both values are independent of the local {@code RowType}, which is exactly what lets + * the tests prove decode is driven by the writer schema. + */ + private static final class StaticGsrReader implements GsrProtobufReader { + private static final long serialVersionUID = 1L; + private final String writerSchemaDefinition; + private final byte[] payload; + + StaticGsrReader(String writerSchemaDefinition, byte[] payload) { + this.writerSchemaDefinition = writerSchemaDefinition; + this.payload = payload; + } + + @Override + public String writerSchemaDefinition(byte[] gsrEncoded) { + return writerSchemaDefinition; + } + + @Override + public byte[] actualData(byte[] gsrEncoded) { + return payload; + } + } +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufRoundTripIntegrationTest.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufRoundTripIntegrationTest.java new file mode 100644 index 000000000..51b91acaf --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufRoundTripIntegrationTest.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.runtime.typeutils.InternalTypeInfo; +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.VarCharType; + +import com.google.protobuf.DescriptorProtos; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import org.junit.jupiter.api.Test; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.HashMap; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Integration tests for Protobuf round-trip serialization/deserialization with mock GSR header + * handling. + * + *

Validates Requirement 8.1. + */ +class ProtobufRoundTripIntegrationTest { + + private static final int GSR_HEADER_SIZE = 18; + private static final String SCHEMA_NAME = "TestMessage"; + + /** + * Tests full Protobuf round-trip: RowData → Protobuf serialize → prepend GSR header → + * GsrProtobufRowDataDeserializationSchema (strips header + deser) → RowData. + */ + @Test + void testProtobufRoundTrip() throws Exception { + RowType rowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField( + "name", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("age", new IntType()))); + + Descriptors.Descriptor descriptor = buildDescriptor(rowType); + + // Serialize: RowData → DynamicMessage → Protobuf bytes → prepend GSR header + GenericRowData original = new GenericRowData(2); + original.setField(0, StringData.fromString("Alice")); + original.setField(1, 30); + + DynamicMessage message = + RowDataToProtobufConverter.convertRowData(original, rowType, descriptor); + byte[] protobufBytes = message.toByteArray(); + byte[] gsrEncoded = prependMockGsrHeader(protobufBytes); + + // Deserialize using GsrProtobufRowDataDeserializationSchema, routed through a fake GSR + // reader that resolves the writer schema and strips the header (mirrors the facade). + GsrProtobufRowDataDeserializationSchema deser = + new GsrProtobufRowDataDeserializationSchema( + rowType, InternalTypeInfo.of(rowType), SCHEMA_NAME, new HashMap<>()); + deser.setReader(new FakeGsrReader(rowType, SCHEMA_NAME)); + deser.open(null); + + RowData deserialized = deser.deserialize(gsrEncoded); + assertThat(deserialized).isNotNull(); + assertThat(deserialized.getString(0).toString()).isEqualTo("Alice"); + assertThat(deserialized.getInt(1)).isEqualTo(30); + } + + /** Tests round-trip with multiple records. */ + @Test + void testProtobufRoundTripMultipleRecords() throws Exception { + RowType rowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField( + "name", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("value", new IntType()))); + + Descriptors.Descriptor descriptor = buildDescriptor(rowType); + + GsrProtobufRowDataDeserializationSchema deser = + new GsrProtobufRowDataDeserializationSchema( + rowType, InternalTypeInfo.of(rowType), SCHEMA_NAME, new HashMap<>()); + deser.setReader(new FakeGsrReader(rowType, SCHEMA_NAME)); + deser.open(null); + + String[] names = {"Alice", "Bob", "Charlie"}; + int[] values = {10, 20, 30}; + + for (int i = 0; i < names.length; i++) { + GenericRowData row = new GenericRowData(2); + row.setField(0, StringData.fromString(names[i])); + row.setField(1, values[i]); + + DynamicMessage message = + RowDataToProtobufConverter.convertRowData(row, rowType, descriptor); + byte[] protobufBytes = message.toByteArray(); + byte[] gsrEncoded = prependMockGsrHeader(protobufBytes); + + RowData result = deser.deserialize(gsrEncoded); + assertThat(result).isNotNull(); + assertThat(result.getString(0).toString()).isEqualTo(names[i]); + assertThat(result.getInt(1)).isEqualTo(values[i]); + } + } + + private Descriptors.Descriptor buildDescriptor(RowType rowType) { + try { + DescriptorProtos.FileDescriptorProto fileProto = + ProtobufSchemaConverter.buildFileDescriptorProto(rowType, SCHEMA_NAME); + Descriptors.FileDescriptor fileDescriptor = + Descriptors.FileDescriptor.buildFrom( + fileProto, new Descriptors.FileDescriptor[] {}); + return fileDescriptor.findMessageTypeByName(SCHEMA_NAME); + } catch (Descriptors.DescriptorValidationException e) { + throw new RuntimeException(e); + } + } + + /** Creates a mock 18-byte GSR header and prepends it to the payload. */ + private byte[] prependMockGsrHeader(byte[] payload) { + UUID schemaId = UUID.randomUUID(); + ByteBuffer buffer = ByteBuffer.allocate(GSR_HEADER_SIZE + payload.length); + buffer.put((byte) 0x03); // header version + buffer.put((byte) 0x00); // no compression + buffer.putLong(schemaId.getMostSignificantBits()); + buffer.putLong(schemaId.getLeastSignificantBits()); + buffer.put(payload); + return buffer.array(); + } + + /** + * Fake {@link GsrProtobufReader} that stands in for the GSR facade in tests: it returns the + * writer schema definition derived from the given RowType and strips the mock 18-byte header + * (no compression) to yield the raw Protobuf payload. + */ + private static final class FakeGsrReader implements GsrProtobufReader { + private static final long serialVersionUID = 1L; + private final String writerSchemaDefinition; + + FakeGsrReader(RowType rowType, String schemaName) { + this.writerSchemaDefinition = + ProtobufSchemaConverter.convertToProtobufSchema(rowType, schemaName); + } + + @Override + public String writerSchemaDefinition(byte[] gsrEncoded) { + return writerSchemaDefinition; + } + + @Override + public byte[] actualData(byte[] gsrEncoded) { + return Arrays.copyOfRange(gsrEncoded, GSR_HEADER_SIZE, gsrEncoded.length); + } + } +} diff --git a/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufTypeCoverageTest.java b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufTypeCoverageTest.java new file mode 100644 index 000000000..2b901dfd9 --- /dev/null +++ b/flink-formats-aws/flink-protobuf-glue-schema-registry/src/test/java/org/apache/flink/formats/protobuf/glue/schema/registry/ProtobufTypeCoverageTest.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.formats.protobuf.glue.schema.registry; + +import org.apache.flink.table.data.DecimalData; +import org.apache.flink.table.data.GenericRowData; +import org.apache.flink.table.data.RowData; +import org.apache.flink.table.data.StringData; +import org.apache.flink.table.data.TimestampData; +import org.apache.flink.table.types.logical.ArrayType; +import org.apache.flink.table.types.logical.DateType; +import org.apache.flink.table.types.logical.DecimalType; +import org.apache.flink.table.types.logical.IntType; +import org.apache.flink.table.types.logical.RowType; +import org.apache.flink.table.types.logical.TimeType; +import org.apache.flink.table.types.logical.TimestampType; +import org.apache.flink.table.types.logical.VarCharType; + +import com.google.protobuf.DescriptorProtos; +import com.google.protobuf.Descriptors; +import com.google.protobuf.DynamicMessage; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.util.Arrays; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** + * Type-coverage tests for the {@code protobuf-glue} format, exercising the review fixes: + * + *

    + *
  • B4 — TIMESTAMP/DATE/TIME/DECIMAL encode via the correct accessors (no {@code + * ClassCastException}) and decode symmetrically. + *
  • C3 — genuinely unsupported complex types (ARRAY/MAP/ROW/...) fail fast instead of + * being silently coerced to {@code string}. + *
  • C4 — column names that are not valid proto identifiers are sanitized so descriptor + * construction does not throw {@code DescriptorValidationException}. + *
+ */ +class ProtobufTypeCoverageTest { + + private static final String SCHEMA_NAME = "TestMessage"; + + private static Descriptors.Descriptor buildDescriptor(RowType rowType) { + try { + DescriptorProtos.FileDescriptorProto fileProto = + ProtobufSchemaConverter.buildFileDescriptorProto(rowType, SCHEMA_NAME); + Descriptors.FileDescriptor fileDescriptor = + Descriptors.FileDescriptor.buildFrom( + fileProto, new Descriptors.FileDescriptor[] {}); + return fileDescriptor.findMessageTypeByName(SCHEMA_NAME); + } catch (Descriptors.DescriptorValidationException e) { + throw new RuntimeException(e); + } + } + + private static RowData roundTrip(RowData in, RowType rowType) throws Exception { + Descriptors.Descriptor descriptor = buildDescriptor(rowType); + DynamicMessage message = RowDataToProtobufConverter.convertRowData(in, rowType, descriptor); + DynamicMessage reparsed = DynamicMessage.parseFrom(descriptor, message.toByteArray()); + return ProtobufToRowDataConverter.convertToRowData(reparsed, rowType); + } + + /** B4: TIMESTAMP (epoch millis int64) round-trips without ClassCastException. */ + @Test + void testTimestampRoundTrip() throws Exception { + RowType rowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField("ts", new TimestampType(3)), + new RowType.RowField("id", new IntType()))); + long millis = 1_700_000_000_123L; + GenericRowData in = new GenericRowData(2); + in.setField(0, TimestampData.fromEpochMillis(millis)); + in.setField(1, 7); + + RowData out = roundTrip(in, rowType); + assertThat(out.getTimestamp(0, 3).getMillisecond()).isEqualTo(millis); + assertThat(out.getInt(1)).isEqualTo(7); + } + + /** B4: DATE (epoch day int32) and TIME (millis-of-day int32) round-trip. */ + @Test + void testDateAndTimeRoundTrip() throws Exception { + RowType rowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField("d", new DateType()), + new RowType.RowField("t", new TimeType(3)))); + int epochDay = 20_000; // 2024-10-04 + int millisOfDay = 45_296_000; // 12:34:56 + GenericRowData in = new GenericRowData(2); + in.setField(0, epochDay); + in.setField(1, millisOfDay); + + RowData out = roundTrip(in, rowType); + assertThat(out.getInt(0)).isEqualTo(epochDay); + assertThat(out.getInt(1)).isEqualTo(millisOfDay); + } + + /** B4: DECIMAL round-trips losslessly via its BigDecimal text form. */ + @Test + void testDecimalRoundTrip() throws Exception { + RowType rowType = + new RowType( + false, + Arrays.asList(new RowType.RowField("amount", new DecimalType(10, 2)))); + BigDecimal value = new BigDecimal("12345.67"); + GenericRowData in = new GenericRowData(1); + in.setField(0, DecimalData.fromBigDecimal(value, 10, 2)); + + RowData out = roundTrip(in, rowType); + assertThat(out.getDecimal(0, 10, 2).toBigDecimal()).isEqualByComparingTo(value); + } + + /** C4: a column name with a space / leading digit is sanitized and still round-trips. */ + @Test + void testFieldNameSanitizationDoesNotThrow() throws Exception { + RowType rowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField( + "user name", new VarCharType(VarCharType.MAX_LENGTH)), + new RowType.RowField("1st_place", new IntType()))); + + Descriptors.Descriptor descriptor = buildDescriptor(rowType); + // Descriptor built without DescriptorValidationException; names are valid proto idents. + assertThat(descriptor.findFieldByName("user_name")).isNotNull(); + assertThat(descriptor.findFieldByName("_1st_place")).isNotNull(); + + GenericRowData in = new GenericRowData(2); + in.setField(0, StringData.fromString("Alice")); + in.setField(1, 42); + RowData out = roundTrip(in, rowType); + assertThat(out.getString(0).toString()).isEqualTo("Alice"); + assertThat(out.getInt(1)).isEqualTo(42); + } + + /** C4: original SQL column name is preserved as the proto field's json_name. */ + @Test + void testOriginalNamePreservedAsJsonName() { + RowType rowType = + new RowType( + false, + Arrays.asList( + new RowType.RowField( + "user name", new VarCharType(VarCharType.MAX_LENGTH)))); + Descriptors.Descriptor descriptor = buildDescriptor(rowType); + Descriptors.FieldDescriptor fd = descriptor.findFieldByName("user_name"); + assertThat(fd).isNotNull(); + assertThat(fd.toProto().getJsonName()).isEqualTo("user name"); + } + + /** C3: an unsupported complex type (ARRAY) fails fast rather than coercing to string. */ + @Test + void testUnsupportedComplexTypeFailsFast() { + RowType rowType = + new RowType( + false, + Arrays.asList(new RowType.RowField("tags", new ArrayType(new IntType())))); + assertThatThrownBy( + () -> + ProtobufSchemaConverter.buildFileDescriptorProto( + rowType, SCHEMA_NAME)) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("does not support"); + } +} diff --git a/flink-formats-aws/flink-sql-avro-glue-schema-registry/pom.xml b/flink-formats-aws/flink-sql-avro-glue-schema-registry/pom.xml new file mode 100644 index 000000000..b6203bf20 --- /dev/null +++ b/flink-formats-aws/flink-sql-avro-glue-schema-registry/pom.xml @@ -0,0 +1,147 @@ + + + + + 4.0.0 + + + org.apache.flink + flink-formats-aws-parent + 6.0-SNAPSHOT + + + flink-sql-avro-glue-schema-registry + Flink : Formats : AWS : SQL : Avro Glue Schema Registry + jar + + + + org.apache.flink + flink-avro-glue-schema-registry + ${project.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-flink + package + + shade + + + + + org.apache.flink:flink-connector-aws-base + org.apache.flink:flink-avro-glue-schema-registry + com.amazonaws:* + software.amazon.awssdk:* + software.amazon.glue:* + org.apache.flink:flink-avro + org.apache.avro:avro + com.fasterxml.jackson.core:* + com.fasterxml.jackson.dataformat:* + org.apache.commons:commons-compress + com.google.guava:guava + com.google.guava:failureaccess + + + software.amazon.glue:schema-registry-build-tools + com.google.guava:listenablefuture + org.checkerframework:checker-qual + com.google.errorprone:error_prone_annotations + com.google.j2objc:j2objc-annotations + com.google.code.findbugs:jsr305 + org.apache.kafka:kafka-clients + org.reactivestreams:reactive-streams + + + + + software.amazon + org.apache.flink.avro.registry.glue.shaded.software.amazon + + + com.amazonaws + org.apache.flink.avro.registry.glue.shaded.com.amazonaws + + + com.google + org.apache.flink.avro.registry.glue.shaded.com.google + + + com.typesafe.netty + org.apache.flink.avro.registry.glue.shaded.com.typesafe.netty + + + org.apache.http + org.apache.flink.avro.registry.glue.shaded.org.apache.http + + + + + com.fasterxml.jackson + org.apache.flink.avro.shaded.com.fasterxml.jackson + + + org.apache.avro + org.apache.flink.avro.shaded.org.apache.avro + + + org.apache.commons.compress + org.apache.flink.avro.shaded.org.apache.commons.compress + + + + + *:* + + **/MavenPackaging.java + **/mime.types + **/VersionInfo.java + codegen-resources/** + mozilla/** + + + + software.amazon.glue:schema-registry-serde + + + additionalTypes/** + java/** + metadata/** + + + + + + + + + + + diff --git a/flink-formats-aws/flink-sql-protobuf-glue-schema-registry/pom.xml b/flink-formats-aws/flink-sql-protobuf-glue-schema-registry/pom.xml new file mode 100644 index 000000000..ae064d381 --- /dev/null +++ b/flink-formats-aws/flink-sql-protobuf-glue-schema-registry/pom.xml @@ -0,0 +1,144 @@ + + + + + 4.0.0 + + + org.apache.flink + flink-formats-aws-parent + 6.0-SNAPSHOT + + + flink-sql-protobuf-glue-schema-registry + Flink : Formats : AWS : SQL : Protobuf Glue Schema Registry + jar + + + + org.apache.flink + flink-protobuf-glue-schema-registry + ${project.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-flink + package + + shade + + + + + org.apache.flink:flink-connector-aws-base + org.apache.flink:flink-protobuf-glue-schema-registry + org.apache.flink:flink-avro-glue-schema-registry + com.amazonaws:* + software.amazon.awssdk:* + software.amazon.glue:* + com.google.protobuf:* + com.fasterxml.jackson.core:* + com.fasterxml.jackson.dataformat:* + com.google.guava:guava + com.google.guava:failureaccess + com.google.code.gson:gson + + + software.amazon.glue:schema-registry-build-tools + com.google.guava:listenablefuture + org.checkerframework:checker-qual + com.google.errorprone:error_prone_annotations + com.google.j2objc:j2objc-annotations + com.google.code.findbugs:jsr305 + org.apache.kafka:kafka-clients + org.reactivestreams:reactive-streams + + + + + software.amazon + org.apache.flink.protobuf.registry.glue.shaded.software.amazon + + + com.amazonaws + org.apache.flink.protobuf.registry.glue.shaded.com.amazonaws + + + com.google + org.apache.flink.protobuf.registry.glue.shaded.com.google + + + com.typesafe.netty + org.apache.flink.protobuf.registry.glue.shaded.com.typesafe.netty + + + org.apache.http + org.apache.flink.protobuf.registry.glue.shaded.org.apache.http + + + com.fasterxml.jackson + org.apache.flink.protobuf.registry.glue.shaded.com.fasterxml.jackson + + + + + *:* + + **/MavenPackaging.java + **/mime.types + **/VersionInfo.java + codegen-resources/** + mozilla/** + + + + + + + + + + + + + + + com.google.errorprone + error_prone_annotations + 2.21.1 + + + org.opentest4j + opentest4j + 1.3.0 + + + + + diff --git a/flink-formats-aws/pom.xml b/flink-formats-aws/pom.xml index 3e571484f..a48974912 100644 --- a/flink-formats-aws/pom.xml +++ b/flink-formats-aws/pom.xml @@ -36,6 +36,9 @@ under the License. flink-avro-glue-schema-registry flink-json-glue-schema-registry + flink-protobuf-glue-schema-registry + flink-sql-avro-glue-schema-registry + flink-sql-protobuf-glue-schema-registry diff --git a/pom.xml b/pom.xml index fe59a298c..1748e2874 100644 --- a/pom.xml +++ b/pom.xml @@ -57,7 +57,7 @@ under the License. 4.1.86.Final 2.0.0 2.14.3 - 1.1.18 + 1.1.25 32.1.3-jre 5.8.1 @@ -348,6 +348,16 @@ under the License. okio-jvm 3.4.0 + + com.squareup.okio + okio-fakefilesystem + 3.9.1 + + + com.squareup.okio + okio-fakefilesystem-jvm + 3.9.1 + org.jetbrains.kotlin kotlin-stdlib-common @@ -373,6 +383,11 @@ under the License. kotlin-scripting-compiler-embeddable ${kotlin.version} + + org.jetbrains.kotlin + kotlin-scripting-compiler-impl-embeddable + ${kotlin.version} + joda-time joda-time