diff --git a/CHANGELOG.md b/CHANGELOG.md index 494dc73f..adf6c4cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,11 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +### Added +- Added additional type conversions for mysql for BIGINT, TINYINT, SMALLINT, and handling of int(10) format +- Add support for composite primary keys during SQL import + ### Fixed - `changelog` command help text now advertises `(url or path)` for V1/V2 arguments, clarifying that HTTP/HTTPS URLs are accepted (#1162) - **breaking:** `test` command now exits non-zero when a server is specified, but soda-core fails to connect or authenticate (#1181) - correct swapped `check_type` labels `model_qualty_sql` and `field_quality_sql` (#1187) - `import spark` now emits a native Spark SQL physicalType (e.g. `string`) instead of Python repr (e.g. `StringType()`). Contracts imported using Spark in v0.11.0–v0.12.1 did not perform type checks and must be re-imported. (#1048) +- Fixed issue where `import sql` would only import a single table from a sql ddl file even if multiple were defined +- Fixed issue where foreign key constraint in sql ddl was creating duplicate table entries during on `import sql` action ## [0.12.1] - 2026-04-21 diff --git a/datacontract/imports/sql_importer.py b/datacontract/imports/sql_importer.py index 01021557..6d40d494 100644 --- a/datacontract/imports/sql_importer.py +++ b/datacontract/imports/sql_importer.py @@ -42,7 +42,7 @@ def import_sql(source: str, import_args: dict = None) -> OpenDataContractStandar dialect = to_dialect(import_args) try: - parsed = sqlglot.parse_one(sql=sql, read=dialect) + parsed_statements = sqlglot.parse(sql=sql, read=dialect) except Exception as e: logging.error(f"Error sqlglot SQL: {str(e)}") raise DataContractException( @@ -65,77 +65,98 @@ def import_sql(source: str, import_args: dict = None) -> OpenDataContractStandar "Update host, port, database, and schema in the output before use." ) - tables = [ - t - for t in parsed.find_all(sqlglot.expressions.Table) - if isinstance(t.find_ancestor(sqlglot.expressions.Create), sqlglot.expressions.Create) - ] - - for table in tables: - table_name = table.this.name - properties = [] - - primary_key_position = 1 - for column in parsed.find_all(sqlglot.exp.ColumnDef): - if column.parent.this.name != table_name: - continue - - col_name = column.this.name - col_type = to_col_type(column, dialect) - logical_type, format = map_type_from_sql(col_type) - col_description = get_description(column) - max_length = get_max_length(column) - precision, scale = get_precision_scale(column) - is_primary_key = get_primary_key(column) - is_required = column.find(sqlglot.exp.NotNullColumnConstraint) is not None or None - tags = get_tags(column) - - prop = create_property( - name=col_name, - logical_type=logical_type, - physical_type=col_type, - description=col_description, - max_length=max_length, - precision=precision, - scale=scale, - format=format, - primary_key=is_primary_key, - primary_key_position=primary_key_position if is_primary_key else None, - required=is_required if is_required else None, - tags=tags, - ) - - if is_primary_key: - primary_key_position += 1 - - properties.append(prop) - - table_comment_property = parsed.find(sqlglot.expressions.SchemaCommentProperty) - - table_description = None - if table_comment_property: - table_description = table_comment_property.this.this - - table_tags = None - table_props = parsed.find(sqlglot.expressions.Properties) - if table_props: - tags = table_props.find(sqlglot.expressions.Tags) - if tags: - table_tags = [str(t) for t in tags.expressions] - - schema_obj = create_schema_object( - name=table_name, - physical_type="table", - description=table_description, - tags=table_tags, - properties=properties, - ) - odcs.schema_.append(schema_obj) + for parsed in parsed_statements: + tables = [ + t + for t in parsed.find_all(sqlglot.expressions.Table) + if isinstance(t.find_ancestor(sqlglot.expressions.Create), sqlglot.expressions.Create) + ] + + for table in tables: + table_name = table.this.name + properties = [] + + columns = [ + column for column in parsed.find_all(sqlglot.exp.ColumnDef) if column.parent.this.name == table_name + ] + + primary_keys = [ + expression.name + for key in parsed.find_all(sqlglot.expressions.PrimaryKey) + for expression in key.expressions + ] + + # table foreign keys were duplicating the referenced tables-- so only include tables with columns + if columns: + column_primary_key_index = 1 + for column in columns: + col_name = column.this.name + col_type = to_col_type(column, dialect) + logical_type, format = map_type_from_sql(col_type) + col_description = get_description(column) + max_length = get_max_length(column) + precision, scale = get_precision_scale(column) + is_primary_key = get_primary_key(column, primary_keys) + is_required = column.find(sqlglot.exp.NotNullColumnConstraint) is not None or None + tags = get_tags(column) + + # Primary key can be defined at the column level OR table level. If defined at the table level + # use the index rather than the position of the columns to define primary key position. + primary_key_position = column_primary_key_index + if is_primary_key and col_name in primary_keys: + primary_key_position = primary_keys.index(col_name) + 1 + + prop = create_property( + name=col_name, + logical_type=logical_type, + physical_type=col_type, + description=col_description, + max_length=max_length, + precision=precision, + scale=scale, + format=format, + primary_key=is_primary_key, + primary_key_position=primary_key_position if is_primary_key else None, + required=is_required if is_required else None, + tags=tags, + ) + + if is_primary_key: + column_primary_key_index += 1 + + properties.append(prop) + + table_comment_property = parsed.find(sqlglot.expressions.SchemaCommentProperty) + + table_description = None + if table_comment_property: + table_description = table_comment_property.this.this + + table_tags = None + table_props = parsed.find(sqlglot.expressions.Properties) + if table_props: + tags = table_props.find(sqlglot.expressions.Tags) + if tags: + table_tags = [str(t) for t in tags.expressions] + + schema_obj = create_schema_object( + name=table_name, + physical_type="table", + description=table_description, + tags=table_tags, + properties=properties, + ) + odcs.schema_.append(schema_obj) return odcs -def get_primary_key(column) -> bool | None: +def get_primary_key(column, table_primary_keys=None) -> bool | None: + col_name = column.this.name + if table_primary_keys is None: + table_primary_keys = [] + if col_name in table_primary_keys: + return True if column.find(sqlglot.exp.PrimaryKeyColumnConstraint) is not None: return True if column.find(sqlglot.exp.PrimaryKey) is not None: @@ -308,6 +329,10 @@ def map_type_from_sql(sql_type: str) -> tuple[str, str | None]: return ("integer", None) elif sql_type_normed.endswith("integer"): return ("integer", None) + elif re.search(r"^(small|big)?int(\(\d+\))?(\sunsigned)?$", sql_type_normed): # Looking for format int(10) + return ("integer", None) + elif sql_type_normed.startswith("tinyint") and not sql_type_normed.startswith("tinyint(1)"): + return ("integer", None) elif sql_type_normed.startswith("float"): return ("number", None) elif sql_type_normed.startswith("double"): @@ -326,6 +351,8 @@ def map_type_from_sql(sql_type: str) -> tuple[str, str | None]: return ("boolean", None) elif sql_type_normed.startswith("bit"): return ("boolean", None) + elif sql_type_normed.startswith("tinyint(1)"): + return ("boolean", None) elif sql_type_normed.startswith("binary"): return ("string", "binary") elif sql_type_normed.startswith("varbinary"): diff --git a/tests/test_import_sql_oracle.py b/tests/test_import_sql_oracle.py index 4fcbe96c..3a346eb7 100644 --- a/tests/test_import_sql_oracle.py +++ b/tests/test_import_sql_oracle.py @@ -179,8 +179,10 @@ def test_import_sql_constraints(): physicalName: customer_location properties: - name: id - logicalType: number physicalType: DECIMAL + primaryKey: true + primaryKeyPosition: 1 + logicalType: number required: true - name: created_by logicalType: string diff --git a/tests/test_import_sql_postgres.py b/tests/test_import_sql_postgres.py index 197f0c5c..0622cdef 100644 --- a/tests/test_import_sql_postgres.py +++ b/tests/test_import_sql_postgres.py @@ -87,6 +87,8 @@ def test_import_sql_constraints(): - name: id logicalType: number physicalType: DECIMAL + primaryKey: true + primaryKeyPosition: 1 required: true - name: created_by logicalType: string diff --git a/tests/test_import_sql_snowflake.py b/tests/test_import_sql_snowflake.py index 926ff008..e3dfdbc9 100644 --- a/tests/test_import_sql_snowflake.py +++ b/tests/test_import_sql_snowflake.py @@ -31,6 +31,8 @@ def test_import_sql_snowflake(): - name: field_primary_key physicalType: DECIMAL(38, 0) description: Primary key + primaryKey: true + primaryKeyPosition: 1 customProperties: - property: precision value: 38