Skip to content
Open
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion dlt/common/normalizers/json/relational.py
Original file line number Diff line number Diff line change
Expand Up @@ -332,7 +332,7 @@ def extend_schema(self, extend_tables: bool = True) -> None:
RelationalNormalizerConfig,
self.schema._normalizers_config["json"].get("config") or {},
)
DataItemNormalizer._validate_normalizer_config(self.schema, config)
self._validate_normalizer_config(self.schema, config)

# add hints, do not compile.
self.schema._merge_hints(
Expand Down
9 changes: 3 additions & 6 deletions dlt/common/schema/normalizers.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
from dlt.common.configuration.inject import with_config
from dlt.common.configuration.specs import known_sections
from dlt.common.schema.configuration import SchemaConfiguration
from dlt.common.normalizers.exceptions import InvalidJsonNormalizer
from dlt.common.normalizers.json import SupportsDataItemNormalizer, DataItemNormalizer
from dlt.common.normalizers.naming import NamingConvention
from dlt.common.normalizers.naming.exceptions import (
Expand Down Expand Up @@ -107,20 +106,18 @@ def import_normalizers(
if "." not in json_module_name:
json_module_name = f"{DEFAULT_JSON_NAMESPACE}.{json_module_name}"
item_normalizer["module"] = json_module_name
json_module = cast(SupportsDataItemNormalizer, import_module(json_module_name))
# if max_table_nesting is set, we need to set the max_table_nesting in the json_normalizer
if destination_capabilities and destination_capabilities.max_table_nesting is not None:
# TODO: this is a hack, we need a better method to do this
from dlt.common.normalizers.json.relational import DataItemNormalizer

try:
DataItemNormalizer.ensure_this_normalizer(item_normalizer)
if issubclass(json_module.DataItemNormalizer, DataItemNormalizer):
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This should keep the ensure_this_normalizer check.

item_normalizer.setdefault("config", {})
item_normalizer["config"]["max_nesting"] = destination_capabilities.max_table_nesting # type: ignore[index]
except InvalidJsonNormalizer:
else:
# not a right normalizer
logger.warning(f"JSON Normalizer {item_normalizer} does not support max_nesting")
pass
json_module = cast(SupportsDataItemNormalizer, import_module(json_module_name))
explicit_normalizers["json"] = item_normalizer
return (
explicit_normalizers,
Expand Down
20 changes: 14 additions & 6 deletions dlt/extract/source.py
Original file line number Diff line number Diff line change
Expand Up @@ -394,25 +394,31 @@ def name(self) -> str:
@property
def max_table_nesting(self) -> int:
"""A schema hint that sets the maximum depth of nested table above which the remaining nodes are loaded as structs or JSON."""
return RelationalNormalizer.get_normalizer_config(self._schema).get("max_nesting")
data_normalizer = self._schema.data_item_normalizer
assert isinstance(data_normalizer, RelationalNormalizer)
return data_normalizer.get_normalizer_config(self._schema).get("max_nesting")

@max_table_nesting.setter
def max_table_nesting(self, value: int) -> None:
data_normalizer = self._schema.data_item_normalizer
assert isinstance(data_normalizer, RelationalNormalizer)
if value is None:
# this also check the normalizer type
config = RelationalNormalizer.get_normalizer_config(self._schema)
config = data_normalizer.get_normalizer_config(self._schema)
config.pop("max_nesting", None)
else:
RelationalNormalizer.update_normalizer_config(self._schema, {"max_nesting": value})
data_normalizer.update_normalizer_config(self._schema, {"max_nesting": value})

@property
def root_key(self) -> Optional[bool]:
"""Enables merging on all resources by propagating root foreign key to nested tables.
This option is most useful if you plan to change write disposition of a resource to disable/enable merge.

"""
data_normalizer = self._schema.data_item_normalizer
assert isinstance(data_normalizer, RelationalNormalizer)
# this also check the normalizer type
config = RelationalNormalizer.get_normalizer_config(self._schema)
config = data_normalizer.get_normalizer_config(self._schema)
is_root_key = config.get("root_key_propagation")
if is_root_key is None:
# if not found get legacy value
Expand All @@ -424,12 +430,14 @@ def root_key(self) -> Optional[bool]:

@root_key.setter
def root_key(self, value: bool) -> None:
data_normalizer = self._schema.data_item_normalizer
assert isinstance(data_normalizer, RelationalNormalizer)
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we don't need this assertion if we correctly validate in dlt.common.normalizers.json.relational

# this also check the normalizer type
config = RelationalNormalizer.get_normalizer_config(self._schema)
config = data_normalizer.get_normalizer_config(self._schema)
if value is None:
value = self._get_root_key_legacy(config)
if value is not None:
RelationalNormalizer.update_normalizer_config(
data_normalizer.update_normalizer_config(
self._schema,
{"root_key_propagation": value},
)
Expand Down