Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion dlt/common/normalizers/json/relational.py
Original file line number Diff line number Diff line change
Expand Up @@ -332,7 +332,7 @@ def extend_schema(self, extend_tables: bool = True) -> None:
RelationalNormalizerConfig,
self.schema._normalizers_config["json"].get("config") or {},
)
DataItemNormalizer._validate_normalizer_config(self.schema, config)
self._validate_normalizer_config(self.schema, config)

# add hints, do not compile.
self.schema._merge_hints(
Expand Down
9 changes: 3 additions & 6 deletions dlt/common/schema/normalizers.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@
from dlt.common.configuration.inject import with_config
from dlt.common.configuration.specs import known_sections
from dlt.common.schema.configuration import SchemaConfiguration
from dlt.common.normalizers.exceptions import InvalidJsonNormalizer
from dlt.common.normalizers.json import SupportsDataItemNormalizer, DataItemNormalizer
from dlt.common.normalizers.naming import NamingConvention
from dlt.common.normalizers.naming.exceptions import (
Expand Down Expand Up @@ -107,20 +106,18 @@ def import_normalizers(
if "." not in json_module_name:
json_module_name = f"{DEFAULT_JSON_NAMESPACE}.{json_module_name}"
item_normalizer["module"] = json_module_name
json_module = cast(SupportsDataItemNormalizer, import_module(json_module_name))
# if max_table_nesting is set, we need to set the max_table_nesting in the json_normalizer
if destination_capabilities and destination_capabilities.max_table_nesting is not None:
# TODO: this is a hack, we need a better method to do this
from dlt.common.normalizers.json.relational import DataItemNormalizer

try:
DataItemNormalizer.ensure_this_normalizer(item_normalizer)
if issubclass(json_module.DataItemNormalizer, DataItemNormalizer):
item_normalizer.setdefault("config", {})
item_normalizer["config"]["max_nesting"] = destination_capabilities.max_table_nesting # type: ignore[index]
except InvalidJsonNormalizer:
else:
# not a right normalizer
logger.warning(f"JSON Normalizer {item_normalizer} does not support max_nesting")
pass
json_module = cast(SupportsDataItemNormalizer, import_module(json_module_name))
explicit_normalizers["json"] = item_normalizer
return (
explicit_normalizers,
Expand Down
20 changes: 14 additions & 6 deletions dlt/extract/source.py
Original file line number Diff line number Diff line change
Expand Up @@ -394,25 +394,31 @@ def name(self) -> str:
@property
def max_table_nesting(self) -> int:
"""A schema hint that sets the maximum depth of nested table above which the remaining nodes are loaded as structs or JSON."""
return RelationalNormalizer.get_normalizer_config(self._schema).get("max_nesting")
data_normalizer = self._schema.data_item_normalizer
assert isinstance(data_normalizer, RelationalNormalizer)
return data_normalizer.get_normalizer_config(self._schema).get("max_nesting")

@max_table_nesting.setter
def max_table_nesting(self, value: int) -> None:
data_normalizer = self._schema.data_item_normalizer
assert isinstance(data_normalizer, RelationalNormalizer)
if value is None:
# this also check the normalizer type
config = RelationalNormalizer.get_normalizer_config(self._schema)
config = data_normalizer.get_normalizer_config(self._schema)
config.pop("max_nesting", None)
else:
RelationalNormalizer.update_normalizer_config(self._schema, {"max_nesting": value})
data_normalizer.update_normalizer_config(self._schema, {"max_nesting": value})

@property
def root_key(self) -> Optional[bool]:
"""Enables merging on all resources by propagating root foreign key to nested tables.
This option is most useful if you plan to change write disposition of a resource to disable/enable merge.

"""
data_normalizer = self._schema.data_item_normalizer
assert isinstance(data_normalizer, RelationalNormalizer)
# this also check the normalizer type
config = RelationalNormalizer.get_normalizer_config(self._schema)
config = data_normalizer.get_normalizer_config(self._schema)
is_root_key = config.get("root_key_propagation")
if is_root_key is None:
# if not found get legacy value
Expand All @@ -424,12 +430,14 @@ def root_key(self) -> Optional[bool]:

@root_key.setter
def root_key(self, value: bool) -> None:
data_normalizer = self._schema.data_item_normalizer
assert isinstance(data_normalizer, RelationalNormalizer)
# this also check the normalizer type
config = RelationalNormalizer.get_normalizer_config(self._schema)
config = data_normalizer.get_normalizer_config(self._schema)
if value is None:
value = self._get_root_key_legacy(config)
if value is not None:
RelationalNormalizer.update_normalizer_config(
data_normalizer.update_normalizer_config(
self._schema,
{"root_key_propagation": value},
)
Expand Down