Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 13 additions & 11 deletions be/src/core/column/variant_v2/column_variant_v2.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -507,31 +507,33 @@ class CompositeVariantShreddedState final : public VariantShreddedState {
if (_segments.empty()) {
return std::nullopt;
}
std::vector<VariantShreddedTypedValue> matches;
std::vector<std::optional<VariantShreddedTypedValue>> matches;
matches.reserve(_segments.size());
bool all_direct = true;
for (const auto& segment : _segments) {
auto match = segment->find_typed_value(path);
if (!match.has_value()) {
all_direct = false;
break;
}
matches.push_back(std::move(*match));
// Visit every segment even after one requests normalized fallback. Format states use
// find_typed_value() as their validation boundary, so stopping early would let a later
// segment publish an unvalidated normalized value and make errors depend on row order.
matches.push_back(std::move(match));
}

const bool homogeneous = all_direct && matches.front().column && matches.front().type &&
const auto& first = matches.front();
const bool homogeneous = all_direct && first->column && first->type &&
std::ranges::all_of(matches, [&](const auto& match) {
return match.column && match.type && !match.normalized &&
exact_typed_identity(matches.front().type, match.type);
return match->column && match->type && !match->normalized &&
exact_typed_identity(first->type, match->type);
});
if (homogeneous) {
MutableColumnPtr combined = matches.front().column->clone_empty();
MutableColumnPtr combined = first->column->clone_empty();
for (const auto& match : matches) {
combined->insert_range_from(*match.column, 0, match.column->size());
combined->insert_range_from(*match->column, 0, match->column->size());
}
return VariantShreddedTypedValue {.column = std::move(combined),
.type = matches.front().type,
.normalized = nullptr};
return VariantShreddedTypedValue {
.column = std::move(combined), .type = first->type, .normalized = nullptr};
}

auto normalized = find_normalized_value(path);
Expand Down
42 changes: 23 additions & 19 deletions be/src/format_v2/column_mapper.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2054,27 +2054,15 @@ static const ColumnDefinition* find_file_child_by_name(
return child_it == children.end() ? nullptr : &*child_it;
}

static bool variant_leaf_type_preserves_physical_identity(const ColumnDefinition& leaf) {
static bool variant_leaf_is_projectable_scalar(const ColumnDefinition& leaf) {
if (!leaf.children.empty() || leaf.type == nullptr) {
return false;
}
// ColumnDefinition does not transport Parquet's raw-binary/UUID and timestamp-unit tags.
// Limit direct leaves to identities fully described by the Doris scalar type; every ambiguous
// identity must retain the complete wrapper so reconstruction can inspect its physical schema.
switch (remove_nullable(leaf.type)->get_primitive_type()) {
case TYPE_BOOLEAN:
case TYPE_TINYINT:
case TYPE_SMALLINT:
case TYPE_INT:
case TYPE_BIGINT:
case TYPE_FLOAT:
case TYPE_DOUBLE:
case TYPE_DECIMAL128I:
case TYPE_DATEV2:
return true;
default:
return false;
}
// Physical identity - raw binary versus UTF-8, UUID, timestamp units - is resolved by the
// reader from ParquetColumnSchema::type_descriptor, which the projected schema copies verbatim.
// The mapper only decides which columns to read, so every scalar leaf is projectable. Complex
// leaves keep the complete wrapper because reconstruction still needs their full shape.
return !is_complex_type(remove_nullable(leaf.type)->get_primitive_type());
}

static bool build_variant_leaf_path_projection(const ColumnMapping& mapping,
Expand All @@ -2098,6 +2086,15 @@ static bool build_variant_leaf_path_projection(const ColumnMapping& mapping,
return false;
}
*root_projection = LocalColumnIndex::partial_local(*mapping.file_local_id);
// Decoding a residual needs the root key dictionary. The root `value` itself stays unprojected:
// it holds every unshredded field, so reading it would give back all the pruned I/O. Shredding
// keeps a present typed object's residual keys disjoint from its shredded fields, and a null
// typed object means the value is not an object, so the requested path is absent either way.
const auto* root_metadata = find_file_child_by_name(mapping.original_file_children, "metadata");
if (root_metadata == nullptr) {
return false;
}
root_projection->children.push_back(LocalColumnIndex::local(root_metadata->file_local_id()));
const auto* root_typed = find_file_child_by_name(mapping.original_file_children, "typed_value");
if (root_typed == nullptr || root_typed->children.empty() || root_typed->type == nullptr ||
remove_nullable(root_typed->type)->get_primitive_type() != TYPE_STRUCT) {
Expand All @@ -2124,9 +2121,16 @@ static bool build_variant_leaf_path_projection(const ColumnMapping& mapping,
if (leaf) {
// Only primitive typed values can be returned as a direct vector. Complex shredded
// values still need their wrapper shape and therefore keep the full Variant fallback.
if (!variant_leaf_type_preserves_physical_identity(*typed)) {
if (!variant_leaf_is_projectable_scalar(*typed)) {
return false;
}
// The residual beside the leaf carries the rows whose value did not match the shredded
// type. Reading it lets the reader merge those rows instead of rejecting the batch.
if (const auto* residual = find_file_child_by_name(wrapper->children, "value");
residual != nullptr) {
current_projection->children.push_back(
LocalColumnIndex::local(residual->file_local_id()));
}
typed_projection.project_all_children = true;
} else if (typed->type == nullptr ||
remove_nullable(typed->type)->get_primitive_type() != TYPE_STRUCT) {
Expand Down
6 changes: 6 additions & 0 deletions be/src/format_v2/parquet/parquet_profile.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ void ParquetProfile::init(RuntimeProfile* profile) {
TUnit::UNIT, parquet_profile, 1);
variant_leaf_projection_row_group_columns = ADD_CHILD_COUNTER_WITH_LEVEL(
profile, "VariantLeafProjectionRowGroupColumns", TUnit::UNIT, parquet_profile, 1);
variant_residual_projection_row_group_columns = ADD_CHILD_COUNTER_WITH_LEVEL(
profile, "VariantResidualProjectionRowGroupColumns", TUnit::UNIT, parquet_profile, 1);
variant_full_projection_row_group_columns = ADD_CHILD_COUNTER_WITH_LEVEL(
profile, "VariantFullProjectionRowGroupColumns", TUnit::UNIT, parquet_profile, 1);
pages_skipped_by_data_page_filter = ADD_CHILD_COUNTER_WITH_LEVEL(
Expand Down Expand Up @@ -148,6 +150,10 @@ void ParquetProfile::init(RuntimeProfile* profile) {
profile, "VariantDirectLeafPathMisses", TUnit::UNIT, parquet_profile);
variant->variant_direct_leaf_residual_fallbacks = add_persistent_counter(
profile, "VariantDirectLeafResidualFallbacks", TUnit::UNIT, parquet_profile);
variant->variant_direct_leaf_residual_merged_rows = add_persistent_counter(
profile, "VariantDirectLeafResidualMergedRows", TUnit::UNIT, parquet_profile);
variant->variant_residual_seek_rows = add_persistent_counter(profile, "VariantResidualSeekRows",
TUnit::UNIT, parquet_profile);
variant->variant_direct_leaf_unsupported_fallbacks = add_persistent_counter(
profile, "VariantDirectLeafUnsupportedFallbacks", TUnit::UNIT, parquet_profile);
variant_counters = std::move(variant);
Expand Down
3 changes: 3 additions & 0 deletions be/src/format_v2/parquet/parquet_profile.h
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ struct ParquetVariantProfile {
std::shared_ptr<RuntimeProfile::Counter> variant_direct_leaf_rows;
std::shared_ptr<RuntimeProfile::Counter> variant_direct_leaf_path_misses;
std::shared_ptr<RuntimeProfile::Counter> variant_direct_leaf_residual_fallbacks;
std::shared_ptr<RuntimeProfile::Counter> variant_direct_leaf_residual_merged_rows;
std::shared_ptr<RuntimeProfile::Counter> variant_residual_seek_rows;
std::shared_ptr<RuntimeProfile::Counter> variant_direct_leaf_unsupported_fallbacks;
};

Expand Down Expand Up @@ -183,6 +185,7 @@ struct ParquetProfile {
RuntimeProfile::Counter* variant_leaf_projections = nullptr;
// Row-group outcomes for candidate Variant projections, counted per projected column opened.
RuntimeProfile::Counter* variant_leaf_projection_row_group_columns = nullptr;
RuntimeProfile::Counter* variant_residual_projection_row_group_columns = nullptr;
RuntimeProfile::Counter* variant_full_projection_row_group_columns = nullptr;

// ======== Page Skip ========
Expand Down
123 changes: 114 additions & 9 deletions be/src/format_v2/parquet/parquet_reader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -117,9 +117,9 @@ bool collect_variant_terminal_fallback_leaf_ids(const ParquetColumnSchema& schem

namespace {

bool variant_leaf_projection_is_safe_for_row_group_impl(
const tparquet::RowGroup& row_group, const ParquetColumnSchema& schema,
const format::LocalColumnIndex& projection) {
bool variant_residual_columns_are_prunable_impl(const tparquet::RowGroup& row_group,
const ParquetColumnSchema& schema,
const format::LocalColumnIndex& projection) {
if (schema.kind != ParquetColumnSchemaKind::VARIANT || schema.max_repetition_level != 0 ||
!format::is_partial_projection(&projection)) {
return false;
Expand Down Expand Up @@ -147,26 +147,130 @@ bool variant_leaf_projection_is_safe_for_row_group_impl(

} // namespace

bool detail::variant_leaf_projection_is_safe_for_row_group(
bool detail::variant_residual_columns_are_prunable_for_row_group(
const tparquet::RowGroup& row_group, const ParquetColumnSchema& schema,
const format::LocalColumnIndex& projection) {
return variant_leaf_projection_is_safe_for_row_group_impl(row_group, schema, projection);
return variant_residual_columns_are_prunable_impl(row_group, schema, projection);
}

namespace {

void remove_child_projection(format::LocalColumnIndex* projection, int32_t local_id) {
std::erase_if(projection->children, [local_id](const format::LocalColumnIndex& child) {
return child.local_id() == local_id;
});
}

void prune_variant_residual_columns_impl(const ParquetColumnSchema& schema,
format::LocalColumnIndex* projection) {
const auto* value = schema_child_by_name(schema, "value");
const auto* typed_value = schema_child_by_name(schema, "typed_value");
if (value != nullptr && typed_value != nullptr &&
typed_value->kind == ParquetColumnSchemaKind::PRIMITIVE) {
// Terminal wrapper: statistics proved its residual is entirely NULL for this row group,
// so the shredded leaf alone answers every row.
remove_child_projection(projection, value->local_id);
return;
}
if (schema.kind == ParquetColumnSchemaKind::VARIANT) {
// The root dictionary is only needed to decode a residual.
if (const auto* metadata = schema_child_by_name(schema, "metadata"); metadata != nullptr) {
remove_child_projection(projection, metadata->local_id);
}
}
for (auto& child_projection : projection->children) {
const auto* child = projected_schema_child(schema, child_projection.local_id());
if (child != nullptr) {
prune_variant_residual_columns_impl(*child, &child_projection);
}
}
}

} // namespace

namespace {

// A leaf projection can only answer rows whose value sits outside the shredded leaf if it actually
// reads the residual beside that leaf, and the root dictionary needed to decode it. A projection
// that omits either cannot serve those rows, and nothing else would notice: the leaf would report
// them as absent.
bool variant_projection_carries_residuals_impl(const ParquetColumnSchema& schema,
const format::LocalColumnIndex& projection) {
if (schema.kind == ParquetColumnSchemaKind::VARIANT && schema.max_repetition_level != 0) {
// A repeated Variant produces one physical entry per array element, not per row. The
// per-row residual merge and the row-group statistics both address rows, so a repeated
// Variant keeps the complete wrapper exactly as it did before leaf projections existed.
return false;
}
const auto* value = schema_child_by_name(schema, "value");
const auto* typed_value = schema_child_by_name(schema, "typed_value");
const auto projects = [&projection](int32_t local_id) {
return std::ranges::any_of(projection.children, [local_id](const auto& child) {
return child.local_id() == local_id;
});
};
if (value != nullptr && typed_value != nullptr) {
if (typed_value->kind == ParquetColumnSchemaKind::PRIMITIVE) {
return projects(value->local_id);
}
if (schema.kind == ParquetColumnSchemaKind::VARIANT) {
const auto* metadata = schema_child_by_name(schema, "metadata");
if (metadata == nullptr || !projects(metadata->local_id)) {
return false;
}
}
}
for (const auto& child_projection : projection.children) {
const auto* child = projected_schema_child(schema, child_projection.local_id());
if (child != nullptr &&
!variant_projection_carries_residuals_impl(*child, child_projection)) {
return false;
}
}
return true;
}

} // namespace

bool detail::variant_projection_carries_residuals(const ParquetColumnSchema& schema,
const format::LocalColumnIndex& projection) {
return variant_projection_carries_residuals_impl(schema, projection);
}

void detail::prune_variant_residual_columns(const ParquetColumnSchema& schema,
format::LocalColumnIndex* projection) {
DORIS_CHECK(projection != nullptr);
if (format::is_partial_projection(projection)) {
prune_variant_residual_columns_impl(schema, projection);
}
}

size_t detail::finalize_variant_leaf_projection_for_row_group(const tparquet::RowGroup& row_group,
const ParquetColumnSchema& schema,
format::LocalColumnIndex* projection,
size_t* residual_projections,
size_t* full_projections) {
DORIS_CHECK(projection != nullptr);
if (!format::is_partial_projection(projection)) {
return 0;
}
if (schema.kind == ParquetColumnSchemaKind::VARIANT) {
if (variant_leaf_projection_is_safe_for_row_group(row_group, schema, *projection)) {
// The residual null_count is row-group scoped, so the decision is per reader.
if (variant_residual_columns_are_prunable_for_row_group(row_group, schema, *projection)) {
detail::prune_variant_residual_columns(schema, projection);
return 1;
}
if (variant_projection_carries_residuals_impl(schema, *projection)) {
// The reader merges the rows stored outside the shredded leaves, so the row group
// keeps reading leaves instead of the complete Variant.
if (residual_projections != nullptr) {
++*residual_projections;
}
return 1;
}
// The residual null_count is row-group scoped. Restore the complete Variant wrapper for
// only this reader so another row group can still keep the typed-leaf projection.
// Nothing proves the residual is empty and the projection cannot read it. Restore the
// complete Variant wrapper for only this reader so another row group can still keep its
// typed-leaf projection.
projection->project_all_children = true;
projection->children.clear();
if (full_projections != nullptr) {
Expand All @@ -180,7 +284,8 @@ size_t detail::finalize_variant_leaf_projection_for_row_group(const tparquet::Ro
const auto* child_schema = projected_schema_child(schema, child_projection.local_id());
DORIS_CHECK(child_schema != nullptr);
retained += finalize_variant_leaf_projection_for_row_group(
row_group, *child_schema, &child_projection, full_projections);
row_group, *child_schema, &child_projection, residual_projections,
full_projections);
}
return retained;
}
Expand Down
Loading
Loading