diff --git a/parquet/metadata/statistics.go b/parquet/metadata/statistics.go index 9e5684030..0aa9ef46d 100644 --- a/parquet/metadata/statistics.go +++ b/parquet/metadata/statistics.go @@ -388,8 +388,8 @@ var ( defaultMaxInt96 parquet.Int96 defaultMaxUInt96 parquet.Int96 - defaultMinFloat16 parquet.FixedLenByteArray = float16.MaxNum.ToLEBytes() - defaultMaxFloat16 parquet.FixedLenByteArray = float16.MinNum.ToLEBytes() + defaultMinFloat16 parquet.FixedLenByteArray = float16.Inf().ToLEBytes() + defaultMaxFloat16 parquet.FixedLenByteArray = float16.Inf().Negate().ToLEBytes() ) func init() { @@ -432,10 +432,10 @@ func (Float16Statistics) defaultMax() parquet.FixedLenByteArray { return defaultMaxFloat16 } -func (Float32Statistics) defaultMin() float32 { return math.MaxFloat32 } -func (Float32Statistics) defaultMax() float32 { return -math.MaxFloat32 } -func (Float64Statistics) defaultMin() float64 { return math.MaxFloat64 } -func (Float64Statistics) defaultMax() float64 { return -math.MaxFloat64 } +func (Float32Statistics) defaultMin() float32 { return float32(math.Inf(1)) } +func (Float32Statistics) defaultMax() float32 { return float32(math.Inf(-1)) } +func (Float64Statistics) defaultMin() float64 { return math.Inf(1) } +func (Float64Statistics) defaultMax() float64 { return math.Inf(-1) } func (ByteArrayStatistics) defaultMin() parquet.ByteArray { return nil } func (ByteArrayStatistics) defaultMax() parquet.ByteArray { return nil } func (FixedLenByteArrayStatistics) defaultMin() parquet.FixedLenByteArray { return nil } @@ -531,7 +531,7 @@ func (Float32Statistics) cleanStat(minMax minmaxPairFloat32) *minmaxPairFloat32 return nil } - if minMax[0] == math.MaxFloat32 && minMax[1] == -math.MaxFloat32 { + if minMax[0] > minMax[1] { return nil } @@ -552,7 +552,7 @@ func (Float64Statistics) cleanStat(minMax minmaxPairFloat64) *minmaxPairFloat64 return nil } - if minMax[0] == math.MaxFloat64 && minMax[1] == -math.MaxFloat64 { + if minMax[0] > minMax[1] { return nil } @@ -576,7 +576,7 @@ func (Float16Statistics) cleanStat(minMax minmaxPairFloat16) *minmaxPairFloat16 return nil } - if min.Equal(float16.MaxNum) && max.Equal(float16.MinNum) { + if min.Greater(max) { return nil } @@ -864,9 +864,9 @@ func (c *floatComparator[T]) defaultMin() T { var z T switch any(z).(type) { case float32: - return math.MaxFloat32 + return T(math.Inf(1)) case float64: - v := math.MaxFloat64 + v := math.Inf(1) return T(v) } panic("unreachable") diff --git a/parquet/metadata/statistics_test.go b/parquet/metadata/statistics_test.go index 6809eeef0..f7fcbda65 100644 --- a/parquet/metadata/statistics_test.go +++ b/parquet/metadata/statistics_test.go @@ -175,6 +175,42 @@ func TestCheckNaNs(t *testing.T) { assertMinMaxAreSpaced(someNanStatsf16, someNansf16, validBitmap, f16Min, f16Max) } +func TestFloatingPointInfinityStatistics(t *testing.T) { + f32Col := schema.NewColumn(schema.NewFloat32Node("f32", parquet.Repetitions.Required, -1), 0, 0) + f64Col := schema.NewColumn(schema.NewFloat64Node("f64", parquet.Repetitions.Required, -1), 0, 0) + f16Col := schema.NewColumn(newFloat16Node("f16", parquet.Repetitions.Required, -1), 0, 0) + + t.Run("float32", func(t *testing.T) { + for _, value := range []float32{float32(math.Inf(1)), float32(math.Inf(-1))} { + stats := metadata.NewStatistics(f32Col, memory.DefaultAllocator).(*metadata.Float32Statistics) + stats.Update([]float32{value, value}, 0) + require.True(t, stats.HasMinMax()) + assert.Equal(t, value, stats.Min()) + assert.Equal(t, value, stats.Max()) + } + }) + + t.Run("float64", func(t *testing.T) { + for _, value := range []float64{math.Inf(1), math.Inf(-1)} { + stats := metadata.NewStatistics(f64Col, memory.DefaultAllocator).(*metadata.Float64Statistics) + stats.Update([]float64{value, value}, 0) + require.True(t, stats.HasMinMax()) + assert.Equal(t, value, stats.Min()) + assert.Equal(t, value, stats.Max()) + } + }) + + t.Run("float16", func(t *testing.T) { + for _, value := range []float16.Num{float16.Inf(), float16.Inf().Negate()} { + stats := metadata.NewStatistics(f16Col, memory.DefaultAllocator).(*metadata.Float16Statistics) + stats.Update([]parquet.FixedLenByteArray{value.ToLEBytes(), value.ToLEBytes()}, 0) + require.True(t, stats.HasMinMax()) + assert.True(t, value.Equal(float16.FromLEBytes(stats.Min()))) + assert.True(t, value.Equal(float16.FromLEBytes(stats.Max()))) + } + }) +} + func TestCheckNegativeZeroStats(t *testing.T) { assertMinMaxZeroesSign := func(stats metadata.TypedStatistics, values interface{}) { switch s := stats.(type) {