Skip to content

Commit

Permalink
fix: underflow with min int64 with floats
Browse files Browse the repository at this point in the history
it doesn't underflow due to float imprecision
  • Loading branch information
ccoVeille committed Nov 13, 2024
1 parent 41a83f8 commit 1e9618e
Show file tree
Hide file tree
Showing 2 changed files with 76 additions and 24 deletions.
91 changes: 71 additions & 20 deletions asserters.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,23 +5,20 @@ func checkUpperBoundary[T Type, T2 Type](value T, boundary T2) error {
return nil
}

var greater bool
var overflow bool
switch f := any(value).(type) {
case float64:
// for float64, everything fits in float64 without overflow.
// We are using a greater or equal because float cannot be compared easily because of precision loss.
greater = f >= float64(boundary)
overflow = isFloatOverflow(f, boundary)

case float32:
// everything fits in float32, except float64 greater than math.MaxFloat32.
// So, we must convert to float64 and check.
// We are using a greater or equal because float cannot be compared easily because of precision loss.
greater = float64(f) >= float64(boundary)
overflow = isFloatOverflow(f, boundary)

default:
// for all other integer types, it fits in an uint64 without overflow as we know value is positive.
greater = uint64(value) > uint64(boundary)
overflow = uint64(value) > uint64(boundary)
}

if greater {
if overflow {
return Error{
value: value,
boundary: boundary,
Expand All @@ -37,23 +34,18 @@ func checkLowerBoundary[T Type, T2 Type](value T, boundary T2) error {
return nil
}

var smaller bool
var underflow bool
switch f := any(value).(type) {
case float64:
// everything fits in float64 without overflow.
// We are using a lower or equal because float cannot be compared easily because of precision loss.
smaller = f <= float64(boundary)
underflow = isFloatUnderOverflow(f, boundary)
case float32:
// everything fits in float32, except float64 smaller than -math.MaxFloat32.
// So, we must convert to float64 and check.
// We are using a lower or equal because float cannot be compared easily because of precision loss.
smaller = float64(f) <= float64(boundary)
underflow = isFloatUnderOverflow(f, boundary)
default:
// for all other integer types, it fits in an int64 without overflow as we know value is negative.
smaller = int64(value) < int64(boundary)
underflow = int64(value) < int64(boundary)
}

if smaller {
if underflow {
return Error{
value: value,
boundary: boundary,
Expand All @@ -63,3 +55,62 @@ func checkLowerBoundary[T Type, T2 Type](value T, boundary T2) error {

return nil
}

func isFloatOverflow[T Type, T2 Type](value T, boundary T2) bool {
// boundary is positive when checking for an overflow

// everything fits in float64 without overflow.
v := float64(value)
b := float64(boundary)

if v > b*1.01 {
// way greater than the maximum value
return true
}

if v < b*0.99 {
// we are way below the maximum value
return false
}
// we are close to the maximum value

// let's try to create the overflow
// by converting back and forth with type juggling
conv := float64(T(T2(v)))

// the number was between 0.99 and 1.01 of the maximum value
// once converted back and forth, we need to check if the value is in the same range
// if not, so it's an overflow
return conv <= b*0.99
}

func isFloatUnderOverflow[T Type, T2 Type](value T, boundary T2) bool {
// everything fits in float64 without overflow.
v := float64(value)
b := float64(boundary)

if b == 0 {
// boundary is 0
// we can check easily
return value < 0
}

if v < b*1.01 { // please note value and boundary are negative here
// way below than the minimum value, it would underflow
return true
}

if v > b*0.99 { // please note value and boundary are negative here
// way greater than the minimum value
return false
}

// we are just above to the minimum value
// let's try to create the underflow
conv := float64(T(T2(v)))

// the number was between 0.99 and 1.01 of the minimum value
// once converted back and forth, we need to check if the value is in the same range
// if not, so it's an underflow
return conv >= b*0.99
}
9 changes: 5 additions & 4 deletions conversion_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1163,7 +1163,7 @@ func TestToInt64(t *testing.T) {
assertInt64Error(t, []caseInt64[float32]{
{name: "out of range math.MaxInt64 + 1", input: math.MaxInt64 + 1},
{name: "out of range math.MaxUint64", input: math.MaxUint64},
{name: "out of range math.MinInt64", input: math.MinInt64},
{name: "out of range math.MinInt64", input: math.MinInt64 * 1.02}, // because of float imprecision, we have to exceed the min int64 to trigger the error
{name: "out of range math.MaxFloat32", input: math.MaxFloat32},
{name: "out of range -math.MaxFloat32", input: -math.MaxFloat32},
})
Expand All @@ -1183,7 +1183,7 @@ func TestToInt64(t *testing.T) {
assertInt64Error(t, []caseInt64[float64]{
{name: "out of range math.MaxInt64 + 1", input: math.MaxInt64 + 1},
{name: "out of range math.MaxUint64", input: math.MaxUint64},
{name: "out of range math.MinInt64", input: math.MinInt64},
{name: "out of range math.MinInt64", input: math.MinInt64 * 1.02}, // because of float imprecision, we have to exceed the min int64 to trigger the error
{name: "out of range math.MaxFloat32", input: math.MaxFloat32},
{name: "out of range -math.MaxFloat32", input: -math.MaxFloat32},
{name: "out of range math.MaxFloat64", input: math.MaxFloat64},
Expand Down Expand Up @@ -1466,7 +1466,7 @@ func TestToInt(t *testing.T) {
assertIntError(t, []caseInt[float32]{
{name: "out of range math.MaxInt64 + 1", input: math.MaxInt64 + 1},
{name: "out of range math.MaxUint64", input: math.MaxUint64},
{name: "out of range math.MinInt64", input: math.MinInt64},
{name: "out of range math.MinInt64", input: math.MinInt64 * 1.02}, // because of float imprecision, we have to exceed the min int64 to trigger the error
{name: "out of range math.MaxFloat32", input: math.MaxFloat32},
{name: "out of range -math.MaxFloat32", input: -math.MaxFloat32},
})
Expand All @@ -1477,12 +1477,13 @@ func TestToInt(t *testing.T) {
{name: "zero", input: 0.0, want: 0},
{name: "rounded value", input: 1.1, want: 1},
{name: "positive within range", input: 10000.9, want: 10000},
{name: "math.MinInt64", input: math.MinInt64, want: math.MinInt64}, // pass because of float imprecision
})

assertIntError(t, []caseInt[float64]{
{name: "out of range math.MaxInt64 + 1", input: math.MaxInt64 + 1},
{name: "out of range math.MaxUint64", input: math.MaxUint64},
{name: "out of range math.MinInt64", input: math.MinInt64},
{name: "out of range math.MinInt64", input: math.MinInt64 * 1.02}, // because of float imprecision, we have to exceed the min int64 to trigger the error
{name: "out of range math.MaxFloat32", input: math.MaxFloat32},
{name: "out of range -math.MaxFloat32", input: -math.MaxFloat32},
{name: "out of range math.MaxFloat64", input: math.MaxFloat64},
Expand Down

0 comments on commit 1e9618e

Please sign in to comment.