Skip to content

Commit e35718d

Browse files
findepifindepi
committed
Round floats but not decimals in SqlLogicTests
- stop rounding decimal values in SLT. It's the very nature of decimal arithmetics that it should in general be exact. - round float values taking into account the bit width of the float. The Float64 carries more information than Float32 or Float16.
1 parent af5b84b commit e35718d

25 files changed

Lines changed: 961 additions & 895 deletions

‎datafusion/sqllogictest/src/engines/conversion.rs‎

Lines changed: 143 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616
// under the License.
1717

1818
use arrow::datatypes::{i256, Decimal128Type, Decimal256Type, DecimalType};
19-
use bigdecimal::BigDecimal;
19+
use bigdecimal::{BigDecimal, RoundingMode};
2020
use half::f16;
2121
use rust_decimal::prelude::*;
2222

@@ -40,34 +40,18 @@ pub(crate) fn varchar_to_str(value: &str) -> String {
4040
}
4141

4242
pub(crate) fn f16_to_str(value: f16) -> String {
43-
if value.is_nan() {
44-
// The sign of NaN can be different depending on platform.
45-
// So the string representation of NaN ignores the sign.
46-
"NaN".to_string()
47-
} else if value == f16::INFINITY {
48-
"Infinity".to_string()
49-
} else if value == f16::NEG_INFINITY {
50-
"-Infinity".to_string()
51-
} else {
52-
big_decimal_to_str(BigDecimal::from_str(&value.to_string()).unwrap())
53-
}
43+
float_to_str(value.into(), 4)
5444
}
5545

5646
pub(crate) fn f32_to_str(value: f32) -> String {
57-
if value.is_nan() {
58-
// The sign of NaN can be different depending on platform.
59-
// So the string representation of NaN ignores the sign.
60-
"NaN".to_string()
61-
} else if value == f32::INFINITY {
62-
"Infinity".to_string()
63-
} else if value == f32::NEG_INFINITY {
64-
"-Infinity".to_string()
65-
} else {
66-
big_decimal_to_str(BigDecimal::from_str(&value.to_string()).unwrap())
67-
}
47+
float_to_str(value.into(), 6)
6848
}
6949

7050
pub(crate) fn f64_to_str(value: f64) -> String {
51+
float_to_str(value, 10)
52+
}
53+
54+
fn float_to_str(value: f64, max_significant_digits: u8) -> String {
7155
if value.is_nan() {
7256
// The sign of NaN can be different depending on platform.
7357
// So the string representation of NaN ignores the sign.
@@ -77,79 +61,161 @@ pub(crate) fn f64_to_str(value: f64) -> String {
7761
} else if value == f64::NEG_INFINITY {
7862
"-Infinity".to_string()
7963
} else {
80-
big_decimal_to_str(BigDecimal::from_str(&value.to_string()).unwrap())
64+
let mut big_decimal = BigDecimal::from_f64(value)
65+
.unwrap()
66+
// Truncate trailing decimal zeros
67+
.normalized();
68+
let precision = big_decimal.digits();
69+
if precision > max_significant_digits as u64 {
70+
let scale = big_decimal.as_bigint_and_exponent().1;
71+
big_decimal = big_decimal
72+
.with_scale_round(
73+
scale + (max_significant_digits as i64 - precision as i64),
74+
RoundingMode::HalfUp,
75+
)
76+
// Truncate trailing decimal zeros
77+
.normalized();
78+
}
79+
big_decimal.to_plain_string()
8180
}
8281
}
8382

8483
pub(crate) fn decimal_128_to_str(value: i128, scale: i8) -> String {
8584
let precision = u8::MAX; // does not matter
86-
big_decimal_to_str(
87-
BigDecimal::from_str(&Decimal128Type::format_decimal(value, precision, scale))
88-
.unwrap(),
89-
)
85+
BigDecimal::from_str(&Decimal128Type::format_decimal(value, precision, scale))
86+
.unwrap()
87+
.normalized() // Truncate trailing decimal zeros
88+
.to_plain_string()
9089
}
9190

9291
pub(crate) fn decimal_256_to_str(value: i256, scale: i8) -> String {
9392
let precision = u8::MAX; // does not matter
94-
big_decimal_to_str(
95-
BigDecimal::from_str(&Decimal256Type::format_decimal(value, precision, scale))
96-
.unwrap(),
97-
)
93+
BigDecimal::from_str(&Decimal256Type::format_decimal(value, precision, scale))
94+
.unwrap()
95+
.normalized() // Truncate trailing decimal zeros
96+
.to_plain_string()
9897
}
9998

10099
#[cfg(feature = "postgres")]
101100
pub(crate) fn decimal_to_str(value: Decimal) -> String {
102-
big_decimal_to_str(BigDecimal::from_str(&value.to_string()).unwrap())
103-
}
104-
105-
pub(crate) fn big_decimal_to_str(value: BigDecimal) -> String {
106-
// Round the value to limit the number of decimal places
107-
let value = value.round(12).normalized();
108-
// Format the value to a string
109-
value.to_plain_string()
101+
BigDecimal::from_str(&value.to_string())
102+
.unwrap()
103+
.normalized() // Truncate trailing decimal zeros
104+
.to_plain_string()
110105
}
111106

112107
#[cfg(test)]
113108
mod tests {
114-
use super::big_decimal_to_str;
115-
use bigdecimal::{num_bigint::BigInt, BigDecimal};
116-
117-
macro_rules! assert_decimal_str_eq {
118-
($integer:expr, $scale:expr, $expected:expr) => {
119-
assert_eq!(
120-
big_decimal_to_str(BigDecimal::from_bigint(
121-
BigInt::from($integer),
122-
$scale
123-
)),
124-
$expected
125-
);
126-
};
109+
use super::*;
110+
use half::f16;
111+
112+
#[test]
113+
fn test_f16_to_str() {
114+
assert_eq!(f16_to_str(f16::from_f32(0.)), "0");
115+
assert_eq!(f16_to_str(f16::from_f32(1.)), "1");
116+
assert_eq!(f16_to_str(f16::from_f32(12.345)), "12.34");
117+
assert_eq!(f16_to_str(f16::from_f32(-12.345)), "-12.34");
118+
assert_eq!(f16_to_str(f16::MAX), "65500");
119+
assert_eq!(f16_to_str(f16::MIN), "-65500");
120+
assert_eq!(f16_to_str(f16::EPSILON), "0.0009766");
121+
assert_eq!(f16_to_str(f16::from_f32(f32::INFINITY)), "Infinity");
122+
assert_eq!(f16_to_str(f16::from_f32(f32::NEG_INFINITY)), "-Infinity");
123+
assert_eq!(f16_to_str(f16::from_f32(f32::NAN)), "NaN");
124+
}
125+
126+
#[test]
127+
fn test_f32_to_str() {
128+
assert_eq!(f32_to_str(0.), "0");
129+
assert_eq!(f32_to_str(1.), "1");
130+
assert_eq!(f32_to_str(12.345), "12.345");
131+
assert_eq!(f32_to_str(-12.345), "-12.345");
132+
assert_eq!(f32_to_str(0.0000012345), "0.0000012345");
133+
assert_eq!(f32_to_str(-0.0000012345), "-0.0000012345");
134+
assert_eq!(f32_to_str(12.345678), "12.3457");
135+
assert_eq!(f32_to_str(-12.345678), "-12.3457");
136+
assert_eq!(
137+
f32_to_str(f32::MAX),
138+
"340282000000000000000000000000000000000"
139+
);
140+
assert_eq!(
141+
f32_to_str(f32::MIN),
142+
"-340282000000000000000000000000000000000"
143+
);
144+
assert_eq!(f32_to_str(f32::EPSILON), "0.000000119209");
145+
assert_eq!(f32_to_str(f32::INFINITY), "Infinity");
146+
assert_eq!(f32_to_str(f32::NEG_INFINITY), "-Infinity");
147+
assert_eq!(f32_to_str(f32::NAN), "NaN");
148+
}
149+
150+
#[test]
151+
fn test_f64_to_str() {
152+
assert_eq!(f64_to_str(0.), "0");
153+
assert_eq!(f64_to_str(1.), "1");
154+
assert_eq!(f64_to_str(12.345), "12.345");
155+
assert_eq!(f64_to_str(-12.345), "-12.345");
156+
assert_eq!(f64_to_str(12.345678), "12.345678");
157+
assert_eq!(f64_to_str(-12.345678), "-12.345678");
158+
assert_eq!(f64_to_str(0.00000000012345678), "0.00000000012345678");
159+
assert_eq!(f64_to_str(-0.00000000012345678), "-0.00000000012345678");
160+
assert_eq!(f64_to_str(12.34567890123456), "12.3456789");
161+
assert_eq!(f64_to_str(-12.34567890123456), "-12.3456789");
162+
assert_eq!(f64_to_str(0.99999999999999999999999), "1");
163+
assert_eq!(f64_to_str(0.0000000000999999999999999), "0.0000000001");
164+
assert_eq!(f64_to_str(f64::MAX), "179769313500000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000");
165+
assert_eq!(f64_to_str(f64::MIN), "-179769313500000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000");
166+
assert_eq!(f64_to_str(f64::EPSILON), "0.0000000000000002220446049");
167+
assert_eq!(f64_to_str(f64::INFINITY), "Infinity");
168+
assert_eq!(f64_to_str(f64::NEG_INFINITY), "-Infinity");
169+
assert_eq!(f64_to_str(f64::NAN), "NaN");
170+
}
171+
172+
#[test]
173+
fn test_decimal_128_to_str() {
174+
assert_eq!(decimal_128_to_str(1, 0), "1");
175+
assert_eq!(decimal_128_to_str(1, 5), "0.00001");
176+
assert_eq!(decimal_128_to_str(1, 20), "0.00000000000000000001");
177+
assert_eq!(
178+
decimal_128_to_str(1, 38),
179+
"0.00000000000000000000000000000000000001"
180+
);
181+
assert_eq!(
182+
decimal_128_to_str(12345678901234567890123456789012345678, 20),
183+
"123456789012345678.90123456789012345678"
184+
);
185+
assert_eq!(
186+
decimal_128_to_str(12345678901234567890123456789012345678, 0),
187+
"12345678901234567890123456789012345678"
188+
);
127189
}
128190

129191
#[test]
130-
fn test_big_decimal_to_str() {
131-
assert_decimal_str_eq!(110, 3, "0.11");
132-
assert_decimal_str_eq!(11, 3, "0.011");
133-
assert_decimal_str_eq!(11, 2, "0.11");
134-
assert_decimal_str_eq!(11, 1, "1.1");
135-
assert_decimal_str_eq!(11, 0, "11");
136-
assert_decimal_str_eq!(11, -1, "110");
137-
assert_decimal_str_eq!(0, 0, "0");
138-
assert_decimal_str_eq!(12345678901234567890123456789012345678_i128, 0, "12345678901234567890123456789012345678");
139-
assert_decimal_str_eq!(12345678901234567890123456789012345678_i128, 38, "0.123456789012");
140-
141-
// Negative cases
142-
assert_decimal_str_eq!(-110, 3, "-0.11");
143-
assert_decimal_str_eq!(-11, 3, "-0.011");
144-
assert_decimal_str_eq!(-11, 2, "-0.11");
145-
assert_decimal_str_eq!(-11, 1, "-1.1");
146-
assert_decimal_str_eq!(-11, 0, "-11");
147-
assert_decimal_str_eq!(-11, -1, "-110");
148-
assert_decimal_str_eq!(-12345678901234567890123456789012345678_i128, 0, "-12345678901234567890123456789012345678");
149-
assert_decimal_str_eq!(-12345678901234567890123456789012345678_i128, 38, "-0.123456789012");
150-
151-
// Round to 12 decimal places
152-
// 1.0000000000011 -> 1.000000000001
153-
assert_decimal_str_eq!(10_i128.pow(13) + 11, 13, "1.000000000001");
192+
fn test_decimal_256_to_str() {
193+
assert_eq!(decimal_256_to_str(i256::from_str("1").unwrap(), 0), "1");
194+
assert_eq!(
195+
decimal_256_to_str(i256::from_str("1").unwrap(), 5),
196+
"0.00001"
197+
);
198+
assert_eq!(
199+
decimal_256_to_str(i256::from_str("1").unwrap(), 20),
200+
"0.00000000000000000001"
201+
);
202+
assert_eq!(
203+
decimal_256_to_str(i256::from_str("1").unwrap(), 38),
204+
"0.00000000000000000000000000000000000001"
205+
);
206+
assert_eq!(
207+
decimal_256_to_str(
208+
i256::from_str("12345678901234567890123456789012345678").unwrap(),
209+
20
210+
),
211+
"123456789012345678.90123456789012345678"
212+
);
213+
assert_eq!(
214+
decimal_256_to_str(
215+
i256::from_str("12345678901234567890123456789012345678").unwrap(),
216+
0
217+
),
218+
"12345678901234567890123456789012345678"
219+
);
154220
}
155221
}

0 commit comments

Comments
 (0)