fix: fix floating point precision bug (#206)

antvis · Nov 23, 2023 · 9d3b30d · 9d3b30d
1 parent 85521ec
commit 9d3b30d
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 2 deletions.
diff --git a/__tests__/unit/tick-methods/r-pretty.spec.ts b/__tests__/unit/tick-methods/r-pretty.spec.ts
@@ -67,7 +67,9 @@ describe('rPretty ticks', () => {
   });
 
   it('rPretty for tiny number', () => {
-    expect(rPretty(9.899999999999999, 9.9)).toStrictEqual([9.899999999999999, 9.899999999999999, 9.9, 9.9]);
+    expect(rPretty(9.89999999999999, 9.9)).toStrictEqual([
+      9.89999999999999, 9.89999999999999, 9.89999999999999, 9.89999999999999, 9.9, 9.9, 9.9,
+    ]);
   });
 
   it('handle decimal tickCount', () => {

diff --git a/__tests__/unit/utils/pretty-number.spec.ts b/__tests__/unit/utils/pretty-number.spec.ts
@@ -7,6 +7,7 @@ describe('prettyNumber', () => {
     expect(prettyNumber(0.1 + 0.2)).toBe(0.3);
     expect(prettyNumber(-1e-16)).toBe(-1e-16);
     expect(prettyNumber(-0.09999999999999998)).toBe(-0.1);
+    expect(prettyNumber(-4.3999999999999995)).toBe(-4.4);
     expect(prettyNumber(-(0.1 + 0.2))).toBe(-0.3);
   });
 });
diff --git a/src/utils/pretty-number.ts b/src/utils/pretty-number.ts
@@ -1,4 +1,4 @@
 // 为了解决 js 运算的精度问题
 export function prettyNumber(n: number) {
-  return Math.abs(n) < 1e-15 ? n : parseFloat(n.toFixed(15));
+  return Math.abs(n) < 1e-14 ? n : parseFloat(n.toFixed(14));
 }