jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix toRational overflow by calculating internally with int64s (fixes #4194)

Browse source
This commit is contained in:
def 2016-05-22 18:09:55 +02:00
commit 834ad57d94
1 changed files with 11 additions and 11 deletions
Download patch file Download diff file Expand all files Collapse all files

22
lib/pure/rationals.nim
View file

@ -41,26 +41,26 @@ proc toRational*[T:SomeInteger](x: T): Rational[T] =
proc toRationalSub(x: float, n: int): Rational[int] = proc toRationalSub(x: float, n: int): Rational[int] =
var var
a = 0 a = 0'i64
b, c, d = 1 b, c, d = 1'i64
result = 0 // 1 # rational 0 result = 0 // 1 # rational 0
while b <= n and d <= n: while b <= n and d <= n:
let ac = (a+c) let ac = (a+c)
let bd = (b+d) let bd = (b+d)
# scale by 1000 so not overflow for high precision # scale by 1000 so not overflow for high precision
let mediant = (ac/1000) / (bd/1000) let mediant = (ac.float/1000) / (bd.float/1000)
if x == mediant: if x == mediant:
if bd <= n: if bd <= n:
result.num = ac result.num = ac.int
result.den = bd result.den = bd.int
return result return result
elif d > b: elif d > b:
result.num = c result.num = c.int
result.den = d result.den = d.int
return result return result
else: else:
result.num = a result.num = a.int
result.den = b result.den = b.int
return result return result
elif x > mediant: elif x > mediant:
a = ac a = ac
@ -69,8 +69,8 @@ proc toRationalSub(x: float, n: int): Rational[int] =
c = ac c = ac
d = bd d = bd
if (b > n): if (b > n):
return initRational(c, d) return initRational(c.int, d.int)
return initRational(a, b) return initRational(a.int, b.int)
proc toRational*(x: float, n: int = high(int)): Rational[int] = proc toRational*(x: float, n: int = high(int)): Rational[int] =
## Calculate the best rational numerator and denominator ## Calculate the best rational numerator and denominator