Sat Mar 10 09:44:47 2018 UTC ()
Fix signed overflow in atan2

As a component of atan2(y, x), the case of x == 1.0 is farmed out to
atan(y). The current implementation of this comparison is vulnerable
to signed integer underflow (that is, undefined behavior), and it's
performed in a somewhat more complicated way than it need be. Change
it to not be quite so cute, rather directly comparing the high/low
bits of x to the specific IEEE-754 bit pattern that encodes 1.0.

ok martin@ pgoyette@ maya@
obtained from FreeBSD


(eadler)
diff -r1.12 -r1.13 src/lib/libm/src/e_atan2.c

cvs diff -r1.12 -r1.13 src/lib/libm/src/e_atan2.c (expand / switch to context diff)
--- src/lib/libm/src/e_atan2.c 2002/05/26 22:01:48 1.12
+++ src/lib/libm/src/e_atan2.c 2018/03/10 09:44:47 1.13
@@ -12,7 +12,7 @@
 
 #include <sys/cdefs.h>
 #if defined(LIBM_SCCS) && !defined(lint)
-__RCSID("$NetBSD: e_atan2.c,v 1.12 2002/05/26 22:01:48 wiz Exp $");
+__RCSID("$NetBSD: e_atan2.c,v 1.13 2018/03/10 09:44:47 eadler Exp $");
 #endif
 
 /* __ieee754_atan2(y,x)
@@ -67,7 +67,7 @@
 	if(((ix|((lx|-lx)>>31))>0x7ff00000)||
 	   ((iy|((ly|-ly)>>31))>0x7ff00000))	/* x or y is NaN */
 	   return x+y;
-	if(((hx-0x3ff00000)|lx)==0) return atan(y);   /* x=1.0 */
+	if(hx==0x3ff00000&&lx==0) return atan(y);   /* x=1.0 */
 	m = ((hy>>31)&1)|((hx>>30)&2);	/* 2*sign(x)+sign(y) */
 
     /* when y = 0 */