BN_gcd(): Avoid shifts of negative values

Fixes openssl#22216

Thanks to Leland Mills for investigation and testing.

Reviewed-by: Tom Cosgrove <[email protected]>
Reviewed-by: Matt Caswell <[email protected]>
Reviewed-by: Paul Dale <[email protected]>
(Merged from openssl#22272)

crypto/bn/bn_gcd.c

@@ -642,9 +642,9 @@ int BN_gcd(BIGNUM *r, const BIGNUM *in_a, const BIGNUM *in_b, BN_CTX *ctx)
 
     for (i = 0; i < m; i++) {
         /* conditionally flip signs if delta is positive and g is odd */
-        cond = (-delta >> (8 * sizeof(delta) - 1)) & g->d[0] & 1
+        cond = ((unsigned int)-delta >> (8 * sizeof(delta) - 1)) & g->d[0] & 1
             /* make sure g->top > 0 (i.e. if top == 0 then g == 0 always) */
-            & (~((g->top - 1) >> (sizeof(g->top) * 8 - 1)));
+            & (~((unsigned int)(g->top - 1) >> (sizeof(g->top) * 8 - 1)));
         delta = (-cond & -delta) | ((cond - 1) & delta);
         r->neg ^= cond;
         /* swap */
@@ -656,7 +656,7 @@ int BN_gcd(BIGNUM *r, const BIGNUM *in_a, const BIGNUM *in_b, BN_CTX *ctx)
             goto err;
         BN_consttime_swap(g->d[0] & 1 /* g is odd */
                 /* make sure g->top > 0 (i.e. if top == 0 then g == 0 always) */
-                & (~((g->top - 1) >> (sizeof(g->top) * 8 - 1))),
+                & (~((unsigned int)(g->top - 1) >> (sizeof(g->top) * 8 - 1))),
                 g, temp, top);
         if (!BN_rshift1(g, g))
             goto err;