sha1sum.c, sha256.c: make code work with -fstrict-aliasing

-fstrict-aliasing is enabled by default when using optimization levels
higher than 1, including -Os. With that option, compiler may assume
that object of one type never resides at the same address as object of
a different type. Both sha1_final() and sha256_final() used to write
message length by casting a pointer to buffer into a pointer to u64,
while surrounding code operated on the buffer directly.

The problem manifests in GCC 11 (and possibly later versions).

To solve this problem, the commit adds message length field to SHA
context structures and accesses it through the structure, without any
casting to different types. Message length is declared volatile,
otherwise compiler doesn't write it before address of buffer is
obtained and passed to sha*_transform().

Signed-off-by: Krystian Hebel <[email protected]>

sha1sum.c

@@ -34,25 +34,29 @@ static inline u32 rol( u32 x, int n)
     return (x << n) | (x >> (-n & 31));
 }
 
+#define SHA1_BLOCK_SIZE     64
+
 typedef struct {
     u32 count;
+    u32 h[5];
     union {
+        unsigned char buf[SHA1_BLOCK_SIZE];
         struct {
-            u32 h0, h1, h2, h3, h4;
+            unsigned char pad[SHA1_BLOCK_SIZE - sizeof(u64)];
+            /* Must be volatile, otherwise compiler over-optimizes it away. */
+            volatile u64 ml;
         };
-        u32 h[5];
     };
-    unsigned char buf[64];
 } SHA1_CONTEXT;
 
 static void sha1_init( SHA1_CONTEXT *hd )
 {
     *hd = (SHA1_CONTEXT){
-        .h0 = 0x67452301,
-        .h1 = 0xefcdab89,
-        .h2 = 0x98badcfe,
-        .h3 = 0x10325476,
-        .h4 = 0xc3d2e1f0,
+        .h[0] = 0x67452301,
+        .h[1] = 0xefcdab89,
+        .h[2] = 0x98badcfe,
+        .h[3] = 0x10325476,
+        .h[4] = 0xc3d2e1f0,
     };
 }
 
@@ -76,11 +80,11 @@ static void sha1_transform(SHA1_CONTEXT *hd, const void *_data)
     int i;
 
     /* get values from the chaining vars */
-    a = hd->h0;
-    b = hd->h1;
-    c = hd->h2;
-    d = hd->h3;
-    e = hd->h4;
+    a = hd->h[0];
+    b = hd->h[1];
+    c = hd->h[2];
+    d = hd->h[3];
+    e = hd->h[4];
 
     for ( i = 0; i < 16; ++i )
         x[i] = cpu_to_be32(data[i]);
@@ -147,18 +151,19 @@ static void sha1_transform(SHA1_CONTEXT *hd, const void *_data)
     }
 
     /* Update chaining vars */
-    hd->h0 += a;
-    hd->h1 += b;
-    hd->h2 += c;
-    hd->h3 += d;
-    hd->h4 += e;
+    hd->h[0] += a;
+    hd->h[1] += b;
+    hd->h[2] += c;
+    hd->h[3] += d;
+    hd->h[4] += e;
 }
 
 
 static void sha1_once(SHA1_CONTEXT *hd, const void *data, u32 len)
 {
     hd->count = len;
-    for ( ; len >= 64; data += 64, len -= 64 )
+    for ( ; len >= SHA1_BLOCK_SIZE;
+          data += SHA1_BLOCK_SIZE, len -= SHA1_BLOCK_SIZE )
         sha1_transform(hd, data);
 
     memcpy(hd->buf, data, len);
@@ -180,19 +185,18 @@ sha1_final(SHA1_CONTEXT *hd, u8 hash[SHA1_DIGEST_SIZE])
     /* Start padding */
     hd->buf[partial++] = 0x80;
 
-    if ( partial > 56 )
+    if ( partial > sizeof(hd->pad) )
     {
         /* Need one extra block - pad to 64 */
-        memset(hd->buf + partial, 0, 64 - partial);
+        memset(hd->buf + partial, 0, sizeof(hd->buf) - partial);
         sha1_transform(hd, hd->buf);
         partial = 0;
     }
     /* Pad to 56 */
-    memset(hd->buf + partial, 0, 56 - partial);
+    memset(hd->pad + partial, 0, sizeof(hd->pad) - partial);
 
     /* append the 64 bit count */
-    u64 *count = (void *)&hd->buf[56];
-    *count = cpu_to_be64((u64)hd->count << 3);
+    hd->ml = cpu_to_be64((u64)hd->count << 3);
     sha1_transform(hd, hd->buf);
 
     u32 *p = (void *)hash;

sha256.c

@@ -25,7 +25,14 @@
 struct sha256_state {
     u32 state[SHA256_DIGEST_SIZE / 4];
     u32 count;
-    u8 buf[SHA256_BLOCK_SIZE];
+    union {
+        u8 buf[SHA256_BLOCK_SIZE];
+        struct {
+            u8 pad[SHA256_BLOCK_SIZE - sizeof(u64)];
+            /* Must be volatile, otherwise compiler over-optimizes it away. */
+            volatile u64 ml;
+        };
+    };
 };
 
 static inline u32 ror32(u32 word, unsigned int shift)
@@ -155,7 +162,8 @@ static void sha256_init(struct sha256_state *sctx)
 static void sha256_once(struct sha256_state *sctx, const void *data, u32 len)
 {
     sctx->count = len;
-    for ( ; len >= 64; data += 64, len -= 64 )
+    for ( ; len >= SHA256_BLOCK_SIZE;
+          data += SHA256_BLOCK_SIZE, len -= SHA256_BLOCK_SIZE )
         sha256_transform(sctx->state, data);
 
     memcpy(sctx->buf, data, len);
@@ -164,25 +172,23 @@ static void sha256_once(struct sha256_state *sctx, const void *data, u32 len)
 static void sha256_final(struct sha256_state *sctx, void *_dst)
 {
     u32 *dst = _dst;
-    u64 *count;
     unsigned int i, partial = sctx->count & 0x3f;
 
     /* Start padding */
     sctx->buf[partial++] = 0x80;
 
-    if ( partial > 56 )
+    if ( partial > sizeof(sctx->pad) )
     {
         /* Need one extra block - pad to 64 */
-        memset(sctx->buf + partial, 0, 64 - partial);
+        memset(sctx->buf + partial, 0, sizeof(sctx->buf) - partial);
         sha256_transform(sctx->state, sctx->buf);
         partial = 0;
     }
     /* Pad to 56 */
-    memset(sctx->buf + partial, 0, 56 - partial);
+    memset(sctx->pad + partial, 0, sizeof(sctx->pad) - partial);
 
     /* Append the 64 bit count */
-    count = (void *)&sctx->buf[56];
-    *count = cpu_to_be64((u64)sctx->count << 3);
+    sctx->ml = cpu_to_be64((u64)sctx->count << 3);
     sha256_transform(sctx->state, sctx->buf);
 
     /* Store state in digest */