Use crc32 hashes for binary delta version 4

Autoupdate/SPUDeltaArchive.m

@@ -10,7 +10,7 @@
 #import "SPUDeltaArchiveProtocol.h"
 #import "SPUSparkleDeltaArchive.h"
 #import "SPUXarDeltaArchive.h"
-#import <CommonCrypto/CommonDigest.h>
+#import "SUBinaryDeltaCommon.h"
 
 
 #include "AppKitPrevention.h"
@@ -80,8 +80,8 @@ - (instancetype)initWithRelativeFilePath:(NSString *)relativeFilePath commands:(
 
 @implementation SPUDeltaArchiveHeader
 {
-    unsigned char _beforeTreeHash[CC_SHA1_DIGEST_LENGTH];
-    unsigned char _afterTreeHash[CC_SHA1_DIGEST_LENGTH];
+    unsigned char _beforeTreeHash[BINARY_DELTA_HASH_LENGTH];
+    unsigned char _afterTreeHash[BINARY_DELTA_HASH_LENGTH];
 }
 
 @synthesize compression = _compression;

Autoupdate/SPUSparkleDeltaArchive.m

@@ -338,18 +338,18 @@ - (nullable SPUDeltaArchiveHeader *)readHeader
         return nil;
     }
 
-    unsigned char beforeTreeHash[CC_SHA1_DIGEST_LENGTH] = {0};
+    unsigned char beforeTreeHash[BINARY_DELTA_HASH_LENGTH] = {0};
     if (![self _readBuffer:beforeTreeHash length:sizeof(beforeTreeHash)]) {
         return nil;
     }
 
-    unsigned char afterTreeHash[CC_SHA1_DIGEST_LENGTH] = {0};
+    unsigned char afterTreeHash[BINARY_DELTA_HASH_LENGTH] = {0};
     if (![self _readBuffer:afterTreeHash length:sizeof(afterTreeHash)]) {
         return nil;
     }
 
     NSDate *bundleCreationDate;
-    if (MAJOR_VERSION_IS_AT_LEAST(majorVersion, SUBinaryDeltaMajorVersion4)) {
+    if (majorVersion >= SUBinaryDeltaMajorVersion4) {
         double bundleCreationTimeInterval = 0;
         if (![self _readBuffer:&bundleCreationTimeInterval length:sizeof(bundleCreationTimeInterval)]) {
             return nil;
@@ -865,10 +865,10 @@ - (void)writeHeader:(SPUDeltaArchiveHeader *)header
     uint16_t minorVersion = header.minorVersion;
     [self _writeBuffer:&minorVersion length:sizeof(minorVersion)];
 
-    [self _writeBuffer:header.beforeTreeHash length:CC_SHA1_DIGEST_LENGTH];
-    [self _writeBuffer:header.afterTreeHash length:CC_SHA1_DIGEST_LENGTH];
+    [self _writeBuffer:header.beforeTreeHash length:BINARY_DELTA_HASH_LENGTH];
+    [self _writeBuffer:header.afterTreeHash length:BINARY_DELTA_HASH_LENGTH];
 
-    if (MAJOR_VERSION_IS_AT_LEAST(majorVersion, SUBinaryDeltaMajorVersion4)) {
+    if (majorVersion >= SUBinaryDeltaMajorVersion4) {
         NSDate *bundleCreationDate = header.bundleCreationDate;
 
         // If bundleCreationDate == nil, we will write out a 0 time interval

Autoupdate/SPUXarDeltaArchive.m

@@ -159,10 +159,10 @@ - (nullable SPUDeltaArchiveHeader *)readHeader
         }
     }
 
-    unsigned char rawExpectedBeforeHash[CC_SHA1_DIGEST_LENGTH] = {0};
+    unsigned char rawExpectedBeforeHash[BINARY_DELTA_HASH_LENGTH] = {0};
     getRawHashFromDisplayHash(rawExpectedBeforeHash, expectedBeforeHash);
 
-    unsigned char rawExpectedAfterHash[CC_SHA1_DIGEST_LENGTH] = {0};
+    unsigned char rawExpectedAfterHash[BINARY_DELTA_HASH_LENGTH] = {0};
     getRawHashFromDisplayHash(rawExpectedAfterHash, expectedAfterHash);
 
     // I wasn't able to figure out how to retrieve the compression options from xar,

Autoupdate/SUBinaryDeltaApply.m

@@ -92,7 +92,7 @@ BOOL applyBinaryDelta(NSString *source, NSString *finalDestination, NSString *pa
 
     progressCallback(1/7.0);
 
-    unsigned char beforeHash[CC_SHA1_DIGEST_LENGTH] = {0};
+    unsigned char beforeHash[BINARY_DELTA_HASH_LENGTH] = {0};
     if (!getRawHashOfTreeWithVersion(beforeHash, source, majorDiffVersion)) {
         if (verbose) {
             fprintf(stderr, "\n");
@@ -102,8 +102,8 @@ BOOL applyBinaryDelta(NSString *source, NSString *finalDestination, NSString *pa
         }
         return NO;
     }
-
-    if (memcmp(beforeHash, expectedBeforeHash, CC_SHA1_DIGEST_LENGTH) != 0) {
+    
+    if (memcmp(beforeHash, expectedBeforeHash, BINARY_DELTA_HASH_LENGTH) != 0) {
         if (verbose) {
             fprintf(stderr, "\n");
         }
@@ -460,7 +460,7 @@ BOOL applyBinaryDelta(NSString *source, NSString *finalDestination, NSString *pa
         fprintf(stderr, "\nVerifying destination...");
     }
 
-    unsigned char afterHash[CC_SHA1_DIGEST_LENGTH] = {0};
+    unsigned char afterHash[BINARY_DELTA_HASH_LENGTH] = {0};
     if (!getRawHashOfTreeWithVersion(afterHash, finalDestination, majorDiffVersion)) {
         if (verbose) {
             fprintf(stderr, "\n");
@@ -471,8 +471,8 @@ BOOL applyBinaryDelta(NSString *source, NSString *finalDestination, NSString *pa
         removeTree(finalDestination);
         return NO;
     }
-
-    if (memcmp(afterHash, expectedAfterHash, CC_SHA1_DIGEST_LENGTH) != 0) {
+    
+    if (memcmp(afterHash, expectedAfterHash, BINARY_DELTA_HASH_LENGTH) != 0) {
         if (verbose) {
             fprintf(stderr, "\n");
         }

Autoupdate/SUBinaryDeltaCommon.h

@@ -32,8 +32,6 @@
 #define VERBOSE_MODIFIED "Modified" // file's metadata is modified
 #define VERBOSE_CLONED "Cloned" // file is cloned in content from a differently named file
 
-#define MAJOR_VERSION_IS_AT_LEAST(actualMajor, expectedMajor) (actualMajor >= expectedMajor)
-
 // Relative path of custom icon data that may be set on a bundle via a resource fork
 #define CUSTOM_ICON_PATH @"/Icon\r"
 
@@ -58,12 +56,16 @@ extern SUBinaryDeltaMajorVersion SUBinaryDeltaMajorVersionFirstSupported;
 
 //#define COMPRESSION_LEVEL_ARGUMENT_DESCRIPTION @"The compression level to use for generating delta updates. This only applies if the compression method used is bzip2 which accepts values from 1 - 9. A special value of 0 will use the default compression level."
 
+// This is the same as CC_SHA1_DIGEST_LENGTH
+// Major versions >= 4 use a crc32 hash (using a subset of these bytes) while older versions use a sha1 hash
+#define BINARY_DELTA_HASH_LENGTH 20
+
 SPUDeltaCompressionMode deltaCompressionModeFromDescription(NSString *description, BOOL *requestValid);
 NSString *deltaCompressionStringFromMode(SPUDeltaCompressionMode mode);
 
 extern int compareFiles(const FTSENT **a, const FTSENT **b);
-BOOL getRawHashOfTreeWithVersion(unsigned char *hashBuffer, NSString *path, uint16_t majorVersion);
-BOOL getRawHashOfTreeAndFileTablesWithVersion(unsigned char *hashBuffer, NSString *path, uint16_t majorVersion, NSMutableDictionary<NSData *, NSMutableArray<NSString *> *> *hashToFileKeyDictionary, NSMutableDictionary<NSString *, NSData *> *fileKeyToHashDictionary);
+BOOL getRawHashOfTreeWithVersion(void *hashBuffer, NSString *path, uint16_t majorVersion);
+BOOL getRawHashOfTreeAndFileTablesWithVersion(void *hashBuffer, NSString *path, uint16_t majorVersion, NSMutableDictionary<NSData *, NSMutableArray<NSString *> *> *hashToFileKeyDictionary, NSMutableDictionary<NSString *, NSData *> *fileKeyToHashDictionary);
 NSString *displayHashFromRawHash(const unsigned char *hash);
 void getRawHashFromDisplayHash(unsigned char *hash, NSString *hexHash);
 extern NSString *hashOfTreeWithVersion(NSString *path, uint16_t majorVersion);

Autoupdate/SUBinaryDeltaCommon.m

@@ -8,6 +8,7 @@
 
 #include "SUBinaryDeltaCommon.h"
 #include <CommonCrypto/CommonDigest.h>
+#include <zlib.h> // for crc32()
 #include <Foundation/Foundation.h>
 #include <fcntl.h>
 #include <stdio.h>
@@ -158,7 +159,7 @@ uint16_t latestMinorVersionForMajorVersion(SUBinaryDeltaMajorVersion majorVersio
     return stringWithFileSystemRepresentation(templateResult);
 }
 
-static void _hashOfBuffer(unsigned char *hash, const char *buffer, ssize_t bufferLength)
+static void _sha1HashOfBuffer(unsigned char *hash, const char *buffer, ssize_t bufferLength)
 {
     assert(bufferLength >= 0 && bufferLength <= UINT32_MAX);
     CC_SHA1_CTX hashContext;
@@ -167,7 +168,59 @@ static void _hashOfBuffer(unsigned char *hash, const char *buffer, ssize_t buffe
     CC_SHA1_Final(hash, &hashContext);
 }
 
-static BOOL _hashOfFileContents(unsigned char *hash, FTSENT *ent, void *tempBuffer, size_t tempBufferSize)
+static BOOL _crc32HashOfFileContents(uLong *outChecksum, FTSENT *ent, void *tempBuffer, size_t tempBufferSize)
+{
+    uLong checksum = *outChecksum;
+
+    if (ent->fts_info == FTS_SL) {
+        char linkDestination[MAXPATHLEN + 1];
+        ssize_t linkDestinationLength = readlink(ent->fts_path, linkDestination, MAXPATHLEN);
+        if (linkDestinationLength < 0) {
+            perror("readlink");
+            return NO;
+        }
+
+        checksum = crc32(checksum, (const void *)linkDestination, (unsigned int)linkDestinationLength);
+    } else if (ent->fts_info == FTS_F) {
+        ssize_t fileSize = ent->fts_statp->st_size;
+
+        uint64_t encodedFileSize = (uint64_t)fileSize;
+        checksum = crc32(checksum, (const void *)&encodedFileSize, sizeof(encodedFileSize));
+
+        if (fileSize > 0) {
+            FILE *file = fopen(ent->fts_path, "rb");
+            if (file == NULL) {
+                perror("fopen");
+                return NO;
+            }
+
+            size_t bytesLeft = (size_t)fileSize;
+            while (bytesLeft > 0) {
+                size_t bytesToConsume = (bytesLeft >= tempBufferSize) ? tempBufferSize : bytesLeft;
+
+                if (fread(tempBuffer, bytesToConsume, 1, file) < 1) {
+                    perror("fread");
+                    fclose(file);
+                    return NO;
+                }
+
+                checksum = crc32(checksum, tempBuffer, (uInt)bytesToConsume);
+
+                bytesLeft -= bytesToConsume;
+            }
+
+            fclose(file);
+        }
+    } else {
+        return NO;
+    }
+
+    *outChecksum = checksum;
+
+    return YES;
+}
+
+static BOOL _sha1HashOfFileContents(unsigned char *hash, FTSENT *ent, void *tempBuffer, size_t tempBufferSize)
 {
     if (ent->fts_info == FTS_SL) {
         char linkDestination[MAXPATHLEN + 1];
@@ -177,11 +230,11 @@ static BOOL _hashOfFileContents(unsigned char *hash, FTSENT *ent, void *tempBuff
             return NO;
         }
 
-        _hashOfBuffer(hash, linkDestination, linkDestinationLength);
+        _sha1HashOfBuffer(hash, linkDestination, linkDestinationLength);
     } else if (ent->fts_info == FTS_F) {
         ssize_t fileSize = ent->fts_statp->st_size;
         if (fileSize <= 0) {
-            _hashOfBuffer(hash, NULL, 0);
+            _sha1HashOfBuffer(hash, NULL, 0);
         } else {
             FILE *file = fopen(ent->fts_path, "rb");
             if (file == NULL) {
@@ -218,12 +271,12 @@ static BOOL _hashOfFileContents(unsigned char *hash, FTSENT *ent, void *tempBuff
     return YES;
 }
 
-BOOL getRawHashOfTreeWithVersion(unsigned char *hashBuffer, NSString *path, uint16_t majorVersion)
+BOOL getRawHashOfTreeWithVersion(void *hashBuffer, NSString *path, uint16_t majorVersion)
 {
     return getRawHashOfTreeAndFileTablesWithVersion(hashBuffer, path, majorVersion, nil, nil);
 }
 
-BOOL getRawHashOfTreeAndFileTablesWithVersion(unsigned char *hashBuffer, NSString *path, uint16_t __unused majorVersion, NSMutableDictionary<NSData *, NSMutableArray<NSString *> *> *hashToFileKeyDictionary, NSMutableDictionary<NSString *, NSData *> *fileKeyToHashDictionary)
+BOOL getRawHashOfTreeAndFileTablesWithVersion(void *hashBuffer, NSString *path, uint16_t majorVersion, NSMutableDictionary<NSData *, NSMutableArray<NSString *> *> *hashToFileKeyDictionary, NSMutableDictionary<NSString *, NSData *> *fileKeyToHashDictionary)
 {
     char pathBuffer[PATH_MAX] = { 0 };
     if (![path getFileSystemRepresentation:pathBuffer maxLength:sizeof(pathBuffer)]) {
@@ -246,7 +299,12 @@ BOOL getRawHashOfTreeAndFileTablesWithVersion(unsigned char *hashBuffer, NSStrin
     }
 
     CC_SHA1_CTX hashContext;
-    CC_SHA1_Init(&hashContext);
+    const uLong initialCrc32Value = crc32(0L, Z_NULL, 0);
+    uLong crc32ChecksumValue = initialCrc32Value;
+
+    if (majorVersion < SUBinaryDeltaMajorVersion4) {
+        CC_SHA1_Init(&hashContext);
+    }
 
     // Ensure the path uses filesystem-specific Unicode normalization #1017
     NSString *normalizedPath = stringWithFileSystemRepresentation(pathBuffer);
@@ -263,32 +321,67 @@ BOOL getRawHashOfTreeAndFileTablesWithVersion(unsigned char *hashBuffer, NSStrin
             continue;
         }
 
-        unsigned char fileHash[CC_SHA1_DIGEST_LENGTH];
-        if (!_hashOfFileContents(fileHash, ent, tempBuffer, tempBufferSize)) {
-            fts_close(fts);
-            free(tempBuffer);
-            return NO;
+        NSData *fileHashKey;
+        if (majorVersion >= SUBinaryDeltaMajorVersion4) {
+            if (ent->fts_info == FTS_D) {
+                // No need to hash any further values for directories
+                // We hash relative file path and file type later
+                fileHashKey = nil;
+            } else {
+                uLong fileContentsChecksum = initialCrc32Value;
+                if (!_crc32HashOfFileContents(&fileContentsChecksum, ent, tempBuffer, tempBufferSize)) {
+                    fts_close(fts);
+                    free(tempBuffer);
+                    return NO;
+                }
+
+                uint64_t encodedFileContentsChecksum = fileContentsChecksum;
+
+                crc32ChecksumValue = crc32(crc32ChecksumValue, (const void *)&encodedFileContentsChecksum, sizeof(encodedFileContentsChecksum));
+
+                if (ent->fts_info == FTS_F) {
+                    fileHashKey = [NSData dataWithBytes:&encodedFileContentsChecksum length:sizeof(encodedFileContentsChecksum)];
+                } else {
+                    fileHashKey = nil;
+                }
+            }
+        } else {
+            unsigned char fileHash[CC_SHA1_DIGEST_LENGTH];
+            if (!_sha1HashOfFileContents(fileHash, ent, tempBuffer, tempBufferSize)) {
+                fts_close(fts);
+                free(tempBuffer);
+                return NO;
+            }
+            CC_SHA1_Update(&hashContext, fileHash, sizeof(fileHash));
+
+            if (ent->fts_info == FTS_F) {
+                fileHashKey = [NSData dataWithBytes:fileHash length:sizeof(fileHash)];
+            } else {
+                fileHashKey = nil;
+            }
         }
-        CC_SHA1_Update(&hashContext, fileHash, sizeof(fileHash));
 
         // For file hash tables we only track regular files
-        if (ent->fts_info == FTS_F) {
-            NSData *hashKey = [NSData dataWithBytes:fileHash length:sizeof(fileHash)];
-
+        if (fileHashKey != nil) {
             if (hashToFileKeyDictionary != nil) {
-                if (hashToFileKeyDictionary[hashKey] == nil) {
-                    hashToFileKeyDictionary[hashKey] = [NSMutableArray array];
+                if (hashToFileKeyDictionary[fileHashKey] == nil) {
+                    hashToFileKeyDictionary[fileHashKey] = [NSMutableArray array];
                 }
-                [hashToFileKeyDictionary[hashKey] addObject:relativePath];
+                [hashToFileKeyDictionary[fileHashKey] addObject:relativePath];
             }
 
             if (fileKeyToHashDictionary != nil) {
-                fileKeyToHashDictionary[relativePath] = hashKey;
+                fileKeyToHashDictionary[relativePath] = fileHashKey;
             }
         }
 
         const char *relativePathBytes = [relativePath fileSystemRepresentation];
-        CC_SHA1_Update(&hashContext, relativePathBytes, (CC_LONG)strlen(relativePathBytes));
+
+        if (majorVersion >= SUBinaryDeltaMajorVersion4) {
+            crc32ChecksumValue = crc32(crc32ChecksumValue, (const void *)relativePathBytes, (uInt)strlen(relativePathBytes));
+        } else {
+            CC_SHA1_Update(&hashContext, relativePathBytes, (CC_LONG)strlen(relativePathBytes));
+        }
 
         uint16_t mode = ent->fts_statp->st_mode;
         uint16_t type = ent->fts_info;
@@ -299,15 +392,26 @@ BOOL getRawHashOfTreeAndFileTablesWithVersion(unsigned char *hashBuffer, NSStrin
         // hardcoding a value helps avoid differences between filesystems.
         uint16_t hashedPermissions = (ent->fts_info == FTS_SL) ? VALID_SYMBOLIC_LINK_PERMISSIONS : permissions;
 
-        CC_SHA1_Update(&hashContext, &type, sizeof(type));
-        CC_SHA1_Update(&hashContext, &hashedPermissions, sizeof(hashedPermissions));
+        if (majorVersion >= SUBinaryDeltaMajorVersion4) {
+            crc32ChecksumValue = crc32(crc32ChecksumValue, (const void *)&type, sizeof(type));
+            crc32ChecksumValue = crc32(crc32ChecksumValue, (const void *)&hashedPermissions, sizeof(hashedPermissions));
+        } else {
+            CC_SHA1_Update(&hashContext, &type, sizeof(type));
+            CC_SHA1_Update(&hashContext, &hashedPermissions, sizeof(hashedPermissions));
+        }
     }
 
     free(tempBuffer);
 
     fts_close(fts);
 
-    CC_SHA1_Final(hashBuffer, &hashContext);
+    if (majorVersion >= SUBinaryDeltaMajorVersion4) {
+        uint64_t encodedCrc32ChecksumValue = crc32ChecksumValue;
+        memset(hashBuffer, 0, BINARY_DELTA_HASH_LENGTH);
+        memcpy(hashBuffer, &encodedCrc32ChecksumValue, sizeof(encodedCrc32ChecksumValue));
+    } else {
+        CC_SHA1_Final(hashBuffer, &hashContext);
+    }
 
     return YES;
 }
@@ -319,7 +423,7 @@ void getRawHashFromDisplayHash(unsigned char *hash, NSString *hexHash)
         return;
     }
 
-    for (size_t blockIndex = 0; blockIndex < CC_SHA1_DIGEST_LENGTH; blockIndex++) {
+    for (size_t blockIndex = 0; blockIndex < BINARY_DELTA_HASH_LENGTH; blockIndex++) {
         const char *currentBlock = hexString + blockIndex * 2;
         char convertedBlock[3] = {currentBlock[0], currentBlock[1], '\0'};
         hash[blockIndex] = (unsigned char)strtol(convertedBlock, NULL, 16);
@@ -328,16 +432,16 @@ void getRawHashFromDisplayHash(unsigned char *hash, NSString *hexHash)
 
 NSString *displayHashFromRawHash(const unsigned char *hash)
 {
-    char hexHash[CC_SHA1_DIGEST_LENGTH * 2 + 1] = {0};
-    for (size_t i = 0; i < CC_SHA1_DIGEST_LENGTH; i++) {
+    char hexHash[BINARY_DELTA_HASH_LENGTH * 2 + 1] = {0};
+    for (size_t i = 0; i < BINARY_DELTA_HASH_LENGTH; i++) {
         snprintf(hexHash + i * 2, 3, "%02x", hash[i]);
     }
     return @(hexHash);
 }
 
 NSString *hashOfTreeWithVersion(NSString *path, uint16_t majorVersion)
 {
-    unsigned char hash[CC_SHA1_DIGEST_LENGTH];
+    unsigned char hash[BINARY_DELTA_HASH_LENGTH] = {0};
     if (!getRawHashOfTreeWithVersion(hash, path, majorVersion)) {
         return nil;
     }