From 447bb4ca42978e45a0ba98dd151ba8bd9f7ab742 Mon Sep 17 00:00:00 2001 From: Zorg Date: Sun, 29 Sep 2024 18:21:03 -0700 Subject: [PATCH 1/9] Use crc32 hashes for delta patches This is more efficient than SHA1. --- Autoupdate/SPUDeltaArchive.m | 6 +- Autoupdate/SPUSparkleDeltaArchive.m | 8 +- Autoupdate/SPUXarDeltaArchive.m | 4 +- Autoupdate/SUBinaryDeltaApply.m | 16 ++- Autoupdate/SUBinaryDeltaCommon.h | 11 +- Autoupdate/SUBinaryDeltaCommon.m | 160 +++++++++++++++++++++++----- Autoupdate/SUBinaryDeltaCreate.m | 4 +- Sparkle.xcodeproj/project.pbxproj | 4 + Tests/SUBinaryDeltaTest.m | 17 +-- 9 files changed, 176 insertions(+), 54 deletions(-) diff --git a/Autoupdate/SPUDeltaArchive.m b/Autoupdate/SPUDeltaArchive.m index ca7ade75ad..a80927ad8b 100644 --- a/Autoupdate/SPUDeltaArchive.m +++ b/Autoupdate/SPUDeltaArchive.m @@ -10,7 +10,7 @@ #import "SPUDeltaArchiveProtocol.h" #import "SPUSparkleDeltaArchive.h" #import "SPUXarDeltaArchive.h" -#import +#import "SUBinaryDeltaCommon.h" #include "AppKitPrevention.h" @@ -80,8 +80,8 @@ - (instancetype)initWithRelativeFilePath:(NSString *)relativeFilePath commands:( @implementation SPUDeltaArchiveHeader { - unsigned char _beforeTreeHash[CC_SHA1_DIGEST_LENGTH]; - unsigned char _afterTreeHash[CC_SHA1_DIGEST_LENGTH]; + unsigned char _beforeTreeHash[BINARY_DELTA_HASH_LENGTH]; + unsigned char _afterTreeHash[BINARY_DELTA_HASH_LENGTH]; } @synthesize compression = _compression; diff --git a/Autoupdate/SPUSparkleDeltaArchive.m b/Autoupdate/SPUSparkleDeltaArchive.m index cb71c73c26..9d65d6fd15 100644 --- a/Autoupdate/SPUSparkleDeltaArchive.m +++ b/Autoupdate/SPUSparkleDeltaArchive.m @@ -338,12 +338,12 @@ - (nullable SPUDeltaArchiveHeader *)readHeader return nil; } - unsigned char beforeTreeHash[CC_SHA1_DIGEST_LENGTH] = {0}; + unsigned char beforeTreeHash[BINARY_DELTA_HASH_LENGTH] = {0}; if (![self _readBuffer:beforeTreeHash length:sizeof(beforeTreeHash)]) { return nil; } - unsigned char afterTreeHash[CC_SHA1_DIGEST_LENGTH] = {0}; + unsigned char afterTreeHash[BINARY_DELTA_HASH_LENGTH] = {0}; if (![self _readBuffer:afterTreeHash length:sizeof(afterTreeHash)]) { return nil; } @@ -853,8 +853,8 @@ - (void)writeHeader:(SPUDeltaArchiveHeader *)header uint16_t minorVersion = header.minorVersion; [self _writeBuffer:&minorVersion length:sizeof(minorVersion)]; - [self _writeBuffer:header.beforeTreeHash length:CC_SHA1_DIGEST_LENGTH]; - [self _writeBuffer:header.afterTreeHash length:CC_SHA1_DIGEST_LENGTH]; + [self _writeBuffer:header.beforeTreeHash length:BINARY_DELTA_HASH_LENGTH]; + [self _writeBuffer:header.afterTreeHash length:BINARY_DELTA_HASH_LENGTH]; } - (void)addItem:(SPUDeltaArchiveItem *)item diff --git a/Autoupdate/SPUXarDeltaArchive.m b/Autoupdate/SPUXarDeltaArchive.m index ab117230d4..c888045711 100644 --- a/Autoupdate/SPUXarDeltaArchive.m +++ b/Autoupdate/SPUXarDeltaArchive.m @@ -159,10 +159,10 @@ - (nullable SPUDeltaArchiveHeader *)readHeader } } - unsigned char rawExpectedBeforeHash[CC_SHA1_DIGEST_LENGTH] = {0}; + unsigned char rawExpectedBeforeHash[BINARY_DELTA_HASH_LENGTH] = {0}; getRawHashFromDisplayHash(rawExpectedBeforeHash, expectedBeforeHash); - unsigned char rawExpectedAfterHash[CC_SHA1_DIGEST_LENGTH] = {0}; + unsigned char rawExpectedAfterHash[BINARY_DELTA_HASH_LENGTH] = {0}; getRawHashFromDisplayHash(rawExpectedAfterHash, expectedAfterHash); // I wasn't able to figure out how to retrieve the compression options from xar, diff --git a/Autoupdate/SUBinaryDeltaApply.m b/Autoupdate/SUBinaryDeltaApply.m index 9e862f1ff2..3d234c69e4 100644 --- a/Autoupdate/SUBinaryDeltaApply.m +++ b/Autoupdate/SUBinaryDeltaApply.m @@ -92,7 +92,9 @@ BOOL applyBinaryDelta(NSString *source, NSString *finalDestination, NSString *pa progressCallback(1/7.0); - unsigned char beforeHash[CC_SHA1_DIGEST_LENGTH] = {0}; + NSDate *beforeHashDate = [NSDate date]; + + unsigned char beforeHash[BINARY_DELTA_HASH_LENGTH] = {0}; if (!getRawHashOfTreeWithVersion(beforeHash, source, majorDiffVersion)) { if (verbose) { fprintf(stderr, "\n"); @@ -102,8 +104,10 @@ BOOL applyBinaryDelta(NSString *source, NSString *finalDestination, NSString *pa } return NO; } + + NSLog(@"Before hash took %f seconds", [[NSDate date] timeIntervalSinceDate:beforeHashDate]); - if (memcmp(beforeHash, expectedBeforeHash, CC_SHA1_DIGEST_LENGTH) != 0) { + if (memcmp(beforeHash, expectedBeforeHash, BINARY_DELTA_HASH_LENGTH) != 0) { if (verbose) { fprintf(stderr, "\n"); } @@ -450,7 +454,9 @@ BOOL applyBinaryDelta(NSString *source, NSString *finalDestination, NSString *pa fprintf(stderr, "\nVerifying destination..."); } - unsigned char afterHash[CC_SHA1_DIGEST_LENGTH] = {0}; + NSDate *afterHashDate = [NSDate date]; + + unsigned char afterHash[BINARY_DELTA_HASH_LENGTH] = {0}; if (!getRawHashOfTreeWithVersion(afterHash, finalDestination, majorDiffVersion)) { if (verbose) { fprintf(stderr, "\n"); @@ -461,8 +467,10 @@ BOOL applyBinaryDelta(NSString *source, NSString *finalDestination, NSString *pa removeTree(finalDestination); return NO; } + + NSLog(@"After hash took %f seconds", [[NSDate date] timeIntervalSinceDate:afterHashDate]); - if (memcmp(afterHash, expectedAfterHash, CC_SHA1_DIGEST_LENGTH) != 0) { + if (memcmp(afterHash, expectedAfterHash, BINARY_DELTA_HASH_LENGTH) != 0) { if (verbose) { fprintf(stderr, "\n"); } diff --git a/Autoupdate/SUBinaryDeltaCommon.h b/Autoupdate/SUBinaryDeltaCommon.h index e70aa1ce40..4abc9a5e01 100644 --- a/Autoupdate/SUBinaryDeltaCommon.h +++ b/Autoupdate/SUBinaryDeltaCommon.h @@ -44,7 +44,8 @@ typedef NS_ENUM(uint16_t, SUBinaryDeltaMajorVersion) // Note: support for creating or applying version 1 deltas have been removed SUBinaryDeltaMajorVersion1 = 1, SUBinaryDeltaMajorVersion2 = 2, - SUBinaryDeltaMajorVersion3 = 3 + SUBinaryDeltaMajorVersion3 = 3, + SUBinaryDeltaMajorVersion4 = 4 }; extern SUBinaryDeltaMajorVersion SUBinaryDeltaMajorVersionDefault; @@ -57,12 +58,16 @@ extern SUBinaryDeltaMajorVersion SUBinaryDeltaMajorVersionFirstSupported; //#define COMPRESSION_LEVEL_ARGUMENT_DESCRIPTION @"The compression level to use for generating delta updates. This only applies if the compression method used is bzip2 which accepts values from 1 - 9. A special value of 0 will use the default compression level." +// This is the same as CC_SHA1_DIGEST_LENGTH +// Major versions >= 4 use a crc32 hash (using a subset of these bytes) while older versions use a sha1 hash +#define BINARY_DELTA_HASH_LENGTH 20 + SPUDeltaCompressionMode deltaCompressionModeFromDescription(NSString *description, BOOL *requestValid); NSString *deltaCompressionStringFromMode(SPUDeltaCompressionMode mode); extern int compareFiles(const FTSENT **a, const FTSENT **b); -BOOL getRawHashOfTreeWithVersion(unsigned char *hashBuffer, NSString *path, uint16_t majorVersion); -BOOL getRawHashOfTreeAndFileTablesWithVersion(unsigned char *hashBuffer, NSString *path, uint16_t majorVersion, NSMutableDictionary *> *hashToFileKeyDictionary, NSMutableDictionary *fileKeyToHashDictionary); +BOOL getRawHashOfTreeWithVersion(void *hashBuffer, NSString *path, uint16_t majorVersion); +BOOL getRawHashOfTreeAndFileTablesWithVersion(void *hashBuffer, NSString *path, uint16_t majorVersion, NSMutableDictionary *> *hashToFileKeyDictionary, NSMutableDictionary *fileKeyToHashDictionary); NSString *displayHashFromRawHash(const unsigned char *hash); void getRawHashFromDisplayHash(unsigned char *hash, NSString *hexHash); extern NSString *hashOfTreeWithVersion(NSString *path, uint16_t majorVersion); diff --git a/Autoupdate/SUBinaryDeltaCommon.m b/Autoupdate/SUBinaryDeltaCommon.m index d1c718c3f1..2d006815c6 100644 --- a/Autoupdate/SUBinaryDeltaCommon.m +++ b/Autoupdate/SUBinaryDeltaCommon.m @@ -8,6 +8,7 @@ #include "SUBinaryDeltaCommon.h" #include +#include // for crc32() #include #include #include @@ -22,7 +23,7 @@ // Note: the framework bundle version must be bumped, and generate_appcast must be updated to compare it, // when we add/change new major versions and defaults. Unit tests need to be updated to use new versions too. SUBinaryDeltaMajorVersion SUBinaryDeltaMajorVersionDefault = SUBinaryDeltaMajorVersion3; -SUBinaryDeltaMajorVersion SUBinaryDeltaMajorVersionLatest = SUBinaryDeltaMajorVersion3; +SUBinaryDeltaMajorVersion SUBinaryDeltaMajorVersionLatest = SUBinaryDeltaMajorVersion4; SUBinaryDeltaMajorVersion SUBinaryDeltaMajorVersionFirst = SUBinaryDeltaMajorVersion1; SUBinaryDeltaMajorVersion SUBinaryDeltaMajorVersionFirstSupported = SUBinaryDeltaMajorVersion2; @@ -115,6 +116,8 @@ uint16_t latestMinorVersionForMajorVersion(SUBinaryDeltaMajorVersion majorVersio return 4; case SUBinaryDeltaMajorVersion3: return 1; + case SUBinaryDeltaMajorVersion4: + return 0; } return 0; } @@ -156,7 +159,7 @@ uint16_t latestMinorVersionForMajorVersion(SUBinaryDeltaMajorVersion majorVersio return stringWithFileSystemRepresentation(templateResult); } -static void _hashOfBuffer(unsigned char *hash, const char *buffer, ssize_t bufferLength) +static void _sha1HashOfBuffer(unsigned char *hash, const char *buffer, ssize_t bufferLength) { assert(bufferLength >= 0 && bufferLength <= UINT32_MAX); CC_SHA1_CTX hashContext; @@ -165,7 +168,59 @@ static void _hashOfBuffer(unsigned char *hash, const char *buffer, ssize_t buffe CC_SHA1_Final(hash, &hashContext); } -static BOOL _hashOfFileContents(unsigned char *hash, FTSENT *ent, void *tempBuffer, size_t tempBufferSize) +static BOOL _crc32HashOfFileContents(uLong *outChecksum, FTSENT *ent, void *tempBuffer, size_t tempBufferSize) +{ + uLong checksum = *outChecksum; + + if (ent->fts_info == FTS_SL) { + char linkDestination[MAXPATHLEN + 1]; + ssize_t linkDestinationLength = readlink(ent->fts_path, linkDestination, MAXPATHLEN); + if (linkDestinationLength < 0) { + perror("readlink"); + return NO; + } + + checksum = crc32(checksum, (const void *)linkDestination, (unsigned int)linkDestinationLength); + } else if (ent->fts_info == FTS_F) { + ssize_t fileSize = ent->fts_statp->st_size; + + uint64_t encodedFileSize = (uint64_t)fileSize; + checksum = crc32(checksum, (const void *)&encodedFileSize, sizeof(encodedFileSize)); + + if (fileSize > 0) { + FILE *file = fopen(ent->fts_path, "rb"); + if (file == NULL) { + perror("fopen"); + return NO; + } + + size_t bytesLeft = (size_t)fileSize; + while (bytesLeft > 0) { + size_t bytesToConsume = (bytesLeft >= tempBufferSize) ? tempBufferSize : bytesLeft; + + if (fread(tempBuffer, bytesToConsume, 1, file) < 1) { + perror("fread"); + fclose(file); + return NO; + } + + checksum = crc32(checksum, tempBuffer, (uInt)bytesToConsume); + + bytesLeft -= bytesToConsume; + } + + fclose(file); + } + } else { + return NO; + } + + *outChecksum = checksum; + + return YES; +} + +static BOOL _sha1HashOfFileContents(unsigned char *hash, FTSENT *ent, void *tempBuffer, size_t tempBufferSize) { if (ent->fts_info == FTS_SL) { char linkDestination[MAXPATHLEN + 1]; @@ -175,11 +230,11 @@ static BOOL _hashOfFileContents(unsigned char *hash, FTSENT *ent, void *tempBuff return NO; } - _hashOfBuffer(hash, linkDestination, linkDestinationLength); + _sha1HashOfBuffer(hash, linkDestination, linkDestinationLength); } else if (ent->fts_info == FTS_F) { ssize_t fileSize = ent->fts_statp->st_size; if (fileSize <= 0) { - _hashOfBuffer(hash, NULL, 0); + _sha1HashOfBuffer(hash, NULL, 0); } else { FILE *file = fopen(ent->fts_path, "rb"); if (file == NULL) { @@ -216,12 +271,12 @@ static BOOL _hashOfFileContents(unsigned char *hash, FTSENT *ent, void *tempBuff return YES; } -BOOL getRawHashOfTreeWithVersion(unsigned char *hashBuffer, NSString *path, uint16_t majorVersion) +BOOL getRawHashOfTreeWithVersion(void *hashBuffer, NSString *path, uint16_t majorVersion) { return getRawHashOfTreeAndFileTablesWithVersion(hashBuffer, path, majorVersion, nil, nil); } -BOOL getRawHashOfTreeAndFileTablesWithVersion(unsigned char *hashBuffer, NSString *path, uint16_t __unused majorVersion, NSMutableDictionary *> *hashToFileKeyDictionary, NSMutableDictionary *fileKeyToHashDictionary) +BOOL getRawHashOfTreeAndFileTablesWithVersion(void *hashBuffer, NSString *path, uint16_t majorVersion, NSMutableDictionary *> *hashToFileKeyDictionary, NSMutableDictionary *fileKeyToHashDictionary) { char pathBuffer[PATH_MAX] = { 0 }; if (![path getFileSystemRepresentation:pathBuffer maxLength:sizeof(pathBuffer)]) { @@ -244,7 +299,11 @@ BOOL getRawHashOfTreeAndFileTablesWithVersion(unsigned char *hashBuffer, NSStrin } CC_SHA1_CTX hashContext; - CC_SHA1_Init(&hashContext); + uLong crc32ChecksumValue = 0; + + if (majorVersion < 4) { + CC_SHA1_Init(&hashContext); + } // Ensure the path uses filesystem-specific Unicode normalization #1017 NSString *normalizedPath = stringWithFileSystemRepresentation(pathBuffer); @@ -261,32 +320,65 @@ BOOL getRawHashOfTreeAndFileTablesWithVersion(unsigned char *hashBuffer, NSStrin continue; } - unsigned char fileHash[CC_SHA1_DIGEST_LENGTH]; - if (!_hashOfFileContents(fileHash, ent, tempBuffer, tempBufferSize)) { - fts_close(fts); - free(tempBuffer); - return NO; + NSData *fileHashKey; + if (majorVersion >= 4) { + if (ent->fts_info == FTS_D) { + fileHashKey = nil; + } else { + uLong fileContentsChecksum = 0; + if (!_crc32HashOfFileContents(&fileContentsChecksum, ent, tempBuffer, tempBufferSize)) { + fts_close(fts); + free(tempBuffer); + return NO; + } + + uint64_t encodedFileContentsChecksum = fileContentsChecksum; + + crc32ChecksumValue = crc32(crc32ChecksumValue, (const void *)&encodedFileContentsChecksum, sizeof(encodedFileContentsChecksum)); + + if (ent->fts_info == FTS_F) { + fileHashKey = [NSData dataWithBytes:&encodedFileContentsChecksum length:sizeof(encodedFileContentsChecksum)]; + } else { + fileHashKey = nil; + } + } + } else { + unsigned char fileHash[CC_SHA1_DIGEST_LENGTH]; + if (!_sha1HashOfFileContents(fileHash, ent, tempBuffer, tempBufferSize)) { + fts_close(fts); + free(tempBuffer); + return NO; + } + CC_SHA1_Update(&hashContext, fileHash, sizeof(fileHash)); + + if (ent->fts_info == FTS_F) { + fileHashKey = [NSData dataWithBytes:fileHash length:sizeof(fileHash)]; + } else { + fileHashKey = nil; + } } - CC_SHA1_Update(&hashContext, fileHash, sizeof(fileHash)); // For file hash tables we only track regular files - if (ent->fts_info == FTS_F) { - NSData *hashKey = [NSData dataWithBytes:fileHash length:sizeof(fileHash)]; - + if (fileHashKey != nil) { if (hashToFileKeyDictionary != nil) { - if (hashToFileKeyDictionary[hashKey] == nil) { - hashToFileKeyDictionary[hashKey] = [NSMutableArray array]; + if (hashToFileKeyDictionary[fileHashKey] == nil) { + hashToFileKeyDictionary[fileHashKey] = [NSMutableArray array]; } - [hashToFileKeyDictionary[hashKey] addObject:relativePath]; + [hashToFileKeyDictionary[fileHashKey] addObject:relativePath]; } if (fileKeyToHashDictionary != nil) { - fileKeyToHashDictionary[relativePath] = hashKey; + fileKeyToHashDictionary[relativePath] = fileHashKey; } } const char *relativePathBytes = [relativePath fileSystemRepresentation]; - CC_SHA1_Update(&hashContext, relativePathBytes, (CC_LONG)strlen(relativePathBytes)); + + if (majorVersion >= 4) { + crc32ChecksumValue = crc32(crc32ChecksumValue, (const void *)relativePathBytes, (uInt)strlen(relativePathBytes)); + } else { + CC_SHA1_Update(&hashContext, relativePathBytes, (CC_LONG)strlen(relativePathBytes)); + } uint16_t mode = ent->fts_statp->st_mode; uint16_t type = ent->fts_info; @@ -297,15 +389,25 @@ BOOL getRawHashOfTreeAndFileTablesWithVersion(unsigned char *hashBuffer, NSStrin // hardcoding a value helps avoid differences between filesystems. uint16_t hashedPermissions = (ent->fts_info == FTS_SL) ? VALID_SYMBOLIC_LINK_PERMISSIONS : permissions; - CC_SHA1_Update(&hashContext, &type, sizeof(type)); - CC_SHA1_Update(&hashContext, &hashedPermissions, sizeof(hashedPermissions)); + if (majorVersion >= 4) { + crc32ChecksumValue = crc32(crc32ChecksumValue, (const void *)&type, sizeof(type)); + crc32ChecksumValue = crc32(crc32ChecksumValue, (const void *)&hashedPermissions, sizeof(hashedPermissions)); + } else { + CC_SHA1_Update(&hashContext, &type, sizeof(type)); + CC_SHA1_Update(&hashContext, &hashedPermissions, sizeof(hashedPermissions)); + } } free(tempBuffer); fts_close(fts); - CC_SHA1_Final(hashBuffer, &hashContext); + if (majorVersion >= 4) { + uint64_t encodedCrc32ChecksumValue = crc32ChecksumValue; + memcpy(hashBuffer, &encodedCrc32ChecksumValue, sizeof(encodedCrc32ChecksumValue)); + } else { + CC_SHA1_Final(hashBuffer, &hashContext); + } return YES; } @@ -317,7 +419,7 @@ void getRawHashFromDisplayHash(unsigned char *hash, NSString *hexHash) return; } - for (size_t blockIndex = 0; blockIndex < CC_SHA1_DIGEST_LENGTH; blockIndex++) { + for (size_t blockIndex = 0; blockIndex < BINARY_DELTA_HASH_LENGTH; blockIndex++) { const char *currentBlock = hexString + blockIndex * 2; char convertedBlock[3] = {currentBlock[0], currentBlock[1], '\0'}; hash[blockIndex] = (unsigned char)strtol(convertedBlock, NULL, 16); @@ -326,8 +428,8 @@ void getRawHashFromDisplayHash(unsigned char *hash, NSString *hexHash) NSString *displayHashFromRawHash(const unsigned char *hash) { - char hexHash[CC_SHA1_DIGEST_LENGTH * 2 + 1] = {0}; - for (size_t i = 0; i < CC_SHA1_DIGEST_LENGTH; i++) { + char hexHash[BINARY_DELTA_HASH_LENGTH * 2 + 1] = {0}; + for (size_t i = 0; i < BINARY_DELTA_HASH_LENGTH; i++) { snprintf(hexHash + i * 2, 3, "%02x", hash[i]); } return @(hexHash); @@ -335,7 +437,7 @@ void getRawHashFromDisplayHash(unsigned char *hash, NSString *hexHash) NSString *hashOfTreeWithVersion(NSString *path, uint16_t majorVersion) { - unsigned char hash[CC_SHA1_DIGEST_LENGTH]; + unsigned char hash[BINARY_DELTA_HASH_LENGTH]; if (!getRawHashOfTreeWithVersion(hash, path, majorVersion)) { return nil; } diff --git a/Autoupdate/SUBinaryDeltaCreate.m b/Autoupdate/SUBinaryDeltaCreate.m index 925688c69a..6448f64c9b 100644 --- a/Autoupdate/SUBinaryDeltaCreate.m +++ b/Autoupdate/SUBinaryDeltaCreate.m @@ -508,7 +508,7 @@ BOOL createBinaryDelta(NSString *source, NSString *destination, NSString *patchF // This dictionary will help us keep track of clones NSMutableDictionary *> *beforeHashToFileKeyDictionary = MAJOR_VERSION_IS_AT_LEAST(majorVersion, SUBinaryDeltaMajorVersion3) ? [NSMutableDictionary dictionary] : nil; - unsigned char beforeHash[CC_SHA1_DIGEST_LENGTH] = {0}; + unsigned char beforeHash[BINARY_DELTA_HASH_LENGTH] = {0}; if (!getRawHashOfTreeAndFileTablesWithVersion(beforeHash, source, majorVersion, beforeHashToFileKeyDictionary, nil)) { if (verbose) { fprintf(stderr, "\n"); @@ -686,7 +686,7 @@ BOOL createBinaryDelta(NSString *source, NSString *destination, NSString *patchF // This dictionary will help us keep track of clones NSMutableDictionary *afterFileKeyToHashDictionary = MAJOR_VERSION_IS_AT_LEAST(majorVersion, SUBinaryDeltaMajorVersion3) ? [NSMutableDictionary dictionary] : nil; - unsigned char afterHash[CC_SHA1_DIGEST_LENGTH] = {0}; + unsigned char afterHash[BINARY_DELTA_HASH_LENGTH] = {0}; if (!getRawHashOfTreeAndFileTablesWithVersion(afterHash, destination, majorVersion, nil, afterFileKeyToHashDictionary)) { if (verbose) { fprintf(stderr, "\n"); diff --git a/Sparkle.xcodeproj/project.pbxproj b/Sparkle.xcodeproj/project.pbxproj index 04afdb88b1..ef9f0d979c 100644 --- a/Sparkle.xcodeproj/project.pbxproj +++ b/Sparkle.xcodeproj/project.pbxproj @@ -206,6 +206,7 @@ 723C8A551E2D60DB00C14942 /* SUTouchBarButtonGroup.m in Sources */ = {isa = PBXBuildFile; fileRef = 723C8A531E2D60DB00C14942 /* SUTouchBarButtonGroup.m */; }; 723C8A561E2D60DB00C14942 /* SUTouchBarButtonGroup.m in Sources */ = {isa = PBXBuildFile; fileRef = 723C8A531E2D60DB00C14942 /* SUTouchBarButtonGroup.m */; }; 723EDC3F26885A8E000BCBA4 /* testappcast_channels.xml in Resources */ = {isa = PBXBuildFile; fileRef = 723EDC3E26885A8E000BCBA4 /* testappcast_channels.xml */; }; + 7240852E2CA11C1400ED2FCD /* libz.tbd in Frameworks */ = {isa = PBXBuildFile; fileRef = 7240852D2CA11C1400ED2FCD /* libz.tbd */; }; 72464F701E1F31E000FB341C /* SUOperatingSystem.m in Sources */ = {isa = PBXBuildFile; fileRef = 726F2CE41BC9C33D001971A4 /* SUOperatingSystem.m */; }; 72464F7C1E2097F600FB341C /* SUOperatingSystem.m in Sources */ = {isa = PBXBuildFile; fileRef = 726F2CE41BC9C33D001971A4 /* SUOperatingSystem.m */; }; 72464F7E1E21ED8C00FB341C /* SUHost.m in Sources */ = {isa = PBXBuildFile; fileRef = 61EF67550E25B58D00F754E0 /* SUHost.m */; }; @@ -1205,6 +1206,7 @@ 723C8A531E2D60DB00C14942 /* SUTouchBarButtonGroup.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = SUTouchBarButtonGroup.m; sourceTree = ""; }; 723DFF962B3DFF6E00628E6C /* Package.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Package.swift; sourceTree = ""; }; 723EDC3E26885A8E000BCBA4 /* testappcast_channels.xml */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xml; path = testappcast_channels.xml; sourceTree = ""; }; + 7240852D2CA11C1400ED2FCD /* libz.tbd */ = {isa = PBXFileReference; lastKnownFileType = "sourcecode.text-based-dylib-definition"; name = libz.tbd; path = usr/lib/libz.tbd; sourceTree = SDKROOT; }; 7246E0A11C83B685003B4E75 /* SPUStandardUpdaterController.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SPUStandardUpdaterController.h; sourceTree = ""; }; 7246E0A21C83B685003B4E75 /* SPUStandardUpdaterController.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = SPUStandardUpdaterController.m; sourceTree = ""; }; 724BB36C1D31D0B7005D534A /* InstallerConnection.xpc */ = {isa = PBXFileReference; explicitFileType = "wrapper.xpc-service"; includeInIndex = 0; path = InstallerConnection.xpc; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -1634,6 +1636,7 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( + 7240852E2CA11C1400ED2FCD /* libz.tbd in Frameworks */, EA1E281722B645AE004AA304 /* libbsdiff.a in Frameworks */, 7267E5C91D3D8C4300D1BF90 /* CoreServices.framework in Frameworks */, 7267E5CA1D3D8C4800D1BF90 /* Foundation.framework in Frameworks */, @@ -1742,6 +1745,7 @@ 0867D69AFE84028FC02AAC07 /* Frameworks */ = { isa = PBXGroup; children = ( + 7240852D2CA11C1400ED2FCD /* libz.tbd */, 654F352629B154AE00B10EEB /* ImageIO.framework */, 725B81F92781AEA40041746F /* libcompression.tbd */, 0867D6A5FE840307C02AAC07 /* AppKit.framework */, diff --git a/Tests/SUBinaryDeltaTest.m b/Tests/SUBinaryDeltaTest.m index ecf19705fb..3416ca476c 100644 --- a/Tests/SUBinaryDeltaTest.m +++ b/Tests/SUBinaryDeltaTest.m @@ -106,24 +106,27 @@ - (BOOL)createAndApplyPatchWithBeforeDiffHandler:(SUDeltaHandler)beforeDiffHandl - (BOOL)createAndApplyPatchWithBeforeDiffHandler:(SUDeltaHandler)beforeDiffHandler afterDiffHandler:(SUDeltaHandler)afterDiffHandler afterPatchHandler:(SUDeltaHandler)afterPatchHandler testingVersion2Delta:(BOOL)testingVersion2Delta { - XCTAssertEqual(SUBinaryDeltaMajorVersion3, SUBinaryDeltaMajorVersionLatest); + XCTAssertEqual(SUBinaryDeltaMajorVersion4, SUBinaryDeltaMajorVersionLatest); - BOOL version3DeltaFormatWithLZMASuccess = [self createAndApplyPatchUsingVersion:SUBinaryDeltaMajorVersion3 compressionMode:SPUDeltaCompressionModeLZMA beforeDiffHandler:beforeDiffHandler afterDiffHandler:afterDiffHandler afterPatchHandler:afterPatchHandler]; + BOOL version4DeltaFormatWithLZMASuccess = [self createAndApplyPatchUsingVersion:SUBinaryDeltaMajorVersion4 compressionMode:SPUDeltaCompressionModeLZMA beforeDiffHandler:beforeDiffHandler afterDiffHandler:afterDiffHandler afterPatchHandler:afterPatchHandler]; #if SPARKLE_BUILD_BZIP2_DELTA_SUPPORT - BOOL version3DeltaFormatWithBZIP2Success = [self createAndApplyPatchUsingVersion:SUBinaryDeltaMajorVersion3 compressionMode:SPUDeltaCompressionModeBzip2 beforeDiffHandler:beforeDiffHandler afterDiffHandler:afterDiffHandler afterPatchHandler:afterPatchHandler]; + BOOL version4DeltaFormatWithBZIP2Success = [self createAndApplyPatchUsingVersion:SUBinaryDeltaMajorVersion4 compressionMode:SPUDeltaCompressionModeBzip2 beforeDiffHandler:beforeDiffHandler afterDiffHandler:afterDiffHandler afterPatchHandler:afterPatchHandler]; #endif - BOOL version3DeltaFormatWithZLIBSuccess = [self createAndApplyPatchUsingVersion:SUBinaryDeltaMajorVersion3 compressionMode:SPUDeltaCompressionModeZLIB beforeDiffHandler:beforeDiffHandler afterDiffHandler:afterDiffHandler afterPatchHandler:afterPatchHandler]; + BOOL version4DeltaFormatWithZLIBSuccess = [self createAndApplyPatchUsingVersion:SUBinaryDeltaMajorVersion4 compressionMode:SPUDeltaCompressionModeZLIB beforeDiffHandler:beforeDiffHandler afterDiffHandler:afterDiffHandler afterPatchHandler:afterPatchHandler]; + + BOOL version3FormatSuccess = [self createAndApplyPatchUsingVersion:SUBinaryDeltaMajorVersion3 compressionMode:SPUDeltaCompressionModeLZMA beforeDiffHandler:beforeDiffHandler afterDiffHandler:afterDiffHandler afterPatchHandler:afterPatchHandler]; BOOL version2FormatSuccess = !testingVersion2Delta || [self createAndApplyPatchUsingVersion:SUBinaryDeltaMajorVersion2 compressionMode:SPUDeltaCompressionModeDefault beforeDiffHandler:beforeDiffHandler afterDiffHandler:afterDiffHandler afterPatchHandler:afterPatchHandler]; return ( - version3DeltaFormatWithLZMASuccess && + version4DeltaFormatWithLZMASuccess && #if SPARKLE_BUILD_BZIP2_DELTA_SUPPORT - version3DeltaFormatWithBZIP2Success && + version4DeltaFormatWithBZIP2Success && #endif - version3DeltaFormatWithZLIBSuccess && + version4DeltaFormatWithZLIBSuccess && + version3FormatSuccess && version2FormatSuccess ); } From 30d21ff868ad333a6dabf364e96fb055c5a8041a Mon Sep 17 00:00:00 2001 From: Zorg Date: Sun, 29 Sep 2024 18:54:02 -0700 Subject: [PATCH 2/9] Remove perf measurement testing --- Autoupdate/SUBinaryDeltaApply.m | 8 -------- 1 file changed, 8 deletions(-) diff --git a/Autoupdate/SUBinaryDeltaApply.m b/Autoupdate/SUBinaryDeltaApply.m index e0b34b4e76..6e3df0ce52 100644 --- a/Autoupdate/SUBinaryDeltaApply.m +++ b/Autoupdate/SUBinaryDeltaApply.m @@ -92,8 +92,6 @@ BOOL applyBinaryDelta(NSString *source, NSString *finalDestination, NSString *pa progressCallback(1/7.0); - NSDate *beforeHashDate = [NSDate date]; - unsigned char beforeHash[BINARY_DELTA_HASH_LENGTH] = {0}; if (!getRawHashOfTreeWithVersion(beforeHash, source, majorDiffVersion)) { if (verbose) { @@ -105,8 +103,6 @@ BOOL applyBinaryDelta(NSString *source, NSString *finalDestination, NSString *pa return NO; } - NSLog(@"Before hash took %f seconds", [[NSDate date] timeIntervalSinceDate:beforeHashDate]); - if (memcmp(beforeHash, expectedBeforeHash, BINARY_DELTA_HASH_LENGTH) != 0) { if (verbose) { fprintf(stderr, "\n"); @@ -464,8 +460,6 @@ BOOL applyBinaryDelta(NSString *source, NSString *finalDestination, NSString *pa fprintf(stderr, "\nVerifying destination..."); } - NSDate *afterHashDate = [NSDate date]; - unsigned char afterHash[BINARY_DELTA_HASH_LENGTH] = {0}; if (!getRawHashOfTreeWithVersion(afterHash, finalDestination, majorDiffVersion)) { if (verbose) { @@ -478,8 +472,6 @@ BOOL applyBinaryDelta(NSString *source, NSString *finalDestination, NSString *pa return NO; } - NSLog(@"After hash took %f seconds", [[NSDate date] timeIntervalSinceDate:afterHashDate]); - if (memcmp(afterHash, expectedAfterHash, BINARY_DELTA_HASH_LENGTH) != 0) { if (verbose) { fprintf(stderr, "\n"); From 0413bc72374abd270e28262c7dc72deb48e8b823 Mon Sep 17 00:00:00 2001 From: Zorg Date: Sun, 29 Sep 2024 19:01:19 -0700 Subject: [PATCH 3/9] Remove MAJOR_VERSION_IS_AT_LEAST() Not a terribly useful macro. --- Autoupdate/SPUSparkleDeltaArchive.m | 4 ++-- Autoupdate/SUBinaryDeltaCommon.h | 2 -- Autoupdate/SUBinaryDeltaCommon.m | 8 ++++---- Autoupdate/SUBinaryDeltaCreate.m | 12 ++++++------ 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/Autoupdate/SPUSparkleDeltaArchive.m b/Autoupdate/SPUSparkleDeltaArchive.m index 9b9832ca19..90da37490d 100644 --- a/Autoupdate/SPUSparkleDeltaArchive.m +++ b/Autoupdate/SPUSparkleDeltaArchive.m @@ -349,7 +349,7 @@ - (nullable SPUDeltaArchiveHeader *)readHeader } NSDate *bundleCreationDate; - if (MAJOR_VERSION_IS_AT_LEAST(majorVersion, SUBinaryDeltaMajorVersion4)) { + if (majorVersion >= SUBinaryDeltaMajorVersion4) { double bundleCreationTimeInterval = 0; if (![self _readBuffer:&bundleCreationTimeInterval length:sizeof(bundleCreationTimeInterval)]) { return nil; @@ -868,7 +868,7 @@ - (void)writeHeader:(SPUDeltaArchiveHeader *)header [self _writeBuffer:header.beforeTreeHash length:BINARY_DELTA_HASH_LENGTH]; [self _writeBuffer:header.afterTreeHash length:BINARY_DELTA_HASH_LENGTH]; - if (MAJOR_VERSION_IS_AT_LEAST(majorVersion, SUBinaryDeltaMajorVersion4)) { + if (majorVersion >= SUBinaryDeltaMajorVersion4) { NSDate *bundleCreationDate = header.bundleCreationDate; // If bundleCreationDate == nil, we will write out a 0 time interval diff --git a/Autoupdate/SUBinaryDeltaCommon.h b/Autoupdate/SUBinaryDeltaCommon.h index 1733abbd2c..7b1bfc9142 100644 --- a/Autoupdate/SUBinaryDeltaCommon.h +++ b/Autoupdate/SUBinaryDeltaCommon.h @@ -32,8 +32,6 @@ #define VERBOSE_MODIFIED "Modified" // file's metadata is modified #define VERBOSE_CLONED "Cloned" // file is cloned in content from a differently named file -#define MAJOR_VERSION_IS_AT_LEAST(actualMajor, expectedMajor) (actualMajor >= expectedMajor) - // Relative path of custom icon data that may be set on a bundle via a resource fork #define CUSTOM_ICON_PATH @"/Icon\r" diff --git a/Autoupdate/SUBinaryDeltaCommon.m b/Autoupdate/SUBinaryDeltaCommon.m index 2d006815c6..03e52d6e1f 100644 --- a/Autoupdate/SUBinaryDeltaCommon.m +++ b/Autoupdate/SUBinaryDeltaCommon.m @@ -321,7 +321,7 @@ BOOL getRawHashOfTreeAndFileTablesWithVersion(void *hashBuffer, NSString *path, } NSData *fileHashKey; - if (majorVersion >= 4) { + if (majorVersion >= SUBinaryDeltaMajorVersion4) { if (ent->fts_info == FTS_D) { fileHashKey = nil; } else { @@ -374,7 +374,7 @@ BOOL getRawHashOfTreeAndFileTablesWithVersion(void *hashBuffer, NSString *path, const char *relativePathBytes = [relativePath fileSystemRepresentation]; - if (majorVersion >= 4) { + if (majorVersion >= SUBinaryDeltaMajorVersion4) { crc32ChecksumValue = crc32(crc32ChecksumValue, (const void *)relativePathBytes, (uInt)strlen(relativePathBytes)); } else { CC_SHA1_Update(&hashContext, relativePathBytes, (CC_LONG)strlen(relativePathBytes)); @@ -389,7 +389,7 @@ BOOL getRawHashOfTreeAndFileTablesWithVersion(void *hashBuffer, NSString *path, // hardcoding a value helps avoid differences between filesystems. uint16_t hashedPermissions = (ent->fts_info == FTS_SL) ? VALID_SYMBOLIC_LINK_PERMISSIONS : permissions; - if (majorVersion >= 4) { + if (majorVersion >= SUBinaryDeltaMajorVersion4) { crc32ChecksumValue = crc32(crc32ChecksumValue, (const void *)&type, sizeof(type)); crc32ChecksumValue = crc32(crc32ChecksumValue, (const void *)&hashedPermissions, sizeof(hashedPermissions)); } else { @@ -402,7 +402,7 @@ BOOL getRawHashOfTreeAndFileTablesWithVersion(void *hashBuffer, NSString *path, fts_close(fts); - if (majorVersion >= 4) { + if (majorVersion >= SUBinaryDeltaMajorVersion4) { uint64_t encodedCrc32ChecksumValue = crc32ChecksumValue; memcpy(hashBuffer, &encodedCrc32ChecksumValue, sizeof(encodedCrc32ChecksumValue)); } else { diff --git a/Autoupdate/SUBinaryDeltaCreate.m b/Autoupdate/SUBinaryDeltaCreate.m index 7e4c5bae25..3fe1834507 100644 --- a/Autoupdate/SUBinaryDeltaCreate.m +++ b/Autoupdate/SUBinaryDeltaCreate.m @@ -506,7 +506,7 @@ BOOL createBinaryDelta(NSString *source, NSString *destination, NSString *patchF fts_close(fts); // This dictionary will help us keep track of clones - NSMutableDictionary *> *beforeHashToFileKeyDictionary = MAJOR_VERSION_IS_AT_LEAST(majorVersion, SUBinaryDeltaMajorVersion3) ? [NSMutableDictionary dictionary] : nil; + NSMutableDictionary *> *beforeHashToFileKeyDictionary = (majorVersion >= SUBinaryDeltaMajorVersion3) ? [NSMutableDictionary dictionary] : nil; unsigned char beforeHash[BINARY_DELTA_HASH_LENGTH] = {0}; if (!getRawHashOfTreeAndFileTablesWithVersion(beforeHash, source, majorVersion, beforeHashToFileKeyDictionary, nil)) { @@ -644,7 +644,7 @@ BOOL createBinaryDelta(NSString *source, NSString *destination, NSString *patchF // If we find any executable files that are using file system compression, that is sufficient // for recording that the applier should re-apply file system compression. // We check for executable files because they are likely candidates to be compressed. - if (!foundFilesystemCompression && MAJOR_VERSION_IS_AT_LEAST(majorVersion, SUBinaryDeltaMajorVersion3) && ent->fts_info == FTS_F && (ent->fts_statp->st_mode & (S_IXUSR | S_IXGRP | S_IXOTH)) != 0 && (ent->fts_statp->st_flags & UF_COMPRESSED) != 0) { + if (!foundFilesystemCompression && (majorVersion >= SUBinaryDeltaMajorVersion3) && ent->fts_info == FTS_F && (ent->fts_statp->st_mode & (S_IXUSR | S_IXGRP | S_IXOTH)) != 0 && (ent->fts_statp->st_flags & UF_COMPRESSED) != 0) { foundFilesystemCompression = true; if (verbose) { @@ -684,7 +684,7 @@ BOOL createBinaryDelta(NSString *source, NSString *destination, NSString *patchF fts_close(fts); // This dictionary will help us keep track of clones - NSMutableDictionary *afterFileKeyToHashDictionary = MAJOR_VERSION_IS_AT_LEAST(majorVersion, SUBinaryDeltaMajorVersion3) ? [NSMutableDictionary dictionary] : nil; + NSMutableDictionary *afterFileKeyToHashDictionary = (majorVersion >= SUBinaryDeltaMajorVersion3) ? [NSMutableDictionary dictionary] : nil; unsigned char afterHash[BINARY_DELTA_HASH_LENGTH] = {0}; if (!getRawHashOfTreeAndFileTablesWithVersion(afterHash, destination, majorVersion, nil, afterFileKeyToHashDictionary)) { @@ -725,7 +725,7 @@ BOOL createBinaryDelta(NSString *source, NSString *destination, NSString *patchF // Record creation date of root bundle item NSDate *bundleCreationDate; - if (MAJOR_VERSION_IS_AT_LEAST(majorVersion, SUBinaryDeltaMajorVersion4)) { + if (majorVersion >= SUBinaryDeltaMajorVersion4) { NSError *fileAttributesError = nil; NSDictionary *fileAttributes = [[NSFileManager defaultManager] attributesOfItemAtPath:destination error:&fileAttributesError]; @@ -769,7 +769,7 @@ BOOL createBinaryDelta(NSString *source, NSString *destination, NSString *patchF // Using a couple of heuristics we track if files have been moved to other locations within the app bundle NSMutableDictionary *frameworkVersionsSubstitutes = [NSMutableDictionary dictionary]; NSMutableDictionary *fileSubstitutes = [NSMutableDictionary dictionary]; - if (MAJOR_VERSION_IS_AT_LEAST(majorVersion, SUBinaryDeltaMajorVersion3)) { + if (majorVersion >= SUBinaryDeltaMajorVersion3) { // Heuristic #1: track if an old framework version was removed and a new framework version was added // Keep track of these prefixes in a dictionary // Eg: /Contents/Frameworks/Foo.framework/Versions/B/ (new) -> /Contents/Frameworks/Foo.framework/Versions/A/ (old) @@ -887,7 +887,7 @@ BOOL createBinaryDelta(NSString *source, NSString *destination, NSString *patchF NSNumber *newPermissions = nil; BOOL clonePermissionsChanged = NO; BOOL clonedBinaryDiff = NO; - NSString *clonedRelativePath = MAJOR_VERSION_IS_AT_LEAST(majorVersion, SUBinaryDeltaMajorVersion3) ? cloneableRelativePath(afterFileKeyToHashDictionary, beforeHashToFileKeyDictionary, frameworkVersionsSubstitutes, fileSubstitutes, originalTreeState, newInfo, key, &newPermissions, &clonePermissionsChanged, &clonedBinaryDiff) : nil; + NSString *clonedRelativePath = (majorVersion >= SUBinaryDeltaMajorVersion3) ? cloneableRelativePath(afterFileKeyToHashDictionary, beforeHashToFileKeyDictionary, frameworkVersionsSubstitutes, fileSubstitutes, originalTreeState, newInfo, key, &newPermissions, &clonePermissionsChanged, &clonedBinaryDiff) : nil; if (clonedRelativePath != nil) { if (clonedBinaryDiff) { NSDictionary *cloneInfo = originalTreeState[clonedRelativePath]; From db939599a275c3c00fd4ff167a40b5d3279f7bbb Mon Sep 17 00:00:00 2001 From: Zorg Date: Sun, 29 Sep 2024 19:03:33 -0700 Subject: [PATCH 4/9] Use SUBinaryDeltaMajorVersion4 --- Autoupdate/SUBinaryDeltaCommon.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Autoupdate/SUBinaryDeltaCommon.m b/Autoupdate/SUBinaryDeltaCommon.m index 03e52d6e1f..b84eec00c0 100644 --- a/Autoupdate/SUBinaryDeltaCommon.m +++ b/Autoupdate/SUBinaryDeltaCommon.m @@ -301,7 +301,7 @@ BOOL getRawHashOfTreeAndFileTablesWithVersion(void *hashBuffer, NSString *path, CC_SHA1_CTX hashContext; uLong crc32ChecksumValue = 0; - if (majorVersion < 4) { + if (majorVersion < SUBinaryDeltaMajorVersion4) { CC_SHA1_Init(&hashContext); } From 868fb646e90b0928e7a68d5acddf7211c20286e2 Mon Sep 17 00:00:00 2001 From: Zorg Date: Sun, 29 Sep 2024 19:37:10 -0700 Subject: [PATCH 5/9] Bump project version for binary delta change --- Configurations/ConfigCommon.xcconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Configurations/ConfigCommon.xcconfig b/Configurations/ConfigCommon.xcconfig index 7e4d688a56..620161cc09 100644 --- a/Configurations/ConfigCommon.xcconfig +++ b/Configurations/ConfigCommon.xcconfig @@ -104,7 +104,7 @@ SPARKLE_VERSION_PATCH = 0 // This should be in SemVer format or empty, ie. "-beta.1" // These variables must have a space after the '=' too SPARKLE_VERSION_SUFFIX = -beta.1 -CURRENT_PROJECT_VERSION = 2040 +CURRENT_PROJECT_VERSION = 2041 MARKETING_VERSION = $(SPARKLE_VERSION_MAJOR).$(SPARKLE_VERSION_MINOR).$(SPARKLE_VERSION_PATCH)$(SPARKLE_VERSION_SUFFIX) ALWAYS_SEARCH_USER_PATHS = NO From 7d6bbe42dc65a6de9f0eb9cbc2b236e9edd32f4b Mon Sep 17 00:00:00 2001 From: Zorg Date: Sun, 29 Sep 2024 20:37:56 -0700 Subject: [PATCH 6/9] Zero hashBuffer entirely before storing 8-byte hash --- Autoupdate/SUBinaryDeltaCommon.m | 1 + 1 file changed, 1 insertion(+) diff --git a/Autoupdate/SUBinaryDeltaCommon.m b/Autoupdate/SUBinaryDeltaCommon.m index b84eec00c0..7edc02bec7 100644 --- a/Autoupdate/SUBinaryDeltaCommon.m +++ b/Autoupdate/SUBinaryDeltaCommon.m @@ -404,6 +404,7 @@ BOOL getRawHashOfTreeAndFileTablesWithVersion(void *hashBuffer, NSString *path, if (majorVersion >= SUBinaryDeltaMajorVersion4) { uint64_t encodedCrc32ChecksumValue = crc32ChecksumValue; + memset(hashBuffer, 0, BINARY_DELTA_HASH_LENGTH); memcpy(hashBuffer, &encodedCrc32ChecksumValue, sizeof(encodedCrc32ChecksumValue)); } else { CC_SHA1_Final(hashBuffer, &hashContext); From 7a08ab62f1870ba414c8ea5157ee2eb6850e24bb Mon Sep 17 00:00:00 2001 From: Zorg Date: Sun, 29 Sep 2024 20:38:26 -0700 Subject: [PATCH 7/9] Fix unarchiver tests to account for new hash function --- Tests/SUUnarchiverTest.swift | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Tests/SUUnarchiverTest.swift b/Tests/SUUnarchiverTest.swift index 754af20305..2db7aa9690 100644 --- a/Tests/SUUnarchiverTest.swift +++ b/Tests/SUUnarchiverTest.swift @@ -34,7 +34,8 @@ class SUUnarchiverTest: XCTestCase let extractedAppURL = tempDirectoryURL.appendingPathComponent(extractedAppName).appendingPathExtension("app") XCTAssertTrue(fileManager.fileExists(atPath: extractedAppURL.path)) - XCTAssertEqual("6a60ab31430cfca8fb499a884f4a29f73e59b472", hashOfTree(extractedAppURL.path)) + XCTAssertEqual("6a60ab31430cfca8fb499a884f4a29f73e59b472", hashOfTreeWithVersion(extractedAppURL.path, 3)) + XCTAssertEqual("52111bc200000000000000000000000000000000", hashOfTree(extractedAppURL.path)) } else if archiveExtension != "pkg" { let extractedPackageURL = tempDirectoryURL.appendingPathComponent(extractedAppName).appendingPathExtension("pkg") XCTAssertTrue(fileManager.fileExists(atPath: extractedPackageURL.path)) From b551ee5a4fd88f76035417f44cea019b0367d97c Mon Sep 17 00:00:00 2001 From: Zorg Date: Sat, 5 Oct 2024 17:44:38 -0700 Subject: [PATCH 8/9] Zero initialize hash buffer in hashOfTreeWithVersion() --- Autoupdate/SUBinaryDeltaCommon.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Autoupdate/SUBinaryDeltaCommon.m b/Autoupdate/SUBinaryDeltaCommon.m index 7edc02bec7..99259c2aec 100644 --- a/Autoupdate/SUBinaryDeltaCommon.m +++ b/Autoupdate/SUBinaryDeltaCommon.m @@ -438,7 +438,7 @@ void getRawHashFromDisplayHash(unsigned char *hash, NSString *hexHash) NSString *hashOfTreeWithVersion(NSString *path, uint16_t majorVersion) { - unsigned char hash[BINARY_DELTA_HASH_LENGTH]; + unsigned char hash[BINARY_DELTA_HASH_LENGTH] = {0}; if (!getRawHashOfTreeWithVersion(hash, path, majorVersion)) { return nil; } From 5d1253de099685fe15e312c7d41933908a7d3538 Mon Sep 17 00:00:00 2001 From: Zorg Date: Sun, 13 Oct 2024 16:18:12 -0700 Subject: [PATCH 9/9] Compute initial crc32 value directly (even though it's zero) --- Autoupdate/SUBinaryDeltaCommon.m | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Autoupdate/SUBinaryDeltaCommon.m b/Autoupdate/SUBinaryDeltaCommon.m index 99259c2aec..90e5d50da7 100644 --- a/Autoupdate/SUBinaryDeltaCommon.m +++ b/Autoupdate/SUBinaryDeltaCommon.m @@ -299,7 +299,8 @@ BOOL getRawHashOfTreeAndFileTablesWithVersion(void *hashBuffer, NSString *path, } CC_SHA1_CTX hashContext; - uLong crc32ChecksumValue = 0; + const uLong initialCrc32Value = crc32(0L, Z_NULL, 0); + uLong crc32ChecksumValue = initialCrc32Value; if (majorVersion < SUBinaryDeltaMajorVersion4) { CC_SHA1_Init(&hashContext); @@ -323,9 +324,11 @@ BOOL getRawHashOfTreeAndFileTablesWithVersion(void *hashBuffer, NSString *path, NSData *fileHashKey; if (majorVersion >= SUBinaryDeltaMajorVersion4) { if (ent->fts_info == FTS_D) { + // No need to hash any further values for directories + // We hash relative file path and file type later fileHashKey = nil; } else { - uLong fileContentsChecksum = 0; + uLong fileContentsChecksum = initialCrc32Value; if (!_crc32HashOfFileContents(&fileContentsChecksum, ent, tempBuffer, tempBufferSize)) { fts_close(fts); free(tempBuffer);