Add support for ZSO format.

.vscode/settings.json

@@ -0,0 +1,5 @@
+{
+    "files.associations": {
+        "global.h": "c"
+    }
+}

kernel/DI.c

@@ -91,11 +91,11 @@ static u8 *NetworkCMDBuffer;
 static u8 *const DIMMMemory = (u8*)0x12B80000;
 
 // Multi-disc filenames.
-static const char disc_filenames[8][16] = {
+static const char disc_filenames[10][16] = {
 	// Disc 1
-	"game.ciso", "game.cso", "game.gcm", "game.iso",
+	"game.ciso", "game.cso", "game.gcm", "game.iso", "game.zso",
 	// Disc 2
-	"disc2.ciso", "disc2.cso", "disc2.gcm", "disc2.iso"
+	"disc2.ciso", "disc2.cso", "disc2.gcm", "disc2.iso", "disc2.zso"
 };
 
 // Filename portions for 2-disc mode.
@@ -192,24 +192,24 @@ void DIinit( bool FirstTime )
 			const char **DI_2disc_otherdisc = NULL;
 			DI_2disc_filenames[0] = NULL;
 			DI_2disc_filenames[1] = NULL;
-			for (i = 0; i < 8; i++)
+			for (i = 0; i < 10; i++)
 			{
 				if (!strcasecmp(TempDiscName+slash_pos, disc_filenames[i]))
 				{
 					// Filename is either:
 					// -  game.(ciso|cso|gcm|iso) (Disc 1)
 					// - disc2.(ciso|cso|gcm|iso) (Disc 2)
-					const int discIdx = i / 4;	// either 0 or 1
+					const int discIdx = i / 5;	// either 0 or 1
 					DI_2disc_filenames[discIdx] = disc_filenames[i];
 
 					// Set variables to check for the other disc.
 					if (discIdx == 0) {
-						checkIdxMin = 4;
-						checkIdxMax = 7;
+						checkIdxMin = 5;
+						checkIdxMax = 9;
 						DI_2disc_otherdisc = &DI_2disc_filenames[1];
 					} else {
 						checkIdxMin = 0;
-						checkIdxMax = 3;
+						checkIdxMax = 4;
 						DI_2disc_otherdisc = &DI_2disc_filenames[0];
 					}
 					break;

kernel/ISO.c

@@ -24,6 +24,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "GCNCard.h"
 #include "debug.h"
 #include "wdvd.h"
+#include "lz4.h"
 
 #include "ff_utf8.h"
 
@@ -33,6 +34,8 @@ extern bool wiiVCInternal;
 
 u32 ISOFileOpen = 0;
 
+#define MIN(x,y) ((x<y)?x:y)
+
 #define CACHE_MAX		0x400
 #define CACHE_START		(u8*)0x11000000
 #define CACHE_SIZE		0x1E80000
@@ -56,6 +59,12 @@ static FIL GameFile;
 static u64 LastOffset64 = ~0ULL;
 bool Datel = false;
 
+enum {
+	TYPE_ISO,
+	TYPE_CSO,
+	TYPE_ZSO,
+};
+
 // CISO: On-disc structure.
 // Temporarily loaded into cache memory.
 #define CISO_MAGIC	0x4349534F /* "CISO" */
@@ -69,10 +78,35 @@ typedef struct _CISO_t {
 	u8 map[CISO_MAP_SIZE];	// Block map;
 } CISO_t;
 
+// ZISO
+#define ZISO_MAGIC 0x5A49534F /* "ZISO" */
+#define ZISO_IDX_CACHE_SIZE 128
+#define ZISO_MAX_BLOCK_SIZE 8192
+typedef struct _ZISO_t {
+    u32 magic;  // 0
+    u32 header_size;  // 4
+    u64 total_bytes; // 8
+    u32 block_size; // 16
+    u8 ver; // 20
+    u8 align;  // 21
+    u8 rsv_06[2];  // 22
+} ZISO_t;
+
 // CISO: Block map.
 // Supports files up to 2 GB when using 2 MB blocks.
-static uint16_t ciso_block_map[CISO_MAP_SIZE];
-static bool ISO_IsCISO = false;	// Set to 1 for CISO mode.
+static union {
+	uint16_t ciso_block_map[CISO_MAP_SIZE];
+	u32 ziso_block_map[ZISO_IDX_CACHE_SIZE]; // we use this as a cache on ZISO
+} block_map;
+
+static unsigned char ISO_Type = TYPE_ISO;	// Set to 1 for CISO mode. 2 for ZISO.
+
+static u64 uncompressed_size = 0;
+static u32 ziso_align = 0;
+static u32 ziso_block_size = 0;
+static int ziso_idx_start_block = -1;
+static u8 ziso_com_buf[ZISO_MAX_BLOCK_SIZE] __attribute__((aligned(64)));
+static u8 ziso_dec_buf[ZISO_MAX_BLOCK_SIZE] __attribute__((aligned(64)));
 
 /**
  * Read directly from the ISO file.
@@ -81,31 +115,143 @@ static bool ISO_IsCISO = false;	// Set to 1 for CISO mode.
  * @param Length Data length.
  * @param Offset ISO file offset. (Must have ISOShift64 added!)
  */
+static UINT read_raw_data(void *Buffer, u32 Length, u64 Offset64){
+	UINT read;
+	if(LastOffset64 != Offset64)
+	{
+		if(wiiVCInternal)
+			WDVD_FST_LSeek( Offset64 );
+		else
+			f_lseek( &GameFile, Offset64 );
+	}
+	if(wiiVCInternal)
+	{
+		sync_before_read( Buffer, Length );
+		read = WDVD_FST_Read( Buffer, Length );
+	}
+	else
+		f_read( &GameFile, Buffer, Length, &read );
+
+	return read;
+}
+
 static inline void ISOReadDirect(void *Buffer, u32 Length, u64 Offset64)
 {
 	if(ISOFileOpen == 0)
 		return;
 
 	UINT read;
-	if (!ISO_IsCISO)
+	if (ISO_Type == TYPE_ISO)
 	{
 		// Standard ISO/GCM file.
-		if(LastOffset64 != Offset64)
-		{
-			if(wiiVCInternal)
-				WDVD_FST_LSeek( Offset64 );
-			else
-				f_lseek( &GameFile, Offset64 );
+		read = read_raw_data(Buffer, Length, Offset64);
+	}
+	else if (ISO_Type == TYPE_ZSO){
+		// ZISO
+		u32 cur_block;
+		u32 pos, read_bytes;
+		u8* com_buf = ziso_com_buf;
+		u8* dec_buf = ziso_dec_buf;
+		u8* c_buf = NULL;
+		u8* addr = (u8*)Buffer;
+		u8* top_addr = addr+Length;
+		u32 size = Length;
+		u64 offset = Offset64;
+		LastOffset64 = ~0ULL;
+
+		if(offset > uncompressed_size) {
+			// return if the offset goes beyond the iso size
+			return;
 		}
-		if(wiiVCInternal)
-		{
-			sync_before_read( Buffer, Length );
-			read = WDVD_FST_Read( Buffer, Length );
+		else if(offset + size > uncompressed_size) {
+			// adjust size if it tries to read beyond the game data
+			size = uncompressed_size - offset;
+		}
+
+		// IO speedup tricks
+		u32 starting_block = offset / ziso_block_size;
+		u32 ending_block = ((offset+size)/ziso_block_size);
+
+		// refresh index table if needed
+		if (ziso_idx_start_block < 0 || starting_block < ziso_idx_start_block || starting_block-ziso_idx_start_block+1 >= ZISO_IDX_CACHE_SIZE-1){
+			read_raw_data(block_map.ziso_block_map, ZISO_IDX_CACHE_SIZE*sizeof(u32), starting_block * sizeof(u32) + sizeof(ZISO_t));
+			ziso_idx_start_block = starting_block;
+		}
+
+		// Calculate total size of compressed data
+		u32 o_start = (
+			bswap32(block_map.ziso_block_map[starting_block-ziso_idx_start_block])
+			&0x7FFFFFFF
+			)<<ziso_align;
+		// last block index might be outside the block offset cache, better read it from disk
+		u32 o_end;
+		if (ending_block-ziso_idx_start_block < ZISO_IDX_CACHE_SIZE-1){
+			o_end = block_map.ziso_block_map[ending_block-ziso_idx_start_block];
+		}
+		else read_raw_data(&o_end, sizeof(u32), ending_block*sizeof(u32)+sizeof(ZISO_t)); // read last two offsets
+		o_end = (bswap32(o_end)&0x7FFFFFFF)<<ziso_align;
+		u32 compressed_size = o_end-o_start;
+
+		// try to read at once as much compressed data as possible
+		if (size > ziso_block_size*2){ // only if going to read more than two blocks
+			if (size < compressed_size) compressed_size = size-ziso_block_size; // adjust chunk size if compressed data is still bigger than uncompressed
+			c_buf = top_addr - compressed_size; // read into the end of the user buffer
+			read_raw_data(c_buf, compressed_size, o_start);
+		}
+
+		while(size > 0) {
+			// calculate block number and offset within block
+			cur_block = offset / ziso_block_size;
+			pos = offset & (ziso_block_size - 1);
+
+			// check if we need to refresh index table
+			if (cur_block-ziso_idx_start_block >= ZISO_IDX_CACHE_SIZE-1){
+				read_raw_data(block_map.ziso_block_map, ZISO_IDX_CACHE_SIZE*sizeof(u32), cur_block*sizeof(u32) + sizeof(ZISO_t));
+				ziso_idx_start_block = cur_block;
+			}
+
+			// read compressed block offset and size
+			u32 b_offset = bswap32(block_map.ziso_block_map[cur_block-ziso_idx_start_block]);
+			u32 b_size = bswap32(block_map.ziso_block_map[cur_block-ziso_idx_start_block+1]);
+			u32 topbit = b_offset&0x80000000; // extract top bit for decompressor
+			b_offset = (b_offset&0x7FFFFFFF) << ziso_align;
+			b_size = (b_size&0x7FFFFFFF) << ziso_align;
+			b_size -= b_offset;
+
+			// check if we need to (and can) read another chunk of data
+			if (c_buf < addr || c_buf+b_size > top_addr){
+				if (size > b_size+ziso_block_size){ // only if more than two blocks left, otherwise just use normal reading
+					compressed_size = o_end-b_offset; // recalculate remaining compressed data
+					if (size < compressed_size) compressed_size = size-ziso_block_size; // adjust if still bigger than uncompressed
+					if (compressed_size >= b_size){
+						c_buf = top_addr - compressed_size; // read into the end of the user buffer
+						read_raw_data(c_buf, compressed_size, b_offset);
+					}
+				}
+			}
+
+			// read block, skipping header if needed
+			if (c_buf >= addr && c_buf+b_size <= top_addr){
+				memcpy(com_buf, c_buf, b_size); // fast read
+				c_buf += b_size;
+			}
+			else{ // slow read
+				b_size = read_raw_data(com_buf, b_size, b_offset);
+				if (c_buf) c_buf += b_size;
+			}
+
+			if (topbit) memcpy(dec_buf, com_buf, ziso_block_size); // check for NC area
+		    else LZ4_decompress_fast((const char*)com_buf, (char*)dec_buf, ziso_block_size); // decompress block
+
+			// read data from block into buffer
+			read_bytes = MIN(size, (ziso_block_size - pos));
+			memcpy(addr, dec_buf + pos, read_bytes);
+			size -= read_bytes;
+			addr += read_bytes;
+			offset += read_bytes;
 		}
-		else
-			f_read( &GameFile, Buffer, Length, &read );
 	}
-	else
+	else if (ISO_Type == TYPE_CSO)
 	{
 		// CISO. Handle individual blocks.
 		// TODO: LastOffset64 optimization?
@@ -129,7 +275,7 @@ static inline void ISOReadDirect(void *Buffer, u32 Length, u64 Offset64)
 				return;
 			}
 
-			const u16 physBlockStartIdx = ciso_block_map[blockStart];
+			const u16 physBlockStartIdx = block_map.ciso_block_map[blockStart];
 			if (physBlockStartIdx == 0xFFFF)
 			{
 				// Empty block.
@@ -176,7 +322,7 @@ static inline void ISOReadDirect(void *Buffer, u32 Length, u64 Offset64)
 				return;
 			}
 
-			const u16 physBlockIdx = ciso_block_map[blockIdx];
+			const u16 physBlockIdx = block_map.ciso_block_map[blockIdx];
 			if (physBlockIdx == 0xFFFF)
 			{
 				// Empty block.
@@ -218,7 +364,7 @@ static inline void ISOReadDirect(void *Buffer, u32 Length, u64 Offset64)
 				return;
 			}
 
-			const u16 physBlockEndIdx = ciso_block_map[blockEnd];
+			const u16 physBlockEndIdx = block_map.ciso_block_map[blockEnd];
 			if (physBlockEndIdx == 0xFFFF)
 			{
 				// Empty block.
@@ -294,11 +440,11 @@ bool ISOInit()
 	/* Setup direct reader */
 	ISOFileOpen = 1;
 	LastOffset64 = ~0ULL;
-	ISO_IsCISO = false;
+	ISO_Type = TYPE_ISO;
 
 	/* Check for CISO format. */
 	CISO_t *tmp_ciso = (CISO_t*)malloca(0x8000, 0x20);
-	ISOReadDirect(tmp_ciso, 0x8000, 0);
+	read_raw_data(tmp_ciso, 0x8000, 0);
 	if (tmp_ciso->magic == CISO_MAGIC)
 	{
 		// Only CISOs with 2 MB block sizes are supported.
@@ -327,20 +473,29 @@ bool ISOInit()
 				if (tmp_ciso->map[i])
 				{
 					// Used block.
-					ciso_block_map[i] = physBlockIdx;
+					block_map.ciso_block_map[i] = physBlockIdx;
 					physBlockIdx++;
 				}
 				else
 				{
 					// Empty block.
-					ciso_block_map[i] = 0xFFFF;
+					block_map.ciso_block_map[i] = 0xFFFF;
 				}
 			}
 
 			// Enable CISO mode.
-			ISO_IsCISO = true;
+			ISO_Type = TYPE_CSO;
 		}
 	}
+	else if (tmp_ciso->magic == ZISO_MAGIC) {
+		ZISO_t* ziso_header = (ZISO_t*)tmp_ciso;
+		uncompressed_size = bswap64(ziso_header->total_bytes);
+		ziso_block_size = bswap32(ziso_header->block_size);
+		ziso_align = ziso_header->align;
+		ziso_idx_start_block = -1;
+		ISO_Type = TYPE_ZSO;
+		// TODO: dynamically allocate buffers based on block size
+	}
 	free(tmp_ciso);
 
 	/* Set Low Mem */
@@ -380,7 +535,7 @@ void ISOClose()
 		}
 	}
 	ISOFileOpen = 0;
-	ISO_IsCISO = false;
+	ISO_Type = TYPE_ISO;
 }
 
 void ISOSetupCache()
@@ -426,13 +581,13 @@ void ISOSeek(u32 Offset)
 	const u64 Offset64 = (u64)Offset + ISOShift64;
 	if(LastOffset64 != Offset64)
 	{
-		if (ISO_IsCISO)
+		if (ISO_Type == TYPE_CSO)
 		{
 			const u32 blockIdx = (u32)(Offset64 / CISO_BLOCK_SIZE);
 			if (blockIdx >= CISO_MAP_SIZE)
 				return;
 
-			const u16 physBlockIdx = ciso_block_map[blockIdx];
+			const u16 physBlockIdx = block_map.ciso_block_map[blockIdx];
 			if (physBlockIdx == 0xFFFF)
 				return;
 
@@ -443,6 +598,9 @@ void ISOSeek(u32 Offset)
 			else
 				f_lseek( &GameFile, physAddr );
 		}
+		else if (ISO_Type == TYPE_ZSO){
+			// TODO (?)
+		}
 		else
 		{
 			if(wiiVCInternal)

kernel/Makefile

@@ -36,7 +36,7 @@ OBJECTS	:= start.o common.o alloc.o GCAM.o JVSIO.o JVSIOMessage.o FST.o DI.o Rea
 	   Patch.o PatchTimers.o TRI.o PatchWidescreen.o ISO.o Stream.o adp.o \
 	   EXI.o SRAM.o GCNCard.o SI.o HID.o diskio.o Config.o utils_asm.o ES.o NAND.o \
 	   main.o syscalls.o ReadSpeed.o vsprintf.o string.o prs.o \
-	   SDI.o usb.o usbstorage.o wdvd.o sock.o
+	   SDI.o usb.o usbstorage.o wdvd.o sock.o lz4.o
 LIBS	:= ../fatfs/libfatfs-arm.a be/libc.a be/libgcc.a
 ZIPFILE	:= ../loader/data/kernel.zip