diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de8f653..08ec089 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,10 @@ jobs: chmod +x llvm.sh sudo ./llvm.sh 18 rm llvm.sh + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '22' - name: Build run: make all - name: Install tools @@ -25,7 +29,6 @@ jobs: run: make test - name: Benchmark run: make benchmark - macos: runs-on: macos-latest diff --git a/docs/fs.md b/docs/fs.md index 3155ff6..e9efb47 100644 --- a/docs/fs.md +++ b/docs/fs.md @@ -8,7 +8,7 @@ one entry file named `main.js`, and ckb-js-vm will load this file system from any cell and execute `main.js` in it. A file system is represented as a binary file in the format described below. We -may use the script [fs.lua](../tools/fs.lua) to create a file system from given +may use the script [fs-packer](../tools/fs-packer) to create a file system from given files or unpack the file system into files. ## How to create a Simple File System @@ -35,9 +35,9 @@ export function fib(n) { If we want ckb-js-vm to execute this code smoothly, we must package them into a file system first. To pack them within the current directory into `fib.fs`, you -may run +may run ```shell -find . -name *.js -type f | lua tools/fs.lua pack fib.fs +find . -name *.js -type f | node tools/fs-packer/dist/index.js pack fib.fs ``` ``` @@ -46,7 +46,7 @@ packing file ./fib_module.js to fib_module.js packing file ./main.js to main.js ``` -Note that all file paths piped into the `fs.lua` must be in the relative path +Note that all file paths piped into the `fs-packer` must be in the relative path format. The absolute path of a file in the current system is usually meaningless in the Simple File System. @@ -88,7 +88,7 @@ JavaScript files. ## Unpack Simple File System to Files -To unpack the files contained within a fs, you may run `lua tools/fs.lua unpack fib.fs .`. +To unpack the files contained within a fs, you may run `node tools/fs-packer/dist/index.js unpack fib.fs .`. ## Simple File System On-disk Representation diff --git a/tests/ckb_js_tests/Makefile b/tests/ckb_js_tests/Makefile index 03b2360..6d90116 100644 --- a/tests/ckb_js_tests/Makefile +++ b/tests/ckb_js_tests/Makefile @@ -24,12 +24,12 @@ module: spawn_caller cargo run --bin module | ${CKB_DEBUGGER} --tx-file=- -s lock build/bytecode/fs_modules.fs: test_data/fs_module/main.js test_data/fs_module/fib_module.js - cd test_data/fs_module && lua ../../../../tools/fs.lua pack ../../../../$@ main.js fib_module.js + cd test_data/fs_module && node ../../../../tools/fs-packer/dist/index.js pack ../../../../$@ main.js fib_module.js fs_bytecode: $(CKB_DEBUGGER) --read-file test_data/fs_module/main.js --bin $(BIN_PATH) -- -c | awk -f $(ROOT_DIR)/../../tools/compile.awk | xxd -r -p > ../../build/bytecode/main.bc $(CKB_DEBUGGER) --read-file test_data/fs_module/fib_module.js --bin $(BIN_PATH) -- -c | awk -f $(ROOT_DIR)/../../tools/compile.awk | xxd -r -p > ../../build/bytecode/fib_module.bc - cd ../../build/bytecode && lua ../../tools/fs.lua pack ../../build/bytecode/fs_modules_bc.fs main.bc fib_module.bc + cd ../../build/bytecode && node ../../tools/fs-packer/dist/index.js pack ../../build/bytecode/fs_modules_bc.fs main.bc fib_module.bc $(CKB_DEBUGGER) --max-cycles $(MAX_CYCLES) --read-file ../../build/bytecode/fs_modules_bc.fs --bin $(BIN_PATH) -- -f -r 2>&1 | fgrep 'Run result: 0' file_system: build/bytecode/fs_modules.fs @@ -39,7 +39,7 @@ syscall: cargo run --bin syscall | $(CKB_DEBUGGER) --tx-file=- -s lock fs_mount: - cd test_data/fs_module_mount && lua ../../../../tools/fs.lua pack ../../../../build/bytecode/fib_module.fs fib_module.js + cd test_data/fs_module_mount && node ../../../../tools/fs-packer/dist/index.js pack ../../../../build/bytecode/fib_module.fs fib_module.js cargo run --bin module_mount | ${CKB_DEBUGGER} --tx-file=- -s lock simple_udt: diff --git a/tools/fs-packer/.gitignore b/tools/fs-packer/.gitignore new file mode 100644 index 0000000..faeabe9 --- /dev/null +++ b/tools/fs-packer/.gitignore @@ -0,0 +1,3 @@ +node_modules/ +test.pak +test-output diff --git a/tools/fs-packer/dist/index.js b/tools/fs-packer/dist/index.js new file mode 100644 index 0000000..7fc0fc2 --- /dev/null +++ b/tools/fs-packer/dist/index.js @@ -0,0 +1,434 @@ +"use strict"; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype); + return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +var __asyncValues = (this && this.__asyncValues) || function (o) { + if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); + var m = o[Symbol.asyncIterator], i; + return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i); + function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; } + function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.pack = pack; +exports.unpack = unpack; +var fs = require("node:fs/promises"); +var fsSync = require("node:fs"); +var path = require("path"); +function getFileSize(filePath) { + return __awaiter(this, void 0, void 0, function () { + var stats; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, fs.stat(filePath)]; + case 1: + stats = _a.sent(); + return [2 /*return*/, stats.size]; + } + }); + }); +} +function writeToFile(data, writeStream) { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_a) { + return [2 /*return*/, new Promise(function (resolve, reject) { + var canContinue = writeStream.write(data); + if (canContinue) { + resolve(); + } + else { + writeStream.once("drain", resolve); + writeStream.once("error", reject); + } + })]; + }); + }); +} +function appendFileToStream(filePath, stream) { + return __awaiter(this, void 0, void 0, function () { + var content; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, fs.readFile(filePath)]; + case 1: + content = _a.sent(); + return [4 /*yield*/, writeToFile(content, stream)]; + case 2: + _a.sent(); + return [2 /*return*/]; + } + }); + }); +} +function appendStringNullToStream(str, stream) { + return __awaiter(this, void 0, void 0, function () { + var buffer; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + buffer = Buffer.from(str + "\0"); + return [4 /*yield*/, writeToFile(buffer, stream)]; + case 1: + _a.sent(); + return [2 /*return*/]; + } + }); + }); +} +function appendIntegerToStream(num, stream) { + return __awaiter(this, void 0, void 0, function () { + var buffer; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + buffer = Buffer.alloc(4); + buffer.writeInt32LE(num); + return [4 /*yield*/, writeToFile(buffer, stream)]; + case 1: + _a.sent(); + return [2 /*return*/]; + } + }); + }); +} +function pack(files, outputStream) { + return __awaiter(this, void 0, void 0, function () { + var numFiles, offset, length, _i, _a, _b, name_1, filePath, _c, _d, _e, name_2, filePath; + return __generator(this, function (_f) { + switch (_f.label) { + case 0: + numFiles = Object.keys(files).length; + return [4 /*yield*/, appendIntegerToStream(numFiles, outputStream)]; + case 1: + _f.sent(); + offset = 0; + length = 0; + _i = 0, _a = Object.entries(files); + _f.label = 2; + case 2: + if (!(_i < _a.length)) return [3 /*break*/, 9]; + _b = _a[_i], name_1 = _b[0], filePath = _b[1]; + console.log("packing file ".concat(filePath, " to ").concat(name_1)); + return [4 /*yield*/, appendIntegerToStream(offset, outputStream)]; + case 3: + _f.sent(); + length = Buffer.byteLength(name_1) + 1; + return [4 /*yield*/, appendIntegerToStream(length, outputStream)]; + case 4: + _f.sent(); + offset += length; + return [4 /*yield*/, appendIntegerToStream(offset, outputStream)]; + case 5: + _f.sent(); + return [4 /*yield*/, getFileSize(filePath)]; + case 6: + length = _f.sent(); + return [4 /*yield*/, appendIntegerToStream(length, outputStream)]; + case 7: + _f.sent(); + offset += length; + _f.label = 8; + case 8: + _i++; + return [3 /*break*/, 2]; + case 9: + _c = 0, _d = Object.entries(files); + _f.label = 10; + case 10: + if (!(_c < _d.length)) return [3 /*break*/, 14]; + _e = _d[_c], name_2 = _e[0], filePath = _e[1]; + return [4 /*yield*/, appendStringNullToStream(name_2, outputStream)]; + case 11: + _f.sent(); + return [4 /*yield*/, appendFileToStream(filePath, outputStream)]; + case 12: + _f.sent(); + _f.label = 13; + case 13: + _c++; + return [3 /*break*/, 10]; + case 14: return [2 /*return*/]; + } + }); + }); +} +function createDirectory(dir) { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_a) { + switch (_a.label) { + case 0: return [4 /*yield*/, fs.mkdir(dir, { recursive: true })]; + case 1: + _a.sent(); + return [2 /*return*/]; + } + }); + }); +} +function unpack(directory, fileContent) { + return __awaiter(this, void 0, void 0, function () { + function readInteger() { + var value = fileContent.readInt32LE(position); + position += 4; + return value; + } + function readStringNull(length) { + var value = fileContent + .toString("utf8", position, position + length) + .replace(/\0$/, ""); + position += length; + return value; + } + function copyToFile(directory, filename, offset, length) { + return __awaiter(this, void 0, void 0, function () { + var normalizedFilename, filePath, dir, content; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + normalizedFilename = filename.replace(/\//g, path.sep); + filePath = path.join(directory, normalizedFilename); + dir = path.dirname(filePath); + return [4 /*yield*/, createDirectory(dir)]; + case 1: + _a.sent(); + console.log("unpacking file ".concat(filename, " to ").concat(filePath)); + content = fileContent.slice(offset, offset + length); + return [4 /*yield*/, fs.writeFile(filePath, content)]; + case 2: + _a.sent(); + return [2 /*return*/]; + } + }); + }); + } + var position, numFiles, metadata, i, blobStart, _i, metadata_1, metadatum, filename; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + position = 0; + numFiles = readInteger(); + metadata = []; + // Read metadata + for (i = 0; i < numFiles; i++) { + metadata.push({ + fileNameOffset: readInteger(), + fileNameLength: readInteger(), + fileContentOffset: readInteger(), + fileContentLength: readInteger(), + }); + } + blobStart = position; + _i = 0, metadata_1 = metadata; + _a.label = 1; + case 1: + if (!(_i < metadata_1.length)) return [3 /*break*/, 4]; + metadatum = metadata_1[_i]; + position = blobStart + metadatum.fileNameOffset; + filename = readStringNull(metadatum.fileNameLength); + position = blobStart + metadatum.fileContentOffset; + return [4 /*yield*/, copyToFile(directory, filename, position, metadatum.fileContentLength)]; + case 2: + _a.sent(); + _a.label = 3; + case 3: + _i++; + return [3 /*break*/, 1]; + case 4: return [2 /*return*/]; + } + }); + }); +} +function isIdealPath(path) { + return (!path.startsWith("/") && !path.startsWith("..") && !/^[a-zA-Z]:/.test(path)); +} +function normalizeRelativePath(path) { + return path.replace(/^\.\//, "").replace(/\/\.\//g, "/"); +} +function mustNormalizePath(path) { + if (!isIdealPath(path)) { + throw new Error("Either not a relative path or a relative path referring to ..: ".concat(path)); + } + return normalizeRelativePath(path); +} +// Add CLI functionality +function usage(msg) { + if (msg) + console.log(msg); + console.log("".concat(process.argv[1], " pack output_file [files] | ").concat(process.argv[1], " unpack input_file [directory]")); +} +function doPack() { + return __awaiter(this, void 0, void 0, function () { + var outfile, stream, files, n, i, file, chunks, _a, _b, _c, chunk, e_1_1, input, _i, _d, file; + var _e, e_1, _f, _g; + return __generator(this, function (_h) { + switch (_h.label) { + case 0: + if (process.argv.length === 2) { + usage("You must specify the output file."); + process.exit(1); + } + outfile = process.argv[3]; + stream = fsSync.createWriteStream(outfile); + files = {}; + n = 0; + if (!(process.argv.length !== 3)) return [3 /*break*/, 1]; + // Read files from command line arguments + for (i = 4; i < process.argv.length; i++) { + n++; + file = process.argv[i]; + files[mustNormalizePath(file)] = file; + } + return [3 /*break*/, 14]; + case 1: + chunks = []; + _h.label = 2; + case 2: + _h.trys.push([2, 7, 8, 13]); + _a = true, _b = __asyncValues(process.stdin); + _h.label = 3; + case 3: return [4 /*yield*/, _b.next()]; + case 4: + if (!(_c = _h.sent(), _e = _c.done, !_e)) return [3 /*break*/, 6]; + _g = _c.value; + _a = false; + chunk = _g; + chunks.push(Buffer.from(chunk)); + _h.label = 5; + case 5: + _a = true; + return [3 /*break*/, 3]; + case 6: return [3 /*break*/, 13]; + case 7: + e_1_1 = _h.sent(); + e_1 = { error: e_1_1 }; + return [3 /*break*/, 13]; + case 8: + _h.trys.push([8, , 11, 12]); + if (!(!_a && !_e && (_f = _b.return))) return [3 /*break*/, 10]; + return [4 /*yield*/, _f.call(_b)]; + case 9: + _h.sent(); + _h.label = 10; + case 10: return [3 /*break*/, 12]; + case 11: + if (e_1) throw e_1.error; + return [7 /*endfinally*/]; + case 12: return [7 /*endfinally*/]; + case 13: + input = Buffer.concat(chunks).toString("utf-8"); + for (_i = 0, _d = input.split("\n").filter(Boolean); _i < _d.length; _i++) { + file = _d[_i]; + n++; + files[mustNormalizePath(file)] = file; + } + _h.label = 14; + case 14: + if (n === 0) { + usage("You must at least specify one file to pack"); + process.exit(1); + } + return [4 /*yield*/, pack(files, stream)]; + case 15: + _h.sent(); + stream.end(); + return [2 /*return*/]; + } + }); + }); +} +function doUnpack() { + return __awaiter(this, void 0, void 0, function () { + var infile, _a, fileContent, directory; + return __generator(this, function (_b) { + switch (_b.label) { + case 0: + if (process.argv.length === 2) { + usage("You must specify the input file when unpacking."); + process.exit(1); + } + infile = process.argv[3]; + _b.label = 1; + case 1: + _b.trys.push([1, 3, , 4]); + return [4 /*yield*/, fs.access(infile)]; + case 2: + _b.sent(); + return [3 /*break*/, 4]; + case 3: + _a = _b.sent(); + console.error("Error: File '".concat(infile, "' does not exist")); + process.exit(1); + return [3 /*break*/, 4]; + case 4: return [4 /*yield*/, fs.readFile(infile)]; + case 5: + fileContent = _b.sent(); + directory = process.argv.length !== 3 ? process.argv[4] : "."; + return [4 /*yield*/, unpack(directory, fileContent)]; + case 6: + _b.sent(); + return [2 /*return*/]; + } + }); + }); +} +// Main program +function main() { + return __awaiter(this, void 0, void 0, function () { + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + if (process.argv.length <= 2 || + !["pack", "unpack"].includes(process.argv[2])) { + usage("Please specify whether to pack or unpack"); + process.exit(1); + } + if (!(process.argv[2] === "pack")) return [3 /*break*/, 2]; + return [4 /*yield*/, doPack()]; + case 1: + _a.sent(); + return [3 /*break*/, 4]; + case 2: return [4 /*yield*/, doUnpack()]; + case 3: + _a.sent(); + _a.label = 4; + case 4: return [2 /*return*/]; + } + }); + }); +} +// Only run if this is the main module +if (require.main === module) { + main(); +} diff --git a/tools/fs-packer/package-lock.json b/tools/fs-packer/package-lock.json new file mode 100644 index 0000000..aed2009 --- /dev/null +++ b/tools/fs-packer/package-lock.json @@ -0,0 +1,68 @@ +{ + "name": "fs-packer", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "fs-packer", + "version": "1.0.0", + "license": "ISC", + "devDependencies": { + "@types/node": "^22.10.2", + "node": "^23.5.0", + "typescript": "^5.7.2" + } + }, + "node_modules/@types/node": { + "version": "22.10.2", + "resolved": "https://registry.npmmirror.com/@types/node/-/node-22.10.2.tgz", + "integrity": "sha512-Xxr6BBRCAOQixvonOye19wnzyDiUtTeqldOOmj3CkeblonbccA12PFwlufvRdrpjXxqnmUaeiU5EOA+7s5diUQ==", + "dev": true, + "dependencies": { + "undici-types": "~6.20.0" + } + }, + "node_modules/node": { + "version": "23.5.0", + "resolved": "https://registry.npmmirror.com/node/-/node-23.5.0.tgz", + "integrity": "sha512-Wco8qYfFUAotVJJoMbB30cYdPbTqFd9QtzC528GvTCYWMldnPUu1pLNz4sKNKxal+dgBuAyUu8tRkeLVx1VT8Q==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "node-bin-setup": "^1.0.0" + }, + "bin": { + "node": "bin/node" + }, + "engines": { + "npm": ">=5.0.0" + } + }, + "node_modules/node-bin-setup": { + "version": "1.1.3", + "resolved": "https://registry.npmmirror.com/node-bin-setup/-/node-bin-setup-1.1.3.tgz", + "integrity": "sha512-opgw9iSCAzT2+6wJOETCpeRYAQxSopqQ2z+N6BXwIMsQQ7Zj5M8MaafQY8JMlolRR6R1UXg2WmhKp0p9lSOivg==", + "dev": true + }, + "node_modules/typescript": { + "version": "5.7.2", + "resolved": "https://registry.npmmirror.com/typescript/-/typescript-5.7.2.tgz", + "integrity": "sha512-i5t66RHxDvVN40HfDd1PsEThGNnlMCMT3jMUuoh9/0TaqWevNontacunWyN02LA9/fIbEWlcHZcgTKb9QoaLfg==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.20.0", + "resolved": "https://registry.npmmirror.com/undici-types/-/undici-types-6.20.0.tgz", + "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", + "dev": true + } + } +} diff --git a/tools/fs-packer/package.json b/tools/fs-packer/package.json new file mode 100644 index 0000000..47e3e8f --- /dev/null +++ b/tools/fs-packer/package.json @@ -0,0 +1,21 @@ +{ + "name": "fs-packer", + "version": "1.0.0", + "main": "index.js", + "scripts": { + "build": "tsc src/index.ts --outDir dist", + "pretty": "prettier --write src/**/*.ts", + "test:pack": "node dist/index.js pack test.pak dist/index.js src/index.ts", + "test:unpack": "node dist/index.js unpack test.pak test-output", + "test:compare": "cmp dist/index.js test-output/dist/index.js && cmp src/index.ts test-output/src/index.ts && echo 'passed'", + "test": "npm run build && npm run test:pack && npm run test:unpack && npm run test:compare" + }, + "author": "", + "license": "ISC", + "description": "", + "devDependencies": { + "typescript": "^5.7.2", + "@types/node": "^22.10.2", + "node": "^23.5.0" + } +} diff --git a/tools/fs-packer/src/index.ts b/tools/fs-packer/src/index.ts new file mode 100644 index 0000000..2250b90 --- /dev/null +++ b/tools/fs-packer/src/index.ts @@ -0,0 +1,266 @@ +import * as fs from "node:fs/promises"; +import * as fsSync from "node:fs"; +import * as path from "path"; + +interface FileMetadata { + fileNameOffset: number; + fileNameLength: number; + fileContentOffset: number; + fileContentLength: number; +} + +interface FileMap { + [key: string]: string; +} + +async function getFileSize(filePath: string): Promise { + const stats = await fs.stat(filePath); + return stats.size; +} + +async function writeToFile( + data: Buffer | string, + writeStream: fsSync.WriteStream, +): Promise { + return new Promise((resolve, reject) => { + const canContinue = writeStream.write(data); + if (canContinue) { + resolve(); + } else { + writeStream.once("drain", resolve); + writeStream.once("error", reject); + } + }); +} + +async function appendFileToStream( + filePath: string, + stream: fsSync.WriteStream, +): Promise { + const content = await fs.readFile(filePath); + await writeToFile(content, stream); +} + +async function appendStringNullToStream( + str: string, + stream: fsSync.WriteStream, +): Promise { + const buffer = Buffer.from(str + "\0"); + await writeToFile(buffer, stream); +} + +async function appendIntegerToStream( + num: number, + stream: fsSync.WriteStream, +): Promise { + const buffer = Buffer.alloc(4); + buffer.writeInt32LE(num); + await writeToFile(buffer, stream); +} + +async function pack( + files: FileMap, + outputStream: fsSync.WriteStream, +): Promise { + const numFiles = Object.keys(files).length; + await appendIntegerToStream(numFiles, outputStream); + + let offset = 0; + let length = 0; + + // Write metadata + for (const [name, filePath] of Object.entries(files)) { + console.log(`packing file ${filePath} to ${name}`); + await appendIntegerToStream(offset, outputStream); + length = Buffer.byteLength(name) + 1; + await appendIntegerToStream(length, outputStream); + offset += length; + await appendIntegerToStream(offset, outputStream); + length = await getFileSize(filePath); + await appendIntegerToStream(length, outputStream); + offset += length; + } + + // Write actual file data + for (const [name, filePath] of Object.entries(files)) { + await appendStringNullToStream(name, outputStream); + await appendFileToStream(filePath, outputStream); + } +} + +async function createDirectory(dir: string): Promise { + await fs.mkdir(dir, { recursive: true }); +} + +async function unpack(directory: string, fileContent: Buffer): Promise { + let position = 0; + + function readInteger(): number { + const value = fileContent.readInt32LE(position); + position += 4; + return value; + } + + function readStringNull(length: number): string { + const value = fileContent + .toString("utf8", position, position + length) + .replace(/\0$/, ""); + position += length; + return value; + } + + async function copyToFile( + directory: string, + filename: string, + offset: number, + length: number, + ): Promise { + const normalizedFilename = filename.replace(/\//g, path.sep); + const filePath = path.join(directory, normalizedFilename); + const dir = path.dirname(filePath); + + await createDirectory(dir); + console.log(`unpacking file ${filename} to ${filePath}`); + + const content = fileContent.slice(offset, offset + length); + await fs.writeFile(filePath, content); + } + + const numFiles = readInteger(); + const metadata: FileMetadata[] = []; + + // Read metadata + for (let i = 0; i < numFiles; i++) { + metadata.push({ + fileNameOffset: readInteger(), + fileNameLength: readInteger(), + fileContentOffset: readInteger(), + fileContentLength: readInteger(), + }); + } + + const blobStart = position; + + for (const metadatum of metadata) { + position = blobStart + metadatum.fileNameOffset; + const filename = readStringNull(metadatum.fileNameLength); + position = blobStart + metadatum.fileContentOffset; + await copyToFile( + directory, + filename, + position, + metadatum.fileContentLength, + ); + } +} + +function isIdealPath(path: string): boolean { + return ( + !path.startsWith("/") && !path.startsWith("..") && !/^[a-zA-Z]:/.test(path) + ); +} + +function normalizeRelativePath(path: string): string { + return path.replace(/^\.\//, "").replace(/\/\.\//g, "/"); +} + +function mustNormalizePath(path: string): string { + if (!isIdealPath(path)) { + throw new Error( + `Either not a relative path or a relative path referring to ..: ${path}`, + ); + } + return normalizeRelativePath(path); +} + +// Add CLI functionality +function usage(msg?: string): void { + if (msg) console.log(msg); + console.log( + `${process.argv[1]} pack output_file [files] | ${process.argv[1]} unpack input_file [directory]`, + ); +} + +async function doPack(): Promise { + if (process.argv.length === 2) { + usage("You must specify the output file."); + process.exit(1); + } + + const outfile = process.argv[3]; + const stream = fsSync.createWriteStream(outfile); + const files: FileMap = {}; + let n = 0; + + if (process.argv.length !== 3) { + // Read files from command line arguments + for (let i = 4; i < process.argv.length; i++) { + n++; + const file = process.argv[i]; + files[mustNormalizePath(file)] = file; + } + } else { + // Read files from stdin + const chunks: Buffer[] = []; + for await (const chunk of process.stdin) { + chunks.push(Buffer.from(chunk)); + } + const input = Buffer.concat(chunks).toString("utf-8"); + for (const file of input.split("\n").filter(Boolean)) { + n++; + files[mustNormalizePath(file)] = file; + } + } + + if (n === 0) { + usage("You must at least specify one file to pack"); + process.exit(1); + } + + await pack(files, stream); + stream.end(); +} + +async function doUnpack(): Promise { + if (process.argv.length === 2) { + usage("You must specify the input file when unpacking."); + process.exit(1); + } + + const infile = process.argv[3]; + try { + await fs.access(infile); + } catch { + console.error(`Error: File '${infile}' does not exist`); + process.exit(1); + } + + const fileContent = await fs.readFile(infile); + const directory = process.argv.length !== 3 ? process.argv[4] : "."; + await unpack(directory, fileContent); +} + +// Main program +async function main(): Promise { + if ( + process.argv.length <= 2 || + !["pack", "unpack"].includes(process.argv[2]) + ) { + usage("Please specify whether to pack or unpack"); + process.exit(1); + } + + if (process.argv[2] === "pack") { + await doPack(); + } else { + await doUnpack(); + } +} + +// Only run if this is the main module +if (require.main === module) { + main(); +} + +// Export the functions for use as a module +export { pack, unpack, FileMap, FileMetadata }; diff --git a/tools/fs-packer/tsconfig.json b/tools/fs-packer/tsconfig.json new file mode 100644 index 0000000..2de67a0 --- /dev/null +++ b/tools/fs-packer/tsconfig.json @@ -0,0 +1,109 @@ +{ + "compilerOptions": { + /* Visit https://aka.ms/tsconfig to read more about this file */ + + /* Projects */ + // "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */ + // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ + // "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */ + // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */ + // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ + // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ + + /* Language and Environment */ + "target": "ES2020", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ + // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ + // "jsx": "preserve", /* Specify what JSX code is generated. */ + // "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */ + // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ + // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */ + // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ + // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */ + // "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */ + // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ + // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ + // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */ + + /* Modules */ + "module": "commonjs", /* Specify what module code is generated. */ + // "rootDir": "./", /* Specify the root folder within your source files. */ + // "moduleResolution": "node10", /* Specify how TypeScript looks up a file from a given module specifier. */ + // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ + // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ + // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ + // "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */ + // "types": [], /* Specify type package names to be included without being referenced in a source file. */ + // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ + // "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */ + // "allowImportingTsExtensions": true, /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */ + // "resolvePackageJsonExports": true, /* Use the package.json 'exports' field when resolving package imports. */ + // "resolvePackageJsonImports": true, /* Use the package.json 'imports' field when resolving imports. */ + // "customConditions": [], /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */ + // "resolveJsonModule": true, /* Enable importing .json files. */ + // "allowArbitraryExtensions": true, /* Enable importing files with any extension, provided a declaration file is present. */ + // "noResolve": true, /* Disallow 'import's, 'require's or ''s from expanding the number of files TypeScript should add to a project. */ + + /* JavaScript Support */ + // "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */ + // "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ + // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */ + + /* Emit */ + // "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ + // "declarationMap": true, /* Create sourcemaps for d.ts files. */ + // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ + // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ + // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ + // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ + // "outDir": "./", /* Specify an output folder for all emitted files. */ + // "removeComments": true, /* Disable emitting comments. */ + // "noEmit": true, /* Disable emitting files from a compilation. */ + // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ + // "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types. */ + // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ + // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ + // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ + // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ + // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ + // "newLine": "crlf", /* Set the newline character for emitting files. */ + // "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */ + // "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */ + // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ + // "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */ + // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ + // "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */ + + /* Interop Constraints */ + // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ + // "verbatimModuleSyntax": true, /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */ + // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ + "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ + // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ + "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ + + /* Type Checking */ + "strict": true, /* Enable all strict type-checking options. */ + // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ + // "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ + // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ + // "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */ + // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ + // "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ + // "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */ + // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ + // "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ + // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ + // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ + // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ + // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ + // "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */ + // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ + // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */ + // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ + // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ + + /* Completeness */ + // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ + "skipLibCheck": true /* Skip type checking all .d.ts files. */ + } +}