Add filesystem support from the assets branch

projs/shadow/shadow-engine/assets/CMakeLists.txt

@@ -1,9 +1,9 @@
 
 FILE(GLOB_RECURSE SOURCES
         ${CMAKE_CURRENT_LIST_DIR}/src/*.cpp
-)
+        )
 
 target_shadow_module(shadow-engine
         SOURCES ${SOURCES}
         INCLUDE_DIR ${CMAKE_CURRENT_LIST_DIR}/inc/
-)
+        )

projs/shadow/shadow-engine/assets/inc/shadow/assets/fs/file.h

@@ -0,0 +1,114 @@
+#pragma once
+#include <memory>
+#include "iostream.h"
+#include "path.h"
+
+template <class T> struct Delegate;
+
+namespace ShadowEngine {
+
+    // An input stream that can read a file on disk.
+    struct FileInput final : InputStream {
+        FileInput();
+        ~FileInput() = default;
+
+        [[nodiscard]] bool open(const std::string& path);
+        void close();
+
+        using InputStream::read;
+        [[nodiscard]] bool read(void* data, size_t size) override;
+        const void* getBuffer() const override { return nullptr; }
+
+        size_t size() const override;
+        size_t pos();
+
+        [[nodiscard]] bool seek(size_t pos);
+
+    private:
+        void* handle;
+    };
+
+    // An output stream that can write to a file on disk.
+    struct FileOutput final : OutputStream {
+        FileOutput();
+        ~FileOutput() = default;
+
+        [[nodiscard]] bool open(const std::string& path);
+        void close();
+        void flush();
+        bool errored() const { return error; }
+        using OutputStream::write;
+        [[nodiscard]] bool write(const void* data, size_t size) override;
+
+    private:
+        FileOutput(const FileOutput&) = delete;
+        void* handle;
+        bool error;
+    };
+
+    struct FileInfo {
+        bool directory;
+        std::string filename;
+    };
+
+
+    /**
+     * A generic Filesystem API.
+     * Allows interacting with files on disk the same as files in our Virtual Package Format.
+     */
+    struct FileSystem {
+        // A function called when the data of a file is updated, such as when an asynchronous operation completes.
+        using ContentCallback = Delegate<void(size_t, const uint8_t*, bool)>;
+        // A handle for asynchronous data movement; such as reading or writing a file.
+        struct AsyncHandle {
+            static AsyncHandle invalid() { return AsyncHandle(0xffffffff); }
+            explicit AsyncHandle(uint32_t val) : value(val) {}
+
+            [[nodiscard]] bool valid() const { return value != 0xffffffff; }
+
+            uint32_t value;
+        };
+
+        // Create a Filesystem that interacts with files on disk.
+        static std::unique_ptr<FileSystem> createDiskFS(const std::string& basePath);
+        // Create a Virtual Filesystem based on the given path.
+        static std::unique_ptr<FileSystem> createVFS(const std::string& basePath);
+
+        virtual ~FileSystem() = default;
+
+        // Open a file for reading.
+        virtual bool open(const std::string& path, FileInput& input) = 0;
+        // Open a file for writing.
+        virtual bool open(const std::string& path, FileOutput& output) = 0;
+        // Check whether a file exists at the given path.
+        virtual bool fileExists(const std::string& path) = 0;
+        // Get the time a file at the given path was last modified.
+        virtual size_t getLastModified(const std::string& path) = 0;
+        // Copy a file from one path to another.
+        virtual bool copyFile(const std::string& from, const std::string& to) = 0;
+        // Move a file from one path to another.
+        virtual bool moveFile(const std::string& from, const std::string& to) = 0;
+        // Disassociate any files at the given path (not an immediate delete)
+        virtual bool deleteFile(const std::string& path) = 0;
+
+        // Get the path that this FileSystem originates at. The default is "/" for VFS, and whatever the Executable Path is for Disk FS.
+        virtual std::string const& getBasePath() const = 0;
+        // Set a new base path for the FileSystem. Any operations involving file paths will be relative to this new path.
+        virtual void setBasePath(const std::string& path) = 0;
+
+        // Process all the callbacks for async file operations.
+        virtual void processCallbacks() = 0;
+        // Check whether there are any outstanding async operations that need work.
+        virtual bool hasWork() = 0;
+
+        // Write new content to a file synchronously. The thread will be blocked when doing this.
+        virtual bool saveSync(const Path& file, const uint8_t* content, size_t size) = 0;
+        // Read content from a file synchronously. The thread will be blocked when doing this.
+        virtual bool readSync(const Path& file, struct OutputMemoryStream& content) = 0;
+
+        // Read a file asynchronously. The given callback will be called with the file content once it is available.
+        virtual AsyncHandle readAsync(const Path& file, const ContentCallback& callback) = 0;
+        // Cancel an asynchronous operation, if it is not already complete. The associated callback will be called with a special flag.
+        virtual void cancelAsync(AsyncHandle& handle) = 0;
+    };
+}

projs/shadow/shadow-engine/assets/inc/shadow/assets/fs/hash.h

@@ -0,0 +1,158 @@
+#pragma once
+#include <string>
+
+namespace ShadowEngine {
+
+    /**
+     * A 64-bit hashing algorithm that uses the state of the allocation heap as a "salt".
+     * Outputs are NOT stable, so do not serialize this.
+     * However, because it uses the heap, it has a very low collision rate.
+     */
+    struct HeapHash {
+        // For if you MUST recreate a hash exactly.
+        // Please only use this for testing.
+        static HeapHash fromLong(size_t hash);
+
+        HeapHash() = default;
+        // Hash a string; for paths and such.
+        explicit HeapHash(const std::string& str);
+        // Hash arbitrary data.
+        HeapHash(const void* data, uint32_t length);
+
+        bool operator!= (const HeapHash& other) const { return hash != other.hash; }
+        bool operator== (const HeapHash& other) const { return hash == other.hash; }
+
+        size_t getHash() const { return hash; }
+    private:
+        size_t hash = 0;
+    };
+
+    /**
+    * A 32-bit hashing algorithm that uses the state of the allocation heap as a "salt".
+    * Outputs are NOT stable, so do not serialize this.
+    * However, because it uses the heap, it has a very low collision rate.
+    */
+    struct HeapHash32 {
+        // For if you MUST recreate a hash exactly.
+        // Please only use this for testing.
+        static HeapHash32 fromInt(uint32_t hash);
+
+        HeapHash32() = default;
+        // Hash a string; for paths and such.
+        explicit HeapHash32(const std::string& str);
+        // Hash arbitrary data.
+        HeapHash32(const void* data, uint32_t length);
+
+        bool operator!= (HeapHash32& other) const { return hash != other.hash; }
+        bool operator== (HeapHash32& other) const { return hash == other.hash; }
+
+        uint32_t getHash() const { return hash; }
+    private:
+        uint32_t hash = 0;
+    };
+
+    /**
+     * A 64-bit hashing algorithm that generates the same hash value per input every time.
+     * A little more likely to generate conflicts than the hash that uses the state of the heap as a salt.
+     * Suitable for serialization.
+     */
+    struct StableHash {
+        static StableHash fromLong(size_t data);
+        StableHash() = default;
+        explicit StableHash(const std::string& str);
+        StableHash(const void* data, uint32_t length);
+
+        bool operator!= (const StableHash& other) const { return hash != other.hash; }
+        bool operator== (const StableHash& other) const { return hash == other.hash; }
+        bool operator< (const StableHash& other) const { return hash < other.hash; }
+
+        [[nodiscard]] size_t getHash() const { return hash; }
+
+    private:
+        size_t hash = 0;
+    };
+
+    /**
+     * A 32-bit hashing algorithm that generates the same hash value per input every time.
+     * A little more likely to generate conflicts than the hash that uses the state of the heap as a salt.
+     * Suitable for serialization.
+     */
+    struct StableHash32 {
+        static StableHash32 fromInt(uint32_t data);
+        StableHash32() = default;
+        StableHash32(const std::string& str);
+        StableHash32(const void* data, uint32_t length);
+
+        bool operator!= (StableHash32& other) const { return hash != other.hash; }
+        bool operator== (StableHash32& other) const { return hash == other.hash; }
+        bool operator< (StableHash32& other) const { return hash < other.hash; }
+
+        uint32_t getHash() const { return hash; }
+
+    private:
+        uint32_t hash = 0;
+    };
+
+    // File Paths are hashed using the 64-bit StableHash system.
+    using PathHash = StableHash;
+
+    /**
+     * A hashing utility that lets you insert data piecemeal before committing to the hash.
+     * Useful for when you're parsing a file and need to wait for more data to be available before hashing.
+     * Generates a Stable Hash.
+     */
+    struct DeferredHash {
+        DeferredHash();
+        // Insert new data to be considered for hashing
+        void insert(const void* data, uint32_t length);
+        // Submit the data to the hashing algorithm, and return a value in 64-bit StableHash
+        StableHash submit();
+        // Submit the data to the hashing algorithm, and return a value in 32-bit StableHash
+        StableHash32 submit32();
+    };
+
+    /**
+     * A hashing utility that lets you insert data piecemeal before committing to the hash.
+     * Useful for when you're parsing a file and need to wait for more data to be available before hashing.
+     * Generates a Heap Hash.
+     */
+    struct DeferredHeapHash {
+        DeferredHeapHash();
+        // Insert new data to be considered for hashing
+        void insert(const void* data, uint32_t length);
+        // Submit the data to the hashing algorithm, and return a value in 64-bit HeapHash
+        HeapHash submit();
+        // Submit the data to the hashing algorithm, and return a value in 32-bit HeapHash
+        HeapHash32 submit32();
+    };
+
+    /** The implementations of these hashing algorithms */
+
+    template <class Hash> struct HashFunc;
+
+    template<> struct HashFunc<HeapHash> {
+        static uint32_t get(const HeapHash& h) {
+            const size_t hash = h.getHash();
+            return uint32_t(hash & (hash >> 16));
+        }
+    };
+
+    template<> struct HashFunc<StableHash> {
+        static uint32_t get(const StableHash& h) {
+            const size_t hash = h.getHash();
+            return uint32_t(hash & (hash >> 16));
+        }
+    };
+
+    template<> struct HashFunc<HeapHash32> {
+        static uint32_t get(const HeapHash32& h) {
+            return h.getHash();
+        }
+    };
+
+    template<> struct HashFunc<StableHash32> {
+        static uint32_t get(const StableHash& h) {
+            return h.getHash();
+        }
+    };
+}