feat: offline mode (#14)

* chore: move mod downloading to cli/cache * feat: data providers, ficsit and local * feat: keep cache in memory, load on init * feat: log invalid cache files instead of returning error * chore: make linter happy * feat: fill cached mod Authors field from CreatedBy * chore: make linter happy again * feat: add icon and size to cached mods * feat: cache the cached file hashes * fix: change to new provider access style --------- Co-authored-by: Vilsol <[email protected]>
satisfactorymodding · Dec 6, 2023 · e4b02a7 · e4b02a7
1 parent ea983cf
commit e4b02a7
Show file tree

Hide file tree

Showing 24 changed files with 909 additions and 141 deletions.
diff --git a/cli/cache/cache.go b/cli/cache/cache.go
@@ -0,0 +1,164 @@
+package cache
+
+import (
+	"archive/zip"
+	"encoding/base64"
+	"encoding/json"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+	"github.com/rs/zerolog/log"
+	"github.com/spf13/viper"
+)
+
+const IconFilename = "Resources/Icon128.png" // This is the path UE expects for the icon
+
+type File struct {
+	Icon         *string
+	ModReference string
+	Hash         string
+	Plugin       UPlugin
+	Size         int64
+}
+
+var loadedCache map[string][]File
+
+func GetCache() (map[string][]File, error) {
+	if loadedCache != nil {
+		return loadedCache, nil
+	}
+	return LoadCache()
+}
+
+func GetCacheMod(mod string) ([]File, error) {
+	cache, err := GetCache()
+	if err != nil {
+		return nil, err
+	}
+	return cache[mod], nil
+}
+
+func LoadCache() (map[string][]File, error) {
+	loadedCache = map[string][]File{}
+	downloadCache := filepath.Join(viper.GetString("cache-dir"), "downloadCache")
+	if _, err := os.Stat(downloadCache); os.IsNotExist(err) {
+		return map[string][]File{}, nil
+	}
+
+	items, err := os.ReadDir(downloadCache)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed reading download cache")
+	}
+
+	for _, item := range items {
+		if item.IsDir() {
+			continue
+		}
+		if item.Name() == integrityFilename {
+			continue
+		}
+
+		_, err = addFileToCache(item.Name())
+		if err != nil {
+			log.Err(err).Str("file", item.Name()).Msg("failed to add file to cache")
+		}
+	}
+	return loadedCache, nil
+}
+
+func addFileToCache(filename string) (*File, error) {
+	cacheFile, err := readCacheFile(filename)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read cache file")
+	}
+
+	loadedCache[cacheFile.ModReference] = append(loadedCache[cacheFile.ModReference], *cacheFile)
+	return cacheFile, nil
+}
+
+func readCacheFile(filename string) (*File, error) {
+	downloadCache := filepath.Join(viper.GetString("cache-dir"), "downloadCache")
+	path := filepath.Join(downloadCache, filename)
+	stat, err := os.Stat(path)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to stat file")
+	}
+
+	zipFile, err := os.Open(path)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to open file")
+	}
+	defer zipFile.Close()
+
+	size := stat.Size()
+	reader, err := zip.NewReader(zipFile, size)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read zip")
+	}
+
+	var upluginFile *zip.File
+	for _, file := range reader.File {
+		if strings.HasSuffix(file.Name, ".uplugin") {
+			upluginFile = file
+			break
+		}
+	}
+	if upluginFile == nil {
+		return nil, errors.New("no uplugin file found in zip")
+	}
+
+	upluginReader, err := upluginFile.Open()
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to open uplugin file")
+	}
+
+	var uplugin UPlugin
+	data, err := io.ReadAll(upluginReader)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to read uplugin file")
+	}
+	if err := json.Unmarshal(data, &uplugin); err != nil {
+		return nil, errors.Wrap(err, "failed to unmarshal uplugin file")
+	}
+
+	modReference := strings.TrimSuffix(upluginFile.Name, ".uplugin")
+
+	hash, err := getFileHash(filename)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to get file hash")
+	}
+
+	var iconFile *zip.File
+	for _, file := range reader.File {
+		if file.Name == IconFilename {
+			iconFile = file
+			break
+		}
+	}
+	var icon *string
+	if iconFile != nil {
+		iconReader, err := iconFile.Open()
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to open icon file")
+		}
+		defer iconReader.Close()
+
+		data, err := io.ReadAll(iconReader)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to read icon file")
+		}
+		iconData := base64.StdEncoding.EncodeToString(data)
+		icon = &iconData
+	}
+
+	return &File{
+		ModReference: modReference,
+		Hash:         hash,
+		Size:         size,
+		Icon:         icon,
+		Plugin:       uplugin,
+	}, nil
+}
diff --git a/cli/cache/download.go b/cli/cache/download.go
@@ -0,0 +1,130 @@
+package cache
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/viper"
+
+	"github.com/satisfactorymodding/ficsit-cli/utils"
+)
+
+type Progresser struct {
+	io.Reader
+	updates chan utils.GenericUpdate
+	total   int64
+	running int64
+}
+
+func (pt *Progresser) Read(p []byte) (int, error) {
+	n, err := pt.Reader.Read(p)
+	pt.running += int64(n)
+
+	if err == nil {
+		if pt.updates != nil {
+			select {
+			case pt.updates <- utils.GenericUpdate{Progress: float64(pt.running) / float64(pt.total)}:
+			default:
+			}
+		}
+	}
+
+	if err == io.EOF {
+		return n, io.EOF
+	}
+
+	return n, errors.Wrap(err, "failed to read")
+}
+
+func DownloadOrCache(cacheKey string, hash string, url string, updates chan utils.GenericUpdate) (io.ReaderAt, int64, error) {
+	downloadCache := filepath.Join(viper.GetString("cache-dir"), "downloadCache")
+	if err := os.MkdirAll(downloadCache, 0o777); err != nil {
+		if !os.IsExist(err) {
+			return nil, 0, errors.Wrap(err, "failed creating download cache")
+		}
+	}
+
+	location := filepath.Join(downloadCache, cacheKey)
+
+	stat, err := os.Stat(location)
+	if err == nil {
+		existingHash := ""
+
+		if hash != "" {
+			f, err := os.Open(location)
+			if err != nil {
+				return nil, 0, errors.Wrap(err, "failed to open file: "+location)
+			}
+
+			existingHash, err = utils.SHA256Data(f)
+			if err != nil {
+				return nil, 0, errors.Wrap(err, "could not compute hash for file: "+location)
+			}
+		}
+
+		if hash == existingHash {
+			f, err := os.Open(location)
+			if err != nil {
+				return nil, 0, errors.Wrap(err, "failed to open file: "+location)
+			}
+
+			return f, stat.Size(), nil
+		}
+
+		if err := os.Remove(location); err != nil {
+			return nil, 0, errors.Wrap(err, "failed to delete file: "+location)
+		}
+	} else if !os.IsNotExist(err) {
+		return nil, 0, errors.Wrap(err, "failed to stat file: "+location)
+	}
+
+	out, err := os.Create(location)
+	if err != nil {
+		return nil, 0, errors.Wrap(err, "failed creating file at: "+location)
+	}
+	defer out.Close()
+
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, 0, errors.Wrap(err, "failed to fetch: "+url)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, 0, fmt.Errorf("bad status: %s on url: %s", resp.Status, url)
+	}
+
+	progresser := &Progresser{
+		Reader:  resp.Body,
+		total:   resp.ContentLength,
+		updates: updates,
+	}
+
+	_, err = io.Copy(out, progresser)
+	if err != nil {
+		return nil, 0, errors.Wrap(err, "failed writing file to disk")
+	}
+
+	f, err := os.Open(location)
+	if err != nil {
+		return nil, 0, errors.Wrap(err, "failed to open file: "+location)
+	}
+
+	if updates != nil {
+		select {
+		case updates <- utils.GenericUpdate{Progress: 1}:
+		default:
+		}
+	}
+
+	_, err = addFileToCache(cacheKey)
+	if err != nil {
+		return nil, 0, errors.Wrap(err, "failed to add file to cache")
+	}
+
+	return f, resp.ContentLength, nil
+}