Initialize Vue.js project in zimui subfolder

.github/workflows/QA.yml

@@ -36,3 +36,27 @@ jobs:
       - name: Check pyright
         working-directory: scraper
         run: inv check-pyright
+
+  check-zimui-qa:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: zimui/.node-version
+
+      - name: Install JS dependencies
+        working-directory: zimui
+        run: |
+          yarn install
+      - name: Check prettier
+        working-directory: zimui
+        run: |
+          yarn format
+      - name: Check eslint
+        working-directory: zimui
+        run: |
+          yarn lint

.github/workflows/Tests.yml

@@ -51,6 +51,25 @@ jobs:
           pip install -U pip build
           python3 -m build --sdist --wheel
 
+  build-zimui:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: zimui/.node-version
+
+      - name: Install dependencies
+        working-directory: zimui
+        run: |
+          yarn install
+      - name: Build
+        working-directory: zimui
+        run: |
+          yarn build
+
   build-docker:
     runs-on: ubuntu-22.04
     steps:

.pre-commit-config.yaml

@@ -6,6 +6,22 @@ repos:
   hooks:
   -   id: trailing-whitespace
   -   id: end-of-file-fixer
+- repo: https://github.com/pre-commit/mirrors-prettier
+  rev: v3.0.3
+  hooks:
+  - id: prettier
+    files: zimui\/.*$ # files in zimui folder
+- repo: https://github.com/pre-commit/mirrors-eslint
+  rev: v8.51.0
+  hooks:
+  - id: eslint
+    types: [file]
+    files: zimui\/src\/.*(?:\.[jt]sx?|\.vue)$ # *.js, *.jsx, *.ts, *.tsx, *.vue in zimui/src folder
+    args:
+    - --ignore-path
+    - zimui/.eslintignore
+    - --config
+    - zimui/.eslintrc.cjs
 - repo: https://github.com/psf/black
   rev: "24.4.2"
   hooks:

CHANGELOG

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Move scraper files to `scraper` subfolder and update workflows
 - Bump `requests` package from 2.32.0 to 2.32.2
+- Initialize new Vue.js project in `zimui` subfolder
 
 ## [2.3.0] - 2024-05-22
 

Dockerfile

@@ -1,3 +1,10 @@
+FROM node:20-alpine as zimui
+
+WORKDIR /src
+COPY zimui /src
+RUN yarn install --frozen-lockfile
+RUN yarn build
+
 FROM python:3.12-bookworm
 LABEL org.opencontainers.image.source https://github.com/openzim/youtube
 
@@ -35,4 +42,9 @@ COPY *.md LICENSE CHANGELOG /src/
 RUN pip install --no-cache-dir /src/scraper \
  && rm -rf /src/scraper
 
+# Copy zimui build output
+COPY --from=zimui /src/dist /src/zimui
+
+ENV KOLIBRI_ZIMUI_DIST=/src/zimui
+
 CMD ["youtube2zim", "--help"]

scraper/src/youtube2zim/entrypoint.py

@@ -3,6 +3,7 @@
 
 import argparse
 import logging
+import os
 import sys
 
 from youtube2zim.constants import CHANNEL, NAME, PLAYLIST, SCRAPER, USER, logger
@@ -86,6 +87,15 @@ def main():
         default=False,
     )
 
+    parser.add_argument(
+        "--zimui-dist",
+        type=str,
+        help=(
+            "Directory containing Vite build output from the Zim UI Vue.JS application"
+        ),
+        default=os.getenv("KOLIBRI_ZIMUI_DIST", "../zimui/dist"),
+    )
+
     parser.add_argument(
         "--zim-file",
         help="ZIM file name (based on --name if not provided). "

scraper/src/youtube2zim/scraper.py

@@ -35,7 +35,7 @@
 from zimscraperlib.image.transformation import resize_image
 from zimscraperlib.inputs import compute_descriptions
 from zimscraperlib.video.presets import VideoMp4Low, VideoWebmLow
-from zimscraperlib.zim import make_zim_file
+from zimscraperlib.zim import Creator
 from zimscraperlib.zim.metadata import (
     validate_description,
     validate_longdescription,
@@ -66,7 +66,7 @@
     credentials_ok,
     extract_playlists_details_from,
     get_channel_json,
-    get_videos_authors_info,
+    # get_videos_authors_info,
     get_videos_json,
     save_channel_branding,
     skip_deleted_videos,
@@ -88,6 +88,7 @@ def __init__(
         all_subtitles,
         autoplay,
         output_dir,
+        zimui_dist,
         no_zim,
         fname,
         debug,
@@ -158,6 +159,7 @@ def __init__(
             tmp_dir = Path(tmp_dir).expanduser().resolve()
             tmp_dir.mkdir(parents=True, exist_ok=True)
         self.build_dir = Path(tempfile.mkdtemp(dir=tmp_dir))
+        self.zimui_dist = Path(zimui_dist)
 
         # process-related
         self.playlists = []
@@ -315,39 +317,42 @@ def run(self):
             )
         logger.info(f"{nb_videos_msg}.")
 
-        # download videos (and recompress)
-        logger.info(
-            "downloading all videos, subtitles and thumbnails "
-            f"(concurrency={self.max_concurrency})"
-        )
-        logger.info(f"  format: {self.video_format}")
-        logger.info(f"  quality: {self.video_quality}")
-        logger.info(f"  generated-subtitles: {self.all_subtitles}")
-        if self.s3_storage:
-            logger.info(
-                f"  using cache: {self.s3_storage.url.netloc} "
-                f"with bucket: {self.s3_storage.bucket_name}"
-            )
-        succeeded, failed = self.download_video_files(
-            max_concurrency=self.max_concurrency
-        )
-        if failed:
-            logger.error(f"{len(failed)} video(s) failed to download: {failed}")
-            if len(failed) >= len(succeeded):
-                logger.critical("More than half of videos failed. exiting")
-                raise OSError("Too much videos failed to download")
-
-        logger.info("retrieve channel-info for all videos (author details)")
-        get_videos_authors_info(succeeded)
+        # Commented out for now,
+        # but we have to rework this part to work with the new vuejs zimui
 
-        logger.info("download all author's profile pictures")
-        self.download_authors_branding()
+        # download videos (and recompress)
+        # logger.info(
+        #     "downloading all videos, subtitles and thumbnails "
+        #     f"(concurrency={self.max_concurrency})"
+        # )
+        # logger.info(f"  format: {self.video_format}")
+        # logger.info(f"  quality: {self.video_quality}")
+        # logger.info(f"  generated-subtitles: {self.all_subtitles}")
+        # if self.s3_storage:
+        #     logger.info(
+        #         f"  using cache: {self.s3_storage.url.netloc} "
+        #         f"with bucket: {self.s3_storage.bucket_name}"
+        #     )
+        # succeeded, failed = self.download_video_files(
+        #     max_concurrency=self.max_concurrency
+        # )
+        # if failed:
+        #     logger.error(f"{len(failed)} video(s) failed to download: {failed}")
+        #     if len(failed) >= len(succeeded):
+        #         logger.critical("More than half of videos failed. exiting")
+        #         raise OSError("Too much videos failed to download")
+
+        # logger.info("retrieve channel-info for all videos (author details)")
+        # get_videos_authors_info(succeeded)
+
+        # logger.info("download all author's profile pictures")
+        # self.download_authors_branding()
 
         logger.info("update general metadata")
         self.update_metadata()
 
-        logger.info("creating HTML files")
-        self.make_html_files(succeeded)
+        # logger.info("creating HTML files")
+        # self.make_html_files(succeeded)
 
         # make zim file
         os.makedirs(self.output_dir, exist_ok=True)
@@ -366,30 +371,78 @@ def run(self):
                 if self.fname
                 else f"{self.name}_{period}.zim"
             )
+
+            # check that build_dir is correct
+            if not self.build_dir.exists() or not self.build_dir.is_dir():
+                raise OSError(f"Incorrect build_dir: {self.build_dir}")
+
+            # check that illustration is correct
+            illustration = "favicon.png"
+            illustration_path = self.build_dir / illustration
+            if not illustration_path.exists() or not illustration_path.is_file():
+                raise OSError(
+                    f"Incorrect illustration: {illustration} ({illustration_path})"
+                )
+            with open(illustration_path, "rb") as fh:
+                illustration_data = fh.read()
+
             logger.info("building ZIM file")
-            make_zim_file(
-                build_dir=self.build_dir,
-                fpath=self.output_dir / self.fname,
-                name=self.name,
-                main_page="home.html",
-                illustration="favicon.png",
-                title=self.title,
-                description=self.description,
-                long_description=self.long_description,  # pyright: ignore[reportArgumentType]
-                language=self.language,
-                creator=self.creator,
-                publisher=self.publisher,
-                tags=self.tags,
-                scraper=SCRAPER,
+            self.zim_file = Creator(
+                filename=self.output_dir / self.fname,
+                main_path="index.html",
+                ignore_duplicates=True,
                 disable_metadata_checks=self.disable_metadata_checks,
             )
+            self.zim_file.config_metadata(
+                Name=self.name,  # pyright: ignore[reportArgumentType]
+                Language=self.language,  # pyright: ignore[reportArgumentType]
+                Title=self.title,
+                Description=self.description,
+                LongDescription=self.long_description,
+                Creator=self.creator,
+                Publisher=self.publisher,
+                tags=";".join(self.tags) if self.tags else "",
+                scraper=SCRAPER,
+                Date=datetime.date.today(),
+                Illustration_48x48_at_1=illustration_data,
+            )
+            self.zim_file.start()
+
+            try:
+                logger.debug(f"Preparing zimfile at {self.zim_file.filename}")
+                logger.debug(f"Recursively adding files from {self.build_dir}")
+                self.add_zimui()
+            except KeyboardInterrupt:
+                self.zim_file.can_finish = False
+                logger.error("KeyboardInterrupt, exiting.")
+            except Exception as exc:
+                # request Creator not to create a ZIM file on finish
+                self.zim_file.can_finish = False
+                logger.error(f"Interrupting process due to error: {exc}")
+                logger.exception(exc)
+            finally:
+                logger.info("Finishing ZIM file…")
+                self.zim_file.finish()
 
             if not self.keep_build_dir:
                 logger.info("removing temp folder")
                 shutil.rmtree(self.build_dir, ignore_errors=True)
 
         logger.info("all done!")
 
+    def add_zimui(self):
+        logger.info(f"Adding files in {self.zimui_dist}")
+        for file in self.zimui_dist.rglob("*"):
+            if file.is_dir():
+                continue
+            path = str(Path(file).relative_to(self.zimui_dist))
+            logger.debug(f"Adding {path} to ZIM")
+            self.zim_file.add_item_for(
+                path,
+                fpath=file,
+                is_front=path == "index.html",
+            )
+
     def s3_credentials_ok(self):
         logger.info("testing S3 Optimization Cache credentials")
         self.s3_storage = KiwixStorage(self.s3_url_with_credentials)

zimui/.eslintrc.cjs

@@ -0,0 +1,18 @@
+/* eslint-env node */
+require('@rushstack/eslint-patch/modern-module-resolution')
+
+module.exports = {
+  root: true,
+  parser: 'vue-eslint-parser',
+  parserOptions: {
+    sourceType: 'module',
+    ecmaVersion: 'latest',
+    parser: '@typescript-eslint/parser'
+  },
+  extends: [
+    'plugin:vue/vue3-recommended',
+    'eslint:recommended',
+    '@vue/eslint-config-typescript/recommended',
+    '@vue/eslint-config-prettier/skip-formatting'
+  ]
+}

zimui/.gitignore

@@ -0,0 +1,30 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+.DS_Store
+dist
+dist-ssr
+coverage
+*.local
+
+/cypress/videos/
+/cypress/screenshots/
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?
+
+*.tsbuildinfo

zimui/.node-version

@@ -0,0 +1 @@
+20.x

zimui/.prettierignore

@@ -0,0 +1,5 @@
+src/assets
+dist
+public
+node_modules
+src/**/*.md

zimui/.prettierrc.json

@@ -0,0 +1,8 @@
+{
+  "$schema": "https://json.schemastore.org/prettierrc",
+  "semi": false,
+  "tabWidth": 2,
+  "singleQuote": true,
+  "printWidth": 100,
+  "trailingComma": "none"
+}