Merge pull request #207 from openzim/new_ui_init

Move scraper to subfolder and initialize new Vue.js project

.github/workflows/Publish.yml

@@ -16,18 +16,19 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version-file: pyproject.toml
+          python-version-file: scraper/pyproject.toml
           architecture: x64
 
       - name: Build packages
+        working-directory: scraper
         run: |
           pip install -U pip build
           python -m build --sdist --wheel
 
       - name: Upload to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1.8
         with:
-          packages-dir: dist/
+          packages-dir: scraper/dist/
 
       - name: Build and push Docker image
         uses: openzim/docker-publish-action@v10

.github/workflows/QA.yml

@@ -16,19 +16,47 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version-file: pyproject.toml
+          python-version-file: scraper/pyproject.toml
           architecture: x64
 
       - name: Install dependencies
+        working-directory: scraper
         run: |
           pip install -U pip
           pip install -e .[lint,check,scripts,test]
 
       - name: Check black formatting
+        working-directory: scraper
         run: inv lint-black
 
       - name: Check ruff
+        working-directory: scraper
         run: inv lint-ruff
 
       - name: Check pyright
-        run: inv check-pyright
+        working-directory: scraper
+        run: inv check-pyright
+
+  check-zimui-qa:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: zimui/.node-version
+
+      - name: Install JS dependencies
+        working-directory: zimui
+        run: |
+          yarn install
+      - name: Check prettier
+        working-directory: zimui
+        run: |
+          yarn format
+      - name: Check eslint
+        working-directory: zimui
+        run: |
+          yarn lint

.github/workflows/Tests.yml

@@ -16,15 +16,17 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version-file: pyproject.toml
+          python-version-file: scraper/pyproject.toml
           architecture: x64
 
       - name: Install dependencies (and project)
+        working-directory: scraper
         run: |
           pip install -U pip
           pip install -e .[test,scripts]
 
       - name: Run the tests
+        working-directory: scraper
         run: inv coverage --args "-vvv"
 
       - name: Upload coverage report to codecov
@@ -40,14 +42,34 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version-file: pyproject.toml
+          python-version-file: scraper/pyproject.toml
           architecture: x64
 
       - name: Ensure we can build Python targets
+        working-directory: scraper
         run: |
           pip install -U pip build
           python3 -m build --sdist --wheel
 
+  build-zimui:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: zimui/.node-version
+
+      - name: Install dependencies
+        working-directory: zimui
+        run: |
+          yarn install
+      - name: Build
+        working-directory: zimui
+        run: |
+          yarn build
+
   build-docker:
     runs-on: ubuntu-22.04
     steps:

.gitignore

@@ -366,13 +366,13 @@ pyrightconfig.json
 # End of https://www.toptal.com/developers/gitignore/api/linux,macos,python,database,visualstudiocode,intellij
 
 # JS deps
-src/youtube2zim/templates/assets/chosen/
-src/youtube2zim/templates/assets/jquery.min.js
-src/youtube2zim/templates/assets/ogvjs/
-src/youtube2zim/templates/assets/videojs-ogvjs.js
-src/youtube2zim/templates/assets/videojs/
-src/youtube2zim/templates/assets/polyfills.js
-src/youtube2zim/templates/assets/webp-hero.bundle.js
+scraper/src/youtube2zim/templates/assets/chosen/
+scraper/src/youtube2zim/templates/assets/jquery.min.js
+scraper/src/youtube2zim/templates/assets/ogvjs/
+scraper/src/youtube2zim/templates/assets/videojs-ogvjs.js
+scraper/src/youtube2zim/templates/assets/videojs/
+scraper/src/youtube2zim/templates/assets/polyfills.js
+scraper/src/youtube2zim/templates/assets/webp-hero.bundle.js
 
 # output dir
 output

.pre-commit-config.yaml

@@ -6,6 +6,22 @@ repos:
   hooks:
   -   id: trailing-whitespace
   -   id: end-of-file-fixer
+- repo: https://github.com/pre-commit/mirrors-prettier
+  rev: v3.0.3
+  hooks:
+  - id: prettier
+    files: zimui\/.*$ # files in zimui folder
+- repo: https://github.com/pre-commit/mirrors-eslint
+  rev: v8.51.0
+  hooks:
+  - id: eslint
+    types: [file]
+    files: zimui\/src\/.*(?:\.[jt]sx?|\.vue)$ # *.js, *.jsx, *.ts, *.tsx, *.vue in zimui/src folder
+    args:
+    - --ignore-path
+    - zimui/.eslintignore
+    - --config
+    - zimui/.eslintrc.cjs
 - repo: https://github.com/psf/black
   rev: "24.4.2"
   hooks:

CHANGELOG

@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Changed
+
+- Move scraper files to `scraper` subfolder and update workflows
+- Bump `requests` package from 2.32.0 to 2.32.2
+- Initialize new Vue.js project in `zimui` subfolder
+
 ## [2.3.0] - 2024-05-22
 
 ### Added

Dockerfile

@@ -1,37 +1,50 @@
+FROM node:20-alpine as zimui
+
+WORKDIR /src
+COPY zimui /src
+RUN yarn install --frozen-lockfile
+RUN yarn build
+
 FROM python:3.12-bookworm
 LABEL org.opencontainers.image.source https://github.com/openzim/youtube
 
 # Install necessary packages
 RUN apt-get update \
- && apt-get install -y --no-install-recommends \
-      locales-all \
-      wget \
-      unzip \
-      ffmpeg \
-      aria2 \
- && rm -rf /var/lib/apt/lists/* \
- && python -m pip install --no-cache-dir -U \
-      pip
+     && apt-get install -y --no-install-recommends \
+     locales-all \
+     wget \
+     unzip \
+     ffmpeg \
+     aria2 \
+     && rm -rf /var/lib/apt/lists/* \
+     && python -m pip install --no-cache-dir -U \
+     pip
 
 # Custom entrypoint
-COPY entrypoint.sh /usr/local/bin/entrypoint.sh
+COPY scraper/entrypoint.sh /usr/local/bin/entrypoint.sh
 ENTRYPOINT ["entrypoint.sh"]
 RUN mkdir -p /output
 WORKDIR /output
 
 # Copy pyproject.toml and its dependencies
-COPY pyproject.toml README.md openzim.toml /src/
-COPY src/youtube2zim/__about__.py /src/src/youtube2zim/__about__.py
+COPY README.md /src/
+COPY scraper/pyproject.toml scraper/openzim.toml /src/scraper/
+COPY scraper/src/youtube2zim/__about__.py /src/scraper/src/youtube2zim/__about__.py
 
 # Install Python dependencies
-RUN pip install --no-cache-dir /src
+RUN pip install --no-cache-dir /src/scraper
 
 # Copy code + associated artifacts
-COPY src /src/src
+COPY scraper/src /src/scraper/src
 COPY *.md LICENSE CHANGELOG /src/
 
 # Install + cleanup
-RUN pip install --no-cache-dir /src \
- && rm -rf /src
+RUN pip install --no-cache-dir /src/scraper \
+ && rm -rf /src/scraper
+
+# Copy zimui build output
+COPY --from=zimui /src/dist /src/zimui
+
+ENV YOUTUBE_ZIMUI_DIST=/src/zimui
 
 CMD ["youtube2zim", "--help"]

README.md

@@ -6,6 +6,8 @@ Youtube2zim
 [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0)
 [![PyPI version shields.io](https://img.shields.io/pypi/v/youtube2zim.svg)](https://pypi.org/project/youtube2zim/)
 
+`⚠️ Disclaimer:` The main branch is currently unstable as the scraper is undergoing significant rework for the 3.0 release. If an important patch needs to be applied before the v3 release, please refer to the [v2](https://github.com/openzim/youtube/tree/v2) branch.
+
 `youtube2zim` allows you to create a [ZIM file](https://openzim.org)
 from a Youtube Channel/Username or one-or-more Playlists.
 
@@ -50,6 +52,7 @@ pip3 install hatch
 Start a hatch shell: this will install software including dependencies in an isolated virtual environment.
 
 ``` bash
+cd scraper
 hatch shell
 ```
 

pyproject.toml → scraper/pyproject.toml

@@ -6,14 +6,14 @@ build-backend = "hatchling.build"
 name = "youtube2zim"
 requires-python = ">=3.12,<3.13"
 description = "Make ZIM file from a Youtube channel, user or playlist(s)"
-readme = "README.md"
+readme = "../README.md"
 dependencies = [
   "python-slugify==3.0.3",
   "yt-dlp",                      # youtube-dl should be updated as frequently as possible
   "python-dateutil==2.9.0.post0",
   "jinja2==3.1.4",
   "zimscraperlib==3.3.2",
-  "requests==2.32.0",
+  "requests==2.32.2",
   "kiwixstorage==0.8.3",
   "pif==0.8.2",
 ]

src/youtube2zim/entrypoint.py → scraper/src/youtube2zim/entrypoint.py

@@ -3,6 +3,7 @@
 
 import argparse
 import logging
+import os
 import sys
 
 from youtube2zim.constants import CHANNEL, NAME, PLAYLIST, SCRAPER, USER, logger
@@ -86,6 +87,15 @@ def main():
         default=False,
     )
 
+    parser.add_argument(
+        "--zimui-dist",
+        type=str,
+        help=(
+            "Directory containing Vite build output from the Zim UI Vue.JS application"
+        ),
+        default=os.getenv("YOUTUBE_ZIMUI_DIST", "../zimui/dist"),
+    )
+
     parser.add_argument(
         "--zim-file",
         help="ZIM file name (based on --name if not provided). "

src/youtube2zim/locale/fr/LC_MESSAGES/messages.po → scraper/src/youtube2zim/locale/fr/LC_MESSAGES/messages.po

@@ -33,4 +33,3 @@ msgstr "Page {current}/{total}"
 #: youtube2zim/scraper.py:872
 msgid "Back to top"
 msgstr "Haut"
-

src/youtube2zim/playlists/scraper.py → scraper/src/youtube2zim/playlists/scraper.py

@@ -198,7 +198,7 @@ def handle_single_zim(self):
 
     @staticmethod
     def compute_format(playlist, fmt):
-        return fmt.format(**playlist.__dict__(), **{"period": "{period}"})
+        return fmt.format(**playlist.to_dict(), **{"period": "{period}"})
 
     def fetch_metadata(self):
         """retrieves and loads metadata from --metadata-from"""