Improvements to docker build setup to reduce image size; removed lega…

…cy migrator code
MESH-Research · May 25, 2024 · 60b1945 · 60b1945
1 parent 20527aa
commit 60b1945
Show file tree

Hide file tree

Showing 32 changed files with 283 additions and 10,417 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -6,4 +6,14 @@
 docker/nginx/samlPrivateKey.key
 docker/nginx_containerized/samlPrivateKey.key
 docker/nginx_staging/samlPrivateKey.key
-docker-compose*
+docker-compose*
+
+.github/
+.pytest_cache/
+**/.pytest_cache/
+.vscode/
+logs/*
+scripts/experiments
+scripts/sample_files
+site/build
+site/kcworks.egg-info
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -2,13 +2,22 @@ name: CI
 
 on:
   push:
-    branches: [main]
+    branches:
+      - "main"
+    tags:
+      - v*
   pull_request:
     branches:
       - main
 
+env:
+  DOCKER_BUILDKIT: 1
+  TARGET_PLATFORMS: linux/amd64,linux/arm64
+  REGISTRY: ghcr.io
+  IMAGE_NAME: kcworks
+
 jobs:
-  build:
+  build_and_release:
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:
@@ -23,26 +32,26 @@ jobs:
           python-version: "3.9"
 
       # Cache docker images so they don't rebuild every time
-      - name: Cache Local Images
-        id: local-images
-        uses: actions/cache@v3
-        with:
-          path: /var/lib/docker/
-          key: local-docker-directory
+      # - name: Cache Local Images
+      #   id: local-images
+      #   uses: actions/cache@v3
+      #   with:
+      #     path: /var/lib/docker/
+      #     key: local-docker-directory
 
       - name: Build Images
         run: |
           docker compose up -d --build
 
-      - name: Set up services
-        run: |
-          docker exec -it knowledgecommonsworks_web-ui_1 bash -c "bash ./scripts/setup-services.sh"
-          exit
+      # - name: Set up services
+      #   run: |
+      #     docker exec -it knowledgecommonsworks_web-ui_1 bash -c "bash ./scripts/setup-services.sh"
+      #     exit
 
-      - name: Build statics and assets
-        run: |
-          docker exec -it knowledgecommonsworks_web-ui_1 bash -c "bash ./scripts/build-assets.sh"
-          exit
+      # - name: Build statics and assets
+      #   run: |
+      #     docker exec -it knowledgecommonsworks_web-ui_1 bash -c "bash ./scripts/build-assets.sh"
+      #     exit
 
       # - name: Run unit tests
       #   run: cd site && pytest

diff --git a/Dockerfile b/Dockerfile
@@ -22,53 +22,47 @@ RUN dnf install libxml2-devel xmlsec1-devel xmlsec1-openssl-devel libtool-ltdl-d
 COPY site ./site
 
 COPY Pipfile Pipfile.lock ./
-# Copy in forked packages to be installed from local
-# COPY ./invenio-communities/ /opt/invenio/src/invenio-communities/
-RUN git clone https://github.com/MESH-Research/invenio-communities.git /opt/invenio/invenio-communities
-# COPY ./invenio-rdm-records/ /opt/invenio/src/invenio-rdm-records/
-RUN git clone https://github.com/MESH-Research/invenio-rdm-records.git /opt/invenio/invenio-rdm-records
-RUN git clone https://github.com/MESH-Research/invenio-group-collections.git /opt/invenio/invenio-group-collections/
-RUN git clone https://github.com/MESH-Research/invenio-modular-deposit-form.git /opt/invenio/invenio-modular-deposit-form/
-RUN git clone https://github.com/MESH-Research/invenio-modular-detail-page.git /opt/invenio/invenio-modular-detail-page/
-RUN git clone https://github.com/MESH-Research/invenio-record-importer.git /opt/invenio/invenio-record-importer/
-RUN git clone https://github.com/MESH-Research/invenio-records-resources.git /opt/invenio/invenio-records-resources/
-RUN git clone https://github.com/MESH-Research/invenio-remote-api-provisioner.git /opt/invenio/invenio-remote-api-provisioner/
-RUN git clone https://github.com/MESH-Research/invenio-remote-user-data.git /opt/invenio/invenio-remote-user-data/
-# Install python requirements with pipenv in container
+
+RUN git clone https://github.com/MESH-Research/invenio-communities.git /opt/invenio/invenio-communities && \
+    git clone https://github.com/MESH-Research/invenio-rdm-records.git /opt/invenio/invenio-rdm-records && \
+    git clone https://github.com/MESH-Research/invenio-group-collections.git /opt/invenio/invenio-group-collections/ && \
+    git clone https://github.com/MESH-Research/invenio-modular-deposit-form.git /opt/invenio/invenio-modular-deposit-form/ && \
+    git clone https://github.com/MESH-Research/invenio-modular-detail-page.git /opt/invenio/invenio-modular-detail-page/ && \
+    git clone https://github.com/MESH-Research/invenio-record-importer.git /opt/invenio/invenio-record-importer/ && \
+    git clone https://github.com/MESH-Research/invenio-records-resources.git /opt/invenio/invenio-records-resources/ && \
+    git clone https://github.com/MESH-Research/invenio-remote-api-provisioner.git /opt/invenio/invenio-remote-api-provisioner/ && \
+    git clone https://github.com/MESH-Research/invenio-remote-user-data.git /opt/invenio/invenio-remote-user-data/
 
 # NOTE: turned off --deploy for dev
 RUN pipenv install --system
-RUN pip install invenio-cli
-
-RUN echo "[cli]" >> .invenio.private
-RUN echo "services_setup=False" >> .invenio.private
-RUN echo "instance_path=/opt/invenio/var/instance" >> .invenio.private
 
+RUN echo "[cli]" >> .invenio.private && \
+    echo "services_setup=False" >> .invenio.private && \
+    echo "instance_path=/opt/invenio/var/instance" >> .invenio.private
 
 # Copying whole app directory into /opt/invenio/src
 # (WORKDIR is set to that folder in base image)
 COPY ./ .
-ENV INVENIO_INSTANCE_PATH=/opt/invenio/var/instance
-ENV INVENIO_SITE_UI_URL=https://localhost
-ENV INVENIO_SITE_API_URL=https://localhost
-ENV MIGRATION_SERVER_DOMAIN=localhost
-ENV MIGRATION_SERVER_PROTOCOL=http
-ENV MIGRATION_API_TOKEN=changeme
+ENV INVENIO_INSTANCE_PATH=/opt/invenio/var/instance \
+    INVENIO_SITE_UI_URL=https://localhost \
+    INVENIO_SITE_API_URL=https://localhost \
+    MIGRATION_SERVER_DOMAIN=localhost \
+    MIGRATION_SERVER_PROTOCOL=http \
+    MIGRATION_API_TOKEN=changeme
 
-RUN cp -r ./docker/uwsgi/uwsgi_rest.ini ${INVENIO_INSTANCE_PATH}/uwsgi_rest.ini
-RUN cp -r ./docker/uwsgi/uwsgi_ui.ini ${INVENIO_INSTANCE_PATH}/uwsgi_ui.ini
-RUN cp ./invenio.cfg ${INVENIO_INSTANCE_PATH}/invenio.cfg
-RUN cp -r ./templates ${INVENIO_INSTANCE_PATH}/templates
-RUN cp -r ./app_data/ ${INVENIO_INSTANCE_PATH}/app_data
+RUN cp -r ./docker/uwsgi/uwsgi_rest.ini ${INVENIO_INSTANCE_PATH}/uwsgi_rest.ini && \
+    cp -r ./docker/uwsgi/uwsgi_ui.ini ${INVENIO_INSTANCE_PATH}/uwsgi_ui.ini && \
+    cp ./invenio.cfg ${INVENIO_INSTANCE_PATH}/invenio.cfg && \
+    cp -r ./templates ${INVENIO_INSTANCE_PATH}/templates && \
+    cp -r ./app_data/ ${INVENIO_INSTANCE_PATH}/app_data
 
-RUN invenio collect --verbose
-RUN invenio webpack clean create
-RUN mkdir -p ${INVENIO_INSTANCE_PATH}/assets/less
-RUN cp ./assets/less/theme.config ${INVENIO_INSTANCE_PATH}/assets/less/theme.config
-RUN mkdir -p ${INVENIO_INSTANCE_PATH}/assets/templates/custom_fields
-RUN mkdir -p ${INVENIO_INSTANCE_PATH}/assets/templates/search
-RUN invenio webpack install
-RUN invenio shell ./scripts/symlink_assets.py
-RUN invenio webpack build
+RUN invenio collect --verbose && invenio webpack clean create && \
+    mkdir -p ${INVENIO_INSTANCE_PATH}/assets/less && \
+    cp ./assets/less/theme.config ${INVENIO_INSTANCE_PATH}/assets/less/theme.config && \
+    mkdir -p ${INVENIO_INSTANCE_PATH}/assets/templates/custom_fields && \
+    mkdir -p ${INVENIO_INSTANCE_PATH}/assets/templates/search && \
+    invenio webpack install && \
+    invenio shell ./scripts/symlink_assets.py && \
+    invenio webpack build
 
 ENTRYPOINT ["bash", "-c"]
diff --git a/Pipfile b/Pipfile
@@ -20,7 +20,7 @@ cchardet = "==2.1.7"
 flask-principal = "*"
 greenlet = {version = "!=0.4.17"}
 halo = "*"
-invenio-app-rdm = {version = "~=12.0.0b3.dev3", extras = ["opensearch2"]}
+invenio-app-rdm = {version = "~=12.0.0b3.dev17", extras = ["opensearch2"]}
 invenio-cli = "==1.0.21"
 invenio-s3 = "*"
 invenio-utilities-tuw = "*"
@@ -54,6 +54,8 @@ invenio-records-resources = {file = "../invenio-records-resources", editable = t
 invenio-communities = {file = "../invenio-communities", editable = true}
 kcworks = {file = "site", editable = true}
 numpy = "*"
+flask-breadcrumbs = "*"
+flask-babelex = "*"
 
 [requires]
 python_version = "3.9"