Merge pull request #361 from SUNET/develop

release v1.6.0

.github/workflows/run-unit-tests.yml

@@ -4,6 +4,26 @@ on:
   workflow_dispatch:
   pull_request:
 jobs:
+  free-disk-space:
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Free Disk Space (Ubuntu)
+      uses: jlumbroso/free-disk-space@main
+      with:
+        # this might remove tools that are actually needed,
+        # if set to "true" but frees about 6 GB
+        tool-cache: false
+
+        # all of these default to true, but feel free to set to
+        # "false" if necessary for your workflow
+        android: true
+        dotnet: true
+        haskell: true
+        large-packages: true
+        docker-images: false
+        swap-storage: false
+
   docker-tests:
     name: "Run unit tests in docker"
     runs-on: ubuntu-latest
@@ -28,11 +48,13 @@ jobs:
           echo "PYTEST_POSTGRES_EXTERNAL=1" >> $GITHUB_ENV
           echo "PYTEST_REDIS_EXTERNAL=1" >> $GITHUB_ENV
           echo "JWT_AUTH_TOKEN=eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiJ9.eyJpYXQiOjE1NzEwNTk2MTgsIm5iZiI6MTU3MTA1OTYxOCwianRpIjoiNTQ2MDk2YTUtZTNmOS00NzFlLWE2NTctZWFlYTZkNzA4NmVhIiwic3ViIjoiYWRtaW4iLCJmcmVzaCI6ZmFsc2UsInR5cGUiOiJhY2Nlc3MifQ.Sfffg9oZg_Kmoq7Oe8IoTcbuagpP6nuUXOQzqJpgDfqDq_GM_4zGzt7XxByD4G0q8g4gZGHQnV14TpDer2hJXw" >> $GITHUB_ENV
+          echo "JWT_SECRET_KEY=unittestsharedsecret" >> $GITHUB_ENV
+          echo "COMPOSE_COMPATIBILITY=1" >> $GITHUB_ENV
 
 #      - name: Create code coverage dir
 #        run: mkdir -p docker/coverage/
 #
-      - uses: jpribyl/action-docker-layer-caching@v0.1.1
+      - uses: satackey/action-docker-layer-caching@v0.0.11
         # Ignore the failure of a step and avoid terminating the job.
         continue-on-error: true
 
@@ -54,28 +76,28 @@ jobs:
         run: echo ${{ env.BRANCH_NAME }} at ${{ env.CNAAS_REPO }}
 
       - name: Build docker
-        run: docker-compose -f docker/docker-compose_test.yaml build --build-arg GITREPO_BASE=${{ env.CNAAS_REPO }} --build-arg BUILDBRANCH=${{ env.BRANCH_NAME }}
+        run: docker compose -f docker/docker-compose_test.yaml build --build-arg GITREPO_BASE=${{ env.CNAAS_REPO }} --build-arg BUILDBRANCH=${{ env.BRANCH_NAME }}
 
       - name: Start docker
-        run: docker-compose -f docker/docker-compose_test.yaml up -d
+        run: docker compose -f docker/docker-compose_test.yaml up -d
 
       - name: Install API certificates
         run: |
           docker cp ./docker/jwt-cert/public.pem docker_cnaas_api_1:/opt/cnaas/jwtcert/public.pem
-          docker-compose -f docker/docker-compose_test.yaml exec -u root -T cnaas_api /bin/chown -R www-data:www-data /opt/cnaas/jwtcert/
-          docker-compose -f docker/docker-compose_test.yaml exec -u root -T cnaas_api /opt/cnaas/createca.sh
+          docker compose -f docker/docker-compose_test.yaml exec -u root -T cnaas_api /bin/chown -R www-data:www-data /opt/cnaas/jwtcert/
+          docker compose -f docker/docker-compose_test.yaml exec -u root -T cnaas_api /opt/cnaas/createca.sh
 
       - name: Check running containers
         run: docker ps -a
 
       - name: Check API running version
-        run: curl -ks --connect-timeout 5 --max-time 10 --retry 5 --retry-delay 0 --retry-max-time 30 https://localhost/api/v1.0/system/version
+        run: curl -ks --connect-timeout 5 --max-time 10 --retry 10 --retry-delay 5 --retry-max-time 120 https://localhost/api/v1.0/system/version
 
       - name: Fix permissions for code coverage
-        run: docker-compose -f docker/docker-compose_test.yaml exec -u root -T cnaas_api chown -R www-data:www-data /opt/cnaas/venv/cnaas-nms/src/
+        run: docker compose -f docker/docker-compose_test.yaml exec -u root -T cnaas_api chown -R www-data:www-data /opt/cnaas/venv/cnaas-nms/src/
 
       - name: Run unit tests in container
-        run: docker-compose -f docker/docker-compose_test.yaml exec -u www-data -T cnaas_api /opt/cnaas/pytest.sh
+        run: docker compose -f docker/docker-compose_test.yaml exec -u www-data -T cnaas_api /opt/cnaas/pytest.sh
 
       - name: Check logs
         if: always()

.jenkins.yaml

@@ -16,7 +16,6 @@ git:
     shallow_clone: true
 
 script:
-  - "ssh [email protected] /root/starttests.sh ${GIT_BRANCH}"
   # - "python3.7 -m venv venv"
   # - ". venv/bin/activate"
   # - "pip install --upgrade setuptools pip wheel mypy"
@@ -45,7 +44,7 @@ extra_jobs:
     docker_name: cnaas/api
     docker_context_dir: docker/api
     docker_tags:
-      - "v1.6"
+      - "v1.7"
       - "develop"
 
   - name: cnaas-api-master
@@ -56,7 +55,6 @@ extra_jobs:
     docker_name: cnaas/api
     docker_context_dir: docker/api
     docker_tags:
-      - "v1.5"
       - "master"
 
   - name: cnaas-postgres

.pre-commit-config.yaml

@@ -1,20 +1,20 @@
-repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.3.0
-    hooks:
-      - id: check-yaml
-      - id: end-of-file-fixer
-      - id: trailing-whitespace
-  - repo: https://github.com/psf/black
-    rev: 22.10.0
-    hooks:
-      - id: black
-  - repo: https://github.com/pycqa/isort
-    rev: 5.11.2
-    hooks:
-      - id: isort
-        name: isort (python)
-  - repo: https://github.com/pycqa/flake8
-    rev: 5.0.4
-    hooks:
-    - id: flake8
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
+    hooks:
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/psf/black
+    rev: 23.11.0
+    hooks:
+      - id: black
+  - repo: https://github.com/pycqa/isort
+    rev: 5.12.0
+    hooks:
+      - id: isort
+        name: isort (python)
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.1.0
+    hooks:
+    - id: flake8

README.md

@@ -1,4 +1,4 @@
-[![Documentation Status](https://readthedocs.org/projects/cnaas-nms/badge/?version=latest)](https://cnaas-nms.readthedocs.io/en/latest/?badge=latest) [![codecov](https://codecov.io/gh/SUNET/cnaas-nms/branch/master/graph/badge.svg)](https://codecov.io/gh/SUNET/cnaas-nms) [![Python 3.7](https://img.shields.io/badge/python-3.7-blue.svg)](https://www.python.org/downloads/release/python-370/)
+[![Documentation Status](https://readthedocs.org/projects/cnaas-nms/badge/?version=latest)](https://cnaas-nms.readthedocs.io/en/latest/?badge=latest) [![codecov](https://codecov.io/gh/SUNET/cnaas-nms/branch/master/graph/badge.svg)](https://codecov.io/gh/SUNET/cnaas-nms) [![Python 3.11](https://img.shields.io/badge/python-3.11-blue.svg)](https://www.python.org/downloads/release/python-3110/)
 
 # CNaaS-NMS
 
@@ -45,6 +45,10 @@ python3 -m cnaas_nms.api.tests.test_api
 python3 -m cnaas_nms.confpush.tests.test_get
 ```
 
+## Authorization
+
+Currently we can use two styles for the authorization. We can use the original style or use OIDC style. For OIDC we need to define some env variables or add a auth_config.yaml in the config. The needed variables are: OIDC_CONF_WELL_KNOWN_URL, OIDC_CLIENT_SECRET, OIDC_CLIENT_ID, FRONTEND_CALLBACK_URL and OIDC_ENABLED. To use the OIDC style the last variable needs to be set to true.
+
 ## License
 
 Copyright (c) 2019 - 2020, SUNET (BSD 2-clause license)

alembic/versions/1327fb92d384_add_config_hash_for_devices.py

@@ -1,7 +1,7 @@
 """add config hash for devices
 
 Revision ID: 1327fb92d384
-Revises: d1e7569c57a2
+Revises: d3aa4454ba7b
 Create Date: 2019-05-23 08:40:31.711177
 
 """
@@ -11,18 +11,14 @@
 
 # revision identifiers, used by Alembic.
 revision = "1327fb92d384"
-down_revision = "a3f3bc390462"
+down_revision = "d3aa4454ba7b"
 branch_labels = None
 depends_on = None
 
 
 def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
     op.add_column("device", sa.Column("confhash", sa.String(length=64), nullable=True))
-    # ### end Alembic commands ###
 
 
 def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
     op.drop_column("device", "confhash")
-    # ### end Alembic commands ###

alembic/versions/395427a732d6_add_aborted_state_for_jobs.py

@@ -5,8 +5,6 @@
 Create Date: 2019-12-06 09:49:26.610811
 
 """
-import sqlalchemy as sa
-
 from alembic import op
 
 # revision identifiers, used by Alembic.
@@ -17,19 +15,15 @@
 
 
 def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
     op.create_index(op.f("ix_job_finish_time"), "job", ["finish_time"], unique=False)
     op.create_index(op.f("ix_job_status"), "job", ["status"], unique=False)
     op.create_index(op.f("ix_job_ticket_ref"), "job", ["ticket_ref"], unique=False)
-    # ### end Alembic commands ###
-    # custom: ALTER TYPE enum_type ADD VALUE 'new_value' AFTER 'old_value';
     op.execute("COMMIT")
     op.execute("ALTER TYPE jobstatus ADD VALUE 'ABORTED' AFTER 'EXCEPTION'")
 
 
 def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
     op.drop_index(op.f("ix_job_ticket_ref"), table_name="job")
     op.drop_index(op.f("ix_job_status"), table_name="job")
     op.drop_index(op.f("ix_job_finish_time"), table_name="job")
-    # ### end Alembic commands ###
+    # removing extra types in an enum can make fields in the database invalid

alembic/versions/5aa2593e147a_save_change_score_in_separate_column_.py

@@ -17,12 +17,8 @@
 
 
 def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
     op.add_column("job", sa.Column("change_score", sa.SmallInteger(), nullable=True))
-    # ### end Alembic commands ###
 
 
 def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
     op.drop_column("job", "change_score")
-    # ### end Alembic commands ###

alembic/versions/65b83c197420_move_job_table_to_postgres_change_.py

@@ -18,7 +18,6 @@
 
 
 def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
     op.create_table(
         "job",
         sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
@@ -46,18 +45,16 @@ def upgrade():
     )
     op.add_column("joblock", sa.Column("job_id", sa.Integer(), nullable=False))
     op.drop_constraint("joblock_jobid_key", "joblock", type_="unique")
-    op.create_unique_constraint(None, "joblock", ["job_id"])
-    op.create_foreign_key(None, "joblock", "job", ["job_id"], ["id"])
+    op.create_unique_constraint("jobid_unique", "joblock", ["job_id"])
+    op.create_foreign_key("fk_joblock_job", "joblock", "job", ["job_id"], ["id"])
     op.drop_column("joblock", "jobid")
-    # ### end Alembic commands ###
 
 
 def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
     op.add_column("joblock", sa.Column("jobid", sa.VARCHAR(length=24), autoincrement=False, nullable=False))
-    op.drop_constraint(None, "joblock", type_="foreignkey")
-    op.drop_constraint(None, "joblock", type_="unique")
+    op.drop_constraint("fk_joblock_job", "joblock", type_="foreignkey")
+    op.drop_constraint("jobid_unique", "joblock", type_="unique")
     op.create_unique_constraint("joblock_jobid_key", "joblock", ["jobid"])
     op.drop_column("joblock", "job_id")
     op.drop_table("job")
-    # ### end Alembic commands ###
+    sa.Enum(name="jobstatus").drop(op.get_bind(), checkfirst=False)

alembic/versions/6627b128bd5c_added_port_field.py

@@ -1,7 +1,7 @@
 """Added port field
 
 Revision ID: 6627b128bd5c
-Revises: 922589188efe
+Revises: 6c6bec879fa8
 Create Date: 2019-08-29 14:36:32.125191
 
 """
@@ -11,19 +11,14 @@
 
 # revision identifiers, used by Alembic.
 revision = "6627b128bd5c"
-# down_revision = '922589188efe'
 down_revision = "6c6bec879fa8"
 branch_labels = None
 depends_on = None
 
 
 def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
     op.add_column("device", sa.Column("port", sa.Integer()))
-    # ### end Alembic commands ###
 
 
 def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
     op.drop_column("device", "port")
-    # ### end Alembic commands ###

alembic/versions/6c6bec879fa8_add_joblock_database.py

@@ -17,21 +17,17 @@
 
 
 def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
     op.create_table(
         "joblock",
         sa.Column("jobid", sa.String(length=24), nullable=False),
         sa.Column("name", sa.String(length=32), nullable=False),
         sa.Column("start_time", sa.DateTime(), nullable=True),
         sa.Column("abort", sa.Boolean(), nullable=True),
         sa.PrimaryKeyConstraint("jobid"),
-        sa.UniqueConstraint("jobid"),
+        sa.UniqueConstraint("jobid", name="joblock_jobid_key"),
         sa.UniqueConstraint("name"),
     )
-    # ### end Alembic commands ###
 
 
 def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
     op.drop_table("joblock")
-    # ### end Alembic commands ###

alembic/versions/8a635012afa7_add_new_interface_config_types.py

@@ -5,8 +5,6 @@
 Create Date: 2020-03-26 09:21:15.439761
 
 """
-import sqlalchemy as sa
-
 from alembic import op
 
 # revision identifiers, used by Alembic.
@@ -17,15 +15,12 @@
 
 
 def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    # ### end Alembic commands ###
     op.execute("COMMIT")
     op.execute("ALTER TYPE interfaceconfigtype ADD VALUE 'TEMPLATE' AFTER 'CUSTOM'")
     op.execute("ALTER TYPE interfaceconfigtype ADD VALUE 'MLAG_PEER' AFTER 'TEMPLATE'")
     op.execute("ALTER TYPE interfaceconfigtype ADD VALUE 'ACCESS_DOWNLINK' AFTER 'ACCESS_UPLINK'")
 
 
 def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
+    # removing extra types in an enum can make fields in the database invalid
     pass
-    # ### end Alembic commands ###

alembic/versions/922589188efe_add_columns_for_dist_switch_vxlan_fabric.py

@@ -18,20 +18,16 @@
 
 
 def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
     op.add_column(
         "device", sa.Column("infra_ip", sqlalchemy_utils.types.ip_address.IPAddressType(length=50), nullable=True)
     )
     op.add_column(
         "device", sa.Column("oob_ip", sqlalchemy_utils.types.ip_address.IPAddressType(length=50), nullable=True)
     )
     op.add_column("mgmtdomain", sa.Column("esi_mac", sa.String(length=12), nullable=True))
-    # ### end Alembic commands ###
 
 
 def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
     op.drop_column("mgmtdomain", "esi_mac")
     op.drop_column("device", "oob_ip")
     op.drop_column("device", "infra_ip")
-    # ### end Alembic commands ###

alembic/versions/9478bbaf8010_add_reservedip_table.py

@@ -31,13 +31,13 @@ def upgrade():
         sa.PrimaryKeyConstraint("device_id"),
     )
     op.create_index(op.f("ix_reservedip_device_id"), "reservedip", ["device_id"], unique=False)
-    op.create_unique_constraint(None, "joblock", ["jobid"])
+    # op.create_unique_constraint("jobid_unique1", "joblock", ["jobid"])
     # ### end Alembic commands ###
 
 
 def downgrade():
     # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_constraint(None, "joblock", type_="unique")
+    # op.drop_constraint("jobid_unique1", "joblock", type_="unique")
     op.drop_index(op.f("ix_reservedip_device_id"), table_name="reservedip")
     op.drop_table("reservedip")
     # ### end Alembic commands ###

alembic/versions/9d01bce3c835_add_job_status_aborting_and_start_.py

@@ -19,13 +19,12 @@
 
 def upgrade():
     # ### commands auto generated by Alembic - please adjust! ###
+
     op.add_column("job", sa.Column("start_arguments", postgresql.JSONB(astext_type=sa.Text()), nullable=True))
     op.execute("COMMIT")
     op.execute("ALTER TYPE jobstatus ADD VALUE 'ABORTING' AFTER 'ABORTED'")
-    # ### end Alembic commands ###
 
 
 def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
     op.drop_column("job", "start_arguments")
-    # ### end Alembic commands ###
+    # removing extra types in an enum can make fields in the database invalid