Merge pull request #116 from datakind/matt-azure-kubernetes-service-d…

…eployment

Azure kubernetes service deployment analysis

.dockerignore

@@ -6,3 +6,4 @@
 !gunicorn_config.py
 !make_celery.py
 !colandr_data/
+!migrations

Dockerfile → Dockerfile.api

@@ -16,25 +16,27 @@ RUN python -m textacy download lang_identifier --version 3.0 \
     && python -m spacy download es_core_news_md \
     && python -m spacy download fr_core_news_md
 
+COPY . .
+
 #####
 FROM base AS dev
 
 RUN python -m pip install -r requirements/dev.txt
 
-COPY . .
-
 # TODO: should we do this instead?
 # RUN python -m pip install -e .[dev]
 
 EXPOSE 5000
 
+# flask --app "colandr.app:create_app()" run --host 0.0.0.0 --port 5000 --debug
 CMD ["flask", "--app", "colandr.app:create_app()", "run", "--host", "0.0.0.0", "--port", "5000", "--debug"]
 
+# Good for debugging
+#CMD ["tail", "-f", "/dev/null"]
+
 #####
 FROM base AS prod
 
-COPY . .
-
 EXPOSE 5000
 
 CMD ["gunicorn", "--config", "./gunicorn.conf.py", "colandr.app:create_app()"]

Dockerfile.worker

@@ -0,0 +1,28 @@
+FROM python:3.10-slim AS base
+
+ENV COLANDR_APP_DIR /app
+RUN mkdir -p ${COLANDR_APP_DIR}
+WORKDIR ${COLANDR_APP_DIR}
+
+RUN apt update \
+    && apt install -y gcc git \
+    && apt clean \
+    && rm -rf /var/lib/apt/lists/* /usr/share/doc /usr/share/man
+
+COPY requirements/ ./requirements/
+RUN python -m pip install --upgrade pip wheel && python -m pip install -r requirements/prod.txt
+RUN python -m textacy download lang_identifier --version 3.0 \
+    && python -m spacy download en_core_web_md \
+    && python -m spacy download es_core_news_md \
+    && python -m spacy download fr_core_news_md
+
+#####
+FROM base AS dev
+
+RUN python -m pip install -r requirements/dev.txt
+
+COPY . .
+
+CMD ["celery", "--app=make_celery.celery_app", "worker", "--loglevel=info"]
+
+#CMD ["tail", "-f", "/dev/null"]

compose.yml

@@ -50,9 +50,10 @@ services:
       retries: 5
   api:
     container_name: colandr-api
+    #image: dkdsprototypesreg01.azurecr.io/colandr-api:api
     build:
       context: "."
-      dockerfile: Dockerfile
+      dockerfile: Dockerfile.api
       target: dev
     depends_on:
       - db
@@ -82,14 +83,17 @@ services:
       - 5001:5000
   worker:
     container_name: colandr-worker
+    #image: dkdsprototypesreg01.azurecr.io/colandr-api:worker
     build:
       context: "."
-      dockerfile: Dockerfile
+      dockerfile: Dockerfile.worker
       target: dev
     depends_on:
       - db
       - broker
     stop_signal: SIGINT
+    ports:
+      - "8000:8000"
     env_file: ".env"
     environment:
       - COLANDR_DATABASE_URI=postgresql+psycopg://${COLANDR_DB_USER}:${COLANDR_DB_PASSWORD}@colandr-db:5432/${COLANDR_DB_NAME}

deploy_azure.py

@@ -0,0 +1,107 @@
+import os
+import sys
+
+import docker
+from dotenv import load_dotenv
+
+
+load_dotenv()
+
+container_registry = "dkdsprototypesreg01.azurecr.io"
+repo = "colandr-api"
+aks_cluster="dkprototypesaks"
+aks_namespace ="colandr-api"
+azure_resource_group="DK-DS-Prototypes"
+
+# Script is run from top directory
+docker_compose_file = "docker-compose-deploy.yml"
+azure_platform = "linux/amd64"
+
+if sys.platform == "darwin":
+    print("Running on Mac")
+    client = docker.DockerClient(
+        base_url=f"unix:///Users/{os.getenv('LOGNAME')}/.docker/run/docker.sock "
+    )
+else:
+    client = docker.from_env()
+
+
+def run_cmd(cmd):
+    """
+    Executes a command in the shell and prints the command before executing.
+
+    Args:
+        cmd (str): The command to be executed.
+
+    Returns:
+        None
+    """
+    print(cmd)
+    os.system(cmd)
+
+
+def deploy():
+    """
+    Deploys the application to Azure using Docker Compose.
+
+    This function performs the following steps:
+    1. Logs into Azure using the 'az login' command.
+    2. Logs into the Azure Container Registry using the 'az acr login' command.
+    3. Stops and removes any existing containers using the 'docker compose down' command.
+    4. Pulls the latest images from the Docker Compose file using the 'docker compose pull' command, for chosen architecture
+    5. Builds the Docker images using the 'docker compose build' command.
+    6. Tags and pushes the Docker images to the Azure Container Registry.
+    7. Deploys any changes AKS services/deployments
+    8. Restarts AKS services (required for when code changes but not AKS deployment files)
+
+    """
+    tags = {
+        "colandr-back-api": [f"{container_registry}/{repo}", "colandr-api"],
+        "colandr-back-worker": [f"{container_registry}/{repo}", "colandr-worker"],
+        # Kubernetes can pull these directly, see deployment files in ./deployment
+        #"postgres:16": [
+        #    f"{container_registry}/{repo}",
+        #    "db",
+        #],
+        #"axllent/mailpit:v1.17": [f"{container_registry}/{repo}", "colandr-email"],
+        #"redis:7.0": [f"{container_registry}/{repo}", "colandr-broker"],
+    }
+
+    # Log in
+    run_cmd("az login")
+    run_cmd(f"az acr login --name {container_registry}")
+    run_cmd(f"az aks get-credentials --resource-group {azure_resource_group} --name {aks_cluster}")
+    run_cmd(f"docker compose -f {docker_compose_file} down")
+
+    # Force build on linux/amd64 for Azure
+    print(f"Building docker images for {azure_platform} ...")
+    run_cmd(
+        f"DOCKER_DEFAULT_PLATFORM={azure_platform} && docker compose -f {docker_compose_file} pull"
+    )
+    run_cmd(
+        f"DOCKER_DEFAULT_PLATFORM={azure_platform} && docker compose -f {docker_compose_file} build"
+    )
+
+    run_cmd(f"docker compose -f {docker_compose_file} up -d --build")
+
+    for image in tags.keys():
+        print(f"Tagging {image} image ... with tag {tags[image][0]}:{tags[image][1]}")
+        client.images.get(image).tag(tags[image][0], tags[image][1])
+        print(f"Pushing {image} image ... to {tags[image][0]}:{tags[image][1]}")
+        client.images.push(tags[image][0], tags[image][1])
+
+    print("Deploying kubernetes any changed services and config ...")
+    run_cmd(f"cd deployment && kubectl apply -f . -n {aks_namespace}")
+    print("Restarting kubernetes services to pick up code changes ...")
+    run_cmd(f"cd deployment && kubectl rollout restart deployment/{aks_namespace} -n {aks_namespace}")
+
+
+    # If you want to bring system back up on host architecture
+    #run_cmd(f"docker compose -f {docker_compose_file} down")
+    #run_cmd(f"docker compose -f {docker_compose_file} pull")
+    #run_cmd(f"docker compose -f {docker_compose_file} build")
+    #run_cmd(f"docker compose -f {docker_compose_file} up -d")
+
+
+if __name__ == "__main__":
+    deploy()

deployment/README.md

@@ -0,0 +1,85 @@
+# Quick start
+
+Install Azure CLI and Kubernetes CLI, as instructed [here](https://learn.microsoft.com/en-us/azure/aks/tutorial-kubernetes-deploy-cluster?tabs=azure-cli).
+
+Then log into Azure ...
+
+`az login`
+
+Then log into [the cluster](https://portal.azure.com/#@DataKindO365.onmicrosoft.com/resource/subscriptions/21fe0672-504b-4b05-b7e1-a154142c9fd4/resourceGroups/dk-ds-prototypes/providers/Microsoft.ContainerService/managedClusters/dkprototypesaks/workloads) ...
+
+`az aks get-credentials --resource-group DK-DS-Prototypes --name dkprototypesaks`
+
+Then all  the commands mentioned the useful commands section below should work. 
+
+So a workflow would look like ...
+
+1. Replace `./deployment/env-configmap.yaml` with the one configured for your host
+2. `git checkout matt-azure-kubernetes-service-deployment` 
+3. Do some work, change some code, or adjust the Kubernetes configuration files in `./deployment`
+4. Run deployment script to push to container registry: `python3 deploy_azure.py`
+5. Go to the URL, to find it click under "Services and Ingresses" in the Azure portal
+
+
+# Useful commands
+
+To Deploy using kubectl ...
+
+One service:
+
+`kubectl apply -f env-configmap.yaml -n colandr-api`
+
+All if in a directory (eg ../deployment)
+
+`kubectl apply -f . -n colandr-api`
+
+Restart all sevices (useful if the code is the only change, ie Kubernetes config files haven't changed) ...
+
+`kubectl rollout restart deployment/colandr-api -n colandr-api`
+
+Get list of pods, you need the name for getting logs and exec commands ...
+
+`kubectl get pods -n colandr-api`
+
+Get logs ...
+
+`kubectl logs <POD>  -n colandr-api`
+
+Get logs in one line of any that have word 'api' in them ...
+
+`kubectl get pods --no-headers -o custom-columns=":metadata.name" -n colandr-api | grep api | xargs -I{} kubectl logs {} -n colandr-api`
+
+Get details on pods ...
+
+```
+kubectl describe pod <POD> -n colandr-api
+kubectl describe configmap env -n colandr-api
+```
+
+Test a pushed image ...
+
+```
+docker compose -f docker-compose-deploy.yml up -d --build 
+docker exec -it colandr-worker /bin/bash
+docker tag colandr-back-worker dkdsprototypesreg01.azurecr.io/colandr-api:worker
+docker exec -it dkdsprototypesreg01.azurecr.io/colandr-api:worker /bin/bash
+docker push dkdsprototypesreg01.azurecr.io/colandr-api:worker
+docker pull dkdsprototypesreg01.azurecr.io/colandr-api:worker
+docker run --env-file .env -it -e "COLANDR_DATABASE_URI=postgresql+psycopg://${COLANDR_DB_USER}:${COLANDR_DB_PASSWORD}@host.docker.internal:5432/${COLANDR_DB_NAME}" --rm dkdsprototypesreg01.azurecr.io/colandr-api:worker /bin/bash
+```
+
+.... Then run the docker compose 'command' or entry point for this container, eg celery --app=make_celery.celery_app worker --loglevel=info
+
+Run a container ...
+
+`kubectl run -i --tty --rm debug --image=dkdsprototypesreg01.azurecr.io/colandr-api:worker -- bash`
+
+To Get running container's env ...
+
+`kubectl exec -it  worker-98b947488-bdhhb  -n colandr-api -- env`
+
+Trick to get a failing container up, so you can exec into it and debug, temporarily change the CMD in the Dockerfile to ...
+
+`CMD ["tail", "-f", "/dev/null"]`
+
+You can then run the command interactively.