Skip to content

Commit

Permalink
change subprocess calls to python api calls
Browse files Browse the repository at this point in the history
  • Loading branch information
@renjith-digicat
renjith-digicat committed Oct 1, 2024
1 parent f41c6d9 commit 9ae4707
Show file tree
Hide file tree
Showing 5 changed files with 150 additions and 240 deletions.
161 changes: 72 additions & 89 deletions src/build_mlflow_docker_image.py
View file
Original file line number Diff line number Diff line change
@@ -1,82 +1,45 @@
"""Build an mlflow deployable docker image."""

import argparse
import json
import os
import subprocess

import mlflow

from src.utils import set_mlflow_tracking_uri


def build_docker_image(model_uri, image_name, use_cli=True):
"""Build mlflow model docker image for deployment."""
print("MLFLow model docker build starting.")
if use_cli:
print("Building using cli...")
subprocess.run(
[
"mlflow",
"models",
"build-docker",
"--model-uri",
model_uri,
"--name",
image_name,
"--enable-mlserver",
],
check=True,
)
else:
print("Building using python api...")
mlflow.models.build_docker(
model_uri=model_uri,
name=image_name,
enable_mlserver=True,
base_image="python:3.12-slim",
)
print(f"MLFlow model docker image `{image_name}` build completed.")

import docker
from mlflow.pyfunc.backend import PyFuncBackend

def generate_docker_file(
model_uri: str, out_dir: str = "./mlflow-dockerfile", use_cli: bool = True
):
from src.utils import envs, set_mlflow_tracking_uri


def generate_docker_file(model_uri: str, out_dir: str = "./mlflow-dockerfile"):
"""Generate mlflow model Dockerfile for deployment."""
if use_cli:
print("Generating Dockerfile using cli...")
subprocess.run(
[
"mlflow",
"models",
"generate-dockerfile",
"--model-uri",
model_uri,
"--output-directory",
out_dir,
"--enable-mlserver",
],
check=True,
)
else:
print("Generating Dockerfile using python api...")
raise NotImplementedError
print("Generating Dockerfile using python api...")
config = {}
env_manager = "conda"
# Create an instance of the PyFuncBackend class
backend = PyFuncBackend(config=config, env_manager=env_manager)
# backend = PyFuncBackend()
backend.generate_dockerfile(
model_uri=model_uri, output_dir=out_dir, enable_mlserver=True
)

print(f"MLFlow model Dockerfile generated in `./{out_dir}/`")


def modify_dockerfile(dockerfile_dir: str = "./mlflow-dockerfile/"):
"""Add dependency installation instructions to the dockerfile."""
# Dependency installation to be added
install_dependencies = "RUN apt-get -y update && apt-get install " \
"-y --no-install-recommends gcc libc-dev\n"
# Command to install dependencies
command_to_install_deps = (
"RUN apt-get -y update && apt-get install "
"-y --no-install-recommends gcc libc-dev\n"
)

# Insert the dependency installation line after the FROM line
dockerfile_path = os.path.join(dockerfile_dir, "Dockerfile")
with open(dockerfile_path, "r") as file:
lines = file.readlines()
for index, line in enumerate(lines):
if line.startswith("FROM"):
lines.insert(index + 1, install_dependencies)
lines.insert(index + 1, command_to_install_deps)
break

# Update Dockerfile
Expand All @@ -88,32 +51,42 @@ def modify_dockerfile(dockerfile_dir: str = "./mlflow-dockerfile/"):

def build_from_dockerfile(
dockerfile_dir: str = "./mlflow-dockerfile/",
docker_registry: str = "localhost:5000",
image_name: str = "mlflow_model",
image_tag: str = "latest",
use_cli: bool = True,
):
"""Build image from given dockerfile."""
dockerfile_path = os.path.join(dockerfile_dir, "Dockerfile")
if use_cli:

print(
f"Building image from Dockerfile - "
f"{dockerfile_path} using python api..."
)

try:
client = docker.from_env()
full_image_name = f"{docker_registry}/{image_name}:{image_tag}"

# Build the Docker image
print(
f"Building image from Dockerfile - {dockerfile_path} using cli..."
)
subprocess.run(
[
"docker",
"build",
"-t",
f"{image_name}:{image_tag}",
f"{dockerfile_dir}",
],
check=True,
f"Building Docker image {full_image_name} from {dockerfile_dir}..."
)
else:
print(
f"Building image from Dockerfile - "
f"{dockerfile_path} using python api..."
image, logs = client.images.build(
path=dockerfile_dir, tag=full_image_name
)
raise NotImplementedError

# Print build logs
for log in logs:
if "stream" in log:
print(log["stream"].strip())

print(f"Docker image {full_image_name} built successfully!")
except docker.errors.BuildError as e:
print(f"Error building Docker image: {str(e)}")
except Exception as e:
print(f"An error occurred: {str(e)}")

return full_image_name


if __name__ == "__main__":
Expand All @@ -125,22 +98,32 @@ def build_from_dockerfile(
help="MLFlow model uri",
)

parser.add_argument(
"--out_dir", type=str, required=False, default="./mlflow-dockerfile"
)

args = parser.parse_args()

mlflow_tracking_uri = os.getenv(
"MLFLOW_TRACKING_URI", "http://mlflow-tracking:80"
)
set_mlflow_tracking_uri(mlflow_tracking_uri)
set_mlflow_tracking_uri(envs.mlflow_tracking_uri)

model_uri = args.model_uri

model_docker_image_name = os.getenv(
"MODEL_DOCKER_IMAGE", "house_price_model"
)
mlflow_build_base_image = os.getenv(
"MODEL_DOCKER_IMAGE", "python:3.12.4-slim"
)
# Generate a docker file with artefacts and dependencies gathered
# from the mlflow remote registry
generate_docker_file(model_uri, out_dir=args.out_dir)

build_docker_image(
model_uri, model_docker_image_name, mlflow_build_base_image
# Add missing dependencies to the generated dockerfile
modify_dockerfile(dockerfile_dir=args.out_dir)

# Build docker image to be used as the base image for deployment
docker_image = build_from_dockerfile(
image_name=envs.mlflow_built_image_name,
docker_registry=envs.docker_registry,
image_tag=envs.mlflow_built_image_tag,
)

# write model_uri to the file checked by Airflow for task XComs
xcom_json = {"docker_image": f"{docker_image}"}

with open("/airflow/xcom/return.json", "w") as f:
json.dump(xcom_json, f)
19 changes: 6 additions & 13 deletions src/get_mlflow_model_details.py
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,11 @@
"""Gather required model and run details of the model we want to deploy."""

import json
import os

from src.utils import set_mlflow_tracking_uri
from src.utils import envs, set_mlflow_tracking_uri


def get_model_run_uri(
mlflow_tracking_uri: str, model_name: str, model_alias: str
):
def get_model_run_uri(model_name: str, model_alias: str):
"""Create an accessible mlflow model uri."""
model_uri = f"models:/{model_name}@{model_alias}"
print(f"MLflow model URI: {model_uri}")
Expand All @@ -17,16 +14,12 @@ def get_model_run_uri(


if __name__ == "__main__":
mlflow_tracking_uri = os.getenv(
"MLFLOW_TRACKING_URI", "http://mlflow-tracking:80"
set_mlflow_tracking_uri(envs.mlflow_tracking_uri)
model_uri = get_model_run_uri(
envs.deploy_model_name, envs.deploy_model_alias
)
model_name = os.getenv("DEPLOY_MODEL_NAME", "house_price_prediction_prod")
model_alias = os.getenv("DEPLOY_MODEL_ALIAS", "champion")

set_mlflow_tracking_uri(mlflow_tracking_uri)
model_uri = get_model_run_uri(mlflow_tracking_uri, model_name, model_alias)

# write to the file checked by Airflow for XComs
# write model_uri to the file checked by Airflow for task XComs
xcom_json = {"model_uri": f"{model_uri}"}

with open("/airflow/xcom/return.json", "w") as f:
Expand Down
109 changes: 24 additions & 85 deletions src/main.py
View file
Original file line number Diff line number Diff line change
@@ -1,100 +1,39 @@
"""Main entry point."""

import os

from src.build_mlflow_docker_image import (build_from_dockerfile,
generate_docker_file,
modify_dockerfile)
from src.get_mlflow_model_details import get_model_run_uri
from src.push_image_to_cr import push_image_to_dockerhub
from src.utils import set_mlflow_tracking_uri
from src.push_image_to_cr import push_image_to_docker_registry
from src.utils import envs, set_mlflow_tracking_uri

if __name__ == "__main__":
mlflow_tracking_uri = os.getenv(
"MLFLOW_TRACKING_URI", "http://mlflow-tracking:80"
)
model_name = os.getenv("DEPLOY_MODEL_NAME", "house_price_prediction_prod")
model_alias = os.getenv("DEPLOY_MODEL_ALIAS", "champion")
model_docker_image_name = os.getenv(
"MODEL_DOCKER_IMAGE", "house_price_model"
)
mlflow_build_base_image = os.getenv(
"MODEL_DOCKER_IMAGE", "python:3.12-slim"
)

image_name = "mlflow_model"
image_tag = "latest"
# Set the output directory for docker files and artefacts
out_dir = "./mlflow-dockerfile"

set_mlflow_tracking_uri(mlflow_tracking_uri)
# Set the mlflow tracking uri before getting the model details
set_mlflow_tracking_uri(envs.mlflow_tracking_uri)

model_uri = get_model_run_uri(mlflow_tracking_uri, model_name, model_alias)

generate_docker_file(model_uri)
# Get the model uri;
# to collect the model and artefacts form the mlflow remote registry
model_uri = get_model_run_uri(
envs.deploy_model_name, envs.deploy_model_alias
)

modify_dockerfile()
build_from_dockerfile(image_name=image_name, image_tag=image_tag)
# Generate a docker file with artefacts and dependencies gathered
# from the mlflow remote registry
generate_docker_file(model_uri, out_dir=out_dir)

# Optional at the moment
push_image_to_dockerhub(image_name, image_tag, use_cli=True)
# Add missing dependencies to the generated dockerfile
modify_dockerfile(dockerfile_dir=out_dir)

"""
# Now manually load the generated image
to local kind cluster using the following command from cli
$ kind load docker-image mlflow_model:\
latest --name bridgeai-gitops-infra
# Then ensure the kserve-inference.yaml
is modified to deploy form docker image
```yaml
apiVersion: "serving.kserve.io/v1beta1"
kind: "InferenceService"
metadata:
name: "house-price"
annotations:
serving.kserve.io/deploymentMode: RawDeployment
serving.kserve.io/gateway-disableIngressCreation: "true"
serving.kserve.io/gateway-disableIstioVirtualHost: "true"
spec:
predictor:
minReplicas: 1
maxReplicas: 1
containers:
- name: "mlflow-regression-model"
image: "mlflow_model:latest"
ports:
- containerPort: 8080
protocol: TCP
env:
- name: PROTOCOL
value: "v2"
resources:
requests:
memory: "4Gi"
cpu: "2"
limits:
memory: "5Gi" # Increase memory limit to avoid OOMKilled
cpu: "3"
```
Note: without `minReplicas: 1` and `maxReplicas: 1`
the pod is restarting because of some error;
```
Warning FailedGetResourceMetric
HorizontalPodAutoscaler/house-price-predictor
failed to get cpu utilization: unable to get metrics for resource cpu:
unable to fetch metrics from resource metrics API:
the server could not find the requested resource (get pods.metrics.k8s.io)
```
Now run
$ kubectl apply -f kserve-inference.yaml
$ kubectl get inferenceservice house-price
NAME URL READY
PREV LATEST PREVROLLEDOUTREVISION LATESTREADYREVISION
AGE
house-price http://house-price-default.example.com True
41m
# Build docker image to be used as the base image for deployment
docker_image = build_from_dockerfile(
image_name=envs.mlflow_built_image_name,
docker_registry=envs.docker_registry,
image_tag=envs.mlflow_built_image_tag,
)

"""
# Push the built image to local container registry
push_image_to_docker_registry(docker_image)
Loading

0 comments on commit 9ae4707

Please sign in to comment.