Release

build.Jenkinsfile

@@ -0,0 +1,93 @@
+@Library('sailpoint/jenkins-release-utils')_
+pipeline {
+
+    parameters {
+        string(name: 'BRANCH', defaultValue: 'master')
+    }
+
+    agent {
+        kubernetes {
+            yaml "${libraryResource 'pods/build-container.yaml'}"
+        }
+    }
+
+    environment {
+
+        SERVICE_NAME = "feast-bytewax"
+        GITHUB_REPO = "[email protected]:sailpoint/feast.git"
+        ECR_REPOSITORY = "${env.AWS_ACCOUNT_ID}.dkr.ecr.us-east-1.amazonaws.com"
+        REPOSITORY_NAME = "sailpoint/${SERVICE_NAME}"
+    }
+
+    stages {
+        stage('Checkout SCM') {
+            steps {
+                checkout(
+                [$class: 'GitSCM',
+                branches: [[name: "origin/${BRANCH}"], [name: "*/master"]],
+                doGenerateSubmoduleConfigurations: false,
+                extensions: [], submoduleCfg: [],
+                userRemoteConfigs: [[credentialsId: 'git-automation-ssh', url: "$GITHUB_REPO"]]])
+            }
+        }
+        stage('Build Docker') {
+            steps {
+                container('kaniko') {
+                    script {
+                        echo "publishing ${SERVICE_NAME}:${BUILD_NUMBER}"
+
+                        sh """
+                            /kaniko/executor \
+                                --context ./ \
+                                --dockerfile ./sailpoint-bytewax.dockerfile \
+                                --build-arg  DOCKER_BUILDKIT=1 \
+                                --destination=${ECR_REPOSITORY}/${REPOSITORY_NAME}:${BUILD_NUMBER}
+                        """
+                    }
+                }
+            }
+        }
+    }
+    post {
+        always {
+            junit allowEmptyResults: true, testResults: '**/cov/*.xml'
+        }
+        success {
+            container('aws-cli') {
+                echo "Rebasing tag 'latest' to be '${BUILD_NUMBER}'"
+                sh '''
+                    MANIFEST=`aws ecr batch-get-image --repository-name ${REPOSITORY_NAME} --image-ids imageTag=${BUILD_NUMBER}  --query images[].imageManifest --output text`
+                    aws ecr put-image --repository-name ${REPOSITORY_NAME} --image-tag latest --image-manifest "\$MANIFEST"
+                '''
+            }
+            slackSend(
+                channel       : "proj-eng-iai-cicd",
+                message       : "${SERVICE_NAME}:${BUILD_NUMBER} Build \n ${JOB_URL} ",
+                replyBroadcast: true,
+                color         : "#33ff33",
+                failOnError   : false
+            )
+        }
+        failure {
+            container('aws-cli') {
+                assumePodRole {
+                    script {
+                       sh '''#!/bin/bash
+                            aws ecr batch-delete-image \
+                                --repository-name $REPOSITORY_NAME \
+                                --image-ids imageTag=$BUILD_NUMBER
+                          '''
+                    }
+                }
+            }
+
+            slackSend(
+                channel       : "proj-eng-iai-cicd",
+                message       : "@iai-tarragon ${SERVICE_NAME}:${BUILD_NUMBER} Build broken \n ${JOB_URL} ",
+                replyBroadcast: true,
+                color         : "#ff3333",
+                failOnError   : false
+            )
+        }
+    }
+}

sailpoint-bytewax.dockerfile

@@ -0,0 +1,30 @@
+# Original File: sdk/python/feast/infra/materialization/contrib/bytewax/Dockerfile
+FROM 406205545357.dkr.ecr.us-east-1.amazonaws.com/sailpoint/python3.10:1 AS build
+
+RUN dnf install --refresh -y git
+
+WORKDIR /bytewax
+
+# Copy dataflow code
+COPY sdk/python/feast/infra/materialization/contrib/bytewax/bytewax_materialization_dataflow.py /bytewax
+COPY sdk/python/feast/infra/materialization/contrib/bytewax/dataflow.py /bytewax
+
+# Copy entrypoint
+COPY sdk/python/feast/infra/materialization/contrib/bytewax/entrypoint.sh /bytewax
+
+# Copy necessary parts of the Feast codebase
+COPY sdk/python sdk/python
+COPY protos protos
+COPY go go
+COPY setup.py setup.py
+COPY pyproject.toml pyproject.toml
+COPY README.md README.md
+
+# Install Feast for AWS with Bytewax dependencies
+# We need this mount thingy because setuptools_scm needs access to the
+# git dir to infer the version of feast we're installing.
+# https://github.com/pypa/setuptools_scm#usage-from-docker
+# I think it also assumes that this dockerfile is being built from the root of the directory.
+RUN --mount=source=.git,target=.git,type=bind SETUPTOOLS_SCM_PRETEND_VERSION=1 \
+pip3 install gunicorn==22.0.0 setuptools==69.* pip==24.* --upgrade --no-cache-dir \
+'.[aws,gcp,bytewax,snowflake,postgres,grpcio]'

sdk/python/feast/infra/materialization/kubernetes/Dockerfile

@@ -5,7 +5,7 @@ RUN apt-get update && \
 
 WORKDIR /app
 
-COPY sdk/python/feast/infra/materialization/kuberentes/main.py /app
+COPY sdk/python/feast/infra/materialization/kubernetes/main.py /app
 
 # Copy necessary parts of the Feast codebase
 COPY sdk/python sdk/python

sdk/python/feast/infra/materialization/kubernetes/k8s_materialization_engine.py

@@ -416,6 +416,14 @@ def _create_job_definition(self, job_id, namespace, pods, env, index_offset=0):
                                 "name": self._configmap_name(job_id),
                             },
                         ],
+                        "tolerations": [
+                            {
+                                "key": "bytewax",
+                                "operator": "Equal",
+                                "value": "instance",
+                                "effect": "NoSchedule",
+                            }
+                        ],
                     },
                 },
             },

sdk/python/feast/repo_config.py

@@ -44,7 +44,7 @@
     "local": "feast.infra.materialization.local_engine.LocalMaterializationEngine",
     "snowflake.engine": "feast.infra.materialization.snowflake_engine.SnowflakeMaterializationEngine",
     "lambda": "feast.infra.materialization.aws_lambda.lambda_engine.LambdaMaterializationEngine",
-    "k8s": "feast.infra.materialization.kubernetes.kubernetes_materialization_engine.KubernetesMaterializationEngine",
+    "k8s": "feast.infra.materialization.kubernetes.k8s_materialization_engine.KubernetesMaterializationEngine",
     "spark.engine": "feast.infra.materialization.contrib.spark.spark_materialization_engine.SparkMaterializationEngine",
 }