Test unification

.github/workflows/go_generate.yml

@@ -11,7 +11,7 @@ on:
         type: string
     secrets:
       FLYTE_BOT_PAT:
-        required: true
+        required: false
 jobs:
   generate:
     runs-on: ubuntu-latest
@@ -22,11 +22,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - uses: arduino/setup-protoc@v1
-        with:
-          repo-token: ${{ secrets.FLYTE_BOT_PAT }}
       - uses: bufbuild/buf-setup-action@v1
-        with:
-          github_token: ${{ secrets.FLYTE_BOT_PAT }}
       - uses: actions/setup-go@v3
         with:
           go-version: ${{ inputs.go-version }}

.github/workflows/single-binary.yml

@@ -156,7 +156,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - uses: unionai/[email protected]
       - name: Setup sandbox
         run: |
@@ -181,17 +181,17 @@ jobs:
         with:
           repository: flyteorg/flytesnacks
           path: flytesnacks
-          # TODO: Enable this once refactored version produces a release tag
-          # ref: ${{ env.FLYTESNACKS_VERSION }}
+          ref: ${{ env.FLYTESNACKS_VERSION }}
       - name: Register specific tests
         run: |
+          flytekit_version=$(pip show flytekit | grep -i version | awk '{ print $2 }')
           while read -r line;
           do
               pyflyte --config ./boilerplate/flyte/end2end/functional-test-config.yaml \
                   register \
                   --project flytesnacks \
                   --domain development \
-                  --image cr.flyte.org/flyteorg/flytekit:py3.11-latest \
+                  --image cr.flyte.org/flyteorg/flytekit:py3.12-${flytekit_version} \
                   --version ${{ env.FLYTESNACKS_VERSION }} \
                   flytesnacks/$line;
           done < flytesnacks/flyte_tests.txt

.github/workflows/validate-helm-charts.yaml

@@ -0,0 +1,89 @@
+name: Validate helm charts & manifests
+
+on:
+  pull_request:
+    branches:
+      - master
+    paths:
+      - "charts/**"
+      - "deployment/**"
+      - "docker/sandbox-bundled/manifests/**"
+
+jobs:
+  lint-and-test-charts:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          sparse-checkout: charts
+
+      - name: Install Helm
+        uses: azure/setup-helm@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          check-latest: true
+
+      - uses: unionai/[email protected]
+        name: Setup flytectl
+
+      - name: Set up chart-testing
+        uses: helm/chart-testing-action@v2
+
+      - name: Detect charts changed (list-changed)
+        id: charts-changed
+        run: |
+          changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }})
+          if [[ -n "$changed" ]]; then
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Run chart-testing (lint)
+        if: steps.charts-changed.outputs.changed == 'true'
+        run: |
+          ct lint \
+            --target-branch ${{ github.event.repository.default_branch }} \
+            --validate-maintainers=false \
+            --check-version-increment=false \
+            --chart-repos spark-operator=https://kubeflow.github.io/spark-operator \
+            --chart-repos dask=https://helm.dask.org \
+            --chart-repos bitnami=https://charts.bitnami.com/bitnami \
+            --chart-repos twuni=https://helm.twun.io \
+            --chart-repos kubernetes-dashboard=https://kubernetes.github.io/dashboard
+
+  validate-manifests:
+    needs:
+      - lint-and-test-charts
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        shell: bash
+    strategy:
+      matrix:
+        k8s_versions: [ "1.29.2", "1.28.7", "1.27.11" ]
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: |
+            deployment
+            docker/sandbox-bundled/manifests
+
+      - name: Install Helm
+        uses: azure/setup-helm@v4
+
+      - name: Install kubeconform
+        run: |
+          curl -L -o kubeconform.tar.gz https://github.com/yannh/kubeconform/releases/download/v0.6.4/kubeconform-linux-amd64.tar.gz
+          tar -zvxf kubeconform.tar.gz
+          chmod +x kubeconform
+          sudo mv kubeconform /usr/local/bin/kubeconform
+
+      - name: Validate manifests
+        run: |
+          kubeconform -strict -summary -skip CustomResourceDefinition -ignore-filename-pattern "deployment/stats/prometheus/*" -kubernetes-version ${{ matrix.k8s_versions }} ./deployment ./docker/sandbox-bundled/manifests

CHANGELOG/CHANGELOG-v1.11.0.md

@@ -0,0 +1,32 @@
+# Flyte v1.11.0 Release Notes
+
+We're excited to announce the release of Flyte v1.11.0! This version brings a host of improvements, bug fixes, and new features designed to enhance your experience with Flyte. From operational enhancements to documentation updates, this release aims to make Flyte more robust, user-friendly, and feature-rich.
+
+## Highlights
+
+- **Agents hit General Availability (GA):** Agents, now in General Availability, are long-running, stateless services that facilitate asynchronous job launches on platforms like Databricks or Snowflake and enable external service calls. They are versatile, supporting implementations in any language through a protobuf interface, enhancing Flyte's flexibility and operational efficiency.
+- **Improved Caching:** Support for loading cached sublists with multiple data types has been introduced, eliminating issues related to cache retrieval across varied data formats.
+- **Tracing and Observability:** The introduction of opentelemetry BlobstoreClientTracer in flyteadmin enhances observability, allowing for better monitoring and troubleshooting.
+- **Security Enhancements:** Added securityContext configuration to Flyte-core charts, strengthening the security posture of Flyte deployments.
+- **Documentation Overhaul:** Continuous improvements and updates have been made to the documentation, fixing broken links and updating content for better clarity and usability.
+- **Operational Improvements:** This release introduces enhancements such as adding a service account for V1 Ray Jobs, caching console assets in a single binary, and conditional mounting of secrets to improve the operational efficiency of Flyte. Additionally, we are removing `kustomize` from our deployment process to simplify the configuration and management of Flyte instances, making it easier for users to maintain and streamline their deployment workflows.
+
+
+## Bug Fixes
+
+- **Fixed Literal in Launchplan:** Added fixed_literal to the launchplan template, addressing issues with hardcoded values in workflows.
+- **Corrected Metadata and Resources:** Fixes have been applied to correct IsParent metadata in ArrayNode eventing and to address invalid "resources" scope issues in deployment configurations.
+- **Enhanced Stability and Performance:** Numerous bug fixes have been implemented to address stability and performance issues, including fixes for data catalog errors, yaml comment errors in pod template examples, and more.
+
+## Documentation and Guides
+
+- **Comprehensive Guides:** New guides and documentation updates have been added, including a ChatGPT Agent Setup guide and an Airflow migration guide. Improvements in documentation for developing agents have been integrated into the broader enhancements for this release.
+- **Updated Troubleshooting and Configuration Docs:** New troubleshooting guides for spark task execution and updates to deployment configuration documents enhance the knowledge base for Flyte users.
+
+## Contributors
+
+We extend our deepest gratitude to all the contributors who made this release possible. Special shoutouts to @neilisaur, @lowc1012, @MortalHappiness, @novahow, and @pryce-turner for making their first contributions!
+
+**For a full list of changes, enhancements, and bug fixes, visit our [changelog](https://github.com/flyteorg/flyte/compare/v1.10.7...v1.11.0).**
+
+Thank you for your continued support of Flyte. We look forward to hearing your feedback on this release!

CHANGELOG/CHANGELOG-v1.11.1-b0.md

@@ -0,0 +1,2 @@
+# Flyte v1.11.1-b0
+

Dockerfile

@@ -2,7 +2,7 @@ ARG FLYTECONSOLE_VERSION=latest
 FROM ghcr.io/flyteorg/flyteconsole:${FLYTECONSOLE_VERSION} AS flyteconsole
 
 
-FROM --platform=${BUILDPLATFORM} golang:1.21.5-bookworm AS flytebuilder
+FROM --platform=${BUILDPLATFORM} golang:1.21-bookworm AS flytebuilder
 
 ARG TARGETARCH
 ENV GOARCH "${TARGETARCH}"

Dockerfile.datacatalog

@@ -1,9 +1,9 @@
 # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
 # ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY:
-# 
+#
 # TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
 
-FROM --platform=${BUILDPLATFORM} golang:1.21.5-alpine3.18 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.21-alpine3.18 as builder
 
 ARG TARGETARCH
 ENV GOARCH "${TARGETARCH}"
@@ -23,6 +23,7 @@ RUN GRPC_HEALTH_PROBE_VERSION=v0.4.11 && \
 
 WORKDIR /go/src/github.com/flyteorg/datacatalog
 
+COPY boilerplate ../boilerplate
 COPY datacatalog .
 COPY flyteadmin ../flyteadmin
 COPY flytecopilot ../flytecopilot

Dockerfile.flyteadmin

@@ -1,9 +1,9 @@
 # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
 # ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY:
-# 
+#
 # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
 
-FROM --platform=${BUILDPLATFORM} golang:1.21.5-alpine3.18 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.21-alpine3.18 as builder
 
 ARG TARGETARCH
 ENV GOARCH "${TARGETARCH}"
@@ -23,6 +23,7 @@ RUN GRPC_HEALTH_PROBE_VERSION=v0.4.11 && \
 
 WORKDIR /go/src/github.com/flyteorg/flyteadmin
 
+COPY boilerplate ../boilerplate
 COPY datacatalog ../datacatalog
 COPY flyteadmin .
 COPY flytecopilot ../flytecopilot

Dockerfile.flytecopilot

@@ -1,9 +1,9 @@
 # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
 # ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY:
-# 
+#
 # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
 
-FROM --platform=${BUILDPLATFORM} golang:1.21.5-alpine3.18 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.21-alpine3.18 as builder
 
 ARG TARGETARCH
 ENV GOARCH "${TARGETARCH}"
@@ -13,6 +13,7 @@ RUN apk add git openssh-client make curl
 
 WORKDIR /go/src/github.com/flyteorg/flytecopilot
 
+COPY boilerplate ../boilerplate
 COPY datacatalog ../datacatalog
 COPY flyteadmin ../flyteadmin
 COPY flytecopilot .

Dockerfile.flytepropeller

@@ -1,9 +1,9 @@
 # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
 # ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY:
-# 
+#
 # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
 
-FROM --platform=${BUILDPLATFORM} golang:1.21.5-alpine3.18 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.21-alpine3.18 as builder
 
 ARG TARGETARCH
 ENV GOARCH "${TARGETARCH}"
@@ -13,6 +13,7 @@ RUN apk add git openssh-client make curl
 
 WORKDIR /go/src/github.com/flyteorg/flytepropeller
 
+COPY boilerplate ../boilerplate
 COPY datacatalog ../datacatalog
 COPY flyteadmin ../flyteadmin
 COPY flytecopilot ../flytecopilot

Dockerfile.flytescheduler

@@ -1,9 +1,9 @@
 # WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
 # ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY:
-# 
+#
 # TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
 
-FROM --platform=${BUILDPLATFORM} golang:1.21.5-alpine3.18 as builder
+FROM --platform=${BUILDPLATFORM} golang:1.21-alpine3.18 as builder
 
 ARG TARGETARCH
 ENV GOARCH "${TARGETARCH}"
@@ -16,6 +16,7 @@ RUN mkdir /artifacts
 
 WORKDIR /go/src/github.com/flyteorg/flyteadmin
 
+COPY boilerplate ../boilerplate
 COPY datacatalog ../datacatalog
 COPY flyteadmin .
 COPY flytecopilot ../flytecopilot

Makefile

@@ -112,3 +112,12 @@ go-tidy:
 	make -C flyteplugins go-tidy
 	make -C flytestdlib go-tidy
 	make -C flytecopilot go-tidy
+
+.PHONY: lint-helm-charts
+lint-helm-charts:
+	# This pressuposes that you have act installed
+	act pull_request -W .github/workflows/validate-helm-charts.yaml --container-architecture linux/amd64 -e charts/event.json
+
+.PHONY: clean
+clean: ## Remove the HTML files related to the Flyteconsole.
+	rm -rf cmd/single/dist

boilerplate/flyte/docker_build/Makefile

@@ -0,0 +1,12 @@
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'FLYTEORG/BOILERPLATE' REPOSITORY:
+# 
+# TO OPT OUT OF UPDATES, SEE https://github.com/flyteorg/boilerplate/blob/master/Readme.rst
+
+.PHONY: docker_build
+docker_build:
+	IMAGE_NAME=$$REPOSITORY ./boilerplate/flyte/docker_build/docker_build.sh
+
+.PHONY: dockerhub_push
+dockerhub_push:
+	IMAGE_NAME=flyteorg/$$REPOSITORY REGISTRY=docker.io ./boilerplate/flyte/docker_build/docker_build.sh

boilerplate/flyte/docker_build/Readme.rst

@@ -0,0 +1,23 @@
+Docker Build and Push
+~~~~~~~~~~~~~~~~~~~~~
+
+Provides a ``make docker_build`` target that builds your image locally.
+
+Provides a ``make dockerhub_push`` target that pushes your final image to Dockerhub.
+
+The Dockerhub image will tagged ``<REPOSITORY>:<GIT COMMIT SHA>``
+
+If git head has a git tag, the Dockerhub image will also be tagged ``<IMAGE>:<GIT_TAG>``.
+
+**To Enable:**
+
+Add ``flyteorg/docker_build`` to your ``boilerplate/update.cfg`` file.
+
+Add ``include boilerplate/flyte/docker_build/Makefile`` in your main ``Makefile`` _after_ your REPOSITORY environment variable
+
+::
+
+    REPOSITORY=<myreponame>
+    include boilerplate/flyte/docker_build/Makefile
+
+(this ensures the extra Make targets get included in your main Makefile)