From 1bf1415e28a8850f67624759487a720bbc1e9ff8 Mon Sep 17 00:00:00 2001 From: Mario Micudaj Date: Fri, 15 Mar 2024 09:00:03 +0100 Subject: [PATCH 01/28] allow redirects in status code --- .../camunda/connectors/kubeflow/KubeflowConnectorFunction.java | 2 +- .../connectors/kubeflow/services/KubeflowConnectorExecutor.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/KubeflowConnectorFunction.java b/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/KubeflowConnectorFunction.java index be3746f..1b5c3bb 100644 --- a/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/KubeflowConnectorFunction.java +++ b/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/KubeflowConnectorFunction.java @@ -32,7 +32,7 @@ public Object execute(final OutboundConnectorContext context) HttpResponse response = connectorExecutor.execute(); // raise error based on status code - if (response.statusCode() >= 300) { + if (response.statusCode() >= 400) { throw new RuntimeException(response.body()); } diff --git a/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/services/KubeflowConnectorExecutor.java b/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/services/KubeflowConnectorExecutor.java index c7e441d..1a1a9be 100644 --- a/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/services/KubeflowConnectorExecutor.java +++ b/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/services/KubeflowConnectorExecutor.java @@ -73,7 +73,7 @@ public KubeflowConnectorExecutor(KubeflowConnectorRequest connectorRequest, long public HttpResponse execute() { try { HttpResponse response = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString()); - if (response.statusCode() >= 300) { + if (response.statusCode() >= 400) { throw new RuntimeException("Error while running request against kubeflow server: " + httpRequest.uri() + " - " + response.body()); } From ad3b29f962658e0c2a07eb59cea479638ab7d9e9 Mon Sep 17 00:00:00 2001 From: Mario Micudaj Date: Fri, 15 Mar 2024 12:10:36 +0100 Subject: [PATCH 02/28] updates --- README.md | 45 +++++++++++++++++++++++++++++++-------- bpmn/get_experiments.bpmn | 8 +++---- bpmn/get_pipelines.bpmn | 9 ++++---- 3 files changed, 43 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index c3ae61b..239fb01 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Kubeflow Connector -A custom Camunda 8 outbound connector to communicate with the [Kubeflow](https://www.kubeflow.org/) API. It supports version 1 aqnd version 2 of the Kubeflow API and supports the functions mentioned below in the [API section](#api-section). +A custom Camunda 8 outbound connector to communicate with the [Kubeflow](https://www.kubeflow.org/) API. It supports standalone as well as multi-suer setup and version 1 and 2 of the Kubeflow API. It currently supports the functions mentioned below in the [API section](#api-section). ## Build @@ -16,7 +16,7 @@ This will create the following artifacts: - An fat JAR containing all dependencies, potentially shaded to avoid classpath conflicts. This will not include the SDK artifacts since those are in scope `provided` and will be brought along by the respective Connector Runtime executing the Connector. ## API -Currenty this connector supports the following methods from the Kubeflow Pipeline API. +Currenty this connector supports the following methods from the Kubeflow Pipeline API in both API versions 1 and 2. - Get Pipelines - Get Experiments @@ -29,17 +29,44 @@ Currenty this connector supports the following methods from the Kubeflow Pipelin ### Get Pipelines -### Input +#### Input +Parameter Filter +API Version 1: + +```json +{ "predicates": + [ + { + "op":"IS_SUBSTRING", + "key": "name", + "string_value": "Control" + } + ] +} +``` + +API Version 2: +```json +{ "predicates": + [ + { + "operation":"IS_SUBSTRING", + "key": "name", + "string_value": "Control" + } + ] +} +``` +For details on filter structure and options check the proto-buffer files: + +Version 1: [https://github.com/kubeflow/pipelines/blob/master/backend/api/v1beta1/filter.proto](https://github.com/kubeflow/pipelines/blob/master/backend/api/v1beta1/filter.proto) + +Version 2: [https://github.com/kubeflow/pipelines/blob/master/backend/api/v2beta1/filter.proto](https://github.com/kubeflow/pipelines/blob/master/backend/api/v2beta1/filter.proto) -| Name | Description | Example | Notes | -|----------|------------------|-------------------|----------------------------------------------------------------------------| -| filter | define the filter to apply to the call | `alice` | Has no effect on the function call outcome. | -| token | Mock token value | `my-secret-token` | Has no effect on the function call outcome. | -| message | Mock message | `Hello World` | Echoed back in the output. If starts with 'fail', an error will be thrown. | -### Output +#### Output ```json { diff --git a/bpmn/get_experiments.bpmn b/bpmn/get_experiments.bpmn index a7e8be0..a7934f4 100644 --- a/bpmn/get_experiments.bpmn +++ b/bpmn/get_experiments.bpmn @@ -1,5 +1,5 @@ - + Flow_0xamqby @@ -9,12 +9,10 @@ - - - + + - diff --git a/bpmn/get_pipelines.bpmn b/bpmn/get_pipelines.bpmn index 61fe149..cc85605 100644 --- a/bpmn/get_pipelines.bpmn +++ b/bpmn/get_pipelines.bpmn @@ -9,16 +9,15 @@ - - - - + + + - + Flow_0xamqby From c68bbc20b1271c36ff4e92a7a2c4d018d8bafcc1 Mon Sep 17 00:00:00 2001 From: Mario Micudaj Date: Fri, 15 Mar 2024 12:12:10 +0100 Subject: [PATCH 03/28] allow redirects in status code --- .../camunda/connectors/kubeflow/KubeflowConnectorFunction.java | 2 +- .../connectors/kubeflow/services/KubeflowConnectorExecutor.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/KubeflowConnectorFunction.java b/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/KubeflowConnectorFunction.java index be3746f..1b5c3bb 100644 --- a/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/KubeflowConnectorFunction.java +++ b/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/KubeflowConnectorFunction.java @@ -32,7 +32,7 @@ public Object execute(final OutboundConnectorContext context) HttpResponse response = connectorExecutor.execute(); // raise error based on status code - if (response.statusCode() >= 300) { + if (response.statusCode() >= 400) { throw new RuntimeException(response.body()); } diff --git a/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/services/KubeflowConnectorExecutor.java b/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/services/KubeflowConnectorExecutor.java index c7e441d..1a1a9be 100644 --- a/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/services/KubeflowConnectorExecutor.java +++ b/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/services/KubeflowConnectorExecutor.java @@ -73,7 +73,7 @@ public KubeflowConnectorExecutor(KubeflowConnectorRequest connectorRequest, long public HttpResponse execute() { try { HttpResponse response = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString()); - if (response.statusCode() >= 300) { + if (response.statusCode() >= 400) { throw new RuntimeException("Error while running request against kubeflow server: " + httpRequest.uri() + " - " + response.body()); } From 8774f868aba63bf149b25d9995e14ae7131f28cd Mon Sep 17 00:00:00 2001 From: Mario Micudaj Date: Fri, 15 Mar 2024 12:12:10 +0100 Subject: [PATCH 04/28] updates --- README.md | 45 +++++++++++++++++++++++++++++++-------- bpmn/get_experiments.bpmn | 8 +++---- bpmn/get_pipelines.bpmn | 9 ++++---- 3 files changed, 43 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index c3ae61b..239fb01 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Kubeflow Connector -A custom Camunda 8 outbound connector to communicate with the [Kubeflow](https://www.kubeflow.org/) API. It supports version 1 aqnd version 2 of the Kubeflow API and supports the functions mentioned below in the [API section](#api-section). +A custom Camunda 8 outbound connector to communicate with the [Kubeflow](https://www.kubeflow.org/) API. It supports standalone as well as multi-suer setup and version 1 and 2 of the Kubeflow API. It currently supports the functions mentioned below in the [API section](#api-section). ## Build @@ -16,7 +16,7 @@ This will create the following artifacts: - An fat JAR containing all dependencies, potentially shaded to avoid classpath conflicts. This will not include the SDK artifacts since those are in scope `provided` and will be brought along by the respective Connector Runtime executing the Connector. ## API -Currenty this connector supports the following methods from the Kubeflow Pipeline API. +Currenty this connector supports the following methods from the Kubeflow Pipeline API in both API versions 1 and 2. - Get Pipelines - Get Experiments @@ -29,17 +29,44 @@ Currenty this connector supports the following methods from the Kubeflow Pipelin ### Get Pipelines -### Input +#### Input +Parameter Filter +API Version 1: + +```json +{ "predicates": + [ + { + "op":"IS_SUBSTRING", + "key": "name", + "string_value": "Control" + } + ] +} +``` + +API Version 2: +```json +{ "predicates": + [ + { + "operation":"IS_SUBSTRING", + "key": "name", + "string_value": "Control" + } + ] +} +``` +For details on filter structure and options check the proto-buffer files: + +Version 1: [https://github.com/kubeflow/pipelines/blob/master/backend/api/v1beta1/filter.proto](https://github.com/kubeflow/pipelines/blob/master/backend/api/v1beta1/filter.proto) + +Version 2: [https://github.com/kubeflow/pipelines/blob/master/backend/api/v2beta1/filter.proto](https://github.com/kubeflow/pipelines/blob/master/backend/api/v2beta1/filter.proto) -| Name | Description | Example | Notes | -|----------|------------------|-------------------|----------------------------------------------------------------------------| -| filter | define the filter to apply to the call | `alice` | Has no effect on the function call outcome. | -| token | Mock token value | `my-secret-token` | Has no effect on the function call outcome. | -| message | Mock message | `Hello World` | Echoed back in the output. If starts with 'fail', an error will be thrown. | -### Output +#### Output ```json { diff --git a/bpmn/get_experiments.bpmn b/bpmn/get_experiments.bpmn index a7e8be0..a7934f4 100644 --- a/bpmn/get_experiments.bpmn +++ b/bpmn/get_experiments.bpmn @@ -1,5 +1,5 @@ - + Flow_0xamqby @@ -9,12 +9,10 @@ - - - + + - diff --git a/bpmn/get_pipelines.bpmn b/bpmn/get_pipelines.bpmn index 61fe149..cc85605 100644 --- a/bpmn/get_pipelines.bpmn +++ b/bpmn/get_pipelines.bpmn @@ -9,16 +9,15 @@ - - - - + + + - + Flow_0xamqby From 14b3c312dc65dbb2f6250a98ebdf61e79ed35f6a Mon Sep 17 00:00:00 2001 From: Mario Micudaj Date: Fri, 15 Mar 2024 12:30:40 +0100 Subject: [PATCH 05/28] allow redirects in status code --- .../camunda/connectors/kubeflow/KubeflowConnectorFunction.java | 2 +- .../connectors/kubeflow/services/KubeflowConnectorExecutor.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/KubeflowConnectorFunction.java b/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/KubeflowConnectorFunction.java index be3746f..1b5c3bb 100644 --- a/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/KubeflowConnectorFunction.java +++ b/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/KubeflowConnectorFunction.java @@ -32,7 +32,7 @@ public Object execute(final OutboundConnectorContext context) HttpResponse response = connectorExecutor.execute(); // raise error based on status code - if (response.statusCode() >= 300) { + if (response.statusCode() >= 400) { throw new RuntimeException(response.body()); } diff --git a/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/services/KubeflowConnectorExecutor.java b/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/services/KubeflowConnectorExecutor.java index c7e441d..1a1a9be 100644 --- a/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/services/KubeflowConnectorExecutor.java +++ b/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/services/KubeflowConnectorExecutor.java @@ -73,7 +73,7 @@ public KubeflowConnectorExecutor(KubeflowConnectorRequest connectorRequest, long public HttpResponse execute() { try { HttpResponse response = httpClient.send(httpRequest, HttpResponse.BodyHandlers.ofString()); - if (response.statusCode() >= 300) { + if (response.statusCode() >= 400) { throw new RuntimeException("Error while running request against kubeflow server: " + httpRequest.uri() + " - " + response.body()); } From d6ebb236a7e424c2451709c8a0f89db40d0488ae Mon Sep 17 00:00:00 2001 From: Mario Micudaj Date: Fri, 15 Mar 2024 12:30:40 +0100 Subject: [PATCH 06/28] updates --- README.md | 45 +++++++++++++++++++++++++++++++-------- bpmn/get_experiments.bpmn | 8 +++---- bpmn/get_pipelines.bpmn | 9 ++++---- 3 files changed, 43 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index c3ae61b..239fb01 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Kubeflow Connector -A custom Camunda 8 outbound connector to communicate with the [Kubeflow](https://www.kubeflow.org/) API. It supports version 1 aqnd version 2 of the Kubeflow API and supports the functions mentioned below in the [API section](#api-section). +A custom Camunda 8 outbound connector to communicate with the [Kubeflow](https://www.kubeflow.org/) API. It supports standalone as well as multi-suer setup and version 1 and 2 of the Kubeflow API. It currently supports the functions mentioned below in the [API section](#api-section). ## Build @@ -16,7 +16,7 @@ This will create the following artifacts: - An fat JAR containing all dependencies, potentially shaded to avoid classpath conflicts. This will not include the SDK artifacts since those are in scope `provided` and will be brought along by the respective Connector Runtime executing the Connector. ## API -Currenty this connector supports the following methods from the Kubeflow Pipeline API. +Currenty this connector supports the following methods from the Kubeflow Pipeline API in both API versions 1 and 2. - Get Pipelines - Get Experiments @@ -29,17 +29,44 @@ Currenty this connector supports the following methods from the Kubeflow Pipelin ### Get Pipelines -### Input +#### Input +Parameter Filter +API Version 1: + +```json +{ "predicates": + [ + { + "op":"IS_SUBSTRING", + "key": "name", + "string_value": "Control" + } + ] +} +``` + +API Version 2: +```json +{ "predicates": + [ + { + "operation":"IS_SUBSTRING", + "key": "name", + "string_value": "Control" + } + ] +} +``` +For details on filter structure and options check the proto-buffer files: + +Version 1: [https://github.com/kubeflow/pipelines/blob/master/backend/api/v1beta1/filter.proto](https://github.com/kubeflow/pipelines/blob/master/backend/api/v1beta1/filter.proto) + +Version 2: [https://github.com/kubeflow/pipelines/blob/master/backend/api/v2beta1/filter.proto](https://github.com/kubeflow/pipelines/blob/master/backend/api/v2beta1/filter.proto) -| Name | Description | Example | Notes | -|----------|------------------|-------------------|----------------------------------------------------------------------------| -| filter | define the filter to apply to the call | `alice` | Has no effect on the function call outcome. | -| token | Mock token value | `my-secret-token` | Has no effect on the function call outcome. | -| message | Mock message | `Hello World` | Echoed back in the output. If starts with 'fail', an error will be thrown. | -### Output +#### Output ```json { diff --git a/bpmn/get_experiments.bpmn b/bpmn/get_experiments.bpmn index a7e8be0..a7934f4 100644 --- a/bpmn/get_experiments.bpmn +++ b/bpmn/get_experiments.bpmn @@ -1,5 +1,5 @@ - + Flow_0xamqby @@ -9,12 +9,10 @@ - - - + + - diff --git a/bpmn/get_pipelines.bpmn b/bpmn/get_pipelines.bpmn index 61fe149..cc85605 100644 --- a/bpmn/get_pipelines.bpmn +++ b/bpmn/get_pipelines.bpmn @@ -9,16 +9,15 @@ - - - - + + + - + Flow_0xamqby From 0a3c3d9fb2571994c78f4fb8b2c5cd1c2c5ee20a Mon Sep 17 00:00:00 2001 From: Mario Micudaj Date: Fri, 15 Mar 2024 13:01:31 +0100 Subject: [PATCH 07/28] getPipiles and GetExperiments documented --- README.md | 125 +++++++++++++++++++++++++++++++++++--- bpmn/get_experiments.bpmn | 4 +- bpmn/get_pipelines.bpmn | 3 +- 3 files changed, 123 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 239fb01..51bf00a 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,9 @@ Currenty this connector supports the following methods from the Kubeflow Pipelin - Start Run and Monitor - Create Experiment +The inputs describes the parameters that can be set in the modeler for the operation. +The output is the complete output you will get written into the variable you enter under Output mapping in the result variable. In the result expression you can more specifically extract the data as required to limit the output. + ### Get Pipelines #### Input @@ -65,22 +68,130 @@ Version 1: [https://github.com/kubeflow/pipelines/blob/master/backend/api/v1beta Version 2: [https://github.com/kubeflow/pipelines/blob/master/backend/api/v2beta1/filter.proto](https://github.com/kubeflow/pipelines/blob/master/backend/api/v2beta1/filter.proto) +#### Output + +API Version 1: +```json +{ + "pipelines":[ + { + "id":"2d974767-3d50-40b8-9bfc-7767d4a13a5c", + "created_at":"2024-03-15T10:43:45Z", + "name":"[Tutorial] DSL - Control structures", + "description":"...", + "default_version":{ + "id":"dadc4c6d-b864-49e6-808e-946424e532ba", + "name":"[Tutorial] DSL - Control structures", + "created_at":"2024-03-15T10:43:45Z", + "resource_references":[ + { + "key":{ + "type":"PIPELINE", + "id":"2d974767-3d50-40b8-9bfc-7767d4a13a5c" + }, + "relationship":"OWNER" + } + ], + "description":"..." + } + } + ], + "total_size":1 +} +``` + +API Version 2: +```json +{ + "pipelines": [ + { + "pipeline_id":"369eea92-e69c-4af4-802a-9f403a362bc5", + "display_name":"[Tutorial] DSL - Control structures", + "description":"...", + "created_at":"2024-03-14T22:09:25Z" + } + ], + "total_size":1 +} +``` + +### Get Experiments + +#### Input + +Parameter Filter + +API Version 1: +```json +{ "predicates": + [ + { + "op":"IS_SUBSTRING", + "key": "name", + "string_value": "test" + } + ] +} +``` + +API Version 2: +```json +{ "predicates": + [ + { + "operation":"IS_SUBSTRING", + "key": "name", + "string_value": "test" + } + ] +} +``` #### Output +API Version 1: +```json +{ + "experiments":[ + { + "id":"f4e404a0-6b83-44a8-9eae-9678706188a0", + "name":"Testexperiment", + "created_at":"2024-03-15T11:47:52Z", + "resource_references":[ + { + "key":{ + "type":"NAMESPACE", + "id":"kubeflow-user-example-com" + }, + "relationship":"OWNER" + } + ], + "storage_state":"STORAGESTATE_AVAILABLE" + } + ], + "total_size":1 +} +``` + +API Version 2: ```json { - "result": { - "myProperty": "Message received: ..." - } + "experiments": [ + { + "experiment_id": "f4e404a0-6b83-44a8-9eae-9678706188a0", + "display_name": "Testexperiment", + "created_at": "2024-03-15T11:47:52Z", + "namespace": "kubeflow-user-example-com", + "storage_state": "AVAILABLE" + } + ], + "total_size": 1 } ``` -### Error codes +### Error handling -| Code | Description | -|------|--------------------------------------------| -| FAIL | Message starts with 'fail' (ignoring case) | +In case a return code >= 400 is returned by the API calls, the connector will raise an exception. ## Test locally diff --git a/bpmn/get_experiments.bpmn b/bpmn/get_experiments.bpmn index a7934f4..fbf7c2b 100644 --- a/bpmn/get_experiments.bpmn +++ b/bpmn/get_experiments.bpmn @@ -10,9 +10,11 @@ + - + + diff --git a/bpmn/get_pipelines.bpmn b/bpmn/get_pipelines.bpmn index cc85605..72445e0 100644 --- a/bpmn/get_pipelines.bpmn +++ b/bpmn/get_pipelines.bpmn @@ -11,8 +11,9 @@ - + + From a4a74dc4b293a7f2f9350607a354b8761598f2c2 Mon Sep 17 00:00:00 2001 From: Mario Micudaj Date: Fri, 15 Mar 2024 13:07:41 +0100 Subject: [PATCH 08/28] getPipiles and GetExperiments documented --- README.md | 28 ++++++++++++++++++---------- bpmn/get_runs.bpmn | 7 +++---- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 51bf00a..c7fa14e 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ A custom Camunda 8 outbound connector to communicate with the [Kubeflow](https://www.kubeflow.org/) API. It supports standalone as well as multi-suer setup and version 1 and 2 of the Kubeflow API. It currently supports the functions mentioned below in the [API section](#api-section). -## Build +# Build You can package the Connector by running the following command: @@ -15,7 +15,7 @@ This will create the following artifacts: - A thin JAR without dependencies. - An fat JAR containing all dependencies, potentially shaded to avoid classpath conflicts. This will not include the SDK artifacts since those are in scope `provided` and will be brought along by the respective Connector Runtime executing the Connector. -## API +# API Currenty this connector supports the following methods from the Kubeflow Pipeline API in both API versions 1 and 2. - Get Pipelines @@ -30,11 +30,11 @@ Currenty this connector supports the following methods from the Kubeflow Pipelin The inputs describes the parameters that can be set in the modeler for the operation. The output is the complete output you will get written into the variable you enter under Output mapping in the result variable. In the result expression you can more specifically extract the data as required to limit the output. -### Get Pipelines +## Get Pipelines -#### Input +### Input -Parameter Filter +#### Parameter Filter (optional) API Version 1: @@ -68,7 +68,11 @@ Version 1: [https://github.com/kubeflow/pipelines/blob/master/backend/api/v1beta Version 2: [https://github.com/kubeflow/pipelines/blob/master/backend/api/v2beta1/filter.proto](https://github.com/kubeflow/pipelines/blob/master/backend/api/v2beta1/filter.proto) -#### Output +#### Parameter Namespace (optional) + +API Version 1 and 2: If omitted, shared pipelines will also be returned. + +### Output API Version 1: ```json @@ -115,11 +119,11 @@ API Version 2: } ``` -### Get Experiments +## Get Experiments -#### Input +### Input -Parameter Filter +#### Parameter Filter API Version 1: ```json @@ -147,7 +151,11 @@ API Version 2: } ``` -#### Output +#### Parameter Namespace (required in multiuser deployment) + +API Version 1 and 2: namespace where the experiments should be retrieved from. + +### Output API Version 1: ```json diff --git a/bpmn/get_runs.bpmn b/bpmn/get_runs.bpmn index 9462b3f..4c6eeb9 100644 --- a/bpmn/get_runs.bpmn +++ b/bpmn/get_runs.bpmn @@ -1,5 +1,5 @@ - + Flow_0xamqby @@ -9,12 +9,11 @@ - + - + - From d76a918382c1651c00063210c5b791c9d44d1198 Mon Sep 17 00:00:00 2001 From: Mario Micudaj Date: Fri, 15 Mar 2024 13:14:00 +0100 Subject: [PATCH 09/28] adeed FEEL annotation for filter parameter --- .../camunda/connectors/kubeflow/entities/input/KubeflowApi.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/entities/input/KubeflowApi.java b/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/entities/input/KubeflowApi.java index f7e7817..ab60005 100644 --- a/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/entities/input/KubeflowApi.java +++ b/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/entities/input/KubeflowApi.java @@ -2,6 +2,7 @@ import java.util.Map; +import io.camunda.connector.feel.annotation.FEEL; import jakarta.validation.constraints.NotEmpty; public record KubeflowApi( @@ -21,6 +22,7 @@ public record KubeflowApi( String experimentId, + @FEEL String filter, String pollingInterval, From a6df18ca224f5b0d99c70d18689e56952fda61ba Mon Sep 17 00:00:00 2001 From: Mario Micudaj Date: Sun, 17 Mar 2024 15:23:25 +0100 Subject: [PATCH 10/28] raised connectors dsk version to 8.4.5 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index cc7033d..adddf8f 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ UTF-8 UTF-8 - 8.3.7 + 8.4.5 3.24.2 5.10.0 From f9b036c380bc17ad95c4666f58985ec1a099b8c6 Mon Sep 17 00:00:00 2001 From: Mario Micudaj Date: Tue, 19 Mar 2024 12:48:03 +0100 Subject: [PATCH 11/28] added more doc --- README.md | 118 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 117 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c7fa14e..dbdff0c 100644 --- a/README.md +++ b/README.md @@ -197,7 +197,123 @@ API Version 2: } ``` -### Error handling +## Get Runs + +### Input + +#### Parameter Filter + +API Version 1: +```json +{ "predicates": + [ + { + "op":"IS_SUBSTRING", + "key": "name", + "string_value": "test" + } + ] +} +``` + +API Version 2: +```json +{ "predicates": + [ + { + "operation":"IS_SUBSTRING", + "key": "name", + "string_value": "test" + } + ] +} +``` + +#### Parameter Namespace (required in multiuser deployment) + +API Version 1 and 2: namespace where the runs should be retrieved from. + +### Output + +API Version 1: [https://www.kubeflow.org/docs/components/pipelines/v1/reference/api/kubeflow-pipeline-api-spec/#/definitions/apiListRunsResponse](https://www.kubeflow.org/docs/components/pipelines/v1/reference/api/kubeflow-pipeline-api-spec/#/definitions/apiListRunsResponse) + +API Version 2: [https://www.kubeflow.org/docs/components/pipelines/v2/reference/api/kubeflow-pipeline-api-spec/#/definitions/v2beta1ListRunsResponse](https://www.kubeflow.org/docs/components/pipelines/v2/reference/api/kubeflow-pipeline-api-spec/#/definitions/v2beta1ListRunsResponse) + + +## Get Run By ID + +### Input + +#### Parameter Run ID +The ID of the run that should be retrieved. + +### Output + +API Version 1: [https://www.kubeflow.org/docs/components/pipelines/v1/reference/api/kubeflow-pipeline-api-spec/#/definitions/apiRunDetail](https://www.kubeflow.org/docs/components/pipelines/v1/reference/api/kubeflow-pipeline-api-spec/#/definitions/apiRunDetail) + +API Version 2: [https://www.kubeflow.org/docs/components/pipelines/v2/reference/api/kubeflow-pipeline-api-spec/#/definitions/v2beta1Run](https://www.kubeflow.org/docs/components/pipelines/v2/reference/api/kubeflow-pipeline-api-spec/#/definitions/v2beta1Run) + +## Get Run By Name +This request tries to retrieve a single run by looking for a run that contains the name defined in the name parameter. + +### Input + +#### Parameter Name +The name to look for in the runs. This looks for the name as a substring not as equal. + +### Output +equal to responses of [Get Runs](#get-runs) + + +## Start Run +This operation starts a run and continues the process without waiting for the pipeline to finish. + +### Input + +#### Parameter Pipeline ID +The ID of the pipeline that should be run. + +#### Parameter Experiment ID +The ID of the experiment in which the run should be started. + +### Output + +API Version 1: [https://www.kubeflow.org/docs/components/pipelines/v1/reference/api/kubeflow-pipeline-api-spec/#/definitions/apiRunDetail](https://www.kubeflow.org/docs/components/pipelines/v1/reference/api/kubeflow-pipeline-api-spec/#/definitions/apiRunDetail) + +API Version 2: [https://www.kubeflow.org/docs/components/pipelines/v2/reference/api/kubeflow-pipeline-api-spec/#/definitions/v2beta1Run](https://www.kubeflow.org/docs/components/pipelines/v2/reference/api/kubeflow-pipeline-api-spec/#/definitions/v2beta1Run) + +## Start Run and Monitor +This operation starts a run and monitors it until it finishes are throws an error. If the connector crashes during the run, the connector will pick up the already started run and monitor it after receiving the job again from Zeebe. + +### Input + +#### Parameter Pipeline ID +The ID of the pipeline that should be run. + +#### Parameter Experiment ID +The ID of the experiment in which the run should be started. + +### Output +see [Start Run](#start-run) + +## Create Experiment +This operation allows to create an experiment, which can e. g. be used in a subsequent run. + +### Input + +#### Parameter Experiment Name +The name of the experiment to create. + +#### Parameter Experiment Description (optional) +The description of the experiment. + +### Output + +API Version 1: [https://www.kubeflow.org/docs/components/pipelines/v1/reference/api/kubeflow-pipeline-api-spec/#/definitions/apiExperiment](https://www.kubeflow.org/docs/components/pipelines/v1/reference/api/kubeflow-pipeline-api-spec/#/definitions/apiExperiment) + +API Version 2: [https://www.kubeflow.org/docs/components/pipelines/v2/reference/api/kubeflow-pipeline-api-spec/#/definitions/v2beta1Experiment](https://www.kubeflow.org/docs/components/pipelines/v2/reference/api/kubeflow-pipeline-api-spec/#/definitions/v2beta1Experiment) + +## Error handling In case a return code >= 400 is returned by the API calls, the connector will raise an exception. From 9679edc80d5cec9320c906a551e21b554c2e3817 Mon Sep 17 00:00:00 2001 From: Mario Micudaj Date: Tue, 19 Mar 2024 14:03:19 +0100 Subject: [PATCH 12/28] added more doc --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index dbdff0c..db07d3e 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,7 @@ Currenty this connector supports the following methods from the Kubeflow Pipelin - Get Pipelines - Get Experiments +- Get Experiment by ID - Get Runs - Get Run by ID - Get Run by Name @@ -197,6 +198,19 @@ API Version 2: } ``` +## Get Experiment By ID + +### Input + +#### Parameter Experiment ID +The ID of the experiment that should be retrieved. + +### Output + +API Version 1: [https://www.kubeflow.org/docs/components/pipelines/v1/reference/api/kubeflow-pipeline-api-spec/#/definitions/apiExperiment](https://www.kubeflow.org/docs/components/pipelines/v1/reference/api/kubeflow-pipeline-api-spec/#/definitions/apiExperiment) + +API Version 2: [https://www.kubeflow.org/docs/components/pipelines/v2/reference/api/kubeflow-pipeline-api-spec/#/definitions/v2beta1Experiment](https://www.kubeflow.org/docs/components/pipelines/v2/reference/api/kubeflow-pipeline-api-spec/#/definitions/v2beta1Experiment) + ## Get Runs ### Input From 521b0d6386a86f93c8d8cfc832ab1ef9bd98604e Mon Sep 17 00:00:00 2001 From: Mario Micudaj Date: Wed, 20 Mar 2024 12:54:24 +0100 Subject: [PATCH 13/28] added more doc --- README.md | 85 +++---------------------------------------------------- 1 file changed, 4 insertions(+), 81 deletions(-) diff --git a/README.md b/README.md index db07d3e..7cece75 100644 --- a/README.md +++ b/README.md @@ -75,50 +75,9 @@ API Version 1 and 2: If omitted, shared pipelines will also be returned. ### Output -API Version 1: -```json -{ - "pipelines":[ - { - "id":"2d974767-3d50-40b8-9bfc-7767d4a13a5c", - "created_at":"2024-03-15T10:43:45Z", - "name":"[Tutorial] DSL - Control structures", - "description":"...", - "default_version":{ - "id":"dadc4c6d-b864-49e6-808e-946424e532ba", - "name":"[Tutorial] DSL - Control structures", - "created_at":"2024-03-15T10:43:45Z", - "resource_references":[ - { - "key":{ - "type":"PIPELINE", - "id":"2d974767-3d50-40b8-9bfc-7767d4a13a5c" - }, - "relationship":"OWNER" - } - ], - "description":"..." - } - } - ], - "total_size":1 -} -``` +API Version 1: [https://www.kubeflow.org/docs/components/pipelines/v1/reference/api/kubeflow-pipeline-api-spec/#/definitions/apiListPipelinesResponse](https://www.kubeflow.org/docs/components/pipelines/v1/reference/api/kubeflow-pipeline-api-spec/#/definitions/apiListPipelinesResponse) -API Version 2: -```json -{ - "pipelines": [ - { - "pipeline_id":"369eea92-e69c-4af4-802a-9f403a362bc5", - "display_name":"[Tutorial] DSL - Control structures", - "description":"...", - "created_at":"2024-03-14T22:09:25Z" - } - ], - "total_size":1 -} -``` +API Version 2: [https://www.kubeflow.org/docs/components/pipelines/v2/reference/api/kubeflow-pipeline-api-spec/#operation--apis-v2beta1-pipelines-get](https://www.kubeflow.org/docs/components/pipelines/v2/reference/api/kubeflow-pipeline-api-spec/#operation--apis-v2beta1-pipelines-get) ## Get Experiments @@ -158,45 +117,9 @@ API Version 1 and 2: namespace where the experiments should be retrieved from. ### Output -API Version 1: -```json -{ - "experiments":[ - { - "id":"f4e404a0-6b83-44a8-9eae-9678706188a0", - "name":"Testexperiment", - "created_at":"2024-03-15T11:47:52Z", - "resource_references":[ - { - "key":{ - "type":"NAMESPACE", - "id":"kubeflow-user-example-com" - }, - "relationship":"OWNER" - } - ], - "storage_state":"STORAGESTATE_AVAILABLE" - } - ], - "total_size":1 -} -``` +API Version 1: [https://www.kubeflow.org/docs/components/pipelines/v1/reference/api/kubeflow-pipeline-api-spec/#/definitions/apiListExperimentsResponse](https://www.kubeflow.org/docs/components/pipelines/v1/reference/api/kubeflow-pipeline-api-spec/#/definitions/apiListExperimentsResponse) -API Version 2: -```json -{ - "experiments": [ - { - "experiment_id": "f4e404a0-6b83-44a8-9eae-9678706188a0", - "display_name": "Testexperiment", - "created_at": "2024-03-15T11:47:52Z", - "namespace": "kubeflow-user-example-com", - "storage_state": "AVAILABLE" - } - ], - "total_size": 1 -} -``` +API Version 2: [https://www.kubeflow.org/docs/components/pipelines/v2/reference/api/kubeflow-pipeline-api-spec/#/definitions/v2beta1ListExperimentsResponse](https://www.kubeflow.org/docs/components/pipelines/v2/reference/api/kubeflow-pipeline-api-spec/#/definitions/v2beta1ListExperimentsResponse) ## Get Experiment By ID From cef00924556281545ca8fbf32b230b21baaa59d2 Mon Sep 17 00:00:00 2001 From: Mario Micudaj Date: Wed, 20 Mar 2024 13:00:09 +0100 Subject: [PATCH 14/28] bpmn models updated to fit test environment --- bpmn/create_experiment.bpmn | 11 +++++------ bpmn/get_experiments.bpmn | 3 ++- bpmn/get_pipelines.bpmn | 3 ++- bpmn/get_run_by_id.bpmn | 8 ++++---- bpmn/get_run_by_name.bpmn | 10 +++++----- bpmn/get_runs.bpmn | 3 ++- bpmn/start_run.bpmn | 8 ++++---- bpmn/start_run_and_monitor.bpmn | 8 ++++---- 8 files changed, 28 insertions(+), 26 deletions(-) diff --git a/bpmn/create_experiment.bpmn b/bpmn/create_experiment.bpmn index 360da8d..8c37784 100644 --- a/bpmn/create_experiment.bpmn +++ b/bpmn/create_experiment.bpmn @@ -1,5 +1,5 @@ - + Flow_0ar4k28 @@ -13,14 +13,13 @@ - - - - + + + + - diff --git a/bpmn/get_experiments.bpmn b/bpmn/get_experiments.bpmn index fbf7c2b..78407b5 100644 --- a/bpmn/get_experiments.bpmn +++ b/bpmn/get_experiments.bpmn @@ -10,10 +10,11 @@ - + + diff --git a/bpmn/get_pipelines.bpmn b/bpmn/get_pipelines.bpmn index 72445e0..5008c3d 100644 --- a/bpmn/get_pipelines.bpmn +++ b/bpmn/get_pipelines.bpmn @@ -10,6 +10,7 @@ + @@ -51,4 +52,4 @@ - + \ No newline at end of file diff --git a/bpmn/get_run_by_id.bpmn b/bpmn/get_run_by_id.bpmn index 1c22247..ea8c18c 100644 --- a/bpmn/get_run_by_id.bpmn +++ b/bpmn/get_run_by_id.bpmn @@ -1,5 +1,5 @@ - + Flow_0xamqby @@ -9,13 +9,13 @@ - - + + + - diff --git a/bpmn/get_run_by_name.bpmn b/bpmn/get_run_by_name.bpmn index cf026ef..7a3b0fe 100644 --- a/bpmn/get_run_by_name.bpmn +++ b/bpmn/get_run_by_name.bpmn @@ -1,5 +1,5 @@ - + Flow_0xamqby @@ -9,13 +9,13 @@ - - - + + + + - diff --git a/bpmn/get_runs.bpmn b/bpmn/get_runs.bpmn index 4c6eeb9..64f33ac 100644 --- a/bpmn/get_runs.bpmn +++ b/bpmn/get_runs.bpmn @@ -10,10 +10,11 @@ - + + diff --git a/bpmn/start_run.bpmn b/bpmn/start_run.bpmn index 2e13bb4..b0b4fdc 100644 --- a/bpmn/start_run.bpmn +++ b/bpmn/start_run.bpmn @@ -1,5 +1,5 @@ - + Flow_18a4hsh @@ -9,15 +9,15 @@ - - + + + - diff --git a/bpmn/start_run_and_monitor.bpmn b/bpmn/start_run_and_monitor.bpmn index ad31b8d..2f3f141 100644 --- a/bpmn/start_run_and_monitor.bpmn +++ b/bpmn/start_run_and_monitor.bpmn @@ -1,5 +1,5 @@ - + Flow_0ppmo2y @@ -27,16 +27,16 @@ - - + + + - From aa7198798f40553b4af5138b0955803a07ef05b8 Mon Sep 17 00:00:00 2001 From: Mario Micudaj Date: Wed, 20 Mar 2024 13:00:36 +0100 Subject: [PATCH 15/28] added example.env --- example.env | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 example.env diff --git a/example.env b/example.env new file mode 100644 index 0000000..1d44563 --- /dev/null +++ b/example.env @@ -0,0 +1,14 @@ +KF_CONNECTOR_URL=http://localhost:30000 + +KF_AUTH_MODE=oauth_cc +KF_AUTH_BASIC_USERNAME= +KF_AUTH_BASIC_PASSWORD= +KF_AUTH_BEARER_TOKEN= +KF_AUTH_OAUTH_TOKEN_ENDPOINT=http://localhost:30000/auth/realms/kubeflow/protocol/openid-connect/token +KF_AUTH_OAUTH_CLIENT_ID=kubeflow +KF_AUTH_OAUTH_CLIENT_SECRET=Jq09L1liFa0UiaXnL3pcnXzlqOKXaoOW +KF_AUTH_OAUTH_SCOPES=openid email profile groups +KF_AUTH_OAUTH_AUDIENCE=kubeflow +KF_AUTH_OAUTH_CLIENT_AUTH=basicAuthHeader +KF_AUTH_OAUTH_USERNAME= +KF_AUTH_OAUTH_PASSWORD= \ No newline at end of file From e3269621b6addf686140018b8b7e79f43cba4819 Mon Sep 17 00:00:00 2001 From: Mario Micudaj Date: Wed, 20 Mar 2024 15:30:52 +0100 Subject: [PATCH 16/28] updated documentation --- README.md | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 7cece75..d2e6ca7 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,9 @@ A custom Camunda 8 outbound connector to communicate with the [Kubeflow](https://www.kubeflow.org/) API. It supports standalone as well as multi-suer setup and version 1 and 2 of the Kubeflow API. It currently supports the functions mentioned below in the [API section](#api-section). -# Build +# Build and Run +## Build a deployable jar file You can package the Connector by running the following command: ```bash @@ -15,6 +16,26 @@ This will create the following artifacts: - A thin JAR without dependencies. - An fat JAR containing all dependencies, potentially shaded to avoid classpath conflicts. This will not include the SDK artifacts since those are in scope `provided` and will be brought along by the respective Connector Runtime executing the Connector. +### Shading dependencies + +You can use the `maven-shade-plugin` defined in the [Maven configuration](./pom.xml) to relocate common dependencies +that are used in other Connectors and the [Connector Runtime](https://github.com/camunda-community-hub/spring-zeebe/tree/master/connector-runtime#building-connector-runtime-bundles). +This helps to avoid classpath conflicts when the Connector is executed. + +Use the `relocations` configuration in the Maven Shade plugin to define the dependencies that should be shaded. +The [Maven Shade documentation](https://maven.apache.org/plugins/maven-shade-plugin/examples/class-relocation.html) +provides more details on relocations. + +## Configure a local environment for the Connector +The example bpmn models provided in the bpmn folder of the repository are set up in a way that they rely on environment variables being defined for the kubeflow and authentication configuration. We provided an [example.env](example.env) file that is tailored to the local development environment. You can copy this file to *.env* so that it is considered when you start the connector runtime locally. + +## Start the connector locally +In order to start the connector locally run +```bash +mvn spring-boot:run +``` + + # API Currenty this connector supports the following methods from the Kubeflow Pipeline API in both API versions 1 and 2. @@ -311,16 +332,4 @@ client-secret: Jq09L1liFa0UiaXnL3pcnXzlqOKXaoOW ``` ## Contact Information -For any queries and further support, please drop us a mail at [camunda8-connector-su-aaaamkuzw6jcci2hm7rscvue7y@viadee.slack.com](mailto:camunda8-connector-su-aaaamkuzw6jcci2hm7rscvue7y@viadee.slack.com) - -# OLD ========== - -### Shading dependencies - -You can use the `maven-shade-plugin` defined in the [Maven configuration](./pom.xml) to relocate common dependencies -that are used in other Connectors and the [Connector Runtime](https://github.com/camunda-community-hub/spring-zeebe/tree/master/connector-runtime#building-connector-runtime-bundles). -This helps to avoid classpath conflicts when the Connector is executed. - -Use the `relocations` configuration in the Maven Shade plugin to define the dependencies that should be shaded. -The [Maven Shade documentation](https://maven.apache.org/plugins/maven-shade-plugin/examples/class-relocation.html) -provides more details on relocations. +For any queries and further support, please drop us a mail at [camunda8-connector-su-aaaamkuzw6jcci2hm7rscvue7y@viadee.slack.com](mailto:camunda8-connector-su-aaaamkuzw6jcci2hm7rscvue7y@viadee.slack.com) \ No newline at end of file From 770c4ae2647baca7aedc87656b891108abc87e7a Mon Sep 17 00:00:00 2001 From: Mario Micudaj Date: Wed, 20 Mar 2024 15:34:30 +0100 Subject: [PATCH 17/28] updated documentation --- README.md | 31 +++++++++++-------------------- 1 file changed, 11 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index d2e6ca7..644bc7b 100644 --- a/README.md +++ b/README.md @@ -26,15 +26,21 @@ Use the `relocations` configuration in the Maven Shade plugin to define the depe The [Maven Shade documentation](https://maven.apache.org/plugins/maven-shade-plugin/examples/class-relocation.html) provides more details on relocations. -## Configure a local environment for the Connector -The example bpmn models provided in the bpmn folder of the repository are set up in a way that they rely on environment variables being defined for the kubeflow and authentication configuration. We provided an [example.env](example.env) file that is tailored to the local development environment. You can copy this file to *.env* so that it is considered when you start the connector runtime locally. +## Test locally + +Run unit tests -## Start the connector locally -In order to start the connector locally run ```bash -mvn spring-boot:run +mvn clean verify ``` +## Test with local runtime +In your IDE you can also simply navigate to the `LocalContainerRuntime` class in test scope and run it via your IDE. +If necessary, you can adjust `application.properties` in test scope. + +### Configure a local environment for the Connector +The example bpmn models provided in the bpmn folder of the repository are set up in a way that they rely on environment variables being defined for the kubeflow and authentication configuration. We provided an [example.env](example.env) file that is tailored to the local development environment. You can copy this file to *.env* so that it is considered when you start the connector runtime locally. + # API Currenty this connector supports the following methods from the Kubeflow Pipeline API in both API versions 1 and 2. @@ -275,21 +281,6 @@ API Version 2: [https://www.kubeflow.org/docs/components/pipelines/v2/reference/ In case a return code >= 400 is returned by the API calls, the connector will raise an exception. -## Test locally - -Run unit tests - -```bash -mvn clean verify -``` - -### Test with local runtime - -Use the [Camunda Connector Runtime](https://github.com/camunda-community-hub/spring-zeebe/tree/master/connector-runtime#building-connector-runtime-bundles) to run your function as a local Java application. - -In your IDE you can also simply navigate to the `LocalContainerRuntime` class in test scope and run it via your IDE. -If necessary, you can adjust `application.properties` in test scope. - ## Element Template The element templates can be found in the [kubeflow-connector.json](./element-templates/kubeflow-connector.json) file. From 06c054595e491c264be768084684db2ca61af59e Mon Sep 17 00:00:00 2001 From: Mario Micudaj Date: Wed, 20 Mar 2024 17:28:50 +0100 Subject: [PATCH 18/28] changed connectors version to 8.3.8 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ea9384f..fc6c412 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ UTF-8 UTF-8 - 8.4.5 + 8.3.8 3.24.2 5.10.0 From 6a8bf54a25a58f545c64effedd98e7e5dfb9259a Mon Sep 17 00:00:00 2001 From: Mario Micudaj Date: Wed, 20 Mar 2024 23:12:59 +0100 Subject: [PATCH 19/28] review comments --- README.md | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 644bc7b..95b3b10 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Kubeflow Connector -A custom Camunda 8 outbound connector to communicate with the [Kubeflow](https://www.kubeflow.org/) API. It supports standalone as well as multi-suer setup and version 1 and 2 of the Kubeflow API. It currently supports the functions mentioned below in the [API section](#api-section). +A custom Camunda 8 outbound connector to communicate with the [Kubeflow](https://www.kubeflow.org/) API. It supports the multi-user setup with profiles and namespaces as well as version 1 and 2 of the Kubeflow Pipeline API. It currently supports the functions mentioned below in the [API section](#api-section). # Build and Run @@ -223,7 +223,10 @@ This request tries to retrieve a single run by looking for a run that contains t ### Input #### Parameter Name -The name to look for in the runs. This looks for the name as a substring not as equal. +The name to look for in the runs. This looks for the name to be equal. + +#### Parameter Namespace +The namespace where to look for runs. ### Output equal to responses of [Get Runs](#get-runs) @@ -240,6 +243,12 @@ The ID of the pipeline that should be run. #### Parameter Experiment ID The ID of the experiment in which the run should be started. +#### Parameter Suffix of run name +The name of the triggered run will be created as such: _. + +#### Parameter Run parameters (optional) +If the pipeline requires parameters to run they can be set here. + ### Output API Version 1: [https://www.kubeflow.org/docs/components/pipelines/v1/reference/api/kubeflow-pipeline-api-spec/#/definitions/apiRunDetail](https://www.kubeflow.org/docs/components/pipelines/v1/reference/api/kubeflow-pipeline-api-spec/#/definitions/apiRunDetail) @@ -257,6 +266,15 @@ The ID of the pipeline that should be run. #### Parameter Experiment ID The ID of the experiment in which the run should be started. +#### Polling interval +You can define the interval in which the connector should regularly check the state of the run using ISO 8601 format. E. g. PT10S means every 10 seconds. + +#### Parameter Suffix of run name +The name of the triggered run will be created as such: _. + +#### Parameter Run parameters (optional) +If the pipeline requires parameters to run they can be set here. + ### Output see [Start Run](#start-run) From 14862535e6cd0ccf3d28e1940e556bf713f034a2 Mon Sep 17 00:00:00 2001 From: Mario Micudaj Date: Wed, 20 Mar 2024 23:13:21 +0100 Subject: [PATCH 20/28] changed connectors version to 8.4.5 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fc6c412..ea9384f 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ UTF-8 UTF-8 - 8.3.8 + 8.4.5 3.24.2 5.10.0 From 006969a90fab8fd64a6c656b5ac63a575ed548f6 Mon Sep 17 00:00:00 2001 From: Mario Micudaj Date: Wed, 20 Mar 2024 23:15:46 +0100 Subject: [PATCH 21/28] review comments --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 95b3b10..fad0b6a 100644 --- a/README.md +++ b/README.md @@ -289,6 +289,9 @@ The name of the experiment to create. #### Parameter Experiment Description (optional) The description of the experiment. +#### Parameter Namespace +The namespace where to create the experiment in. + ### Output API Version 1: [https://www.kubeflow.org/docs/components/pipelines/v1/reference/api/kubeflow-pipeline-api-spec/#/definitions/apiExperiment](https://www.kubeflow.org/docs/components/pipelines/v1/reference/api/kubeflow-pipeline-api-spec/#/definitions/apiExperiment) @@ -304,7 +307,7 @@ The element templates can be found in the [kubeflow-connector.json](./element-te # Local development environment setup -Start a local kubernets cluster e.g. with mimikube or kind. +Start a local kubernets cluster e.g. with minikube or kind. Run the following command from the root folder in order to deploy the dev environment into the kubernetes cluster. ```bash From a0c2cdce91ed139c295d67e414e9e9c0dd4e0d48 Mon Sep 17 00:00:00 2001 From: Mario Micudaj Date: Wed, 20 Mar 2024 23:16:24 +0100 Subject: [PATCH 22/28] changed connectors version to 8.3.8 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ea9384f..fc6c412 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ UTF-8 UTF-8 - 8.4.5 + 8.3.8 3.24.2 5.10.0 From fe6bb9a38f8e7f4a7b22d12e3f503e28c05fb1a2 Mon Sep 17 00:00:00 2001 From: Mario Micudaj Date: Thu, 21 Mar 2024 10:59:24 +0100 Subject: [PATCH 23/28] reset hostname in yaml file after deployment --- deployEnv.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/deployEnv.sh b/deployEnv.sh index 3100d43..414f580 100755 --- a/deployEnv.sh +++ b/deployEnv.sh @@ -33,4 +33,6 @@ else printf "\nKeycloak Admin UI: \x1b[6;37;42mhttp://localhost:30000/auth/\x1b[0m\n\n" printf "\n\n\x1b[6;30;42mKeep this terminal open! Press \x1b[6;37;42mCtrl+C \x1b[6;30;42mto stop the port-forward.\x1b[0m\n\n" kubectl port-forward -n istio-system svc/istio-ingressgateway 30000:80 -fi \ No newline at end of file +fi + +sed -i '' 's/'"$KUBEFLOW_HOST"':3000/${KUBEFLOW_HOST}:3000/g' ./src/test/resources/kubeflow-environment/oauth2_proxy.yml \ No newline at end of file From f97fbce4a97b57d39e2dc1036787006858575f4d Mon Sep 17 00:00:00 2001 From: Mario Micudaj Date: Thu, 21 Mar 2024 11:00:39 +0100 Subject: [PATCH 24/28] reset hostname in yaml file after deployment --- deployEnv.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/deployEnv.sh b/deployEnv.sh index 414f580..068bc58 100755 --- a/deployEnv.sh +++ b/deployEnv.sh @@ -19,6 +19,7 @@ done kubectl wait --for=condition=available --timeout=300s deployment --all --all-namespaces printf "\n\nDeployment finished!\n\n" +sed -i '' 's/'"$KUBEFLOW_HOST"':3000/${KUBEFLOW_HOST}:3000/g' ./src/test/resources/kubeflow-environment/oauth2_proxy.yml printf "Checking if NodePort is reachable on localhost...\n\n" if nc -v -z -w2 localhost 30000 &> /dev/null; then @@ -33,6 +34,4 @@ else printf "\nKeycloak Admin UI: \x1b[6;37;42mhttp://localhost:30000/auth/\x1b[0m\n\n" printf "\n\n\x1b[6;30;42mKeep this terminal open! Press \x1b[6;37;42mCtrl+C \x1b[6;30;42mto stop the port-forward.\x1b[0m\n\n" kubectl port-forward -n istio-system svc/istio-ingressgateway 30000:80 -fi - -sed -i '' 's/'"$KUBEFLOW_HOST"':3000/${KUBEFLOW_HOST}:3000/g' ./src/test/resources/kubeflow-environment/oauth2_proxy.yml \ No newline at end of file +fi \ No newline at end of file From b2ee1c60d62f30836b560cc4c2f381d44787d5b8 Mon Sep 17 00:00:00 2001 From: Mario Micudaj Date: Fri, 22 Mar 2024 08:48:42 +0100 Subject: [PATCH 25/28] added optional namespace to get pipelines --- bpmn/get_pipelines.bpmn | 4 +-- element-templates/kubeflow-connector.json | 25 +++++++++++++++++++ .../services/KubeflowConnectorExecutor.java | 11 +++++++- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/bpmn/get_pipelines.bpmn b/bpmn/get_pipelines.bpmn index 5008c3d..f2a5a2d 100644 --- a/bpmn/get_pipelines.bpmn +++ b/bpmn/get_pipelines.bpmn @@ -14,7 +14,7 @@ - + @@ -52,4 +52,4 @@ - \ No newline at end of file + diff --git a/element-templates/kubeflow-connector.json b/element-templates/kubeflow-connector.json index 52bf60e..22fb8cf 100644 --- a/element-templates/kubeflow-connector.json +++ b/element-templates/kubeflow-connector.json @@ -450,6 +450,31 @@ ] } }, + { + "id": "kubeflowapi.namespace", + "label": "Namespace", + "description": "Specify the name of the namespace.", + "type": "String", + "group": "kubeflowapi", + "feel": "optional", + "optional": true, + "binding": { + "type": "zeebe:input", + "name": "kubeflowapi.namespace" + }, + "condition": { + "allMatch": [ + { + "property": "configuration.typeOfUserMode", + "equals": "multiUserMode" + }, + { + "property": "kubeflowapi.operation", + "oneOf": ["get_pipelines"] + } + ] + } + }, { "id": "kubeflowapi.pipelineId", "label": "Pipeline ID", diff --git a/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/services/KubeflowConnectorExecutor.java b/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/services/KubeflowConnectorExecutor.java index 3f0d249..c2dc4c1 100644 --- a/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/services/KubeflowConnectorExecutor.java +++ b/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/services/KubeflowConnectorExecutor.java @@ -190,8 +190,17 @@ private void addFilter(URIBuilder uriBuilder) throws UnsupportedEncodingExceptio protected void addNamespaceFilter(URIBuilder uriBuilder) { if (isMultiUserMode) { + var namespace = connectorRequest.getKubeflowapi().namespace(); if (kubeflowApiOperationsEnum.isNamespaceFilterRequired()) { - var namespace = connectorRequest.getKubeflowapi().namespace(); + if (KubeflowApisEnum.PIPELINES_V1.equals(kubeflowApisEnum)) { + uriBuilder.addParameter(URI_PARAMETER_PAIR_V1_TYPE_NS.getKey(), + URI_PARAMETER_PAIR_V1_TYPE_NS.getValue()); + uriBuilder.addParameter(URI_PARAMETER_V1_ID, namespace); + } else { + uriBuilder.addParameter(URI_PARAMETER_V2_NS, namespace); + } + } else if (KubeflowApiOperationsEnum.GET_PIPELINES.equals(kubeflowApiOperationsEnum) + && namespace != null) { if (KubeflowApisEnum.PIPELINES_V1.equals(kubeflowApisEnum)) { uriBuilder.addParameter(URI_PARAMETER_PAIR_V1_TYPE_NS.getKey(), URI_PARAMETER_PAIR_V1_TYPE_NS.getValue()); From da62da812dfa8981924fd55b519fd7e304b75a4b Mon Sep 17 00:00:00 2001 From: Mario Micudaj Date: Fri, 22 Mar 2024 08:52:45 +0100 Subject: [PATCH 26/28] added hint about http headers --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index fad0b6a..aa2e105 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,8 @@ Currenty this connector supports the following methods from the Kubeflow Pipelin The inputs describes the parameters that can be set in the modeler for the operation. The output is the complete output you will get written into the variable you enter under Output mapping in the result variable. In the result expression you can more specifically extract the data as required to limit the output. +In every operation additional HTTP headers can be set under Kubeflow API -> HTTP Headers if necessary. + ## Get Pipelines ### Input From 0685368581ef44c589e55870e6f3d22111f2525b Mon Sep 17 00:00:00 2001 From: Andre Strothmann Date: Fri, 22 Mar 2024 09:12:10 +0100 Subject: [PATCH 27/28] fix: testCreateAndGetRuns it-test --- .../connectors/kubeflow/integration/BaseIntegrationTest.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/it/de/viadee/bpm/camunda/connectors/kubeflow/integration/BaseIntegrationTest.java b/src/it/de/viadee/bpm/camunda/connectors/kubeflow/integration/BaseIntegrationTest.java index 42d227e..7679117 100644 --- a/src/it/de/viadee/bpm/camunda/connectors/kubeflow/integration/BaseIntegrationTest.java +++ b/src/it/de/viadee/bpm/camunda/connectors/kubeflow/integration/BaseIntegrationTest.java @@ -82,7 +82,10 @@ protected Configuration getConfiguration() { protected KubeflowConnectorExecutor getExecutor(String pipelineVersion, String operation, String experimentName, String pipelineId, String experimentId, String runName) throws Exception { - var namespace = getEnvOrDefault(KUBEFLOW_NAMESPACE_ENV_KEY, DEFAULT_KUBEFLOW_NAMESPACE); + var namespace = ""; + if (!operation.equals("get_pipelines")) { + namespace = getEnvOrDefault(KUBEFLOW_NAMESPACE_ENV_KEY, DEFAULT_KUBEFLOW_NAMESPACE); + } KubeflowApi kubeflowApi = new KubeflowApi(pipelineVersion, operation, null, runName, null, pipelineId, experimentId, null, null, experimentName, null, null, namespace); KubeflowConnectorRequest kubeflowConnectorRequest = new KubeflowConnectorRequest( From 927b3371b0b0e73412ad7184ef0a656d7e240ddb Mon Sep 17 00:00:00 2001 From: Andre Strothmann Date: Fri, 22 Mar 2024 09:17:16 +0100 Subject: [PATCH 28/28] chore: simplify code --- .../services/KubeflowConnectorExecutor.java | 30 ++++++++----------- 1 file changed, 13 insertions(+), 17 deletions(-) diff --git a/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/services/KubeflowConnectorExecutor.java b/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/services/KubeflowConnectorExecutor.java index c2dc4c1..185e63c 100644 --- a/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/services/KubeflowConnectorExecutor.java +++ b/src/main/java/de/viadee/bpm/camunda/connectors/kubeflow/services/KubeflowConnectorExecutor.java @@ -191,27 +191,23 @@ private void addFilter(URIBuilder uriBuilder) throws UnsupportedEncodingExceptio protected void addNamespaceFilter(URIBuilder uriBuilder) { if (isMultiUserMode) { var namespace = connectorRequest.getKubeflowapi().namespace(); - if (kubeflowApiOperationsEnum.isNamespaceFilterRequired()) { - if (KubeflowApisEnum.PIPELINES_V1.equals(kubeflowApisEnum)) { - uriBuilder.addParameter(URI_PARAMETER_PAIR_V1_TYPE_NS.getKey(), - URI_PARAMETER_PAIR_V1_TYPE_NS.getValue()); - uriBuilder.addParameter(URI_PARAMETER_V1_ID, namespace); - } else { - uriBuilder.addParameter(URI_PARAMETER_V2_NS, namespace); - } - } else if (KubeflowApiOperationsEnum.GET_PIPELINES.equals(kubeflowApiOperationsEnum) - && namespace != null) { - if (KubeflowApisEnum.PIPELINES_V1.equals(kubeflowApisEnum)) { - uriBuilder.addParameter(URI_PARAMETER_PAIR_V1_TYPE_NS.getKey(), - URI_PARAMETER_PAIR_V1_TYPE_NS.getValue()); - uriBuilder.addParameter(URI_PARAMETER_V1_ID, namespace); - } else { - uriBuilder.addParameter(URI_PARAMETER_V2_NS, namespace); - } + if (kubeflowApiOperationsEnum.isNamespaceFilterRequired() || + (KubeflowApiOperationsEnum.GET_PIPELINES.equals(kubeflowApiOperationsEnum) && namespace != null)) { + setNamespaceParameter(uriBuilder, namespace); } } } + private void setNamespaceParameter(URIBuilder uriBuilder, String namespace) { + if (KubeflowApisEnum.PIPELINES_V1.equals(kubeflowApisEnum)) { + uriBuilder.addParameter(URI_PARAMETER_PAIR_V1_TYPE_NS.getKey(), + URI_PARAMETER_PAIR_V1_TYPE_NS.getValue()); + uriBuilder.addParameter(URI_PARAMETER_V1_ID, namespace); + } else { + uriBuilder.addParameter(URI_PARAMETER_V2_NS, namespace); + } + } + public static HttpRequest.BodyPublisher ofFormData(Map data) { var builder = new StringBuilder(); for (Map.Entry entry : data.entrySet()) {