From ae5c60ce9970e1fbf74ac17dbd036f24dd574bc8 Mon Sep 17 00:00:00 2001 From: Jack Tysoe Date: Mon, 23 Sep 2024 16:26:20 +0100 Subject: [PATCH 1/4] fix(ai-gemini): should not be recommending people to use API key in the query params because it logs in plaintext --- .../machine-learning-platform-integration-guides/gemini.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/_hub/kong-inc/ai-proxy/how-to/machine-learning-platform-integration-guides/gemini.md b/app/_hub/kong-inc/ai-proxy/how-to/machine-learning-platform-integration-guides/gemini.md index 3830a3cd2dff..17420eaf4705 100644 --- a/app/_hub/kong-inc/ai-proxy/how-to/machine-learning-platform-integration-guides/gemini.md +++ b/app/_hub/kong-inc/ai-proxy/how-to/machine-learning-platform-integration-guides/gemini.md @@ -29,9 +29,8 @@ curl -i -X POST http://localhost:8001/routes \ ```sh curl -i -X POST http://localhost:8001/services/gemini-service/plugins \ --data 'name=ai-proxy' \ ---data 'config.auth.param_name=key' \ ---data 'config.auth.param_value=' \ ---data 'config.auth.param_location=query' \ +--data 'config.auth.header_name=x-goog-api-key' \ +--data 'config.auth.header_value=' \ --data 'config.route_type=llm/v1/chat' \ --data 'config.model.provider=gemini' \ --data 'config.model.name=gemini-1.5-flash' From 1b12f67a6cefca6cf29d4a6b9a22e2c54d071813 Mon Sep 17 00:00:00 2001 From: Jack Tysoe Date: Mon, 23 Sep 2024 17:20:33 +0100 Subject: [PATCH 2/4] feat(ai-proxy): added Gemini cloud identity instructions --- .../how-to/_cloud-provider-authentication.md | 99 +++++++++++++++++-- .../gemini.md | 7 +- .../how-to/_cloud-provider-authentication.md | 99 +++++++++++++++++-- 3 files changed, 190 insertions(+), 15 deletions(-) diff --git a/app/_hub/kong-inc/ai-proxy-advanced/how-to/_cloud-provider-authentication.md b/app/_hub/kong-inc/ai-proxy-advanced/how-to/_cloud-provider-authentication.md index 07c404d7a688..973052dffdf1 100644 --- a/app/_hub/kong-inc/ai-proxy-advanced/how-to/_cloud-provider-authentication.md +++ b/app/_hub/kong-inc/ai-proxy-advanced/how-to/_cloud-provider-authentication.md @@ -26,11 +26,13 @@ LLM-based services using those same methods. Kong's AI Gateway currently supports the following cloud authentication: -| AI-Proxy Advanced LLM Provider | Cloud Provider | Type | -|-----------------------|----------------|-----------------------------------------| -| `azure` | Azure OpenAI | [Entra / Managed Identity Authentication](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) | +| AI-Proxy Advanced LLM Provider | Cloud Provider | Type | +|--------------------------------|-------------------------------------------------|-----------------------------------------| +| `azure` (Kong Enterprise Only) | Azure OpenAI | [Entra / Managed Identity Authentication](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) | +| `gemini` | Gemini Enterprise (on Vertex or Workspace) | [GCP Service Account](https://cloud.google.com/iam/docs/service-account-overview) | +| `bedrock` | AWS Bedrock Converse-API | [AWS IAM Identity](https://docs.aws.amazon.com/IAM/latest/UserGuide/id.html) | -## Azure OpenAI +## Azure OpenAI (Kong Enterprise Only) When hosting your LLMs with [Azure OpenAI Service](https://azure.microsoft.com/en-gb/products/ai-services/openai-service) and running them through AI Proxy Advanced, it is possible to use the assigned @@ -171,7 +173,7 @@ Plugin configuration: {% plugin_example %} -plugin: kong-inc/ai-proxy +plugin: kong-inc/ai-proxy-advanced name: ai-proxy-advanced config: route_type: "llm/v1/chat" @@ -196,4 +198,89 @@ formats: - kubernetes - terraform {% endplugin_example %} - \ No newline at end of file + + +## Google Gemini + +When hosting your LLMs with [Google Gemini Vertex](https://cloud.google.com/vertex-ai?hl=en) in a business or enterprise plan, and running them through AI Proxy Advanced, +it is possible to use a [GCP Workload Identity](https://cloud.google.com/iam/docs/workload-identity-federation) that can be assigned to a currently running instance, +a Container Platform deployment, or just used via the [gcloud CLI](https://cloud.google.com/sdk/gcloud) sign-in on the local machine. + +How you do this depends on where and how you are running {{site.base_gateway}}. + +### Prerequisites + +You must be running a {{site.ee_product_name}} instance. + +Ensure that the GCP Virtual Machine, Container Deployment, Container Application (or a combination of these) has been assigned the Service Account principal, +configurable from the Google Cloud IAM portal. + +Assign the `'Vertex AI User'` role to the Service Account. + +### Configuring the AI Proxy Advanced Plugin to use GCP Workload Identity + +When running Kong inside of your GCP subscription, AI Proxy Advanced is usually able to detect the designated Service Account automatically, based on the +`GCP_SERVICE_ACCOUNT` JSON that is automatically injected into an environment variable in your Kong deployment (or the Kong Virtual Machine(s)). + +#### GCP-Assigned Workload Identity + +To use a GCP-Assigned Workload Identity, set up your plugin config like this example: + + +{% plugin_example %} +plugin: kong-inc/ai-proxy-advanced +name: ai-proxy-advanced +config: + route_type: "llm/v1/chat" + auth: + use_gcp_service_account: true + logging: + log_statistics: true + log_payloads: false + model: + provider: "gemini" + name: "gemini-1.5-flash" + options: + gemini: + api_endpoint: "us-central1-aiplatform.googleapis.com" + project_id: "sample-project-123456" + location_id: "us-central1" +targets: + - route + - consumer_group + - global +formats: + - konnect + - curl + - yaml + - kubernetes + - terraform +{% endplugin_example %} + + +#### Environment variables + +You can also specify your own GCP Service Account JSON; simply set this environment variable in the Kong workload or deployment configuration: + +Environment variable: +```sh +GCP_SERVICE_ACCOUNT='{ "type": "service_account", "project_id": "sample-project-123456", "private_key_id": "...", "private_key": "..."...}' +``` + +or set it directly in the plugin configuration: + +```yaml +config: + auth: + use_gcp_service_account: true + gcp_service_account_json: '{ "type": "service_account", "project_id": "sample-project-123456", "private_key_id": "...", "private_key": "..."...}' +``` + +or, more securely, use a vault reference to e.g. GCP Secrets Manager: + +```yaml +config: + auth: + use_gcp_service_account: true + gcp_service_account_json: '{vault://gcp/VERTEX_SERVICE_ACCOUNT_JSON}' +``` \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy-advanced/how-to/machine-learning-platform-integrations/gemini.md b/app/_hub/kong-inc/ai-proxy-advanced/how-to/machine-learning-platform-integrations/gemini.md index a6e9e3ac413c..422257a2efbc 100644 --- a/app/_hub/kong-inc/ai-proxy-advanced/how-to/machine-learning-platform-integrations/gemini.md +++ b/app/_hub/kong-inc/ai-proxy-advanced/how-to/machine-learning-platform-integrations/gemini.md @@ -38,9 +38,8 @@ curl -i -X POST http://localhost:8001/services/gemini-service/plugins \ { "route_type": "llm/v1/chat", "auth": { - "param_name": "key", - "param_value": "", - "param_location": "query" + "header_name": "key", + "header_value": "", }, "model": { "provider": "gemini", @@ -55,6 +54,8 @@ curl -i -X POST http://localhost:8001/services/gemini-service/plugins \ Be sure to replace `GEMINI_API_TOKEN` with your API token. +**To use Gemini with a GCP Service Account or workload identity** (i.e. in [Vertex AI Studio](https://cloud.google.com/vertex-ai?hl=en)), follow [the integration guide](/hub/kong-inc/ai-proxy/how-to/cloud-provider-authentication), and adjust your plugin configuration accordingly. + ### Test the configuration Make an `llm/v1/chat` type request to test your new endpoint: diff --git a/app/_hub/kong-inc/ai-proxy/how-to/_cloud-provider-authentication.md b/app/_hub/kong-inc/ai-proxy/how-to/_cloud-provider-authentication.md index d287d5700fe1..419618b79ff7 100644 --- a/app/_hub/kong-inc/ai-proxy/how-to/_cloud-provider-authentication.md +++ b/app/_hub/kong-inc/ai-proxy/how-to/_cloud-provider-authentication.md @@ -26,11 +26,13 @@ LLM-based services using those same methods. Kong's AI Gateway currently supports the following cloud authentication: -| AI-Proxy LLM Provider | Cloud Provider | Type | -|-----------------------|----------------|-----------------------------------------| -| `azure` | Azure OpenAI | [Entra / Managed Identity Authentication](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) | +| AI-Proxy LLM Provider | Cloud Provider | Type | +|--------------------------------|-------------------------------------------------|-----------------------------------------| +| `azure` (Kong Enterprise Only) | Azure OpenAI | [Entra / Managed Identity Authentication](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) | +| `gemini` | Gemini Enterprise (on Vertex or Workspace) | [GCP Service Account](https://cloud.google.com/iam/docs/service-account-overview) | +| `bedrock` | AWS Bedrock Converse-API | [AWS IAM Identity](https://docs.aws.amazon.com/IAM/latest/UserGuide/id.html) | -## Azure OpenAI +## Azure OpenAI (Kong Enterprise Only) When hosting your LLMs with [Azure OpenAI Service](https://azure.microsoft.com/en-gb/products/ai-services/openai-service) and running them through AI Proxy, it is possible to use the assigned @@ -55,7 +57,7 @@ Entra or IAM permissions to execute commands on the desired OpenAI instances. It See [Azure's documentation on managed identity](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/managed-identity) to set this up. -### Configuring the AI Proxy Plugin to use Azure Identity +### Configuring the AI Proxy Plugin to use Azure Identity When running Kong inside of your Azure subscription, AI Proxy is usually able to detect the designated Managed Identity or User-Assigned Identity of that Azure Compute resource, and use it accordingly. @@ -196,4 +198,89 @@ formats: - kubernetes - terraform {% endplugin_example %} - \ No newline at end of file + + +## Google Gemini + +When hosting your LLMs with [Google Gemini Vertex](https://cloud.google.com/vertex-ai?hl=en) in a business or enterprise plan, and running them through AI Proxy, +it is possible to use a [GCP Workload Identity](https://cloud.google.com/iam/docs/workload-identity-federation) that can be assigned to a currently running instance, +a Container Platform deployment, or just used via the [gcloud CLI](https://cloud.google.com/sdk/gcloud) sign-in on the local machine. + +How you do this depends on where and how you are running {{site.base_gateway}}. + +### Prerequisites + +You must be running a {{site.ee_product_name}} instance. + +Ensure that the GCP Virtual Machine, Container Deployment, Container Application (or a combination of these) has been assigned the Service Account principal, +configurable from the Google Cloud IAM portal. + +Assign the `'Vertex AI User'` role to the Service Account. + +### Configuring the AI Proxy Plugin to use GCP Workload Identity + +When running Kong inside of your GCP subscription, AI Proxy is usually able to detect the designated Service Account automatically, based on the +`GCP_SERVICE_ACCOUNT` JSON that is automatically injected into an environment variable in your Kong deployment (or the Kong Virtual Machine(s)). + +#### GCP-Assigned Workload Identity + +To use a GCP-Assigned Workload Identity, set up your plugin config like this example: + + +{% plugin_example %} +plugin: kong-inc/ai-proxy +name: ai-proxy +config: + route_type: "llm/v1/chat" + auth: + use_gcp_service_account: true + logging: + log_statistics: true + log_payloads: false + model: + provider: "gemini" + name: "gemini-1.5-flash" + options: + gemini: + api_endpoint: "us-central1-aiplatform.googleapis.com" + project_id: "sample-project-123456" + location_id: "us-central1" +targets: + - route + - consumer_group + - global +formats: + - konnect + - curl + - yaml + - kubernetes + - terraform +{% endplugin_example %} + + +#### Environment variables + +You can also specify your own GCP Service Account JSON; simply set this environment variable in the Kong workload or deployment configuration: + +Environment variable: +```sh +GCP_SERVICE_ACCOUNT='{ "type": "service_account", "project_id": "sample-project-123456", "private_key_id": "...", "private_key": "..."...}' +``` + +or set it directly in the plugin configuration: + +```yaml +config: + auth: + use_gcp_service_account: true + gcp_service_account_json: '{ "type": "service_account", "project_id": "sample-project-123456", "private_key_id": "...", "private_key": "..."...}' +``` + +or, more securely, use a vault reference to e.g. GCP Secrets Manager: + +```yaml +config: + auth: + use_gcp_service_account: true + gcp_service_account_json: '{vault://gcp/VERTEX_SERVICE_ACCOUNT_JSON}' +``` \ No newline at end of file From bf05c2fec7f9b96919c0b41a57536b67965a4a34 Mon Sep 17 00:00:00 2001 From: Jack Tysoe Date: Mon, 23 Sep 2024 17:47:12 +0100 Subject: [PATCH 3/4] feat(ai-proxy): added AWS IAM cloud identity instructions --- .../how-to/_cloud-provider-authentication.md | 111 +++++++++++++++++- .../how-to/_cloud-provider-authentication.md | 111 +++++++++++++++++- 2 files changed, 210 insertions(+), 12 deletions(-) diff --git a/app/_hub/kong-inc/ai-proxy-advanced/how-to/_cloud-provider-authentication.md b/app/_hub/kong-inc/ai-proxy-advanced/how-to/_cloud-provider-authentication.md index 973052dffdf1..1859c64cbf26 100644 --- a/app/_hub/kong-inc/ai-proxy-advanced/how-to/_cloud-provider-authentication.md +++ b/app/_hub/kong-inc/ai-proxy-advanced/how-to/_cloud-provider-authentication.md @@ -26,11 +26,11 @@ LLM-based services using those same methods. Kong's AI Gateway currently supports the following cloud authentication: -| AI-Proxy Advanced LLM Provider | Cloud Provider | Type | -|--------------------------------|-------------------------------------------------|-----------------------------------------| -| `azure` (Kong Enterprise Only) | Azure OpenAI | [Entra / Managed Identity Authentication](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) | -| `gemini` | Gemini Enterprise (on Vertex or Workspace) | [GCP Service Account](https://cloud.google.com/iam/docs/service-account-overview) | -| `bedrock` | AWS Bedrock Converse-API | [AWS IAM Identity](https://docs.aws.amazon.com/IAM/latest/UserGuide/id.html) | +| AI-Proxy Advanced LLM Provider | Cloud Provider | Type | +|--------------------------------------------|-------------------------------------------------|-----------------------------------------| +| `azure` ('{{site.ee_product_name}}' Only) | Azure OpenAI | [Entra / Managed Identity Authentication](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) | +| `gemini` | Gemini Enterprise (on Vertex or Workspace) | [GCP Service Account](https://cloud.google.com/iam/docs/service-account-overview) | +| `bedrock` | AWS Bedrock Converse-API | [AWS IAM Identity](https://docs.aws.amazon.com/IAM/latest/UserGuide/id.html) | ## Azure OpenAI (Kong Enterprise Only) @@ -283,4 +283,103 @@ config: auth: use_gcp_service_account: true gcp_service_account_json: '{vault://gcp/VERTEX_SERVICE_ACCOUNT_JSON}' -``` \ No newline at end of file +``` + +## AWS Bedrock + +When hosting your LLMs with [AWS Bedrock Converse API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html) in a business or enterprise plan, +and running them through AI Proxy, it is possible to use an [IAM Identity](https://docs.aws.amazon.com/IAM/latest/UserGuide/id.html) that can be assigned to a currently running EC2 instance, +an EKS deployment, ECS deployment, or just used via the [AWS CLI](https://aws.amazon.com/cli/) credential context on the local machine. + +How you do this depends on where and how you are running {{site.base_gateway}}. + +### Prerequisites + +You must be running a {{site.ee_product_name}} instance. + +Ensure that the EC2 instance, EKS deployment, ECS deployment, etcetera, has been assigned the IAM principal, +configurable from the AWS IAM portal. + +If the role requires crossing permission boundaries, ensure that the correct Assume-Role Policy is applied. + +Assign the correct permissions to the identity's IAM Policy: + +* `bedrock:InvokeModel` +* `bedrock:InvokeModelWithResponseStream` + +respective to the `Resource ARNs` that corresponds to the models that Kong is allowed to call on the user's behalf. + +### Configuring the AI Proxy Advanced Plugin to use AWS IAM + +When running Kong inside of your AWS subscription, AI Proxy Advanced is usually able to detect the designated IAM Principal automatically, based on the +assigned identity. + +Kong will use the same **authentication credentials chain** as with most AWS SDKs (and the AWS CLI). See the [Java credentials chain](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/credentials-chain.html) +precedence order, for an example. + +#### AWS IAM Identity + +To use an AWS-assigned IAM Identity, set up your plugin config like this example: + + +{% plugin_example %} +plugin: kong-inc/ai-proxy-advanced +name: ai-proxy +config: + route_type: "llm/v1/chat" + logging: + log_statistics: true + log_payloads: false + model: + provider: "bedrock" + name: "amazon.titan-text-express-v1" +targets: + - route + - consumer_group + - global +formats: + - konnect + - curl + - yaml + - kubernetes + - terraform +{% endplugin_example %} + + +In most workloads, this is **zero-configuration** and you should not need to instruct Kong AI Proxy plugin with any credentials of +Bedrock-specific configuration - Kong will find the correct IAM credentials automatically, upon **first invocation of the model**. + +#### Environment variables + +You can also specify your own AWS IAM credentials; simply set this environment variables in the Kong workload or deployment configuration: + +Environment variable: +```sh +AWS_ACCESS_KEY_ID=AKAI... +AWS_SECRET_ACCESS_KEY=... +AWS_REGION=eu-west-1 +``` + +or set it directly in the plugin configuration: + +```yaml +config: + auth: + aws_access_key_id: 'AKAI...' + aws_secret_access_key: '...' + options: + bedrock: + aws_region: 'eu-west-1' +``` + +or, more securely, use a vault reference to e.g. AWS Secrets Manager: + +```yaml +config: + auth: + aws_access_key_id: 'AKAI...' + aws_secret_access_key: '{vault://aws/BEDROCK_SECRET_ACCESS_KEY}' + options: + bedrock: + aws_region: 'eu-west-1' +``` diff --git a/app/_hub/kong-inc/ai-proxy/how-to/_cloud-provider-authentication.md b/app/_hub/kong-inc/ai-proxy/how-to/_cloud-provider-authentication.md index 419618b79ff7..852ac3570243 100644 --- a/app/_hub/kong-inc/ai-proxy/how-to/_cloud-provider-authentication.md +++ b/app/_hub/kong-inc/ai-proxy/how-to/_cloud-provider-authentication.md @@ -26,11 +26,11 @@ LLM-based services using those same methods. Kong's AI Gateway currently supports the following cloud authentication: -| AI-Proxy LLM Provider | Cloud Provider | Type | -|--------------------------------|-------------------------------------------------|-----------------------------------------| -| `azure` (Kong Enterprise Only) | Azure OpenAI | [Entra / Managed Identity Authentication](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) | -| `gemini` | Gemini Enterprise (on Vertex or Workspace) | [GCP Service Account](https://cloud.google.com/iam/docs/service-account-overview) | -| `bedrock` | AWS Bedrock Converse-API | [AWS IAM Identity](https://docs.aws.amazon.com/IAM/latest/UserGuide/id.html) | +| AI-Proxy LLM Provider | Cloud Provider | Type | +|--------------------------------------------|-------------------------------------------------|-----------------------------------------| +| `azure` ('{{site.ee_product_name}}' Only) | Azure OpenAI | [Entra / Managed Identity Authentication](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) | +| `gemini` | Gemini Enterprise (on Vertex or Workspace) | [GCP Service Account](https://cloud.google.com/iam/docs/service-account-overview) | +| `bedrock` | AWS Bedrock Converse-API | [AWS IAM Identity](https://docs.aws.amazon.com/IAM/latest/UserGuide/id.html) | ## Azure OpenAI (Kong Enterprise Only) @@ -283,4 +283,103 @@ config: auth: use_gcp_service_account: true gcp_service_account_json: '{vault://gcp/VERTEX_SERVICE_ACCOUNT_JSON}' -``` \ No newline at end of file +``` + +## AWS Bedrock + +When hosting your LLMs with [AWS Bedrock Converse API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html) in a business or enterprise plan, +and running them through AI Proxy, it is possible to use an [IAM Identity](https://docs.aws.amazon.com/IAM/latest/UserGuide/id.html) that can be assigned to a currently running EC2 instance, +an EKS deployment, ECS deployment, or just used via the [AWS CLI](https://aws.amazon.com/cli/) credential context on the local machine. + +How you do this depends on where and how you are running {{site.base_gateway}}. + +### Prerequisites + +You must be running a {{site.ee_product_name}} instance. + +Ensure that the EC2 instance, EKS deployment, ECS deployment, etcetera, has been assigned the IAM principal, +configurable from the AWS IAM portal. + +If the role requires crossing permission boundaries, ensure that the correct Assume-Role Policy is applied. + +Assign the correct permissions to the identity's IAM Policy: + +* `bedrock:InvokeModel` +* `bedrock:InvokeModelWithResponseStream` + +respective to the `Resource ARNs` that corresponds to the models that Kong is allowed to call on the user's behalf. + +### Configuring the AI Proxy Plugin to use AWS IAM + +When running Kong inside of your AWS subscription, AI Proxy is usually able to detect the designated IAM Principal automatically, based on the +assigned identity. + +Kong will use the same **authentication credentials chain** as with most AWS SDKs (and the AWS CLI). See the [Java credentials chain](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/credentials-chain.html) +precedence order, for an example. + +#### AWS IAM Identity + +To use an AWS-assigned IAM Identity, set up your plugin config like this example: + + +{% plugin_example %} +plugin: kong-inc/ai-proxy +name: ai-proxy +config: + route_type: "llm/v1/chat" + logging: + log_statistics: true + log_payloads: false + model: + provider: "bedrock" + name: "amazon.titan-text-express-v1" +targets: + - route + - consumer_group + - global +formats: + - konnect + - curl + - yaml + - kubernetes + - terraform +{% endplugin_example %} + + +In most workloads, this is **zero-configuration** and you should not need to instruct Kong AI Proxy plugin with any credentials of +Bedrock-specific configuration - Kong will find the correct IAM credentials automatically, upon **first invocation of the model**. + +#### Environment variables + +You can also specify your own AWS IAM credentials; simply set this environment variables in the Kong workload or deployment configuration: + +Environment variable: +```sh +AWS_ACCESS_KEY_ID=AKAI... +AWS_SECRET_ACCESS_KEY=... +AWS_REGION=eu-west-1 +``` + +or set it directly in the plugin configuration: + +```yaml +config: + auth: + aws_access_key_id: 'AKAI...' + aws_secret_access_key: '...' + options: + bedrock: + aws_region: 'eu-west-1' +``` + +or, more securely, use a vault reference to e.g. AWS Secrets Manager: + +```yaml +config: + auth: + aws_access_key_id: 'AKAI...' + aws_secret_access_key: '{vault://aws/BEDROCK_SECRET_ACCESS_KEY}' + options: + bedrock: + aws_region: 'eu-west-1' +``` From 0d7d0379362072c6362fbe8a2cf459b48c8ad748 Mon Sep 17 00:00:00 2001 From: Diana <75819066+cloudjumpercat@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:47:39 -0500 Subject: [PATCH 4/4] Break up the cloud provider docs into their own section, revise the Azure docs Signed-off-by: Diana <75819066+cloudjumpercat@users.noreply.github.com> --- .../how-to/_cloud-provider-authentication.md | 385 ------------------ .../cloud-provider-authentication/_azure.md | 175 ++++++++ .../cloud-provider-authentication/_bedrock.md | 104 +++++ .../cloud-provider-authentication/_gemini.md | 90 ++++ .../cloud-provider-authentication/_index.md | 38 ++ .../how-to/_cloud-provider-authentication.md | 385 ------------------ .../cloud-provider-authentication/_azure.md | 175 ++++++++ .../cloud-provider-authentication/_bedrock.md | 104 +++++ .../cloud-provider-authentication/_gemini.md | 90 ++++ .../cloud-provider-authentication/_index.md | 33 ++ 10 files changed, 809 insertions(+), 770 deletions(-) delete mode 100644 app/_hub/kong-inc/ai-proxy-advanced/how-to/_cloud-provider-authentication.md create mode 100644 app/_hub/kong-inc/ai-proxy-advanced/how-to/cloud-provider-authentication/_azure.md create mode 100644 app/_hub/kong-inc/ai-proxy-advanced/how-to/cloud-provider-authentication/_bedrock.md create mode 100644 app/_hub/kong-inc/ai-proxy-advanced/how-to/cloud-provider-authentication/_gemini.md create mode 100644 app/_hub/kong-inc/ai-proxy-advanced/how-to/cloud-provider-authentication/_index.md delete mode 100644 app/_hub/kong-inc/ai-proxy/how-to/_cloud-provider-authentication.md create mode 100644 app/_hub/kong-inc/ai-proxy/how-to/cloud-provider-authentication/_azure.md create mode 100644 app/_hub/kong-inc/ai-proxy/how-to/cloud-provider-authentication/_bedrock.md create mode 100644 app/_hub/kong-inc/ai-proxy/how-to/cloud-provider-authentication/_gemini.md create mode 100644 app/_hub/kong-inc/ai-proxy/how-to/cloud-provider-authentication/_index.md diff --git a/app/_hub/kong-inc/ai-proxy-advanced/how-to/_cloud-provider-authentication.md b/app/_hub/kong-inc/ai-proxy-advanced/how-to/_cloud-provider-authentication.md deleted file mode 100644 index 1859c64cbf26..000000000000 --- a/app/_hub/kong-inc/ai-proxy-advanced/how-to/_cloud-provider-authentication.md +++ /dev/null @@ -1,385 +0,0 @@ ---- -nav_title: Cloud Provider Authentication -title: Authenticate to Cloud-Hosted Models Using their Native Authentication -minimum_version: 3.7.x ---- - -{:.note} -> This feature requires {{site.ee_product_name}}. - -This guide walks you through setting up the AI Proxy Advanced plugin with a cloud-hosted model, -using the cloud's native authentication mechanism. - -## Overview - -When running software on a cloud-hosted virtual machine or container instance, the provider -offers a keyless role-based access mechanism, allowing you to call services native to that cloud -provider without having to store any keys inside the running instance (or in the Kong configuration). - -This operates like a single-sign-on (SSO) mechanism for your cloud applications. - -Kong's AI Gateway (AI Proxy Advanced) can take advantage of the authentication mechanisms for -many different cloud providers and, where available, can also use this authentication to call -LLM-based services using those same methods. - -## Supported providers - -Kong's AI Gateway currently supports the following cloud authentication: - -| AI-Proxy Advanced LLM Provider | Cloud Provider | Type | -|--------------------------------------------|-------------------------------------------------|-----------------------------------------| -| `azure` ('{{site.ee_product_name}}' Only) | Azure OpenAI | [Entra / Managed Identity Authentication](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) | -| `gemini` | Gemini Enterprise (on Vertex or Workspace) | [GCP Service Account](https://cloud.google.com/iam/docs/service-account-overview) | -| `bedrock` | AWS Bedrock Converse-API | [AWS IAM Identity](https://docs.aws.amazon.com/IAM/latest/UserGuide/id.html) | - -## Azure OpenAI (Kong Enterprise Only) - -When hosting your LLMs with [Azure OpenAI Service](https://azure.microsoft.com/en-gb/products/ai-services/openai-service) -and running them through AI Proxy Advanced, it is possible to use the assigned -[Azure Managed Identity or User-Assigned Identity](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) -of the VM, Kubernetes service-account, or ACS container, to call the Azure OpenAI models. - -You can also use an Entra principal or App Registration (`client_id`, `client_secret`, and `tenant_id` triplet) when -Kong is hosted outside of Azure. - -How you do this depends on where and how you are running {{site.base_gateway}}. - -### Prerequisites - -You must be running a {{site.ee_product_name}} instance. - -Ensure that the Azure principal that you have assigned to the Compute resource (that is running your {{site.base_gateway}}) has the necessary -Entra or IAM permissions to execute commands on the desired OpenAI instances. It must have one of the following permissions: - -* Cognitive Services OpenAI User -* Cognitive Services OpenAI Contributor - -See [Azure's documentation on managed identity](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/managed-identity) -to set this up. - -### Configuring the AI Proxy Advanced Plugin to use Azure Identity - -When running Kong inside of your Azure subscription, AI Proxy Advanced is usually able to detect the designated Managed Identity or User-Assigned Identity -of that Azure Compute resource, and use it accordingly. - -#### Azure-Assigned Managed Identity - -To use an Azure-Assigned Managed Identity, set up your plugin config like this: - - -{% plugin_example %} -plugin: kong-inc/ai-proxy-advanced -name: ai-proxy-advanced -config: - route_type: "llm/v1/chat" - auth: - azure_use_managed_identity: true - model: - provider: "azure" - name: "gpt-35-turbo" - options: - azure_instance: "my-openai-instance" - azure_deployment_id: "kong-gpt-3-5" -targets: - - route - - consumer_group - - global -formats: - - konnect - - curl - - yaml - - kubernetes - - terraform -{% endplugin_example %} - - -#### User-Assigned Identity - -To use a User-Assigned Identity, specify its client ID like this: - - -{% plugin_example %} -plugin: kong-inc/ai-proxy-advanced -name: ai-proxy-advanced -config: - route_type: "llm/v1/chat" - auth: - azure_use_managed_identity: true - azure_client_id: "aabdecea-fc38-40ca-9edd-263878b290fe" - model: - provider: "azure" - name: "gpt-35-turbo" - options: - azure_instance: "my-openai-instance" - azure_deployment_id: "kong-gpt-3-5" -targets: - - route - - consumer_group - - global -formats: - - konnect - - curl - - yaml - - kubernetes - - terraform -{% endplugin_example %} - - -#### Using Entra or app registration - -If running outside of Azure, to use an Entra principal or app registration, specify all properties like this: - - -{% plugin_example %} -plugin: kong-inc/ai-proxy-advanced -name: ai-proxy-advanced -config: - route_type: "llm/v1/chat" - auth: - azure_use_managed_identity: true - azure_client_id: "aabdecea-fc38-40ca-9edd-263878b290fe" - azure_client_secret: "be0c34b6-b5f1-4343-99a3-140df73e0c1c" - azure_tenant_id: "1e583ecd-9293-4db1-b1c0-2b6126cb5fdd" - model: - provider: "azure" - name: "gpt-35-turbo" - options: - azure_instance: "my-openai-instance" - azure_deployment_id: "kong-gpt-3-5" -targets: - - route - - consumer_group - - global -formats: - - konnect - - curl - - yaml - - kubernetes - - terraform -{% endplugin_example %} - - -#### Environment variables - -You can also specify some (or all) of these properties as environment variables. For example: - - -Environment variable: -```sh -AZURE_CLIENT_SECRET="be0c34b6-b5f1-4343-99a3-140df73e0c1c" -``` - -Plugin configuration: - - -{% plugin_example %} -plugin: kong-inc/ai-proxy-advanced -name: ai-proxy-advanced -config: - route_type: "llm/v1/chat" - auth: - azure_use_managed_identity: true - azure_client_id: "aabdecea-fc38-40ca-9edd-263878b290fe" - azure_tenant_id: "1e583ecd-9293-4db1-b1c0-2b6126cb5fdd" - model: - provider: "azure" - name: "gpt-35-turbo" - options: - azure_instance: "my-openai-instance" - azure_deployment_id: "kong-gpt-3-5" -targets: - - route - - consumer_group - - global -formats: - - konnect - - curl - - yaml - - kubernetes - - terraform -{% endplugin_example %} - - -## Google Gemini - -When hosting your LLMs with [Google Gemini Vertex](https://cloud.google.com/vertex-ai?hl=en) in a business or enterprise plan, and running them through AI Proxy Advanced, -it is possible to use a [GCP Workload Identity](https://cloud.google.com/iam/docs/workload-identity-federation) that can be assigned to a currently running instance, -a Container Platform deployment, or just used via the [gcloud CLI](https://cloud.google.com/sdk/gcloud) sign-in on the local machine. - -How you do this depends on where and how you are running {{site.base_gateway}}. - -### Prerequisites - -You must be running a {{site.ee_product_name}} instance. - -Ensure that the GCP Virtual Machine, Container Deployment, Container Application (or a combination of these) has been assigned the Service Account principal, -configurable from the Google Cloud IAM portal. - -Assign the `'Vertex AI User'` role to the Service Account. - -### Configuring the AI Proxy Advanced Plugin to use GCP Workload Identity - -When running Kong inside of your GCP subscription, AI Proxy Advanced is usually able to detect the designated Service Account automatically, based on the -`GCP_SERVICE_ACCOUNT` JSON that is automatically injected into an environment variable in your Kong deployment (or the Kong Virtual Machine(s)). - -#### GCP-Assigned Workload Identity - -To use a GCP-Assigned Workload Identity, set up your plugin config like this example: - - -{% plugin_example %} -plugin: kong-inc/ai-proxy-advanced -name: ai-proxy-advanced -config: - route_type: "llm/v1/chat" - auth: - use_gcp_service_account: true - logging: - log_statistics: true - log_payloads: false - model: - provider: "gemini" - name: "gemini-1.5-flash" - options: - gemini: - api_endpoint: "us-central1-aiplatform.googleapis.com" - project_id: "sample-project-123456" - location_id: "us-central1" -targets: - - route - - consumer_group - - global -formats: - - konnect - - curl - - yaml - - kubernetes - - terraform -{% endplugin_example %} - - -#### Environment variables - -You can also specify your own GCP Service Account JSON; simply set this environment variable in the Kong workload or deployment configuration: - -Environment variable: -```sh -GCP_SERVICE_ACCOUNT='{ "type": "service_account", "project_id": "sample-project-123456", "private_key_id": "...", "private_key": "..."...}' -``` - -or set it directly in the plugin configuration: - -```yaml -config: - auth: - use_gcp_service_account: true - gcp_service_account_json: '{ "type": "service_account", "project_id": "sample-project-123456", "private_key_id": "...", "private_key": "..."...}' -``` - -or, more securely, use a vault reference to e.g. GCP Secrets Manager: - -```yaml -config: - auth: - use_gcp_service_account: true - gcp_service_account_json: '{vault://gcp/VERTEX_SERVICE_ACCOUNT_JSON}' -``` - -## AWS Bedrock - -When hosting your LLMs with [AWS Bedrock Converse API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html) in a business or enterprise plan, -and running them through AI Proxy, it is possible to use an [IAM Identity](https://docs.aws.amazon.com/IAM/latest/UserGuide/id.html) that can be assigned to a currently running EC2 instance, -an EKS deployment, ECS deployment, or just used via the [AWS CLI](https://aws.amazon.com/cli/) credential context on the local machine. - -How you do this depends on where and how you are running {{site.base_gateway}}. - -### Prerequisites - -You must be running a {{site.ee_product_name}} instance. - -Ensure that the EC2 instance, EKS deployment, ECS deployment, etcetera, has been assigned the IAM principal, -configurable from the AWS IAM portal. - -If the role requires crossing permission boundaries, ensure that the correct Assume-Role Policy is applied. - -Assign the correct permissions to the identity's IAM Policy: - -* `bedrock:InvokeModel` -* `bedrock:InvokeModelWithResponseStream` - -respective to the `Resource ARNs` that corresponds to the models that Kong is allowed to call on the user's behalf. - -### Configuring the AI Proxy Advanced Plugin to use AWS IAM - -When running Kong inside of your AWS subscription, AI Proxy Advanced is usually able to detect the designated IAM Principal automatically, based on the -assigned identity. - -Kong will use the same **authentication credentials chain** as with most AWS SDKs (and the AWS CLI). See the [Java credentials chain](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/credentials-chain.html) -precedence order, for an example. - -#### AWS IAM Identity - -To use an AWS-assigned IAM Identity, set up your plugin config like this example: - - -{% plugin_example %} -plugin: kong-inc/ai-proxy-advanced -name: ai-proxy -config: - route_type: "llm/v1/chat" - logging: - log_statistics: true - log_payloads: false - model: - provider: "bedrock" - name: "amazon.titan-text-express-v1" -targets: - - route - - consumer_group - - global -formats: - - konnect - - curl - - yaml - - kubernetes - - terraform -{% endplugin_example %} - - -In most workloads, this is **zero-configuration** and you should not need to instruct Kong AI Proxy plugin with any credentials of -Bedrock-specific configuration - Kong will find the correct IAM credentials automatically, upon **first invocation of the model**. - -#### Environment variables - -You can also specify your own AWS IAM credentials; simply set this environment variables in the Kong workload or deployment configuration: - -Environment variable: -```sh -AWS_ACCESS_KEY_ID=AKAI... -AWS_SECRET_ACCESS_KEY=... -AWS_REGION=eu-west-1 -``` - -or set it directly in the plugin configuration: - -```yaml -config: - auth: - aws_access_key_id: 'AKAI...' - aws_secret_access_key: '...' - options: - bedrock: - aws_region: 'eu-west-1' -``` - -or, more securely, use a vault reference to e.g. AWS Secrets Manager: - -```yaml -config: - auth: - aws_access_key_id: 'AKAI...' - aws_secret_access_key: '{vault://aws/BEDROCK_SECRET_ACCESS_KEY}' - options: - bedrock: - aws_region: 'eu-west-1' -``` diff --git a/app/_hub/kong-inc/ai-proxy-advanced/how-to/cloud-provider-authentication/_azure.md b/app/_hub/kong-inc/ai-proxy-advanced/how-to/cloud-provider-authentication/_azure.md new file mode 100644 index 000000000000..22143ec9bb07 --- /dev/null +++ b/app/_hub/kong-inc/ai-proxy-advanced/how-to/cloud-provider-authentication/_azure.md @@ -0,0 +1,175 @@ +--- +nav_title: Azure +title: Azure +minimum_version: 3.7.x +--- + +When hosting your LLMs with [Azure OpenAI Service](https://azure.microsoft.com/en-gb/products/ai-services/openai-service) +and running them through AI Proxy Advanced, you can use the assigned +[Azure Managed Identity or User-Assigned Identity](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) +of the VM, Kubernetes service-account, or ACS container, to call the Azure OpenAI models. + +You can also use an Entra principal or App Registration (`client_id`, `client_secret`, and `tenant_id` triplet) when +Kong is hosted outside of Azure. + +How you do this depends on where and how you are running {{site.base_gateway}}. + +You can use the following table to help you determine which method to use: + +| Where is {{site.base_gateway}} hosted? | Then use... | +| -------------------------------------- | ----------- | +| Inside Azure | [Azure-managed identity](#azure-assigned-managed-identity) | +| Inside Azure | [User-assigned identity](#user-assigned-identity) | +| Outside Azure | [Azure-managed identity with Entra](#using-entra-or-app-registration) | + +## Prerequisites + +* You must be running a {{site.ee_product_name}} instance. +* Ensure that the Azure principal that you have assigned to the Compute resource (that is running your {{site.base_gateway}}) has one of the following Entra or IAM permissions to execute commands on the desired OpenAI instances: + * [Cognitive Services OpenAI User](https://learn.microsoft.com/azure/ai-services/openai/how-to/role-based-access-control#cognitive-services-openai-user) + * [Cognitive Services OpenAI Contributor](https://learn.microsoft.com/azure/ai-services/openai/how-to/role-based-access-control#cognitive-services-openai-contributor) + + See [Azure's documentation on managed identity](https://learn.microsoft.com/azure/ai-services/openai/how-to/managed-identity) to set this up. + +## Configure the AI Proxy Advanced plugin to use Azure Identity + +When running {{site.base_gateway}} inside of your Azure subscription, AI Proxy Advanced is usually able to detect the designated Managed Identity or User-Assigned Identity +of that Azure Compute resource and use it accordingly. + +### Azure-assigned managed identity + +To use an Azure-assigned managed identity, set up your plugin config like the following: + + +{% plugin_example %} +plugin: kong-inc/ai-proxy-advanced +name: ai-proxy-advanced +config: + route_type: "llm/v1/chat" + auth: + azure_use_managed_identity: true + model: + provider: "azure" + name: "gpt-35-turbo" + options: + azure_instance: "my-openai-instance" + azure_deployment_id: "kong-gpt-3-5" +targets: + - route + - consumer_group + - global +formats: + - konnect + - curl + - yaml + - kubernetes + - terraform +{% endplugin_example %} + + +### User-assigned identity + +To use a User-assigned identity, specify its client ID like the following: + + +{% plugin_example %} +plugin: kong-inc/ai-proxy-advanced +name: ai-proxy-advanced +config: + route_type: "llm/v1/chat" + auth: + azure_use_managed_identity: true + azure_client_id: "aabdecea-fc38-40ca-9edd-263878b290fe" + model: + provider: "azure" + name: "gpt-35-turbo" + options: + azure_instance: "my-openai-instance" + azure_deployment_id: "kong-gpt-3-5" +targets: + - route + - consumer_group + - global +formats: + - konnect + - curl + - yaml + - kubernetes + - terraform +{% endplugin_example %} + + +### Using Entra or app registration + +If you're running {{site.base_gateway}} outside of Azure, use an Entra principal or app registration by specifing all properties like the following: + + +{% plugin_example %} +plugin: kong-inc/ai-proxy-advanced +name: ai-proxy-advanced +config: + route_type: "llm/v1/chat" + auth: + azure_use_managed_identity: true + azure_client_id: "aabdecea-fc38-40ca-9edd-263878b290fe" + azure_client_secret: "be0c34b6-b5f1-4343-99a3-140df73e0c1c" + azure_tenant_id: "1e583ecd-9293-4db1-b1c0-2b6126cb5fdd" + model: + provider: "azure" + name: "gpt-35-turbo" + options: + azure_instance: "my-openai-instance" + azure_deployment_id: "kong-gpt-3-5" +targets: + - route + - consumer_group + - global +formats: + - konnect + - curl + - yaml + - kubernetes + - terraform +{% endplugin_example %} + + +#### Environment variables + +You can also specify some, or all, of these properties as environment variables. For example: + + +Environment variable: +```sh +AZURE_CLIENT_SECRET="be0c34b6-b5f1-4343-99a3-140df73e0c1c" +``` + +You can then omit that value from the plugin configuration like the following: + + +{% plugin_example %} +plugin: kong-inc/ai-proxy-advanced +name: ai-proxy-advanced +config: + route_type: "llm/v1/chat" + auth: + azure_use_managed_identity: true + azure_client_id: "aabdecea-fc38-40ca-9edd-263878b290fe" + azure_tenant_id: "1e583ecd-9293-4db1-b1c0-2b6126cb5fdd" + model: + provider: "azure" + name: "gpt-35-turbo" + options: + azure_instance: "my-openai-instance" + azure_deployment_id: "kong-gpt-3-5" +targets: + - route + - consumer_group + - global +formats: + - konnect + - curl + - yaml + - kubernetes + - terraform +{% endplugin_example %} + \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy-advanced/how-to/cloud-provider-authentication/_bedrock.md b/app/_hub/kong-inc/ai-proxy-advanced/how-to/cloud-provider-authentication/_bedrock.md new file mode 100644 index 000000000000..a380f4a4845a --- /dev/null +++ b/app/_hub/kong-inc/ai-proxy-advanced/how-to/cloud-provider-authentication/_bedrock.md @@ -0,0 +1,104 @@ +--- +nav_title: Bedrock +title: Bedrock +minimum_version: 3.8.x +--- + +## AWS Bedrock + +When hosting your LLMs with [AWS Bedrock Converse API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html) in a business or enterprise plan, +and running them through AI Proxy, it is possible to use an [IAM Identity](https://docs.aws.amazon.com/IAM/latest/UserGuide/id.html) that can be assigned to a currently running EC2 instance, +an EKS deployment, ECS deployment, or just used via the [AWS CLI](https://aws.amazon.com/cli/) credential context on the local machine. + +How you do this depends on where and how you are running {{site.base_gateway}}. + +### Prerequisites + +You must be running a {{site.ee_product_name}} instance. + +Ensure that the EC2 instance, EKS deployment, ECS deployment, etcetera, has been assigned the IAM principal, +configurable from the AWS IAM portal. + +If the role requires crossing permission boundaries, ensure that the correct Assume-Role Policy is applied. + +Assign the correct permissions to the identity's IAM Policy: + +* `bedrock:InvokeModel` +* `bedrock:InvokeModelWithResponseStream` + +respective to the `Resource ARNs` that corresponds to the models that Kong is allowed to call on the user's behalf. + +### Configuring the AI Proxy Advanced Plugin to use AWS IAM + +When running Kong inside of your AWS subscription, AI Proxy Advanced is usually able to detect the designated IAM Principal automatically, based on the +assigned identity. + +Kong will use the same **authentication credentials chain** as with most AWS SDKs (and the AWS CLI). See the [Java credentials chain](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/credentials-chain.html) +precedence order, for an example. + +#### AWS IAM Identity + +To use an AWS-assigned IAM Identity, set up your plugin config like this example: + + +{% plugin_example %} +plugin: kong-inc/ai-proxy-advanced +name: ai-proxy +config: + route_type: "llm/v1/chat" + logging: + log_statistics: true + log_payloads: false + model: + provider: "bedrock" + name: "amazon.titan-text-express-v1" +targets: + - route + - consumer_group + - global +formats: + - konnect + - curl + - yaml + - kubernetes + - terraform +{% endplugin_example %} + + +In most workloads, this is **zero-configuration** and you should not need to instruct Kong AI Proxy plugin with any credentials of +Bedrock-specific configuration - Kong will find the correct IAM credentials automatically, upon **first invocation of the model**. + +#### Environment variables + +You can also specify your own AWS IAM credentials; simply set this environment variables in the Kong workload or deployment configuration: + +Environment variable: +```sh +AWS_ACCESS_KEY_ID=AKAI... +AWS_SECRET_ACCESS_KEY=... +AWS_REGION=eu-west-1 +``` + +or set it directly in the plugin configuration: + +```yaml +config: + auth: + aws_access_key_id: 'AKAI...' + aws_secret_access_key: '...' + options: + bedrock: + aws_region: 'eu-west-1' +``` + +or, more securely, use a vault reference to e.g. AWS Secrets Manager: + +```yaml +config: + auth: + aws_access_key_id: 'AKAI...' + aws_secret_access_key: '{vault://aws/BEDROCK_SECRET_ACCESS_KEY}' + options: + bedrock: + aws_region: 'eu-west-1' +``` diff --git a/app/_hub/kong-inc/ai-proxy-advanced/how-to/cloud-provider-authentication/_gemini.md b/app/_hub/kong-inc/ai-proxy-advanced/how-to/cloud-provider-authentication/_gemini.md new file mode 100644 index 000000000000..d5319a1b3736 --- /dev/null +++ b/app/_hub/kong-inc/ai-proxy-advanced/how-to/cloud-provider-authentication/_gemini.md @@ -0,0 +1,90 @@ +--- +nav_title: Gemini +title: Gemini +minimum_version: 3.8.x +--- + +## Google Gemini + +When hosting your LLMs with [Google Gemini Vertex](https://cloud.google.com/vertex-ai?hl=en) in a business or enterprise plan, and running them through AI Proxy Advanced, +it is possible to use a [GCP Workload Identity](https://cloud.google.com/iam/docs/workload-identity-federation) that can be assigned to a currently running instance, +a Container Platform deployment, or just used via the [gcloud CLI](https://cloud.google.com/sdk/gcloud) sign-in on the local machine. + +How you do this depends on where and how you are running {{site.base_gateway}}. + +### Prerequisites + +You must be running a {{site.ee_product_name}} instance. + +Ensure that the GCP Virtual Machine, Container Deployment, Container Application (or a combination of these) has been assigned the Service Account principal, +configurable from the Google Cloud IAM portal. + +Assign the `'Vertex AI User'` role to the Service Account. + +### Configuring the AI Proxy Advanced Plugin to use GCP Workload Identity + +When running Kong inside of your GCP subscription, AI Proxy Advanced is usually able to detect the designated Service Account automatically, based on the +`GCP_SERVICE_ACCOUNT` JSON that is automatically injected into an environment variable in your Kong deployment (or the Kong Virtual Machine(s)). + +#### GCP-Assigned Workload Identity + +To use a GCP-Assigned Workload Identity, set up your plugin config like this example: + + +{% plugin_example %} +plugin: kong-inc/ai-proxy-advanced +name: ai-proxy-advanced +config: + route_type: "llm/v1/chat" + auth: + use_gcp_service_account: true + logging: + log_statistics: true + log_payloads: false + model: + provider: "gemini" + name: "gemini-1.5-flash" + options: + gemini: + api_endpoint: "us-central1-aiplatform.googleapis.com" + project_id: "sample-project-123456" + location_id: "us-central1" +targets: + - route + - consumer_group + - global +formats: + - konnect + - curl + - yaml + - kubernetes + - terraform +{% endplugin_example %} + + +#### Environment variables + +You can also specify your own GCP Service Account JSON; simply set this environment variable in the Kong workload or deployment configuration: + +Environment variable: +```sh +GCP_SERVICE_ACCOUNT='{ "type": "service_account", "project_id": "sample-project-123456", "private_key_id": "...", "private_key": "..."...}' +``` + +or set it directly in the plugin configuration: + +```yaml +config: + auth: + use_gcp_service_account: true + gcp_service_account_json: '{ "type": "service_account", "project_id": "sample-project-123456", "private_key_id": "...", "private_key": "..."...}' +``` + +or, more securely, use a vault reference to e.g. GCP Secrets Manager: + +```yaml +config: + auth: + use_gcp_service_account: true + gcp_service_account_json: '{vault://gcp/VERTEX_SERVICE_ACCOUNT_JSON}' +``` \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy-advanced/how-to/cloud-provider-authentication/_index.md b/app/_hub/kong-inc/ai-proxy-advanced/how-to/cloud-provider-authentication/_index.md new file mode 100644 index 000000000000..edb7f7d53073 --- /dev/null +++ b/app/_hub/kong-inc/ai-proxy-advanced/how-to/cloud-provider-authentication/_index.md @@ -0,0 +1,38 @@ +--- +nav_title: Overview +title: Overview +minimum_version: 3.7.x +--- + +{:.note} +> This feature requires {{site.ee_product_name}}. + +This guide walks you through setting up the AI Proxy Advanced plugin with a cloud-hosted model, +using the cloud's native authentication mechanism. + +Cloud native authentication has the following benefits over regular authentication: +* Additional security because you aren't passing sensitive information (like API keys) through the plugin configuration +* Use the roles and permissions from the cloud provider instead of hardcoding them + +When running software on a cloud-hosted virtual machine or container instance, the provider +offers a keyless role-based access mechanism, allowing you to call services native to that cloud +provider without having to store any keys inside the running instance (or in the Kong configuration). + +This operates like a single-sign-on (SSO) mechanism for your cloud applications. + +Kong's AI Gateway (AI Proxy Advanced) can take advantage of the authentication mechanisms for +many different cloud providers and, where available, can also use this authentication to call +LLM-based services using those same methods. + +## Supported providers + +Kong's AI Gateway currently supports the following cloud authentication: + +| AI Proxy Advanced LLM provider | Cloud provider | Type | +|--------------------------------------------|-------------------------------------------------|-----------------------------------------| +| `azure` ('{{site.ee_product_name}}' Only) | Azure OpenAI | [Entra/Managed Identity Authentication](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) | + +{% if_version gte:3.8.x %} +| `gemini` | Gemini Enterprise (on Vertex or Workspace) | [GCP Service Account](https://cloud.google.com/iam/docs/service-account-overview) | +| `bedrock` | AWS Bedrock Converse-API | [AWS IAM Identity](https://docs.aws.amazon.com/IAM/latest/UserGuide/id.html) | +{% endif_version %} \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy/how-to/_cloud-provider-authentication.md b/app/_hub/kong-inc/ai-proxy/how-to/_cloud-provider-authentication.md deleted file mode 100644 index 852ac3570243..000000000000 --- a/app/_hub/kong-inc/ai-proxy/how-to/_cloud-provider-authentication.md +++ /dev/null @@ -1,385 +0,0 @@ ---- -nav_title: Cloud Provider Authentication -title: Authenticate to Cloud-Hosted Models Using their Native Authentication -minimum_version: 3.7.x ---- - -{:.note} -> This feature requires {{site.ee_product_name}}. - -This guide walks you through setting up the AI Proxy plugin with a cloud-hosted model, -using the cloud's native authentication mechanism. - -## Overview - -When running software on a cloud-hosted virtual machine or container instance, the provider -offers a keyless role-based access mechanism, allowing you to call services native to that cloud -provider without having to store any keys inside the running instance (or in the Kong configuration). - -This operates like a single-sign-on (SSO) mechanism for your cloud applications. - -Kong's AI Gateway (AI Proxy) can take advantage of the authentication mechanisms for -many different cloud providers and, where available, can also use this authentication to call -LLM-based services using those same methods. - -## Supported providers - -Kong's AI Gateway currently supports the following cloud authentication: - -| AI-Proxy LLM Provider | Cloud Provider | Type | -|--------------------------------------------|-------------------------------------------------|-----------------------------------------| -| `azure` ('{{site.ee_product_name}}' Only) | Azure OpenAI | [Entra / Managed Identity Authentication](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) | -| `gemini` | Gemini Enterprise (on Vertex or Workspace) | [GCP Service Account](https://cloud.google.com/iam/docs/service-account-overview) | -| `bedrock` | AWS Bedrock Converse-API | [AWS IAM Identity](https://docs.aws.amazon.com/IAM/latest/UserGuide/id.html) | - -## Azure OpenAI (Kong Enterprise Only) - -When hosting your LLMs with [Azure OpenAI Service](https://azure.microsoft.com/en-gb/products/ai-services/openai-service) -and running them through AI Proxy, it is possible to use the assigned -[Azure Managed Identity or User-Assigned Identity](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) -of the VM, Kubernetes service-account, or ACS container, to call the Azure OpenAI models. - -You can also use an Entra principal or App Registration (`client_id`, `client_secret`, and `tenant_id` triplet) when -Kong is hosted outside of Azure. - -How you do this depends on where and how you are running {{site.base_gateway}}. - -### Prerequisites - -You must be running a {{site.ee_product_name}} instance. - -Ensure that the Azure principal that you have assigned to the Compute resource (that is running your {{site.base_gateway}}) has the necessary -Entra or IAM permissions to execute commands on the desired OpenAI instances. It must have one of the following permissions: - -* Cognitive Services OpenAI User -* Cognitive Services OpenAI Contributor - -See [Azure's documentation on managed identity](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/managed-identity) -to set this up. - -### Configuring the AI Proxy Plugin to use Azure Identity - -When running Kong inside of your Azure subscription, AI Proxy is usually able to detect the designated Managed Identity or User-Assigned Identity -of that Azure Compute resource, and use it accordingly. - -#### Azure-Assigned Managed Identity - -To use an Azure-Assigned Managed Identity, set up your plugin config like this: - - -{% plugin_example %} -plugin: kong-inc/ai-proxy -name: ai-proxy -config: - route_type: "llm/v1/chat" - auth: - azure_use_managed_identity: true - model: - provider: "azure" - name: "gpt-35-turbo" - options: - azure_instance: "my-openai-instance" - azure_deployment_id: "kong-gpt-3-5" -targets: - - route - - consumer_group - - global -formats: - - konnect - - curl - - yaml - - kubernetes - - terraform -{% endplugin_example %} - - -#### User-Assigned Identity - -To use a User-Assigned Identity, specify its client ID like this: - - -{% plugin_example %} -plugin: kong-inc/ai-proxy -name: ai-proxy -config: - route_type: "llm/v1/chat" - auth: - azure_use_managed_identity: true - azure_client_id: "aabdecea-fc38-40ca-9edd-263878b290fe" - model: - provider: "azure" - name: "gpt-35-turbo" - options: - azure_instance: "my-openai-instance" - azure_deployment_id: "kong-gpt-3-5" -targets: - - route - - consumer_group - - global -formats: - - konnect - - curl - - yaml - - kubernetes - - terraform -{% endplugin_example %} - - -#### Using Entra or app registration - -If running outside of Azure, to use an Entra principal or app registration, specify all properties like this: - - -{% plugin_example %} -plugin: kong-inc/ai-proxy -name: ai-proxy -config: - route_type: "llm/v1/chat" - auth: - azure_use_managed_identity: true - azure_client_id: "aabdecea-fc38-40ca-9edd-263878b290fe" - azure_client_secret: "be0c34b6-b5f1-4343-99a3-140df73e0c1c" - azure_tenant_id: "1e583ecd-9293-4db1-b1c0-2b6126cb5fdd" - model: - provider: "azure" - name: "gpt-35-turbo" - options: - azure_instance: "my-openai-instance" - azure_deployment_id: "kong-gpt-3-5" -targets: - - route - - consumer_group - - global -formats: - - konnect - - curl - - yaml - - kubernetes - - terraform -{% endplugin_example %} - - -#### Environment variables - -You can also specify some (or all) of these properties as environment variables. For example: - - -Environment variable: -```sh -AZURE_CLIENT_SECRET="be0c34b6-b5f1-4343-99a3-140df73e0c1c" -``` - -Plugin configuration: - - -{% plugin_example %} -plugin: kong-inc/ai-proxy -name: ai-proxy -config: - route_type: "llm/v1/chat" - auth: - azure_use_managed_identity: true - azure_client_id: "aabdecea-fc38-40ca-9edd-263878b290fe" - azure_tenant_id: "1e583ecd-9293-4db1-b1c0-2b6126cb5fdd" - model: - provider: "azure" - name: "gpt-35-turbo" - options: - azure_instance: "my-openai-instance" - azure_deployment_id: "kong-gpt-3-5" -targets: - - route - - consumer_group - - global -formats: - - konnect - - curl - - yaml - - kubernetes - - terraform -{% endplugin_example %} - - -## Google Gemini - -When hosting your LLMs with [Google Gemini Vertex](https://cloud.google.com/vertex-ai?hl=en) in a business or enterprise plan, and running them through AI Proxy, -it is possible to use a [GCP Workload Identity](https://cloud.google.com/iam/docs/workload-identity-federation) that can be assigned to a currently running instance, -a Container Platform deployment, or just used via the [gcloud CLI](https://cloud.google.com/sdk/gcloud) sign-in on the local machine. - -How you do this depends on where and how you are running {{site.base_gateway}}. - -### Prerequisites - -You must be running a {{site.ee_product_name}} instance. - -Ensure that the GCP Virtual Machine, Container Deployment, Container Application (or a combination of these) has been assigned the Service Account principal, -configurable from the Google Cloud IAM portal. - -Assign the `'Vertex AI User'` role to the Service Account. - -### Configuring the AI Proxy Plugin to use GCP Workload Identity - -When running Kong inside of your GCP subscription, AI Proxy is usually able to detect the designated Service Account automatically, based on the -`GCP_SERVICE_ACCOUNT` JSON that is automatically injected into an environment variable in your Kong deployment (or the Kong Virtual Machine(s)). - -#### GCP-Assigned Workload Identity - -To use a GCP-Assigned Workload Identity, set up your plugin config like this example: - - -{% plugin_example %} -plugin: kong-inc/ai-proxy -name: ai-proxy -config: - route_type: "llm/v1/chat" - auth: - use_gcp_service_account: true - logging: - log_statistics: true - log_payloads: false - model: - provider: "gemini" - name: "gemini-1.5-flash" - options: - gemini: - api_endpoint: "us-central1-aiplatform.googleapis.com" - project_id: "sample-project-123456" - location_id: "us-central1" -targets: - - route - - consumer_group - - global -formats: - - konnect - - curl - - yaml - - kubernetes - - terraform -{% endplugin_example %} - - -#### Environment variables - -You can also specify your own GCP Service Account JSON; simply set this environment variable in the Kong workload or deployment configuration: - -Environment variable: -```sh -GCP_SERVICE_ACCOUNT='{ "type": "service_account", "project_id": "sample-project-123456", "private_key_id": "...", "private_key": "..."...}' -``` - -or set it directly in the plugin configuration: - -```yaml -config: - auth: - use_gcp_service_account: true - gcp_service_account_json: '{ "type": "service_account", "project_id": "sample-project-123456", "private_key_id": "...", "private_key": "..."...}' -``` - -or, more securely, use a vault reference to e.g. GCP Secrets Manager: - -```yaml -config: - auth: - use_gcp_service_account: true - gcp_service_account_json: '{vault://gcp/VERTEX_SERVICE_ACCOUNT_JSON}' -``` - -## AWS Bedrock - -When hosting your LLMs with [AWS Bedrock Converse API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html) in a business or enterprise plan, -and running them through AI Proxy, it is possible to use an [IAM Identity](https://docs.aws.amazon.com/IAM/latest/UserGuide/id.html) that can be assigned to a currently running EC2 instance, -an EKS deployment, ECS deployment, or just used via the [AWS CLI](https://aws.amazon.com/cli/) credential context on the local machine. - -How you do this depends on where and how you are running {{site.base_gateway}}. - -### Prerequisites - -You must be running a {{site.ee_product_name}} instance. - -Ensure that the EC2 instance, EKS deployment, ECS deployment, etcetera, has been assigned the IAM principal, -configurable from the AWS IAM portal. - -If the role requires crossing permission boundaries, ensure that the correct Assume-Role Policy is applied. - -Assign the correct permissions to the identity's IAM Policy: - -* `bedrock:InvokeModel` -* `bedrock:InvokeModelWithResponseStream` - -respective to the `Resource ARNs` that corresponds to the models that Kong is allowed to call on the user's behalf. - -### Configuring the AI Proxy Plugin to use AWS IAM - -When running Kong inside of your AWS subscription, AI Proxy is usually able to detect the designated IAM Principal automatically, based on the -assigned identity. - -Kong will use the same **authentication credentials chain** as with most AWS SDKs (and the AWS CLI). See the [Java credentials chain](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/credentials-chain.html) -precedence order, for an example. - -#### AWS IAM Identity - -To use an AWS-assigned IAM Identity, set up your plugin config like this example: - - -{% plugin_example %} -plugin: kong-inc/ai-proxy -name: ai-proxy -config: - route_type: "llm/v1/chat" - logging: - log_statistics: true - log_payloads: false - model: - provider: "bedrock" - name: "amazon.titan-text-express-v1" -targets: - - route - - consumer_group - - global -formats: - - konnect - - curl - - yaml - - kubernetes - - terraform -{% endplugin_example %} - - -In most workloads, this is **zero-configuration** and you should not need to instruct Kong AI Proxy plugin with any credentials of -Bedrock-specific configuration - Kong will find the correct IAM credentials automatically, upon **first invocation of the model**. - -#### Environment variables - -You can also specify your own AWS IAM credentials; simply set this environment variables in the Kong workload or deployment configuration: - -Environment variable: -```sh -AWS_ACCESS_KEY_ID=AKAI... -AWS_SECRET_ACCESS_KEY=... -AWS_REGION=eu-west-1 -``` - -or set it directly in the plugin configuration: - -```yaml -config: - auth: - aws_access_key_id: 'AKAI...' - aws_secret_access_key: '...' - options: - bedrock: - aws_region: 'eu-west-1' -``` - -or, more securely, use a vault reference to e.g. AWS Secrets Manager: - -```yaml -config: - auth: - aws_access_key_id: 'AKAI...' - aws_secret_access_key: '{vault://aws/BEDROCK_SECRET_ACCESS_KEY}' - options: - bedrock: - aws_region: 'eu-west-1' -``` diff --git a/app/_hub/kong-inc/ai-proxy/how-to/cloud-provider-authentication/_azure.md b/app/_hub/kong-inc/ai-proxy/how-to/cloud-provider-authentication/_azure.md new file mode 100644 index 000000000000..ff86f8765851 --- /dev/null +++ b/app/_hub/kong-inc/ai-proxy/how-to/cloud-provider-authentication/_azure.md @@ -0,0 +1,175 @@ +--- +nav_title: Azure +title: Azure +minimum_version: 3.7.x +--- + +When hosting your LLMs with [Azure OpenAI Service](https://azure.microsoft.com/en-gb/products/ai-services/openai-service) +and running them through AI Proxy, you can use the assigned +[Azure Managed Identity or User-Assigned Identity](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) +of the VM, Kubernetes service-account, or ACS container, to call the Azure OpenAI models. + +You can also use an Entra principal or App Registration (`client_id`, `client_secret`, and `tenant_id` triplet) when +Kong is hosted outside of Azure. + +How you do this depends on where and how you are running {{site.base_gateway}}. + +You can use the following table to help you determine which method to use: + +| Where is {{site.base_gateway}} hosted? | Then use... | +| -------------------------------------- | ----------- | +| Inside Azure | [Azure-managed identity](#azure-assigned-managed-identity) | +| Inside Azure | [User-assigned identity](#user-assigned-identity) | +| Outside Azure | [Azure-managed identity with Entra](#using-entra-or-app-registration) | + +## Prerequisites + +* You must be running a {{site.ee_product_name}} instance. +* Ensure that the Azure principal that you have assigned to the Compute resource (that is running your {{site.base_gateway}}) has one of the following Entra or IAM permissions to execute commands on the desired OpenAI instances: + * [Cognitive Services OpenAI User](https://learn.microsoft.com/azure/ai-services/openai/how-to/role-based-access-control#cognitive-services-openai-user) + * [Cognitive Services OpenAI Contributor](https://learn.microsoft.com/azure/ai-services/openai/how-to/role-based-access-control#cognitive-services-openai-contributor) + + See [Azure's documentation on managed identity](https://learn.microsoft.com/azure/ai-services/openai/how-to/managed-identity) to set this up. + +## Configure the AI Proxy plugin to use Azure Identity + +When running {{site.base_gateway}} inside of your Azure subscription, AI Proxy is usually able to detect the designated Managed Identity or User-Assigned Identity +of that Azure Compute resource and use it accordingly. + +### Azure-assigned managed identity + +To use an Azure-assigned managed identity, set up your plugin config like the following: + + +{% plugin_example %} +plugin: kong-inc/ai-proxy +name: ai-proxy +config: + route_type: "llm/v1/chat" + auth: + azure_use_managed_identity: true + model: + provider: "azure" + name: "gpt-35-turbo" + options: + azure_instance: "my-openai-instance" + azure_deployment_id: "kong-gpt-3-5" +targets: + - route + - consumer_group + - global +formats: + - konnect + - curl + - yaml + - kubernetes + - terraform +{% endplugin_example %} + + +### User-assigned identity + +To use a User-assigned identity, specify its client ID like the following: + + +{% plugin_example %} +plugin: kong-inc/ai-proxy +name: ai-proxy +config: + route_type: "llm/v1/chat" + auth: + azure_use_managed_identity: true + azure_client_id: "aabdecea-fc38-40ca-9edd-263878b290fe" + model: + provider: "azure" + name: "gpt-35-turbo" + options: + azure_instance: "my-openai-instance" + azure_deployment_id: "kong-gpt-3-5" +targets: + - route + - consumer_group + - global +formats: + - konnect + - curl + - yaml + - kubernetes + - terraform +{% endplugin_example %} + + +### Using Entra or app registration + +If you're running {{site.base_gateway}} outside of Azure, use an Entra principal or app registration by specifing all properties like the following: + + +{% plugin_example %} +plugin: kong-inc/ai-proxy +name: ai-proxy +config: + route_type: "llm/v1/chat" + auth: + azure_use_managed_identity: true + azure_client_id: "aabdecea-fc38-40ca-9edd-263878b290fe" + azure_client_secret: "be0c34b6-b5f1-4343-99a3-140df73e0c1c" + azure_tenant_id: "1e583ecd-9293-4db1-b1c0-2b6126cb5fdd" + model: + provider: "azure" + name: "gpt-35-turbo" + options: + azure_instance: "my-openai-instance" + azure_deployment_id: "kong-gpt-3-5" +targets: + - route + - consumer_group + - global +formats: + - konnect + - curl + - yaml + - kubernetes + - terraform +{% endplugin_example %} + + +#### Environment variables + +You can also specify some, or all, of these properties as environment variables. For example: + + +Environment variable: +```sh +AZURE_CLIENT_SECRET="be0c34b6-b5f1-4343-99a3-140df73e0c1c" +``` + +You can then omit that value from the plugin configuration like the following: + + +{% plugin_example %} +plugin: kong-inc/ai-proxy +name: ai-proxy +config: + route_type: "llm/v1/chat" + auth: + azure_use_managed_identity: true + azure_client_id: "aabdecea-fc38-40ca-9edd-263878b290fe" + azure_tenant_id: "1e583ecd-9293-4db1-b1c0-2b6126cb5fdd" + model: + provider: "azure" + name: "gpt-35-turbo" + options: + azure_instance: "my-openai-instance" + azure_deployment_id: "kong-gpt-3-5" +targets: + - route + - consumer_group + - global +formats: + - konnect + - curl + - yaml + - kubernetes + - terraform +{% endplugin_example %} + \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy/how-to/cloud-provider-authentication/_bedrock.md b/app/_hub/kong-inc/ai-proxy/how-to/cloud-provider-authentication/_bedrock.md new file mode 100644 index 000000000000..0ebfcab823f5 --- /dev/null +++ b/app/_hub/kong-inc/ai-proxy/how-to/cloud-provider-authentication/_bedrock.md @@ -0,0 +1,104 @@ +--- +nav_title: Bedrock +title: +minimum_version: 3.8.x +--- + +## AWS Bedrock + +When hosting your LLMs with [AWS Bedrock Converse API](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html) in a business or enterprise plan, +and running them through AI Proxy, it is possible to use an [IAM Identity](https://docs.aws.amazon.com/IAM/latest/UserGuide/id.html) that can be assigned to a currently running EC2 instance, +an EKS deployment, ECS deployment, or just used via the [AWS CLI](https://aws.amazon.com/cli/) credential context on the local machine. + +How you do this depends on where and how you are running {{site.base_gateway}}. + +### Prerequisites + +You must be running a {{site.ee_product_name}} instance. + +Ensure that the EC2 instance, EKS deployment, ECS deployment, etcetera, has been assigned the IAM principal, +configurable from the AWS IAM portal. + +If the role requires crossing permission boundaries, ensure that the correct Assume-Role Policy is applied. + +Assign the correct permissions to the identity's IAM Policy: + +* `bedrock:InvokeModel` +* `bedrock:InvokeModelWithResponseStream` + +respective to the `Resource ARNs` that corresponds to the models that Kong is allowed to call on the user's behalf. + +### Configuring the AI Proxy Plugin to use AWS IAM + +When running Kong inside of your AWS subscription, AI Proxy is usually able to detect the designated IAM Principal automatically, based on the +assigned identity. + +Kong will use the same **authentication credentials chain** as with most AWS SDKs (and the AWS CLI). See the [Java credentials chain](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/credentials-chain.html) +precedence order, for an example. + +#### AWS IAM Identity + +To use an AWS-assigned IAM Identity, set up your plugin config like this example: + + +{% plugin_example %} +plugin: kong-inc/ai-proxy +name: ai-proxy +config: + route_type: "llm/v1/chat" + logging: + log_statistics: true + log_payloads: false + model: + provider: "bedrock" + name: "amazon.titan-text-express-v1" +targets: + - route + - consumer_group + - global +formats: + - konnect + - curl + - yaml + - kubernetes + - terraform +{% endplugin_example %} + + +In most workloads, this is **zero-configuration** and you should not need to instruct Kong AI Proxy plugin with any credentials of +Bedrock-specific configuration - Kong will find the correct IAM credentials automatically, upon **first invocation of the model**. + +#### Environment variables + +You can also specify your own AWS IAM credentials; simply set this environment variables in the Kong workload or deployment configuration: + +Environment variable: +```sh +AWS_ACCESS_KEY_ID=AKAI... +AWS_SECRET_ACCESS_KEY=... +AWS_REGION=eu-west-1 +``` + +or set it directly in the plugin configuration: + +```yaml +config: + auth: + aws_access_key_id: 'AKAI...' + aws_secret_access_key: '...' + options: + bedrock: + aws_region: 'eu-west-1' +``` + +or, more securely, use a vault reference to e.g. AWS Secrets Manager: + +```yaml +config: + auth: + aws_access_key_id: 'AKAI...' + aws_secret_access_key: '{vault://aws/BEDROCK_SECRET_ACCESS_KEY}' + options: + bedrock: + aws_region: 'eu-west-1' +``` \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy/how-to/cloud-provider-authentication/_gemini.md b/app/_hub/kong-inc/ai-proxy/how-to/cloud-provider-authentication/_gemini.md new file mode 100644 index 000000000000..b6b7cc413746 --- /dev/null +++ b/app/_hub/kong-inc/ai-proxy/how-to/cloud-provider-authentication/_gemini.md @@ -0,0 +1,90 @@ +--- +nav_title: Gemini +title: +minimum_version: 3.8.x +--- + +## Google Gemini + +When hosting your LLMs with [Google Gemini Vertex](https://cloud.google.com/vertex-ai?hl=en) in a business or enterprise plan, and running them through AI Proxy, +it is possible to use a [GCP Workload Identity](https://cloud.google.com/iam/docs/workload-identity-federation) that can be assigned to a currently running instance, +a Container Platform deployment, or just used via the [gcloud CLI](https://cloud.google.com/sdk/gcloud) sign-in on the local machine. + +How you do this depends on where and how you are running {{site.base_gateway}}. + +### Prerequisites + +You must be running a {{site.ee_product_name}} instance. + +Ensure that the GCP Virtual Machine, Container Deployment, Container Application (or a combination of these) has been assigned the Service Account principal, +configurable from the Google Cloud IAM portal. + +Assign the `'Vertex AI User'` role to the Service Account. + +### Configuring the AI Proxy Plugin to use GCP Workload Identity + +When running Kong inside of your GCP subscription, AI Proxy is usually able to detect the designated Service Account automatically, based on the +`GCP_SERVICE_ACCOUNT` JSON that is automatically injected into an environment variable in your Kong deployment (or the Kong Virtual Machine(s)). + +#### GCP-Assigned Workload Identity + +To use a GCP-Assigned Workload Identity, set up your plugin config like this example: + + +{% plugin_example %} +plugin: kong-inc/ai-proxy +name: ai-proxy +config: + route_type: "llm/v1/chat" + auth: + use_gcp_service_account: true + logging: + log_statistics: true + log_payloads: false + model: + provider: "gemini" + name: "gemini-1.5-flash" + options: + gemini: + api_endpoint: "us-central1-aiplatform.googleapis.com" + project_id: "sample-project-123456" + location_id: "us-central1" +targets: + - route + - consumer_group + - global +formats: + - konnect + - curl + - yaml + - kubernetes + - terraform +{% endplugin_example %} + + +#### Environment variables + +You can also specify your own GCP Service Account JSON; simply set this environment variable in the Kong workload or deployment configuration: + +Environment variable: +```sh +GCP_SERVICE_ACCOUNT='{ "type": "service_account", "project_id": "sample-project-123456", "private_key_id": "...", "private_key": "..."...}' +``` + +or set it directly in the plugin configuration: + +```yaml +config: + auth: + use_gcp_service_account: true + gcp_service_account_json: '{ "type": "service_account", "project_id": "sample-project-123456", "private_key_id": "...", "private_key": "..."...}' +``` + +or, more securely, use a vault reference to e.g. GCP Secrets Manager: + +```yaml +config: + auth: + use_gcp_service_account: true + gcp_service_account_json: '{vault://gcp/VERTEX_SERVICE_ACCOUNT_JSON}' +``` \ No newline at end of file diff --git a/app/_hub/kong-inc/ai-proxy/how-to/cloud-provider-authentication/_index.md b/app/_hub/kong-inc/ai-proxy/how-to/cloud-provider-authentication/_index.md new file mode 100644 index 000000000000..4b08987d03a0 --- /dev/null +++ b/app/_hub/kong-inc/ai-proxy/how-to/cloud-provider-authentication/_index.md @@ -0,0 +1,33 @@ +--- +nav_title: Overview +title: Authenticate to Cloud-Hosted Models Using their Native Authentication +minimum_version: 3.7.x +--- + +{:.note} +> This feature requires {{site.ee_product_name}}. + +This guide walks you through setting up the AI Proxy plugin with a cloud-hosted model, +using the cloud's native authentication mechanism. + +## Overview + +When running software on a cloud-hosted virtual machine or container instance, the provider +offers a keyless role-based access mechanism, allowing you to call services native to that cloud +provider without having to store any keys inside the running instance (or in the Kong configuration). + +This operates like a single-sign-on (SSO) mechanism for your cloud applications. + +Kong's AI Gateway (AI Proxy) can take advantage of the authentication mechanisms for +many different cloud providers and, where available, can also use this authentication to call +LLM-based services using those same methods. + +## Supported providers + +Kong's AI Gateway currently supports the following cloud authentication: + +| AI-Proxy LLM Provider | Cloud Provider | Type | +|--------------------------------------------|-------------------------------------------------|-----------------------------------------| +| `azure` ('{{site.ee_product_name}}' Only) | Azure OpenAI | [Entra / Managed Identity Authentication](https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/overview) | +| `gemini` | Gemini Enterprise (on Vertex or Workspace) | [GCP Service Account](https://cloud.google.com/iam/docs/service-account-overview) | +| `bedrock` | AWS Bedrock Converse-API | [AWS IAM Identity](https://docs.aws.amazon.com/IAM/latest/UserGuide/id.html) | \ No newline at end of file