Add Incoming Request Ingest service with configuration, Docker suppor…

…t, and tests
OneUptime · Nov 21, 2024 · 945cef6 · 945cef6
1 parent 93154aa
commit 945cef6
Show file tree

Hide file tree

Showing 33 changed files with 5,689 additions and 7 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -255,6 +255,21 @@ jobs:
       - name: build docker image
         run: sudo docker build -f ./Ingestor/Dockerfile .
 
+  docker-build-incoming-request-ingest:
+    runs-on: ubuntu-latest
+    env:
+      CI_PIPELINE_ID: ${{github.run_number}}
+    steps:
+      - name: Checkout 
+        uses: actions/checkout@v2
+
+      - name: Preinstall 
+        run: npm run prerun
+
+      # build image probe api
+      - name: build docker image
+        run: sudo docker build -f ./IncomingRequestIngest/Dockerfile .
+
   docker-build-fluent-ingest:
     runs-on: ubuntu-latest
     env:

diff --git a/.github/workflows/compile.yml b/.github/workflows/compile.yml
@@ -216,6 +216,18 @@ jobs:
       - run: cd Common && npm install
       - run: cd Ingestor && npm install && npm run compile && npm run dep-check
 
+  compile-incoming-request-ingest: 
+    runs-on: ubuntu-latest
+    env:
+      CI_PIPELINE_ID: ${{github.run_number}}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 18.3.0
+      - run: cd Common && npm install
+      - run: cd IncomingRequestIngest && npm install && npm run compile && npm run dep-check
+
   compile-fluent-ingest:
     runs-on: ubuntu-latest
     env:

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -613,6 +613,67 @@ jobs:
             GIT_SHA=${{ github.sha }}
             APP_VERSION=7.0.${{needs.generate-build-number.outputs.build_number}}
 
+
+  incoming-request-ingest-docker-image-deploy:
+    needs: [generate-build-number]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Docker Meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            oneuptime/incoming-request-ingest
+            ghcr.io/oneuptime/incoming-request-ingest
+          tags: |
+            type=raw,value=release,enable=true
+            type=semver,value=7.0.${{needs.generate-build-number.outputs.build_number}},pattern={{version}},enable=true
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.ref }}
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 18.3.0
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Generate Dockerfile from Dockerfile.tpl
+        run: npm run prerun
+
+      # Build and deploy incoming-request-ingest.
+
+      - name: Login to Docker Hub 
+        uses: docker/[email protected]
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/[email protected]
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v4
+        with:
+          file: ./IncomingRequestIngest/Dockerfile
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            GIT_SHA=${{ github.sha }}
+            APP_VERSION=7.0.${{needs.generate-build-number.outputs.build_number}}
+
   fluent-ingest-docker-image-deploy:
     needs: [generate-build-number]
     runs-on: ubuntu-latest
@@ -1469,7 +1530,7 @@ jobs:
 
   test-e2e-release-saas: 
     runs-on: ubuntu-latest
-    needs: [copilot-docker-image-deploy, fluent-ingest-docker-image-deploy, docs-docker-image-deploy, api-reference-docker-image-deploy, workflow-docker-image-deploy,  llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, haraka-docker-image-deploy, ingestor-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy,  otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, nginx-docker-image-deploy]
+    needs: [copilot-docker-image-deploy, fluent-ingest-docker-image-deploy, docs-docker-image-deploy, api-reference-docker-image-deploy, workflow-docker-image-deploy,  llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, haraka-docker-image-deploy, ingestor-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy,  otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, nginx-docker-image-deploy, incoming-request-ingest-docker-image-deploy]
     env:
       CI_PIPELINE_ID: ${{github.run_number}}
     steps:
@@ -1522,7 +1583,7 @@ jobs:
   test-e2e-release-self-hosted: 
     runs-on: ubuntu-latest
     # After all the jobs runs
-    needs: [copilot-docker-image-deploy, fluent-ingest-docker-image-deploy, docs-docker-image-deploy,  api-reference-docker-image-deploy, workflow-docker-image-deploy, llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, haraka-docker-image-deploy, ingestor-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, nginx-docker-image-deploy]
+    needs: [copilot-docker-image-deploy, incoming-request-ingest-docker-image-deploy, fluent-ingest-docker-image-deploy, docs-docker-image-deploy,  api-reference-docker-image-deploy, workflow-docker-image-deploy, llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, haraka-docker-image-deploy, ingestor-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, nginx-docker-image-deploy]
     env:
       CI_PIPELINE_ID: ${{github.run_number}}
     steps:

diff --git a/.github/workflows/test-release.yaml b/.github/workflows/test-release.yaml
@@ -660,6 +660,67 @@ jobs:
             GIT_SHA=${{ github.sha }}
             APP_VERSION=7.0.${{needs.generate-build-number.outputs.build_number}}
 
+  incoming-request-ingest-docker-image-deploy:
+    needs: generate-build-number
+    runs-on: ubuntu-latest
+    steps:
+      - name: Docker Meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            oneuptime/incoming-request-ingest
+            ghcr.io/oneuptime/incoming-request-ingest
+          tags: |
+            type=raw,value=test,enable=true
+            type=semver,value=7.0.${{needs.generate-build-number.outputs.build_number}}-test,pattern={{version}},enable=true
+            
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.ref }}
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 18.3.0
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Generate Dockerfile from Dockerfile.tpl
+        run: npm run prerun
+
+      # Build and deploy incoming-request-ingest.
+
+      - name: Login to Docker Hub 
+        uses: docker/[email protected]
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/[email protected]
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v4
+        with:
+          file: ./IncomingRequestIngest/Dockerfile
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            GIT_SHA=${{ github.sha }}
+            APP_VERSION=7.0.${{needs.generate-build-number.outputs.build_number}}
+
   fluent-ingest-docker-image-deploy:
     needs: generate-build-number
     runs-on: ubuntu-latest
@@ -1404,7 +1465,7 @@ jobs:
 
   test-helm-chart:
     runs-on: ubuntu-latest
-    needs: [llm-docker-image-deploy, copilot-docker-image-deploy, docs-docker-image-deploy, worker-docker-image-deploy,  workflow-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, api-reference-docker-image-deploy,  test-server-docker-image-deploy, test-docker-image-deploy, ingestor-docker-image-deploy, probe-docker-image-deploy, haraka-docker-image-deploy, dashboard-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, accounts-docker-image-deploy, otel-collector-docker-image-deploy, status-page-docker-image-deploy, nginx-docker-image-deploy, e2e-docker-image-deploy, fluent-ingest-docker-image-deploy]
+    needs: [llm-docker-image-deploy, copilot-docker-image-deploy, docs-docker-image-deploy, worker-docker-image-deploy,  workflow-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, api-reference-docker-image-deploy,  test-server-docker-image-deploy, test-docker-image-deploy, ingestor-docker-image-deploy, probe-docker-image-deploy, haraka-docker-image-deploy, dashboard-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, accounts-docker-image-deploy, otel-collector-docker-image-deploy, status-page-docker-image-deploy, nginx-docker-image-deploy, e2e-docker-image-deploy, fluent-ingest-docker-image-deploy, incoming-request-ingest-docker-image-deploy]
     env:
       CI_PIPELINE_ID: ${{github.run_number}}
     steps:

diff --git a/.github/workflows/test.incoming-request-ingest.yml b/.github/workflows/test.incoming-request-ingest.yml
@@ -0,0 +1,21 @@
+name: Incoming Request Ingest Test
+
+on:
+  pull_request: 
+  push:
+    branches-ignore:
+      - 'hotfix-*'   # excludes hotfix branches
+      - 'release'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    env:
+      CI_PIPELINE_ID: ${{github.run_number}}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 18.3.0
+      - run: cd IncomingRequestIngest && npm install && npm run test
+
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -175,6 +175,20 @@
             "restart": true,
             "autoAttachChildProcesses": true
         },
+        {
+            "address": "127.0.0.1",
+            "localRoot": "${workspaceFolder}/IncomingRequestIngest",
+            "name": "IncomingRequestIngest: Debug with Docker",
+            "port": 9933,
+            "remoteRoot": "/usr/src/app",
+            "request": "attach",
+            "skipFiles": [
+                "<node_internals>/**"
+            ],
+            "type": "node",
+            "restart": true,
+            "autoAttachChildProcesses": true
+        },
         {
             "address": "127.0.0.1",
             "localRoot": "${workspaceFolder}/FluentIngest",

diff --git a/Common/Server/EnvironmentConfig.ts b/Common/Server/EnvironmentConfig.ts
@@ -94,6 +94,12 @@ export const IngestorHostname: Hostname = Hostname.fromString(
   }`,
 );
 
+export const IncomingRequestIngestHostname: Hostname = Hostname.fromString(
+  `${process.env["SERVER_INCOMING_REQUEST_INGEST_HOSTNAME"] || "localhost"}:${
+    process.env["INCOMING_REQUEST_INGEST_PORT"] || 80
+  }`,
+);
+
 
 export const FluentIngestHostname: Hostname = Hostname.fromString(
   `${process.env["SERVER_FLUENT_INGEST_HOSTNAME"] || "localhost"}:${

diff --git a/Common/ServiceRoute.ts b/Common/ServiceRoute.ts
@@ -28,6 +28,8 @@ export const AdminDashboardRoute: Route = new Route("/admin");
 
 export const IngestorRoute: Route = new Route("/ingestor");
 
+export const IncomingRequestIngestRoute: Route = new Route("/incoming-request-ingest");
+
 export const FluentIngestRoute:  Route = new Route("/fluent-ingest");
 
 export const RealtimeRoute: Route = new Route("/realtime/socket");

diff --git a/Common/UI/Config.ts b/Common/UI/Config.ts
@@ -15,7 +15,8 @@ import {
   StatusPageApiRoute,
   StatusPageRoute,
   WorkflowRoute,
-  FluentIngestRoute
+  FluentIngestRoute,
+  IncomingRequestIngestRoute
 } from "Common/ServiceRoute";
 import Hostname from "Common/Types/API/Hostname";
 import Protocol from "Common/Types/API/Protocol";
@@ -68,6 +69,8 @@ export const WORKFLOW_HOSTNAME: Hostname = Hostname.fromString(HOST);
 
 export const INGESTOR_HOSTNAME: Hostname = Hostname.fromString(HOST);
 
+export const INCOMING_REQUEST_INGEST_HOSTNAME: Hostname = Hostname.fromString(HOST);
+
 export const FLUENT_INGEST_HOSTNAME: Hostname = Hostname.fromString(HOST);
 
 export const HELM_HOSTNAME: Hostname = Hostname.fromString(HOST);
@@ -134,6 +137,12 @@ export const INGESTOR_URL: URL = new URL(
   IngestorRoute,
 );
 
+export const INCOMING_REQUEST_INGEST_URL: URL = new URL(
+  HTTP_PROTOCOL,
+  INCOMING_REQUEST_INGEST_HOSTNAME,
+  IncomingRequestIngestRoute,
+);
+
 export const STATUS_PAGE_URL: URL = new URL(
   HTTP_PROTOCOL,
   STATUS_PAGE_HOSTNAME,

diff --git a/E2E/Tests/FluentIngest/StatusCheck.spec.ts b/E2E/Tests/FluentIngest/StatusCheck.spec.ts
@@ -0,0 +1,35 @@
+import { BASE_URL } from "../../Config";
+import { Page, expect, test } from "@playwright/test";
+import URL from "Common/Types/API/URL";
+
+test.describe("check live and health check of the fluent ingest", () => {
+  test("check if fluent ingest status is ok", async ({ page }: { page: Page }) => {
+    await page.goto(
+      `${URL.fromString(BASE_URL.toString())
+        .addRoute("/fluent-ingest/status")
+        .toString()}`,
+    );
+    const content: string = await page.content();
+    expect(content).toContain('{"status":"ok"}');
+  });
+
+  test("check if fluent ingest is ready", async ({ page }: { page: Page }) => {
+    await page.goto(
+      `${URL.fromString(BASE_URL.toString())
+        .addRoute("/fluent-ingest/status/ready")
+        .toString()}`,
+    );
+    const content: string = await page.content();
+    expect(content).toContain('{"status":"ok"}');
+  });
+
+  test("check if fluent ingest is live", async ({ page }: { page: Page }) => {
+    await page.goto(
+      `${URL.fromString(BASE_URL.toString())
+        .addRoute("/fluent-ingest/status/live")
+        .toString()}`,
+    );
+    const content: string = await page.content();
+    expect(content).toContain('{"status":"ok"}');
+  });
+});
diff --git a/E2E/Tests/IncomingRequestIngest/StatusCheck.spec.ts b/E2E/Tests/IncomingRequestIngest/StatusCheck.spec.ts
@@ -0,0 +1,35 @@
+import { BASE_URL } from "../../Config";
+import { Page, expect, test } from "@playwright/test";
+import URL from "Common/Types/API/URL";
+
+test.describe("check live and health check of the incoming request ingest", () => {
+  test("check if incoming request ingest status is ok", async ({ page }: { page: Page }) => {
+    await page.goto(
+      `${URL.fromString(BASE_URL.toString())
+        .addRoute("/incoming-request-ingest/status")
+        .toString()}`,
+    );
+    const content: string = await page.content();
+    expect(content).toContain('{"status":"ok"}');
+  });
+
+  test("check if incoming request ingest is ready", async ({ page }: { page: Page }) => {
+    await page.goto(
+      `${URL.fromString(BASE_URL.toString())
+        .addRoute("/incoming-request-ingest/status/ready")
+        .toString()}`,
+    );
+    const content: string = await page.content();
+    expect(content).toContain('{"status":"ok"}');
+  });
+
+  test("check if incoming request ingest is live", async ({ page }: { page: Page }) => {
+    await page.goto(
+      `${URL.fromString(BASE_URL.toString())
+        .addRoute("/incoming-request-ingest/status/live")
+        .toString()}`,
+    );
+    const content: string = await page.content();
+    expect(content).toContain('{"status":"ok"}');
+  });
+});
diff --git a/Fluentd/fluent.conf b/Fluentd/fluent.conf
@@ -22,7 +22,7 @@
 <match **>
   @type http
 
-  # endpoint http://fluent-ingest:3401/ingestor/fluentd/v1/logs # This is if you're testing in local development
+  # endpoint http://fluent-ingest:3401/fluent-ingest/fluentd/v1/logs # This is if you're testing in local development
   endpoint https://oneuptime.com/fluentd/logs # This is for prod environment
   open_timeout 2
   # Please make sure you change the token and service name to your own

diff --git a/HelmChart/Public/oneuptime/templates/_helpers.tpl b/HelmChart/Public/oneuptime/templates/_helpers.tpl
@@ -60,6 +60,8 @@ Usage:
   value: {{ $.Release.Name }}-app.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }}
 - name: SERVER_INGESTOR_HOSTNAME
   value: {{ $.Release.Name }}-ingestor.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }}
+- name: SERVER_INCOMING_REQUEST_INGEST_HOSTNAME
+  value: {{ $.Release.Name }}-incoming-request-ingest.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }}
 - name: SERVER_FLUENT_INGEST_HOSTNAME
   value: {{ $.Release.Name }}-fluent-ingest.{{ $.Release.Namespace }}.svc.{{ $.Values.global.clusterDomain }}
 - name: SERVER_TEST_SERVER_HOSTNAME
@@ -79,6 +81,8 @@ Usage:
   value: {{ $.Values.port.app | squote }}
 - name: INGESTOR_PORT
   value: {{ $.Values.port.ingestor | squote }}
+- name: INCOMING_REQUEST_INGEST_PORT
+  value: {{ $.Values.port.incomingRequestIngest | squote }}
 - name: FLUENT_INGEST_PORT
   value: {{ $.Values.port.fluentIngest | squote }}
 - name: PROBE_PORT