Add Fluent Ingest service with configuration, Docker support, and tests

.github/workflows/build.yml

@@ -255,6 +255,21 @@ jobs:
       - name: build docker image
         run: sudo docker build -f ./Ingestor/Dockerfile .
 
+  docker-build-fluent-ingest:
+    runs-on: ubuntu-latest
+    env:
+      CI_PIPELINE_ID: ${{github.run_number}}
+    steps:
+      - name: Checkout 
+        uses: actions/checkout@v2
+
+      - name: Preinstall 
+        run: npm run prerun
+
+      # build image probe api
+      - name: build docker image
+        run: sudo docker build -f ./FluentIngest/Dockerfile .
+
   docker-build-status-page:
     runs-on: ubuntu-latest
     env:

.github/workflows/compile.yml

@@ -215,6 +215,18 @@ jobs:
           node-version: 18.3.0
       - run: cd Common && npm install
       - run: cd Ingestor && npm install && npm run compile && npm run dep-check
+
+  compile-fluent-ingest:
+    runs-on: ubuntu-latest
+    env:
+      CI_PIPELINE_ID: ${{github.run_number}}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 18.3.0
+      - run: cd Common && npm install
+      - run: cd FluentIngest && npm install && npm run compile && npm run dep-check
 
 
   compile-status-page:

.github/workflows/release.yml

@@ -613,6 +613,66 @@ jobs:
             GIT_SHA=${{ github.sha }}
             APP_VERSION=7.0.${{needs.generate-build-number.outputs.build_number}}
 
+  fluent-ingest-docker-image-deploy:
+    needs: [generate-build-number]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Docker Meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            oneuptime/fluent-ingest
+            ghcr.io/oneuptime/fluent-ingest
+          tags: |
+            type=raw,value=release,enable=true
+            type=semver,value=7.0.${{needs.generate-build-number.outputs.build_number}},pattern={{version}},enable=true
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.ref }}
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 18.3.0
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Generate Dockerfile from Dockerfile.tpl
+        run: npm run prerun
+
+      # Build and deploy fluent-ingest.
+
+      - name: Login to Docker Hub 
+        uses: docker/[email protected]
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/[email protected]
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v4
+        with:
+          file: ./FluentIngest/Dockerfile
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            GIT_SHA=${{ github.sha }}
+            APP_VERSION=7.0.${{needs.generate-build-number.outputs.build_number}}
+
   probe-docker-image-deploy:
     needs: [generate-build-number]
     runs-on: ubuntu-latest
@@ -1409,7 +1469,7 @@ jobs:
 
   test-e2e-release-saas: 
     runs-on: ubuntu-latest
-    needs: [copilot-docker-image-deploy, docs-docker-image-deploy, api-reference-docker-image-deploy, workflow-docker-image-deploy,  llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, haraka-docker-image-deploy, ingestor-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy,  otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, nginx-docker-image-deploy]
+    needs: [copilot-docker-image-deploy, fluent-ingest-docker-image-deploy, docs-docker-image-deploy, api-reference-docker-image-deploy, workflow-docker-image-deploy,  llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, haraka-docker-image-deploy, ingestor-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy,  otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, nginx-docker-image-deploy]
     env:
       CI_PIPELINE_ID: ${{github.run_number}}
     steps:
@@ -1462,7 +1522,7 @@ jobs:
   test-e2e-release-self-hosted: 
     runs-on: ubuntu-latest
     # After all the jobs runs
-    needs: [copilot-docker-image-deploy, docs-docker-image-deploy,  api-reference-docker-image-deploy, workflow-docker-image-deploy, llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, haraka-docker-image-deploy, ingestor-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, nginx-docker-image-deploy]
+    needs: [copilot-docker-image-deploy, fluent-ingest-docker-image-deploy, docs-docker-image-deploy,  api-reference-docker-image-deploy, workflow-docker-image-deploy, llm-docker-image-deploy, accounts-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, dashboard-docker-image-deploy, haraka-docker-image-deploy, ingestor-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, worker-docker-image-deploy, otel-collector-docker-image-deploy, probe-docker-image-deploy, status-page-docker-image-deploy, test-docker-image-deploy, test-server-docker-image-deploy, publish-npm-packages, e2e-docker-image-deploy, helm-chart-deploy, generate-build-number, nginx-docker-image-deploy]
     env:
       CI_PIPELINE_ID: ${{github.run_number}}
     steps:

.github/workflows/test-release.yaml

@@ -660,6 +660,67 @@ jobs:
             GIT_SHA=${{ github.sha }}
             APP_VERSION=7.0.${{needs.generate-build-number.outputs.build_number}}
 
+  fluent-ingest-docker-image-deploy:
+    needs: generate-build-number
+    runs-on: ubuntu-latest
+    steps:
+      - name: Docker Meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            oneuptime/fluent-ingest
+            ghcr.io/oneuptime/fluent-ingest
+          tags: |
+            type=raw,value=test,enable=true
+            type=semver,value=7.0.${{needs.generate-build-number.outputs.build_number}}-test,pattern={{version}},enable=true
+            
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.ref }}
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 18.3.0
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Generate Dockerfile from Dockerfile.tpl
+        run: npm run prerun
+
+      # Build and deploy ingestor.
+
+      - name: Login to Docker Hub 
+        uses: docker/[email protected]
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/[email protected]
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v4
+        with:
+          file: ./FluentIngest/Dockerfile
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          build-args: |
+            GIT_SHA=${{ github.sha }}
+            APP_VERSION=7.0.${{needs.generate-build-number.outputs.build_number}}
+
   probe-docker-image-deploy:
     needs: generate-build-number
     runs-on: ubuntu-latest
@@ -1343,7 +1404,7 @@ jobs:
 
   test-helm-chart:
     runs-on: ubuntu-latest
-    needs: [llm-docker-image-deploy, copilot-docker-image-deploy, docs-docker-image-deploy, worker-docker-image-deploy,  workflow-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, api-reference-docker-image-deploy,  test-server-docker-image-deploy, test-docker-image-deploy, ingestor-docker-image-deploy, probe-docker-image-deploy, haraka-docker-image-deploy, dashboard-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, accounts-docker-image-deploy, otel-collector-docker-image-deploy, status-page-docker-image-deploy, nginx-docker-image-deploy, e2e-docker-image-deploy]
+    needs: [llm-docker-image-deploy, copilot-docker-image-deploy, docs-docker-image-deploy, worker-docker-image-deploy,  workflow-docker-image-deploy, isolated-vm-docker-image-deploy, home-docker-image-deploy, api-reference-docker-image-deploy,  test-server-docker-image-deploy, test-docker-image-deploy, ingestor-docker-image-deploy, probe-docker-image-deploy, haraka-docker-image-deploy, dashboard-docker-image-deploy, admin-dashboard-docker-image-deploy, app-docker-image-deploy, accounts-docker-image-deploy, otel-collector-docker-image-deploy, status-page-docker-image-deploy, nginx-docker-image-deploy, e2e-docker-image-deploy, fluent-ingest-docker-image-deploy]
     env:
       CI_PIPELINE_ID: ${{github.run_number}}
     steps:

.github/workflows/test.fluent-ingest.yaml

@@ -0,0 +1,20 @@
+name: Fluent Ingest Test
+
+on:
+  pull_request: 
+  push:
+    branches-ignore:
+      - 'hotfix-*'   # excludes hotfix branches
+      - 'release'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    env:
+      CI_PIPELINE_ID: ${{github.run_number}}
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2
+        with:
+          node-version: 18.3.0
+      - run: cd FluentIngest && npm install && npm run test

.vscode/launch.json

@@ -175,6 +175,20 @@
             "restart": true,
             "autoAttachChildProcesses": true
         },
+        {
+            "address": "127.0.0.1",
+            "localRoot": "${workspaceFolder}/FluentIngest",
+            "name": "Fluent Ingest: Debug with Docker",
+            "port": 9937,
+            "remoteRoot": "/usr/src/app",
+            "request": "attach",
+            "skipFiles": [
+                "<node_internals>/**"
+            ],
+            "type": "node",
+            "restart": true,
+            "autoAttachChildProcesses": true
+        },
         {
             "address": "127.0.0.1",
             "localRoot": "${workspaceFolder}/IsolatedVM",

App/FeatureSet/BaseAPI/Index.ts

@@ -10,7 +10,7 @@ import GlobalConfigAPI from "Common/Server/API/GlobalConfigAPI";
 import MonitorGroupAPI from "Common/Server/API/MonitorGroupAPI";
 import NotificationAPI from "Common/Server/API/NotificationAPI";
 import TelemetryAPI from "Common/Server/API/TelemetryAPI";
-import Ingestor from "Common/Server/API/ProbeAPI";
+import ProbeAPI from "Common/Server/API/ProbeAPI";
 import ProjectAPI from "Common/Server/API/ProjectAPI";
 import ProjectSsoAPI from "Common/Server/API/ProjectSSO";
 
@@ -1359,7 +1359,7 @@ const BaseAPIFeatureSet: FeatureSet = {
     );
     app.use(`/${APP_NAME.toLocaleLowerCase()}`, new UserEmailAPI().getRouter());
     app.use(`/${APP_NAME.toLocaleLowerCase()}`, new UserSMSAPI().getRouter());
-    app.use(`/${APP_NAME.toLocaleLowerCase()}`, new Ingestor().getRouter());
+    app.use(`/${APP_NAME.toLocaleLowerCase()}`, new ProbeAPI().getRouter());
 
     app.use(
       `/${APP_NAME.toLocaleLowerCase()}`,

Common/Server/API/ProbeAPI.ts

@@ -13,7 +13,7 @@ import LIMIT_MAX from "Common/Types/Database/LimitMax";
 import PositiveNumber from "Common/Types/PositiveNumber";
 import Probe from "Common/Models/DatabaseModels/Probe";
 
-export default class Ingestor extends BaseAPI<Probe, ProbeServiceType> {
+export default class ProbeAPI extends BaseAPI<Probe, ProbeServiceType> {
   public constructor() {
     super(Probe, ProbeService);
 

Common/Server/EnvironmentConfig.ts

@@ -94,6 +94,13 @@ export const IngestorHostname: Hostname = Hostname.fromString(
   }`,
 );
 
+
+export const FluentIngestHostname: Hostname = Hostname.fromString(
+  `${process.env["SERVER_FLUENT_INGEST_HOSTNAME"] || "localhost"}:${
+    process.env["FLUENT_INGEST_PORT"] || 80
+  }`,
+);
+
 export const IsolatedVMHostname: Hostname = Hostname.fromString(
   `${process.env["SERVER_ISOLATED_VM_HOSTNAME"] || "localhost"}:${
     process.env["ISOLATED_VM_PORT"] || 80

Common/ServiceRoute.ts

@@ -28,6 +28,8 @@ export const AdminDashboardRoute: Route = new Route("/admin");
 
 export const IngestorRoute: Route = new Route("/ingestor");
 
+export const FluentIngestRoute:  Route = new Route("/fluent-ingest");
+
 export const RealtimeRoute: Route = new Route("/realtime/socket");
 
 export const DocsRoute: Route = new Route("/docs");

Common/Tests/Server/API/ProbeAPI.test.ts

@@ -1,4 +1,4 @@
-import Ingestor from "../../../Server/API/ProbeAPI";
+import ProbeAPI from "../../../Server/API/ProbeAPI";
 import ProbeService from "../../../Server/Services/ProbeService";
 import {
   ExpressRequest,
@@ -37,13 +37,13 @@ jest.mock("../../../Server/Utils/Response", () => {
 
 jest.mock("../../../Server/Services/ProbeService");
 
-describe("Ingestor", () => {
+describe("ProbeAPI", () => {
   let mockRequest: ExpressRequest;
   let mockResponse: ExpressResponse;
   let nextFunction: NextFunction;
 
   beforeEach(() => {
-    new Ingestor();
+    new ProbeAPI();
     mockRequest = {} as ExpressRequest;
     mockResponse = {
       send: jest.fn(),

Common/UI/Config.ts

@@ -15,6 +15,7 @@ import {
   StatusPageApiRoute,
   StatusPageRoute,
   WorkflowRoute,
+  FluentIngestRoute
 } from "Common/ServiceRoute";
 import Hostname from "Common/Types/API/Hostname";
 import Protocol from "Common/Types/API/Protocol";
@@ -67,6 +68,8 @@ export const WORKFLOW_HOSTNAME: Hostname = Hostname.fromString(HOST);
 
 export const INGESTOR_HOSTNAME: Hostname = Hostname.fromString(HOST);
 
+export const FLUENT_INGEST_HOSTNAME: Hostname = Hostname.fromString(HOST);
+
 export const HELM_HOSTNAME: Hostname = Hostname.fromString(HOST);
 
 export const API_DOCS_HOSTNAME: Hostname = Hostname.fromString(HOST);
@@ -101,6 +104,12 @@ export const STATUS_PAGE_API_URL: URL = new URL(
   StatusPageApiRoute,
 );
 
+export const FLUENT_INGEST_URL: URL = new URL(
+  HTTP_PROTOCOL,
+  FLUENT_INGEST_HOSTNAME,
+  FluentIngestRoute,
+);
+
 export const IDENTITY_URL: URL = new URL(
   HTTP_PROTOCOL,
   IDENTITY_HOSTNAME,

Docs/Content/installation/docker-compose.md

@@ -97,4 +97,4 @@ npm run update
 
 ### Things to consider
 
-- In our Docker setup, we employ a local logging driver. OneUptime, particularly within the probe and ingestor containers, generates a substantial amount of logs. To prevent your storage from becoming full, it's crucial to limit the logging storage in Docker. For detailed instructions on how to do this, please refer to the official Docker documentation [here](https://docs.docker.com/config/containers/logging/local/).
+- In our Docker setup, we employ a local logging driver. OneUptime, particularly within the probe and ingest containers, generates a substantial amount of logs. To prevent your storage from becoming full, it's crucial to limit the logging storage in Docker. For detailed instructions on how to do this, please refer to the official Docker documentation [here](https://docs.docker.com/config/containers/logging/local/).