diff --git a/Common/Models/DatabaseModels/Alert.ts b/Common/Models/DatabaseModels/Alert.ts index 2166d438ce..edf1008353 100644 --- a/Common/Models/DatabaseModels/Alert.ts +++ b/Common/Models/DatabaseModels/Alert.ts @@ -801,7 +801,6 @@ export default class Alert extends BaseModel { ], update: [], }) - @Index() @TableColumn({ type: TableColumnType.Markdown, required: false, diff --git a/Common/Models/DatabaseModels/Incident.ts b/Common/Models/DatabaseModels/Incident.ts index b49f5db110..68eba3c0e8 100644 --- a/Common/Models/DatabaseModels/Incident.ts +++ b/Common/Models/DatabaseModels/Incident.ts @@ -855,7 +855,6 @@ export default class Incident extends BaseModel { ], update: [], }) - @Index() @TableColumn({ type: TableColumnType.Markdown, required: false, diff --git a/Common/Models/DatabaseModels/IncidentStateTimeline.ts b/Common/Models/DatabaseModels/IncidentStateTimeline.ts index 612bf6a150..377612982c 100644 --- a/Common/Models/DatabaseModels/IncidentStateTimeline.ts +++ b/Common/Models/DatabaseModels/IncidentStateTimeline.ts @@ -507,7 +507,6 @@ export default class IncidentStateTimeline extends BaseModel { ], update: [], }) - @Index() @TableColumn({ type: TableColumnType.Markdown, required: false, diff --git a/Common/Models/DatabaseModels/MonitorStatusTimeline.ts b/Common/Models/DatabaseModels/MonitorStatusTimeline.ts index 6b48bf1954..e377818e2f 100644 --- a/Common/Models/DatabaseModels/MonitorStatusTimeline.ts +++ b/Common/Models/DatabaseModels/MonitorStatusTimeline.ts @@ -455,7 +455,6 @@ export default class MonitorStatusTimeline extends BaseModel { ], update: [], }) - @Index() @TableColumn({ type: TableColumnType.Markdown, required: false, diff --git a/Common/Server/Infrastructure/Postgres/SchemaMigrations/1731433043136-MigrationName.ts b/Common/Server/Infrastructure/Postgres/SchemaMigrations/1731433043136-MigrationName.ts new file mode 100644 index 0000000000..f8eb265ba4 --- /dev/null +++ b/Common/Server/Infrastructure/Postgres/SchemaMigrations/1731433043136-MigrationName.ts @@ -0,0 +1,17 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class MigrationName1731433043136 implements MigrationInterface { + public name = "MigrationName1731433043136"; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `DROP INDEX "public"."IDX_5218e92f700d91afe6a8db79cb"`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE INDEX "IDX_5218e92f700d91afe6a8db79cb" ON "Incident" ("rootCause") `, + ); + } +} diff --git a/Common/Server/Infrastructure/Postgres/SchemaMigrations/1731433309124-MigrationName.ts b/Common/Server/Infrastructure/Postgres/SchemaMigrations/1731433309124-MigrationName.ts new file mode 100644 index 0000000000..55906f3924 --- /dev/null +++ b/Common/Server/Infrastructure/Postgres/SchemaMigrations/1731433309124-MigrationName.ts @@ -0,0 +1,17 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class MigrationName1731433309124 implements MigrationInterface { + public name = "MigrationName1731433309124"; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `DROP INDEX "public"."IDX_fc40ea6a9ad55f29bca4f4a15d"`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE INDEX "IDX_fc40ea6a9ad55f29bca4f4a15d" ON "Alert" ("rootCause") `, + ); + } +} diff --git a/Common/Server/Infrastructure/Postgres/SchemaMigrations/1731435267537-MigrationName.ts b/Common/Server/Infrastructure/Postgres/SchemaMigrations/1731435267537-MigrationName.ts new file mode 100644 index 0000000000..8b3e6b5cb2 --- /dev/null +++ b/Common/Server/Infrastructure/Postgres/SchemaMigrations/1731435267537-MigrationName.ts @@ -0,0 +1,17 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class MigrationName1731435267537 implements MigrationInterface { + public name = "MigrationName1731435267537"; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `DROP INDEX "public"."IDX_01ac1d1ef9e72aeb6dac6575dd"`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE INDEX "IDX_01ac1d1ef9e72aeb6dac6575dd" ON "MonitorStatusTimeline" ("rootCause") `, + ); + } +} diff --git a/Common/Server/Infrastructure/Postgres/SchemaMigrations/1731435514287-MigrationName.ts b/Common/Server/Infrastructure/Postgres/SchemaMigrations/1731435514287-MigrationName.ts new file mode 100644 index 0000000000..f36d19c3fe --- /dev/null +++ b/Common/Server/Infrastructure/Postgres/SchemaMigrations/1731435514287-MigrationName.ts @@ -0,0 +1,17 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class MigrationName1731435514287 implements MigrationInterface { + public name = "MigrationName1731435514287"; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `DROP INDEX "public"."IDX_7db6b1a8fbbc9eb44c2e7f5047"`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `CREATE INDEX "IDX_7db6b1a8fbbc9eb44c2e7f5047" ON "IncidentStateTimeline" ("rootCause") `, + ); + } +} diff --git a/Common/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts b/Common/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts index 419387356b..251e639e15 100644 --- a/Common/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts +++ b/Common/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts @@ -78,6 +78,10 @@ import { MigrationName1729682875503 } from "./1729682875503-MigrationName"; import { MigrationName1730117995642 } from "./1730117995642-MigrationName"; import { MigrationName1730209089495 } from "./1730209089495-MigrationName"; import { MigrationName1730223198692 } from "./1730223198692-MigrationName"; +import { MigrationName1731433043136 } from "./1731433043136-MigrationName"; +import { MigrationName1731433309124 } from "./1731433309124-MigrationName"; +import { MigrationName1731435267537 } from "./1731435267537-MigrationName"; +import { MigrationName1731435514287 } from "./1731435514287-MigrationName"; export default [ InitialMigration, @@ -160,4 +164,8 @@ export default [ MigrationName1730117995642, MigrationName1730209089495, MigrationName1730223198692, + MigrationName1731433043136, + MigrationName1731433309124, + MigrationName1731435267537, + MigrationName1731435514287, ]; diff --git a/Common/Server/Utils/Browser.ts b/Common/Server/Utils/Browser.ts new file mode 100644 index 0000000000..a4fa450007 --- /dev/null +++ b/Common/Server/Utils/Browser.ts @@ -0,0 +1,192 @@ +import { + Page as PlaywrightPage, + Browser as PlaywrightBrowser, + chromium, + firefox, +} from "playwright"; +import LocalFile from "./LocalFile"; +import BadDataException from "../../Types/Exception/BadDataException"; +import ScreenSizeType from "../../Types/ScreenSizeType"; +import BrowserType from "../../Types/BrowserType"; +import logger from "./Logger"; + +export type Page = PlaywrightPage; +export type Browser = PlaywrightBrowser; + +export default class BrowserUtil { + public static async convertHtmlToBase64Screenshot(data: { + html: string; + }): Promise { + try { + const html: string = data.html; + + const pageAndBrowser: { + page: Page; + browser: Browser; + } = await BrowserUtil.getPageByBrowserType({ + browserType: BrowserType.Chromium, + screenSizeType: ScreenSizeType.Desktop, + }); + + const page: Page = pageAndBrowser.page; + const browser: Browser = pageAndBrowser.browser; + await page.setContent(html); + const screenshot: Buffer = await page.screenshot({ type: "png" }); + + await browser.close(); + + return screenshot.toString("base64"); + } catch (e) { + logger.debug(e); + return null; + } + } + + public static async getPageByBrowserType(data: { + browserType: BrowserType; + screenSizeType: ScreenSizeType; + }): Promise<{ + page: Page; + browser: Browser; + }> { + const viewport: { + height: number; + width: number; + } = BrowserUtil.getViewportHeightAndWidth({ + screenSizeType: data.screenSizeType, + }); + + let page: Page | null = null; + let browser: Browser | null = null; + + if (data.browserType === BrowserType.Chromium) { + browser = await chromium.launch({ + executablePath: await BrowserUtil.getChromeExecutablePath(), + }); + page = await browser.newPage(); + } + + if (data.browserType === BrowserType.Firefox) { + browser = await firefox.launch({ + executablePath: await BrowserUtil.getFirefoxExecutablePath(), + }); + page = await browser.newPage(); + } + + // if (data.browserType === BrowserType.Webkit) { + // browser = await webkit.launch(); + // page = await browser.newPage(); + // } + + await page?.setViewportSize({ + width: viewport.width, + height: viewport.height, + }); + + if (!browser) { + throw new BadDataException("Invalid Browser Type."); + } + + if (!page) { + // close the browser if page is not created + await browser.close(); + throw new BadDataException("Invalid Browser Type."); + } + + return { + page: page, + browser: browser, + }; + } + + public static getViewportHeightAndWidth(options: { + screenSizeType: ScreenSizeType; + }): { + height: number; + width: number; + } { + let viewPortHeight: number = 0; + let viewPortWidth: number = 0; + + switch (options.screenSizeType) { + case ScreenSizeType.Desktop: + viewPortHeight = 1080; + viewPortWidth = 1920; + break; + case ScreenSizeType.Mobile: + viewPortHeight = 640; + viewPortWidth = 360; + break; + case ScreenSizeType.Tablet: + viewPortHeight = 768; + viewPortWidth = 1024; + break; + default: + viewPortHeight = 1080; + viewPortWidth = 1920; + break; + } + + return { height: viewPortHeight, width: viewPortWidth }; + } + + public static async getChromeExecutablePath(): Promise { + const doesDirectoryExist: boolean = await LocalFile.doesDirectoryExist( + "/root/.cache/ms-playwright", + ); + if (!doesDirectoryExist) { + throw new BadDataException("Chrome executable path not found."); + } + + // get list of files in the directory + const directories: string[] = await LocalFile.getListOfDirectories( + "/root/.cache/ms-playwright", + ); + + if (directories.length === 0) { + throw new BadDataException("Chrome executable path not found."); + } + + const chromeInstallationName: string | undefined = directories.find( + (directory: string) => { + return directory.includes("chromium"); + }, + ); + + if (!chromeInstallationName) { + throw new BadDataException("Chrome executable path not found."); + } + + return `/root/.cache/ms-playwright/${chromeInstallationName}/chrome-linux/chrome`; + } + + public static async getFirefoxExecutablePath(): Promise { + const doesDirectoryExist: boolean = await LocalFile.doesDirectoryExist( + "/root/.cache/ms-playwright", + ); + if (!doesDirectoryExist) { + throw new BadDataException("Firefox executable path not found."); + } + + // get list of files in the directory + const directories: string[] = await LocalFile.getListOfDirectories( + "/root/.cache/ms-playwright", + ); + + if (directories.length === 0) { + throw new BadDataException("Firefox executable path not found."); + } + + const firefoxInstallationName: string | undefined = directories.find( + (directory: string) => { + return directory.includes("firefox"); + }, + ); + + if (!firefoxInstallationName) { + throw new BadDataException("Firefox executable path not found."); + } + + return `/root/.cache/ms-playwright/${firefoxInstallationName}/firefox/firefox`; + } +} diff --git a/Common/Server/Utils/Monitor/Criteria/CompareCriteria.ts b/Common/Server/Utils/Monitor/Criteria/CompareCriteria.ts index 35590655b7..07abf9eedd 100644 --- a/Common/Server/Utils/Monitor/Criteria/CompareCriteria.ts +++ b/Common/Server/Utils/Monitor/Criteria/CompareCriteria.ts @@ -544,6 +544,9 @@ export default class CompareCriteria { case FilterType.NotEqualTo: message += ` not equal to ${data.threshold}. `; break; + case FilterType.EqualTo: + message += ` equal to ${data.threshold}. `; + break; case FilterType.Contains: message += ` contains ${data.threshold}. `; break; diff --git a/Common/Server/Utils/Monitor/MonitorIncident.ts b/Common/Server/Utils/Monitor/MonitorIncident.ts index a9a66c81b6..073da331f4 100644 --- a/Common/Server/Utils/Monitor/MonitorIncident.ts +++ b/Common/Server/Utils/Monitor/MonitorIncident.ts @@ -129,8 +129,6 @@ export default class MonitorIncident { continue; } - // create incident here. - logger.debug(`${input.monitor.id?.toString()} - Create incident.`); const incident: Incident = new Incident(); diff --git a/Common/Types/Html.ts b/Common/Types/Html.ts index ab41fcfb50..ed4aa7f43f 100644 --- a/Common/Types/Html.ts +++ b/Common/Types/Html.ts @@ -14,4 +14,15 @@ export default class HTML { public toString(): string { return this.html; } + + public static isHtml(text: string): boolean { + // Check if the text is HTML + + // Example usage const htmlString = '
Hello, World!
'; const notHtmlString = 'Just a regular string' + // console.log(HTML.isHtml(htmlString)); // true + // console.log(HTML.isHtml(notHtmlString)); // false + + const htmlPattern: RegExp = /<\/?[a-z][\s\S]*>/i; + return htmlPattern.test(text); + } } diff --git a/Common/UI/Components/ModelTable/TableView.tsx b/Common/UI/Components/ModelTable/TableView.tsx index 96cb5e34e3..bc5d91e249 100644 --- a/Common/UI/Components/ModelTable/TableView.tsx +++ b/Common/UI/Components/ModelTable/TableView.tsx @@ -199,6 +199,7 @@ const TableViewElement: ( elements.push( | ReactElement | undefined; className?: string | undefined; iconClassName?: string | undefined; + key: string | number; } const MoreMenuItem: FunctionComponent = ( @@ -16,6 +17,7 @@ const MoreMenuItem: FunctionComponent = ( ): ReactElement => { return ( { diff --git a/Common/package-lock.json b/Common/package-lock.json index 6718db0510..265c87ded9 100644 --- a/Common/package-lock.json +++ b/Common/package-lock.json @@ -65,6 +65,7 @@ "nodemailer": "^6.9.10", "otpauth": "^9.3.1", "pg": "^8.7.3", + "playwright": "^1.48.2", "posthog-js": "^1.139.6", "prop-types": "^15.8.1", "qrcode": "^1.5.3", @@ -11053,6 +11054,50 @@ "node": ">=8" } }, + "node_modules/playwright": { + "version": "1.48.2", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.48.2.tgz", + "integrity": "sha512-NjYvYgp4BPmiwfe31j4gHLa3J7bD2WiBz8Lk2RoSsmX38SVIARZ18VYjxLjAcDsAhA+F4iSEXTSGgjua0rrlgQ==", + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.48.2" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.48.2", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.48.2.tgz", + "integrity": "sha512-sjjw+qrLFlriJo64du+EK0kJgZzoQPsabGF4lBvsid+3CNIZIYLgnMj9V6JY5VhM2Peh20DJWIVpVljLLnlawA==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/pngjs": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-5.0.0.tgz", diff --git a/Common/package.json b/Common/package.json index 4856810ef8..9c38ec7c1b 100644 --- a/Common/package.json +++ b/Common/package.json @@ -100,6 +100,7 @@ "nodemailer": "^6.9.10", "otpauth": "^9.3.1", "pg": "^8.7.3", + "playwright": "^1.48.2", "posthog-js": "^1.139.6", "prop-types": "^15.8.1", "qrcode": "^1.5.3", diff --git a/Dashboard/src/Components/Dashboard/Toolbar/DashboardToolbar.tsx b/Dashboard/src/Components/Dashboard/Toolbar/DashboardToolbar.tsx index 61f008f67a..c7d145515a 100644 --- a/Dashboard/src/Components/Dashboard/Toolbar/DashboardToolbar.tsx +++ b/Dashboard/src/Components/Dashboard/Toolbar/DashboardToolbar.tsx @@ -36,18 +36,21 @@ const DashboardToolbar: FunctionComponent = ( { props.onAddComponentClick(DashboardComponentType.Chart); }} /> { props.onAddComponentClick(DashboardComponentType.Value); }} /> { props.onAddComponentClick(DashboardComponentType.Text); }} diff --git a/Ingestor/Dockerfile.tpl b/Ingestor/Dockerfile.tpl index 8a0382341c..8992e1e637 100644 --- a/Ingestor/Dockerfile.tpl +++ b/Ingestor/Dockerfile.tpl @@ -1,9 +1,9 @@ # -# OneUptime-ingestor Dockerfile +# OneUptime-Ingestor Dockerfile # # Pull base image nodejs image. -FROM public.ecr.aws/docker/library/node:21.7.3-alpine3.18 +FROM public.ecr.aws/docker/library/node:22.9 RUN mkdir /tmp/npm && chmod 2777 /tmp/npm && chown 1000:1000 /tmp/npm && npm config set cache /tmp/npm --global RUN npm config set fetch-retries 5 @@ -11,30 +11,40 @@ RUN npm config set fetch-retry-mintimeout 100000 RUN npm config set fetch-retry-maxtimeout 600000 - ARG GIT_SHA ARG APP_VERSION ENV GIT_SHA=${GIT_SHA} ENV APP_VERSION=${APP_VERSION} +ENV NODE_OPTIONS="--use-openssl-ca" + +## Add Intermediate Certs +COPY ./SslCertificates /usr/local/share/ca-certificates +RUN update-ca-certificates # IF APP_VERSION is not set, set it to 1.0.0 RUN if [ -z "$APP_VERSION" ]; then export APP_VERSION=1.0.0; fi -# Install bash. -RUN apk add bash && apk add curl +RUN apt-get update +# Install bash. +RUN apt-get install bash -y && apt-get install curl -y && apt-get install iputils-ping -y # Install python -RUN apk update && apk add --no-cache --virtual .gyp python3 make g++ +RUN apt-get update && apt-get install -y .gyp python3 make g++ + +# Install playwright dependencies +RUN apt-get install -y libnss3 libnspr4 libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 libxkbcommon0 libxcomposite1 libxdamage1 libxfixes3 libxrandr2 libgbm1 libgtk-3-0 libpango-1.0-0 libcairo2 libgdk-pixbuf2.0-0 libasound2 libatspi2.0-0 #Use bash shell by default SHELL ["/bin/bash", "-c"] +# Install iputils +RUN apt-get install net-tools -y -RUN mkdir /usr/src +RUN mkdir -p /usr/src WORKDIR /usr/src/Common COPY ./Common/package*.json /usr/src/Common/ @@ -45,24 +55,16 @@ COPY ./Common /usr/src/Common - - - - - - - - ENV PRODUCTION=true WORKDIR /usr/src/app +RUN npx playwright install --with-deps + # Install app dependencies COPY ./Ingestor/package*.json /usr/src/app/ RUN npm install -ENV NODE_OPTIONS="--max-old-space-size=8192" - # Expose ports. # - 3400: OneUptime-ingestor EXPOSE 3400 diff --git a/Ingestor/package-lock.json b/Ingestor/package-lock.json index 7e559b81d8..1cd958869a 100644 --- a/Ingestor/package-lock.json +++ b/Ingestor/package-lock.json @@ -10,9 +10,7 @@ "license": "Apache-2.0", "dependencies": { "Common": "file:../Common", - "ejs": "^3.1.10", - "Model": "file:../Model", "protobufjs": "^7.3.2", "ts-node": "^10.9.1" }, @@ -29,30 +27,131 @@ "version": "1.0.0", "license": "Apache-2.0", "dependencies": { + "@bull-board/express": "^5.21.4", + "@clickhouse/client": "^0.2.10", + "@elastic/elasticsearch": "^8.12.1", + "@monaco-editor/react": "^4.4.6", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/api-logs": "^0.52.1", + "@opentelemetry/context-zone": "^1.25.1", + "@opentelemetry/exporter-logs-otlp-http": "^0.52.1", + "@opentelemetry/exporter-metrics-otlp-proto": "^0.52.1", + "@opentelemetry/exporter-trace-otlp-http": "^0.52.1", + "@opentelemetry/exporter-trace-otlp-proto": "^0.52.1", + "@opentelemetry/id-generator-aws-xray": "^1.2.2", + "@opentelemetry/instrumentation": "^0.52.1", + "@opentelemetry/instrumentation-fetch": "^0.52.1", + "@opentelemetry/instrumentation-xml-http-request": "^0.52.1", + "@opentelemetry/resources": "^1.25.1", + "@opentelemetry/sdk-logs": "^0.52.1", + "@opentelemetry/sdk-metrics": "^1.25.1", + "@opentelemetry/sdk-node": "^0.52.1", + "@opentelemetry/sdk-trace-node": "^1.25.1", + "@opentelemetry/sdk-trace-web": "^1.25.1", + "@opentelemetry/semantic-conventions": "^1.26.0", + "@remixicon/react": "^4.2.0", + "@tippyjs/react": "^4.2.6", "@types/crypto-js": "^4.2.2", + "@types/qrcode": "^1.5.5", + "@types/react-highlight": "^0.12.8", + "@types/react-syntax-highlighter": "^15.5.13", "@types/uuid": "^8.3.4", + "acme-client": "^5.3.0", + "airtable": "^0.12.2", "axios": "^1.7.2", - "crypto-js": "^4.1.1", + "bullmq": "^5.3.3", + "Common": "file:../Common", + "cookie-parser": "^1.4.7", + "cors": "^2.8.5", + "cron-parser": "^4.8.1", + "crypto-js": "^4.2.0", + "dotenv": "^16.4.4", + "ejs": "^3.1.10", + "express": "^4.21.1", + "formik": "^2.4.6", + "history": "^5.3.0", + "ioredis": "^5.3.2", + "json2csv": "^5.0.7", "json5": "^2.2.3", + "jsonwebtoken": "^9.0.0", + "jwt-decode": "^4.0.0", + "lodash": "^4.17.21", + "marked": "^12.0.2", "moment": "^2.30.1", "moment-timezone": "^0.5.45", + "node-cron": "^3.0.3", + "nodemailer": "^6.9.10", + "otpauth": "^9.3.1", + "pg": "^8.7.3", + "playwright": "^1.48.2", "posthog-js": "^1.139.6", + "prop-types": "^15.8.1", + "qrcode": "^1.5.3", + "react": "^18.3.1", + "react-beautiful-dnd": "^13.1.1", + "react-big-calendar": "^1.13.0", + "react-color": "^2.19.3", + "react-dom": "^18.3.1", + "react-dropzone": "^14.2.2", + "react-error-boundary": "^4.0.13", + "react-highlight": "^0.15.0", + "react-markdown": "^8.0.3", + "react-router-dom": "^6.24.1", + "react-select": "^5.4.0", + "react-spinners": "^0.14.1", + "react-syntax-highlighter": "^15.5.0", + "react-toggle": "^4.1.3", + "reactflow": "^11.11.4", + "recharts": "^2.12.7", + "redis-semaphore": "^5.5.1", "reflect-metadata": "^0.2.2", + "remark-gfm": "^3.0.1", "slugify": "^1.6.5", + "socket.io": "^4.7.4", + "socket.io-client": "^4.7.5", + "stripe": "^10.17.0", + "tailwind-merge": "^2.5.2", + "tippy.js": "^6.3.7", + "twilio": "^4.22.0", "typeorm": "^0.3.20", + "typeorm-extension": "^2.2.13", + "universal-cookie": "^7.2.1", + "use-async-effect": "^2.2.6", "uuid": "^8.3.2" }, "devDependencies": { "@faker-js/faker": "^8.0.2", - "@types/jest": "^27.5.2", - "@types/node": "^17.0.22", - "jest": "^27.5.1", - "ts-jest": "^27.1.4" + "@testing-library/jest-dom": "^5.16.5", + "@testing-library/react": "^13.3.0", + "@testing-library/user-event": "^14.4.3", + "@types/cookie-parser": "^1.4.4", + "@types/cors": "^2.8.12", + "@types/ejs": "^3.1.1", + "@types/express": "^4.17.13", + "@types/jest": "^28.1.4", + "@types/json2csv": "^5.0.3", + "@types/jsonwebtoken": "^8.5.9", + "@types/lodash": "^4.14.202", + "@types/node": "^17.0.45", + "@types/node-cron": "^3.0.7", + "@types/nodemailer": "^6.4.7", + "@types/react": "^18.2.38", + "@types/react-beautiful-dnd": "^13.1.2", + "@types/react-big-calendar": "^1.8.5", + "@types/react-color": "^3.0.6", + "@types/react-test-renderer": "^18.0.0", + "@types/react-toggle": "^4.0.3", + "jest": "^28.1.1", + "jest-environment-jsdom": "^29.7.0", + "jest-mock-extended": "^3.0.5", + "react-test-renderer": "^18.2.0", + "ts-jest": "^28.0.5" } }, "../CommonServer": { "name": "@oneuptime/common-server", "version": "1.0.0", + "extraneous": true, "license": "Apache-2.0", "dependencies": { "@clickhouse/client": "^0.2.10", @@ -116,6 +215,7 @@ "../Model": { "name": "@oneuptime/model", "version": "1.0.0", + "extraneous": true, "license": "Apache-2.0", "dependencies": { "Common": "file:../Common", @@ -1811,10 +1911,6 @@ "resolved": "../Common", "link": true }, - "node_modules/CommonServer": { - "resolved": "../CommonServer", - "link": true - }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -3668,10 +3764,6 @@ "node": "*" } }, - "node_modules/Model": { - "resolved": "../Model", - "link": true - }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", diff --git a/Ingestor/package.json b/Ingestor/package.json index 00b90fd48d..aa8ecb91da 100644 --- a/Ingestor/package.json +++ b/Ingestor/package.json @@ -16,7 +16,6 @@ "license": "Apache-2.0", "dependencies": { "Common": "file:../Common", - "ejs": "^3.1.10", "protobufjs": "^7.3.2", "ts-node": "^10.9.1" @@ -28,4 +27,4 @@ "nodemon": "^2.0.20", "ts-jest": "^28.0.2" } -} \ No newline at end of file +} diff --git a/Probe/Utils/Monitors/Monitor.ts b/Probe/Utils/Monitors/Monitor.ts index e5ba4d8e53..2a6792b588 100644 --- a/Probe/Utils/Monitors/Monitor.ts +++ b/Probe/Utils/Monitors/Monitor.ts @@ -360,7 +360,6 @@ export default class MonitorUtil { const response: ProbeWebsiteResponse | null = await WebsiteMonitor.ping( monitorStep.data?.monitorDestination as URL, { - isHeadRequest: MonitorUtil.isHeadRequest(monitorStep), monitorId: monitorId, retry: 10, timeout: new PositiveNumber(60000), // 60 seconds diff --git a/Probe/Utils/Monitors/MonitorTypes/SyntheticMonitor.ts b/Probe/Utils/Monitors/MonitorTypes/SyntheticMonitor.ts index b8c0b579ff..db90bdcb99 100644 --- a/Probe/Utils/Monitors/MonitorTypes/SyntheticMonitor.ts +++ b/Probe/Utils/Monitors/MonitorTypes/SyntheticMonitor.ts @@ -1,5 +1,4 @@ import { PROBE_SYNTHETIC_MONITOR_SCRIPT_TIMEOUT_IN_MS } from "../../../Config"; -import BadDataException from "Common/Types/Exception/BadDataException"; import ReturnResult from "Common/Types/IsolatedVM/ReturnResult"; import BrowserType from "Common/Types/Monitor/SyntheticMonitors/BrowserType"; import ScreenSizeType from "Common/Types/Monitor/SyntheticMonitors/ScreenSizeType"; @@ -7,8 +6,7 @@ import SyntheticMonitorResponse from "Common/Types/Monitor/SyntheticMonitors/Syn import ObjectID from "Common/Types/ObjectID"; import logger from "Common/Server/Utils/Logger"; import VMRunner from "Common/Server/Utils/VM/VMRunner"; -import { Browser, Page, chromium, firefox } from "playwright"; -import LocalFile from "Common/Server/Utils/LocalFile"; +import BrowserUtil, { Browser, Page } from "Common/Server/Utils/Browser"; export interface SyntheticMonitorOptions { monitorId?: ObjectID | undefined; @@ -82,7 +80,7 @@ export default class SyntheticMonitor { try { const startTime: [number, number] = process.hrtime(); - pageAndBrowser = await SyntheticMonitor.getPageByBrowserType({ + pageAndBrowser = await BrowserUtil.getPageByBrowserType({ browserType: options.browserType, screenSizeType: options.screenSizeType, }); @@ -155,152 +153,4 @@ export default class SyntheticMonitor { return scriptResult; } - - private static getViewportHeightAndWidth(options: { - screenSizeType: ScreenSizeType; - }): { - height: number; - width: number; - } { - let viewPortHeight: number = 0; - let viewPortWidth: number = 0; - - switch (options.screenSizeType) { - case ScreenSizeType.Desktop: - viewPortHeight = 1080; - viewPortWidth = 1920; - break; - case ScreenSizeType.Mobile: - viewPortHeight = 640; - viewPortWidth = 360; - break; - case ScreenSizeType.Tablet: - viewPortHeight = 768; - viewPortWidth = 1024; - break; - default: - viewPortHeight = 1080; - viewPortWidth = 1920; - break; - } - - return { height: viewPortHeight, width: viewPortWidth }; - } - - public static async getChromeExecutablePath(): Promise { - const doesDirectoryExist: boolean = await LocalFile.doesDirectoryExist( - "/root/.cache/ms-playwright", - ); - if (!doesDirectoryExist) { - throw new BadDataException("Chrome executable path not found."); - } - - // get list of files in the directory - const directories: string[] = await LocalFile.getListOfDirectories( - "/root/.cache/ms-playwright", - ); - - if (directories.length === 0) { - throw new BadDataException("Chrome executable path not found."); - } - - const chromeInstallationName: string | undefined = directories.find( - (directory: string) => { - return directory.includes("chromium"); - }, - ); - - if (!chromeInstallationName) { - throw new BadDataException("Chrome executable path not found."); - } - - return `/root/.cache/ms-playwright/${chromeInstallationName}/chrome-linux/chrome`; - } - - public static async getFirefoxExecutablePath(): Promise { - const doesDirectoryExist: boolean = await LocalFile.doesDirectoryExist( - "/root/.cache/ms-playwright", - ); - if (!doesDirectoryExist) { - throw new BadDataException("Firefox executable path not found."); - } - - // get list of files in the directory - const directories: string[] = await LocalFile.getListOfDirectories( - "/root/.cache/ms-playwright", - ); - - if (directories.length === 0) { - throw new BadDataException("Firefox executable path not found."); - } - - const firefoxInstallationName: string | undefined = directories.find( - (directory: string) => { - return directory.includes("firefox"); - }, - ); - - if (!firefoxInstallationName) { - throw new BadDataException("Firefox executable path not found."); - } - - return `/root/.cache/ms-playwright/${firefoxInstallationName}/firefox/firefox`; - } - - private static async getPageByBrowserType(data: { - browserType: BrowserType; - screenSizeType: ScreenSizeType; - }): Promise<{ - page: Page; - browser: Browser; - }> { - const viewport: { - height: number; - width: number; - } = SyntheticMonitor.getViewportHeightAndWidth({ - screenSizeType: data.screenSizeType, - }); - - let page: Page | null = null; - let browser: Browser | null = null; - - if (data.browserType === BrowserType.Chromium) { - browser = await chromium.launch({ - executablePath: await this.getChromeExecutablePath(), - }); - page = await browser.newPage(); - } - - if (data.browserType === BrowserType.Firefox) { - browser = await firefox.launch({ - executablePath: await this.getFirefoxExecutablePath(), - }); - page = await browser.newPage(); - } - - // if (data.browserType === BrowserType.Webkit) { - // browser = await webkit.launch(); - // page = await browser.newPage(); - // } - - await page?.setViewportSize({ - width: viewport.width, - height: viewport.height, - }); - - if (!browser) { - throw new BadDataException("Invalid Browser Type."); - } - - if (!page) { - // close the browser if page is not created - await browser.close(); - throw new BadDataException("Invalid Browser Type."); - } - - return { - page: page, - browser: browser, - }; - } } diff --git a/Probe/Utils/Monitors/MonitorTypes/WebsiteMonitor.ts b/Probe/Utils/Monitors/MonitorTypes/WebsiteMonitor.ts index c76d05a043..eb1d673a71 100644 --- a/Probe/Utils/Monitors/MonitorTypes/WebsiteMonitor.ts +++ b/Probe/Utils/Monitors/MonitorTypes/WebsiteMonitor.ts @@ -29,7 +29,6 @@ export default class WebsiteMonitor { url: URL, options: { retry?: number | undefined; - isHeadRequest?: boolean | undefined; currentRetryCount?: number | undefined; monitorId?: ObjectID | undefined; isOnlineCheckRequest?: boolean | undefined; @@ -45,11 +44,7 @@ export default class WebsiteMonitor { options.currentRetryCount = 1; } - let requestType: HTTPMethod = HTTPMethod.GET; - - if (options.isHeadRequest) { - requestType = HTTPMethod.HEAD; - } + const requestType: HTTPMethod = HTTPMethod.GET; try { logger.debug( @@ -58,26 +53,13 @@ export default class WebsiteMonitor { }`, ); - let startTime: [number, number] = process.hrtime(); - let result: WebsiteResponse = await WebsiteRequest.fetch(url, { - isHeadRequest: options.isHeadRequest, + const startTime: [number, number] = process.hrtime(); + const result: WebsiteResponse = await WebsiteRequest.fetch(url, { + isHeadRequest: false, timeout: options.timeout?.toNumber() || 5000, doNotFollowRedirects: options.doNotFollowRedirects || false, }); - if ( - result.responseStatusCode >= 400 && - result.responseStatusCode < 600 && - requestType === HTTPMethod.HEAD - ) { - startTime = process.hrtime(); - result = await WebsiteRequest.fetch(url, { - isHeadRequest: false, - timeout: options.timeout?.toNumber() || 5000, - doNotFollowRedirects: options.doNotFollowRedirects || false, - }); - } - const endTime: [number, number] = process.hrtime(startTime); const responseTimeInMS: PositiveNumber = new PositiveNumber( (endTime[0] * 1000000000 + endTime[1]) / 1000000, diff --git a/Probe/package-lock.json b/Probe/package-lock.json index a18d7d0f41..88129dbd60 100644 --- a/Probe/package-lock.json +++ b/Probe/package-lock.json @@ -14,7 +14,6 @@ "Common": "file:../Common", "ejs": "^3.1.10", "ping": "^0.4.4", - "playwright": "^1.45.1", "ts-node": "^10.9.1" }, "devDependencies": { @@ -30,30 +29,131 @@ "version": "1.0.0", "license": "Apache-2.0", "dependencies": { + "@bull-board/express": "^5.21.4", + "@clickhouse/client": "^0.2.10", + "@elastic/elasticsearch": "^8.12.1", + "@monaco-editor/react": "^4.4.6", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/api-logs": "^0.52.1", + "@opentelemetry/context-zone": "^1.25.1", + "@opentelemetry/exporter-logs-otlp-http": "^0.52.1", + "@opentelemetry/exporter-metrics-otlp-proto": "^0.52.1", + "@opentelemetry/exporter-trace-otlp-http": "^0.52.1", + "@opentelemetry/exporter-trace-otlp-proto": "^0.52.1", + "@opentelemetry/id-generator-aws-xray": "^1.2.2", + "@opentelemetry/instrumentation": "^0.52.1", + "@opentelemetry/instrumentation-fetch": "^0.52.1", + "@opentelemetry/instrumentation-xml-http-request": "^0.52.1", + "@opentelemetry/resources": "^1.25.1", + "@opentelemetry/sdk-logs": "^0.52.1", + "@opentelemetry/sdk-metrics": "^1.25.1", + "@opentelemetry/sdk-node": "^0.52.1", + "@opentelemetry/sdk-trace-node": "^1.25.1", + "@opentelemetry/sdk-trace-web": "^1.25.1", + "@opentelemetry/semantic-conventions": "^1.26.0", + "@remixicon/react": "^4.2.0", + "@tippyjs/react": "^4.2.6", "@types/crypto-js": "^4.2.2", + "@types/qrcode": "^1.5.5", + "@types/react-highlight": "^0.12.8", + "@types/react-syntax-highlighter": "^15.5.13", "@types/uuid": "^8.3.4", + "acme-client": "^5.3.0", + "airtable": "^0.12.2", "axios": "^1.7.2", - "crypto-js": "^4.1.1", + "bullmq": "^5.3.3", + "Common": "file:../Common", + "cookie-parser": "^1.4.7", + "cors": "^2.8.5", + "cron-parser": "^4.8.1", + "crypto-js": "^4.2.0", + "dotenv": "^16.4.4", + "ejs": "^3.1.10", + "express": "^4.21.1", + "formik": "^2.4.6", + "history": "^5.3.0", + "ioredis": "^5.3.2", + "json2csv": "^5.0.7", "json5": "^2.2.3", + "jsonwebtoken": "^9.0.0", + "jwt-decode": "^4.0.0", + "lodash": "^4.17.21", + "marked": "^12.0.2", "moment": "^2.30.1", "moment-timezone": "^0.5.45", + "node-cron": "^3.0.3", + "nodemailer": "^6.9.10", + "otpauth": "^9.3.1", + "pg": "^8.7.3", + "playwright": "^1.48.2", "posthog-js": "^1.139.6", + "prop-types": "^15.8.1", + "qrcode": "^1.5.3", + "react": "^18.3.1", + "react-beautiful-dnd": "^13.1.1", + "react-big-calendar": "^1.13.0", + "react-color": "^2.19.3", + "react-dom": "^18.3.1", + "react-dropzone": "^14.2.2", + "react-error-boundary": "^4.0.13", + "react-highlight": "^0.15.0", + "react-markdown": "^8.0.3", + "react-router-dom": "^6.24.1", + "react-select": "^5.4.0", + "react-spinners": "^0.14.1", + "react-syntax-highlighter": "^15.5.0", + "react-toggle": "^4.1.3", + "reactflow": "^11.11.4", + "recharts": "^2.12.7", + "redis-semaphore": "^5.5.1", "reflect-metadata": "^0.2.2", + "remark-gfm": "^3.0.1", "slugify": "^1.6.5", + "socket.io": "^4.7.4", + "socket.io-client": "^4.7.5", + "stripe": "^10.17.0", + "tailwind-merge": "^2.5.2", + "tippy.js": "^6.3.7", + "twilio": "^4.22.0", "typeorm": "^0.3.20", + "typeorm-extension": "^2.2.13", + "universal-cookie": "^7.2.1", + "use-async-effect": "^2.2.6", "uuid": "^8.3.2" }, "devDependencies": { "@faker-js/faker": "^8.0.2", - "@types/jest": "^27.5.2", - "@types/node": "^17.0.22", - "jest": "^27.5.1", - "ts-jest": "^27.1.4" + "@testing-library/jest-dom": "^5.16.5", + "@testing-library/react": "^13.3.0", + "@testing-library/user-event": "^14.4.3", + "@types/cookie-parser": "^1.4.4", + "@types/cors": "^2.8.12", + "@types/ejs": "^3.1.1", + "@types/express": "^4.17.13", + "@types/jest": "^28.1.4", + "@types/json2csv": "^5.0.3", + "@types/jsonwebtoken": "^8.5.9", + "@types/lodash": "^4.14.202", + "@types/node": "^17.0.45", + "@types/node-cron": "^3.0.7", + "@types/nodemailer": "^6.4.7", + "@types/react": "^18.2.38", + "@types/react-beautiful-dnd": "^13.1.2", + "@types/react-big-calendar": "^1.8.5", + "@types/react-color": "^3.0.6", + "@types/react-test-renderer": "^18.0.0", + "@types/react-toggle": "^4.0.3", + "jest": "^28.1.1", + "jest-environment-jsdom": "^29.7.0", + "jest-mock-extended": "^3.0.5", + "react-test-renderer": "^18.2.0", + "ts-jest": "^28.0.5" } }, "../CommonServer": { "name": "@oneuptime/common-server", "version": "1.0.0", + "extraneous": true, "license": "Apache-2.0", "dependencies": { "@clickhouse/client": "^0.2.10", @@ -115,6 +215,7 @@ "../Model": { "name": "@oneuptime/model", "version": "1.0.0", + "extraneous": true, "license": "Apache-2.0", "dependencies": { "Common": "file:../Common", @@ -1788,10 +1889,6 @@ "resolved": "../Common", "link": true }, - "node_modules/CommonServer": { - "resolved": "../CommonServer", - "link": true - }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -3699,10 +3796,6 @@ "node": "*" } }, - "node_modules/Model": { - "resolved": "../Model", - "link": true - }, "node_modules/ms": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", @@ -4003,49 +4096,6 @@ "node": ">=8" } }, - "node_modules/playwright": { - "version": "1.45.1", - "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.45.1.tgz", - "integrity": "sha512-Hjrgae4kpSQBr98nhCj3IScxVeVUixqj+5oyif8TdIn2opTCPEzqAqNMeK42i3cWDCVu9MI+ZsGWw+gVR4ISBg==", - "license": "Apache-2.0", - "dependencies": { - "playwright-core": "1.45.1" - }, - "bin": { - "playwright": "cli.js" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "fsevents": "2.3.2" - } - }, - "node_modules/playwright-core": { - "version": "1.45.1", - "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.45.1.tgz", - "integrity": "sha512-LF4CUUtrUu2TCpDw4mcrAIuYrEjVDfT1cHbJMfwnE2+1b8PZcFzPNgvZCvq2JfQ4aTjRCCHw5EJ2tmr2NSzdPg==", - "license": "Apache-2.0", - "bin": { - "playwright-core": "cli.js" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/playwright/node_modules/fsevents": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", - "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", - "hasInstallScript": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, "node_modules/pretty-format": { "version": "27.5.1", "resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-27.5.1.tgz", diff --git a/Probe/package.json b/Probe/package.json index 5e1ca1ab82..6da0b3cfc3 100644 --- a/Probe/package.json +++ b/Probe/package.json @@ -20,11 +20,8 @@ "@types/ping": "^0.4.4", "axios": "^1.7.2", "Common": "file:../Common", - - "ejs": "^3.1.10", "ping": "^0.4.4", - "playwright": "1.45.1", "ts-node": "^10.9.1" }, "devDependencies": {