From c18ceab17554f99099391c797eac654dcf10b961 Mon Sep 17 00:00:00 2001 From: Stephen Fraser Date: Tue, 26 Nov 2024 14:24:56 +0000 Subject: [PATCH 1/3] Added slow request tracking - with flag to disable --- services/madoc-ts/src/app.ts | 12 +++-- services/madoc-ts/src/config.ts | 1 + .../admin/pages/system/system-status.tsx | 28 +++++++++++- services/madoc-ts/src/gateway/api.ts | 2 +- .../madoc-ts/src/middleware/slow-requests.ts | 45 +++++++++++++++++++ services/madoc-ts/src/routes/admin/pm2.ts | 2 + 6 files changed, 84 insertions(+), 6 deletions(-) create mode 100644 services/madoc-ts/src/middleware/slow-requests.ts diff --git a/services/madoc-ts/src/app.ts b/services/madoc-ts/src/app.ts index a9bc6405d..8ca4b374e 100644 --- a/services/madoc-ts/src/app.ts +++ b/services/madoc-ts/src/app.ts @@ -34,6 +34,7 @@ import cookieParser from 'cookie-parser'; import schedule from 'node-schedule'; import passport from 'koa-passport'; import { WebhookServerExtension } from './webhooks/webhook-server-extension'; +import { slowRequests } from './middleware/slow-requests'; const { readFile, readdir } = promises; @@ -124,17 +125,20 @@ export async function createApp(config: ExternalConfig, env: EnvConfig) { } if (enabled) { - passport.serializeUser(function(user: any, done) { + passport.serializeUser(function (user: any, done) { done(null, user); }); - passport.deserializeUser(function(user: any, done) { + passport.deserializeUser(function (user: any, done) { done(null, user); }); app.use(passport.initialize()); } + if (!env.flags.disable_slow_request_tracking) { + app.use(slowRequests); + } app.use(errorHandler); app.use(staticPage); app.use(setJwt); @@ -147,7 +151,7 @@ export async function createApp(config: ExternalConfig, env: EnvConfig) { (app.context.cron as CronJobs).addJob( 'check-expired-manifests', 'Check expired manifests', - schedule.scheduleJob('*/15 * * * *', async function(fireDate) { + schedule.scheduleJob('*/15 * * * *', async function (fireDate) { await pool.connect(async connection => { await checkExpiredManifests({ ...app.context, connection } as any, fireDate); }); @@ -157,7 +161,7 @@ export async function createApp(config: ExternalConfig, env: EnvConfig) { (app.context.cron as CronJobs).addJob( 'restart-queue', 'Restart queue 3am once a day', - schedule.scheduleJob('0 3 * * *', async function() { + schedule.scheduleJob('0 3 * * *', async function () { await bounceQueue(); }) ); diff --git a/services/madoc-ts/src/config.ts b/services/madoc-ts/src/config.ts index 4353de638..39037b987 100644 --- a/services/madoc-ts/src/config.ts +++ b/services/madoc-ts/src/config.ts @@ -30,6 +30,7 @@ export const config: EnvConfig = { }, flags: { capture_model_api_migrated: castBool(process.env.CAPTURE_MODEL_API_MIGRATED, false), + disable_slow_request_tracking: castBool(process.env.DISABLE_SLOW_REQUEST_TRACKING, false), }, postgres: { host: process.env.DATABASE_HOST as string, diff --git a/services/madoc-ts/src/frontend/admin/pages/system/system-status.tsx b/services/madoc-ts/src/frontend/admin/pages/system/system-status.tsx index 0e4e322b5..7ae5b7faf 100644 --- a/services/madoc-ts/src/frontend/admin/pages/system/system-status.tsx +++ b/services/madoc-ts/src/frontend/admin/pages/system/system-status.tsx @@ -16,9 +16,11 @@ import { createUniversalComponent } from '../../../shared/utility/create-univers import { HrefLink } from '../../../shared/utility/href-link'; import { UniversalComponent } from '../../../types'; import { AdminHeader } from '../../molecules/AdminHeader'; +import { SlowRequests } from '../../../../middleware/slow-requests'; +import { TableContainer, TableRow, TableRowLabel } from '../../../shared/layout/Table'; type SystemStatusType = { - data: { list: Pm2Status[]; build: EnvConfig['build'] }; + data: { list: Pm2Status[]; build: EnvConfig['build']; slowRequests: SlowRequests }; query: unknown; params: unknown; variables: unknown; @@ -70,6 +72,8 @@ export const SystemStatus: UniversalComponent = createUniversa ) : { memory: 0, cpu: 0 }; + const slowestRequests = Object.values(data?.slowRequests || {}).sort((a, b) => b.slowest - a.slowest); + return ( <> @@ -250,6 +254,28 @@ export const SystemStatus: UniversalComponent = createUniversa }) : null} + +
+

Slow requests

+ + + + + + + + + + {slowestRequests.map(slow => ( + + + + + + + ))} +
RouteCountAverage timeMax time
{slow.key}{slow.count}{(slow.avg / 1000).toFixed(2)}s{(slow.slowest / 1000).toFixed(2)}s
+
); diff --git a/services/madoc-ts/src/gateway/api.ts b/services/madoc-ts/src/gateway/api.ts index 40d76880d..63dcfbaf2 100644 --- a/services/madoc-ts/src/gateway/api.ts +++ b/services/madoc-ts/src/gateway/api.ts @@ -561,7 +561,7 @@ export class ApiClient { } async getPm2Status() { - return this.request<{ list: Pm2Status[]; build: any }>(`/api/madoc/pm2/list`); + return this.request<{ list: Pm2Status[]; build: any; slowRequests: any }>(`/api/madoc/pm2/list`); } async pm2Restart(service: 'auth' | 'queue' | 'madoc' | 'scheduler') { diff --git a/services/madoc-ts/src/middleware/slow-requests.ts b/services/madoc-ts/src/middleware/slow-requests.ts new file mode 100644 index 000000000..1fc410dd1 --- /dev/null +++ b/services/madoc-ts/src/middleware/slow-requests.ts @@ -0,0 +1,45 @@ +import { RouteMiddleware } from '../types/route-middleware'; + +export type SlowRequests = Record< + string, + { + key: string; + count: number; + avg: number; + slowest: number; + } +>; + +const slowRequestStore: SlowRequests = {}; + +export const slowRequests: RouteMiddleware<{ slug: string }> = async (context, next) => { + const requestStart = Date.now(); + await next(); + + const routeKey = `${context.method} ${context._matchedRoute}`; + + const requestEnd = Date.now(); + const requestTime = requestEnd - requestStart; + if (!slowRequestStore[routeKey]) { + slowRequestStore[routeKey] = { key: routeKey, count: 0, avg: 0, slowest: 0 }; + } + + const existinRoute = slowRequestStore[routeKey]; + + existinRoute.count++; + existinRoute.avg = (existinRoute.avg * (existinRoute.count - 1) + requestTime) / existinRoute.count; + existinRoute.slowest = Math.max(existinRoute.slowest, requestTime); + + if (Object.keys(slowRequestStore).length > 50) { + // Sort by slowest. + const sorted = Object.values(slowRequestStore).sort((a, b) => b.slowest - a.slowest); + const slowest = sorted.pop(); + if (slowest && slowest.key) { + delete slowRequestStore[slowest.key]; + } + } +}; + +export function getSlowRequests() { + return slowRequestStore; +} diff --git a/services/madoc-ts/src/routes/admin/pm2.ts b/services/madoc-ts/src/routes/admin/pm2.ts index 5f17c04ba..5d1f8df4e 100644 --- a/services/madoc-ts/src/routes/admin/pm2.ts +++ b/services/madoc-ts/src/routes/admin/pm2.ts @@ -2,6 +2,7 @@ import pm2, { ProcessDescription } from 'pm2'; import { config } from '../../config'; import { RouteMiddleware } from '../../types/route-middleware'; import { onlyGlobalAdmin } from '../../utility/user-with-scope'; +import { getSlowRequests } from '../../middleware/slow-requests'; async function pm2Connect() { await new Promise((resolve, reject) => @@ -66,6 +67,7 @@ export const pm2Status: RouteMiddleware = async context => { uptime: (item as any)?.pm2_env?.pm_uptime, }; }), + slowRequests: getSlowRequests(), }; pm2.disconnect(); From 0b3c4acf5d0f7579a6fbb438bfb65baea04f9229 Mon Sep 17 00:00:00 2001 From: Stephen Fraser Date: Tue, 26 Nov 2024 14:25:20 +0000 Subject: [PATCH 2/3] Added SWR caching to project task stats and annotation styles --- .../src/routes/projects/get-project.ts | 22 +++++--- services/madoc-ts/src/utility/cache-helper.ts | 52 +++++++++++++++++++ 2 files changed, 68 insertions(+), 6 deletions(-) create mode 100644 services/madoc-ts/src/utility/cache-helper.ts diff --git a/services/madoc-ts/src/routes/projects/get-project.ts b/services/madoc-ts/src/routes/projects/get-project.ts index b12bc7414..37c2b5364 100644 --- a/services/madoc-ts/src/routes/projects/get-project.ts +++ b/services/madoc-ts/src/routes/projects/get-project.ts @@ -2,6 +2,7 @@ import { api } from '../../gateway/api.server'; import { RouteMiddleware } from '../../types/route-middleware'; import { ProjectConfiguration } from '../../types/schemas/project-configuration'; +import { cachePromise, cachePromiseSWR } from '../../utility/cache-helper'; import { castBool } from '../../utility/cast-bool'; import { NotFound } from '../../utility/errors/not-found'; import { parseProjectId } from '../../utility/parse-project-id'; @@ -33,13 +34,22 @@ export const getProject: RouteMiddleware<{ id: string }> = async context => { const [content, taskStats, projectConfiguration, annotationStyles] = await Promise.all([ userApi.getCollectionStatistics(project.collection_id), - userApi.getTaskStats(project.task_id, { - type: 'crowdsourcing-task', - root: true, - distinct_subjects: true, - }), + cachePromiseSWR( + `task-stats:${project.task_id}`, + () => + userApi.getTaskStats(project.task_id, { + type: 'crowdsourcing-task', + root: true, + distinct_subjects: true, + }), + 1000 * 60 * 15 // 15 minutes. + ), userApi.getConfiguration('madoc', [siteUrn, `urn:madoc:project:${project.id}`]), - context.annotationStyles.getProjectAnnotationStyle(project.id, siteId), + cachePromiseSWR( + `annotation-style:${project.id}`, + () => context.annotationStyles.getProjectAnnotationStyle(project.id, siteId), + 1000 * 60 * 15 // 15 minutes. + ), ]); const taskStatuses = taskStats.statuses || {}; diff --git a/services/madoc-ts/src/utility/cache-helper.ts b/services/madoc-ts/src/utility/cache-helper.ts new file mode 100644 index 000000000..cadb75e94 --- /dev/null +++ b/services/madoc-ts/src/utility/cache-helper.ts @@ -0,0 +1,52 @@ +import cache from 'memory-cache'; + +export async function cachePromise( + key: string, + getter: () => Promise, + timeInMs: number +): Promise { + const resource = cache.get(key); + if (resource) { + return resource as T; + } + + const result = await getter(); + cache.put(key, result, timeInMs); + return result; +} + +const swcPromises: Record | null> = {}; +const DEFAULT_STALE_TIME = 1000 * 60 * 60 * 8; // 8 hours + +export async function cachePromiseSWR( + key: string, + getter: () => Promise, + timeInMs: number, + staleTimeInMs: number = DEFAULT_STALE_TIME +): Promise { + if (swcPromises[key]) { + await swcPromises[key]; + } + const resource = cache.get(key); + const staleResource = cache.get(`@stale/${key}`); + if (!resource) { + const promise = getter(); + swcPromises[key] = promise; + promise.then(result => { + cache.put(key, result, timeInMs); + cache.put(`@stale/${key}`, result, staleTimeInMs); + + delete swcPromises[key]; + + return result; + }); + + if (staleResource) { + return staleResource; + } + + return promise; + } + + return resource; +} From 1bf3ce08d08e117363ddca3daf8d3eade6afc09f Mon Sep 17 00:00:00 2001 From: Stephen Fraser Date: Tue, 26 Nov 2024 14:25:45 +0000 Subject: [PATCH 3/3] Added noSsr option to server renderer - and applied to task pages --- .../shared/plugins/external/server-renderer-for.ts | 2 ++ .../frontend/shared/utility/create-server-renderer.tsx | 4 ++-- .../frontend/shared/utility/create-universal-component.ts | 2 ++ services/madoc-ts/src/frontend/site/pages/all-tasks.tsx | 8 +++++++- .../pages/tasks/review-listing/review-listing-page.tsx | 1 + 5 files changed, 14 insertions(+), 3 deletions(-) diff --git a/services/madoc-ts/src/frontend/shared/plugins/external/server-renderer-for.ts b/services/madoc-ts/src/frontend/shared/plugins/external/server-renderer-for.ts index dc5735609..b7a95ffa5 100644 --- a/services/madoc-ts/src/frontend/shared/plugins/external/server-renderer-for.ts +++ b/services/madoc-ts/src/frontend/shared/plugins/external/server-renderer-for.ts @@ -10,8 +10,10 @@ export function serverRendererFor( getData?: (key: string, vars: TVariables, api: ApiClient, pathname: string) => Promise; hooks?: AdditionalHooks[]; theme?: { name: string } & Partial; + noSsr?: boolean; } ) { + (component as any).noSsr = config.noSsr; (component as any).getKey = config.getKey; (component as any).getData = config.getData; (component as any).hooks = config.hooks; diff --git a/services/madoc-ts/src/frontend/shared/utility/create-server-renderer.tsx b/services/madoc-ts/src/frontend/shared/utility/create-server-renderer.tsx index 452025027..390740e7a 100644 --- a/services/madoc-ts/src/frontend/shared/utility/create-server-renderer.tsx +++ b/services/madoc-ts/src/frontend/shared/utility/create-server-renderer.tsx @@ -138,7 +138,7 @@ export function createServerRenderer( routeContext.project = match.params.slug ? match.params.slug : undefined; } - if (route.component && route.component.getKey && route.component.getData) { + if (route.component && route.component.getKey && route.component.getData && route.component.noSsr !== true) { requests.push( prefetchCache.prefetchQuery( route.component.getKey(match.params, queryString, path), @@ -362,7 +362,7 @@ export function createServerRenderer( : `