diff --git a/extensions/populate-metadata/src/getProperties.ts b/extensions/populate-metadata/src/getProperties.ts index 17ed80354..42615d873 100644 --- a/extensions/populate-metadata/src/getProperties.ts +++ b/extensions/populate-metadata/src/getProperties.ts @@ -8,8 +8,8 @@ import type { PoolDBName, DocsetsDocument, ReposBranchesDocument, - clusterZeroConnectionInfo, - ProjectMetadataDocument, + ClusterZeroConnectionInfo, + ProjectsDocument, ClusterZeroDBName, } from 'util/databaseConnection/types'; const EXTENSION_NAME = 'populate-metadata-extension'; @@ -23,7 +23,7 @@ const getDocsetEntry = async ({ projectName, environment, }: { - docsetsConnectionInfo: clusterZeroConnectionInfo; + docsetsConnectionInfo: ClusterZeroConnectionInfo; projectName: string; environment: Environments; }): Promise => { @@ -56,7 +56,7 @@ const getRepoEntry = async ({ }: { repoName: string; branchName: string; - connectionInfo: clusterZeroConnectionInfo; + connectionInfo: ClusterZeroConnectionInfo; }): Promise => { const reposBranches = await getReposBranchesCollection(connectionInfo); @@ -88,13 +88,13 @@ const getRepoEntry = async ({ return repo; }; -const getMetadataEntry = async ({ +const getProjectsEntry = async ({ projectName, connectionInfo, }: { projectName: string; - connectionInfo: clusterZeroConnectionInfo; -}): Promise => { + connectionInfo: ClusterZeroConnectionInfo; +}): Promise => { const projects = await getProjectsCollection(connectionInfo); const query = { name: projectName, @@ -109,7 +109,7 @@ const getMetadataEntry = async ({ }, }; - const projectMetadata = await projects.findOne( + const projectMetadata = await projects.findOne( query, projection, ); @@ -139,7 +139,7 @@ export const getProperties = async ({ }): Promise<{ repo: ReposBranchesDocument; docsetEntry: DocsetsDocument; - metadataEntry: ProjectMetadataDocument; + projectsEntry: ProjectsDocument; }> => { const repoBranchesConnectionInfo = { clusterZeroURI: dbEnvVars.ATLAS_CLUSTER0_URI, @@ -174,12 +174,12 @@ export const getProperties = async ({ extensionName: EXTENSION_NAME, }; - const metadataEntry = await getMetadataEntry({ + const projectsEntry = await getProjectsEntry({ connectionInfo: projectMetadataConnectionInfo, projectName: repo.project, }); await closeClusterZeroDb(); - return { repo, docsetEntry, metadataEntry }; + return { repo, docsetEntry, projectsEntry }; }; diff --git a/extensions/populate-metadata/src/index.ts b/extensions/populate-metadata/src/index.ts index 421d740e7..d6b46087a 100644 --- a/extensions/populate-metadata/src/index.ts +++ b/extensions/populate-metadata/src/index.ts @@ -8,10 +8,11 @@ const extension = new Extension({ extension.addBuildEventHandler( 'onPreBuild', - async ({ netlifyConfig, dbEnvVars }) => { + async ({ netlifyConfig, dbEnvVars, utils: { run } }) => { await updateConfig({ configEnvironment: netlifyConfig?.build?.environment, dbEnvVars, + run, }); }, ); diff --git a/extensions/populate-metadata/src/updateConfig.ts b/extensions/populate-metadata/src/updateConfig.ts index 3f99467a9..9659a3174 100644 --- a/extensions/populate-metadata/src/updateConfig.ts +++ b/extensions/populate-metadata/src/updateConfig.ts @@ -3,10 +3,19 @@ import type { PoolDBName, SnootyDBName, SearchDBName, + OrganizationName, } from 'util/databaseConnection/types'; import { getProperties } from './getProperties'; import type { ConfigEnvironmentVariables } from 'util/extension'; import type { StaticEnvVars } from 'util/assertDbEnvVars'; +import type { NetlifyPluginUtils } from '@netlify/build'; +import * as fs from 'node:fs'; + +const FRONTEND_SITES = [ + 'docs-frontend-stg', + 'docs-frontend-dotcomstg', + 'docs-frontend-dotcomprd', +]; const getDbNames = ( env: Environments, @@ -47,15 +56,9 @@ const determineEnvironment = ({ siteName, }: { isBuildHookDeploy: boolean; siteName: string }): Environments => { // Check if this was an engineer's build or writer's build - const frontendSites = [ - 'docs-frontend-stg', - 'docs-frontend-dotcomstg', - 'docs-frontend-dotcomprd', - ]; - const isFrontendBuild = frontendSites.includes(siteName); + const isFrontendBuild = FRONTEND_SITES.includes(siteName); //Writer's builds = prd, everything not built on a frontend site (a site with 'Snooty' as git source) - if (!isFrontendBuild) { return 'prd'; } @@ -69,17 +72,75 @@ const determineEnvironment = ({ } return 'stg'; }; + +const cloneContentRepo = async ({ + run, + repoName, + branchName, + orgName, +}: { + run: NetlifyPluginUtils['run']; + repoName: string; + branchName: string; + orgName: string; +}) => { + if (fs.existsSync(`${process.cwd()}/${repoName}`)) { + await run.command(`rm -r ${repoName}`); + } + + await run.command( + `git clone -b ${branchName} https://${process.env.GITHUB_BOT_USERNAME}:${process.env.GITHUB_BOT_PWD}@github.com/${orgName}/${repoName}.git -s`, + ); + + if (fs.existsSync(`${repoName}/.git/config`)) { + // Remove git config as it stores the connection string in plain text + await run.command(`rm -r ${repoName}/.git/config`); + } + await run.command(`touch ${repoName}/${branchName}`); + await run.command( + `echo "${repoName}, ${branchName}" > ${repoName}/${branchName}`, + ); + await run.command('ls'); + console.log(fs.readFileSync(`${repoName}/${branchName}`)); +}; + export const updateConfig = async ({ configEnvironment, dbEnvVars, + run, }: { configEnvironment: ConfigEnvironmentVariables; dbEnvVars: StaticEnvVars; + run: NetlifyPluginUtils['run']; }): Promise => { + // Check if this was a build triggered by a frontend change or a content repo change + const isFrontendBuild = FRONTEND_SITES.includes( + configEnvironment.SITE_NAME as string, + ); // Checks if build was triggered by a webhook const isBuildHookDeploy = !!( configEnvironment.INCOMING_HOOK_URL && configEnvironment.INCOMING_HOOK_TITLE ); + + // Determine which repository and branch to build + // Check if repo name and branch name have been set as environment variables through Netlify UI, allows overwriting of database name values + const repoName = isBuildHookDeploy + ? JSON.parse(configEnvironment?.INCOMING_HOOK_BODY as string)?.repoName + : (process.env.REPO_NAME ?? + (process.env.REPOSITORY_URL?.split('/')?.pop() as string)); + + const branchName: string = isBuildHookDeploy + ? JSON.parse(configEnvironment?.INCOMING_HOOK_BODY as string)?.branchName + : (process.env.BRANCH_NAME ?? (configEnvironment.BRANCH as string)); + + if (!repoName || !branchName) { + throw new Error('Repo name or branch name missing from deploy'); + } + + configEnvironment.BRANCH_NAME = branchName; + configEnvironment.REPO_NAME = repoName; + + // Determine which environment the build will run in const env = determineEnvironment({ isBuildHookDeploy, siteName: configEnvironment.SITE_NAME as string, @@ -89,6 +150,7 @@ export const updateConfig = async ({ configEnvironment.ENV = buildEnvironment; + // Get the names of the databases associated with given build environment const { snootyDb, searchDb, poolDb } = getDbNames(buildEnvironment); // Check if values for the database names have been set as environment variables through Netlify UI @@ -102,51 +164,34 @@ export const updateConfig = async ({ configEnvironment.SNOOTY_DB_NAME = (process.env.SNOOTY_DB_NAME as SnootyDBName) ?? snootyDb; - // Check if repo name and branch name have been set as environment variables through Netlify UI - // Allows overwriting of database name values for testing - let branchName: string; - let repoName: string; - if (buildEnvironment !== 'dotcomstg' && buildEnvironment !== 'dotcomprd') { - branchName = - process.env.BRANCH_NAME ?? (configEnvironment.BRANCH as string); - repoName = - process.env.REPO_NAME ?? - (process.env.REPOSITORY_URL?.split('/')?.pop() as string); - } else { - console.log(`Incoming hook body ${configEnvironment?.INCOMING_HOOK_BODY}`); - repoName = JSON.parse( - configEnvironment?.INCOMING_HOOK_BODY as string, - )?.repoName; - branchName = JSON.parse( - configEnvironment?.INCOMING_HOOK_BODY as string, - )?.branchName; - } - - if (!repoName) { - throw new Error('Repo name missing from deploy'); - } - if (!branchName) { - throw new Error('Branch name missing from deploy'); - } - configEnvironment.BRANCH_NAME = branchName; - configEnvironment.REPO_NAME = repoName; - - const { repo, docsetEntry, metadataEntry } = await getProperties({ + const { repo, docsetEntry, projectsEntry } = await getProperties({ branchName, repoName, dbEnvVars, poolDbName: configEnvironment.POOL_DB_NAME, environment: buildEnvironment, }); - configEnvironment.ORG = metadataEntry.github.organization; - // Set process.env SNOOTY_ENV and PREFIX_PATH environment variables for frontend to retrieve at build time - process.env.SNOOTY_ENV = buildEnvironment; - process.env.PATH_PREFIX = docsetEntry.prefix[buildEnvironment]; const { branches: branch, ...repoEntry } = repo; configEnvironment.REPO_ENTRY = repoEntry; configEnvironment.DOCSET_ENTRY = docsetEntry; configEnvironment.BRANCH_ENTRY = branch?.pop(); + configEnvironment.PROJECTS_ENTRY = projectsEntry; + + const orgName = projectsEntry.github.organization; + configEnvironment.ORG = orgName as OrganizationName; + + // Set process.env SNOOTY_ENV and PREFIX_PATH environment variables for frontend to retrieve at build time + process.env.SNOOTY_ENV = buildEnvironment; + process.env.PATH_PREFIX = docsetEntry.prefix[buildEnvironment]; + + // Prep for Snooty frontend build by cloning content repo + if (isFrontendBuild) { + console.log( + `Cloning content repo \n Repo: ${repoName}, branch: ${branchName}, github organization: ${orgName}`, + ); + await cloneContentRepo({ run, repoName, branchName, orgName }); + } console.info( 'BUILD ENVIRONMENT: ', @@ -158,7 +203,7 @@ export const updateConfig = async ({ '\n BRANCH ENTRY: ', configEnvironment.BRANCH_ENTRY, '\n METADATA ENTRY: ', - configEnvironment.METADATA_ENTRY, + configEnvironment.PROJECTS_ENTRY, '\n POOL DB NAME: ', configEnvironment.POOL_DB_NAME, '\n SEARCH DB NAME: ', diff --git a/extensions/search-manifest/src/generateAndUpload.ts b/extensions/search-manifest/src/generateAndUpload.ts new file mode 100644 index 000000000..7861b570c --- /dev/null +++ b/extensions/search-manifest/src/generateAndUpload.ts @@ -0,0 +1,103 @@ +import type { NetlifyPluginUtils } from '@netlify/build'; +import type { StaticEnvVars } from 'util/assertDbEnvVars'; +import type { ConfigEnvironmentVariables } from 'util/extension'; +import { getSearchProperties } from './uploadToAtlas/getProperties'; +import type { + SearchDBName, + BranchEntry, + DocsetsDocument, + ReposBranchesDocument, + S3UploadParams, +} from 'util/databaseConnection/types'; +import { generateManifest } from './generateManifest'; +import { uploadManifest } from './uploadToAtlas/uploadManifest'; +import { uploadManifestToS3 } from './uploadToS3/uploadManifest'; + +const EXTENSION_NAME = 'search-manifest'; + +export const generateAndUploadManifests = async ({ + configEnvironment, + run, + dbEnvVars, +}: { + configEnvironment: ConfigEnvironmentVariables; + run: NetlifyPluginUtils['run']; + dbEnvVars: StaticEnvVars; +}) => { + // Get content repo zipfile as AST representation + await run.command('unzip -o bundle.zip'); + + const branchName = configEnvironment.BRANCH; + const repoName = configEnvironment.SITE_NAME; + if (!repoName || !branchName) { + // Check that an environment variable for repo name was set + throw new Error( + 'Repo or branch name was not found, manifest cannot be uploaded to Atlas or S3 ', + ); + } + + const manifest = await generateManifest(); + + console.log('=========== Finished generating manifests ================'); + + // TODO: this should be made into its own type + const searchConnectionInfo = { + searchURI: dbEnvVars.ATLAS_SEARCH_URI, + databaseName: configEnvironment.SEARCH_DB_NAME as SearchDBName, + collectionName: dbEnvVars.DOCUMENTS_COLLECTION, + extensionName: EXTENSION_NAME, + }; + + const { + url, + searchProperty, + includeInGlobalSearch, + }: { + url: string; + searchProperty: string; + includeInGlobalSearch: boolean; + } = await getSearchProperties({ + branchEntry: configEnvironment.BRANCH_ENTRY as BranchEntry, + docsetEntry: configEnvironment.DOCSET_ENTRY as DocsetsDocument, + repoEntry: configEnvironment.REPO_ENTRY as ReposBranchesDocument, + connectionInfo: searchConnectionInfo, + }); + + const projectName = configEnvironment.REPO_ENTRY?.project; + + console.log('=========== Uploading Manifests to S3================='); + const uploadParams: S3UploadParams = { + //TODO: change based on environments + bucket: + configEnvironment.ENV === 'dotcomstg' + ? 'docs-search-indexes-test/preprd' + : 'docs-search-indexes-test/prd', + prefix: 'search-indexes/', + fileName: `${projectName}-${branchName}.json`, + manifest: manifest.export(), + }; + + const s3Status = await uploadManifestToS3({ + uploadParams, + AWS_S3_ACCESS_KEY_ID: dbEnvVars.AWS_S3_ACCESS_KEY_ID, + AWS_S3_SECRET_ACCESS_KEY: dbEnvVars.AWS_S3_SECRET_ACCESS_KEY, + }); + + console.log(`S3 upload status: ${JSON.stringify(s3Status)}`); + console.log('=========== Finished Uploading to S3 ================'); + + try { + manifest.setUrl(url); + manifest.setGlobalSearchValue(includeInGlobalSearch); + console.log('=========== Uploading Manifests to Atlas ================='); + const status = await uploadManifest({ + manifest, + searchProperty, + connectionInfo: searchConnectionInfo, + }); + console.log(status); + console.log('=========== Manifests uploaded to Atlas ================='); + } catch (e) { + console.log('Manifest could not be uploaded', e); + } +}; diff --git a/extensions/slack/src/functions/deploy-repos.mts b/extensions/slack/src/functions/deploy-repos.mts index a40c6e81f..b9f17cfbd 100644 --- a/extensions/slack/src/functions/deploy-repos.mts +++ b/extensions/slack/src/functions/deploy-repos.mts @@ -7,6 +7,7 @@ export default async (req: Request) => { return new Response('Request received without a body', { status: 401 }); } const requestBody = await new Response(req.body).text(); + console.log(requestBody); const dbEnvVars = getDbConfig(); if ( @@ -51,29 +52,53 @@ export default async (req: Request) => { // parsed?.user?.id, // ); - console.log(`Selected repos: ${JSON.stringify(selectedRepos)}`); + console.log( + `${selectedRepos.length} selected repos: ${JSON.stringify(selectedRepos)} `, + ); for (const individualRepo of selectedRepos) { const [repoName, branchName] = individualRepo.value.split('/'); const jobTitle = `Slack deploy: repoName ${repoName}, branchName ${branchName}, by ${user}`; - if (repoName && branchName) { - // TODO: DOP-5214, change value of the build hooks to env vars retrieved from dbEnvVars - console.log(`Deploying branch ${branchName} of repo ${repoName}`); - const TEST_WEBHOOK_URL = - 'https://api.netlify.com/build_hooks/673bd8c7938ade69f9530ec5?trigger_branch=main&trigger_title=deployHook+'; - const PROD_WEBHOOK_URL = - 'https://api.netlify.com/build_hooks/6744e9fd3344dd3955ccf135?trigger_branch=main&trigger_title=deployHook+'; + console.log('timeout starting', Date.now()); + await asyncTimeout(10, slackCommand, repoName, branchName, jobTitle); + console.log('timeout finished', Date.now()); + await deployRepos(slackCommand, repoName, branchName, jobTitle); + } +}; - console.log(`Deploying branch ${branchName} of repo ${repoName}`); - // Trigger build on a frontend site ('docs-frontend-dotcomstg' or 'docs-frontend-dotcomprd') depending on which modal the request was received from - const resp = await axios.post( - slackCommand === '/netlify-test-deploy' - ? `${TEST_WEBHOOK_URL}${jobTitle}` - : `${PROD_WEBHOOK_URL}${jobTitle}`, - { repoName: repoName, branchName: branchName }, - ); - return; - } +const asyncTimeout = async ( + ms: number, + slackCommand: string, + repoName: string, + branchName: string, + jobTitle: string, +) => { + return new Promise((resolve) => setTimeout(resolve, ms)); +}; + +const deployRepos = async ( + slackCommand: string, + repoName: string, + branchName: string, + jobTitle: string, +) => { + console.log('time in deploy repos', Date.now()); + + if (repoName && branchName) { + // TODO: DOP-5214, change value of the build hooks to env vars retrieved from dbEnvVars + console.log(`deploying job ${jobTitle} at ${Date.now()}`); + const TEST_WEBHOOK_URL = + 'https://api.netlify.com/build_hooks/673bd8c7938ade69f9530ec5?trigger_branch=main&trigger_title=deployHook+'; + const PROD_WEBHOOK_URL = + 'https://api.netlify.com/build_hooks/6744e9fd3344dd3955ccf135?trigger_branch=main&trigger_title=deployHook+'; + // Trigger build on a frontend site ('docs-frontend-dotcomstg' or 'docs-frontend-dotcomprd') depending on which modal the request was received from + const resp = await axios.post( + slackCommand === '/netlify-test-deploy' + ? `${TEST_WEBHOOK_URL}${jobTitle}` + : `${PROD_WEBHOOK_URL}${jobTitle}`, + { repoName: repoName, branchName: branchName }, + ); + } else { throw new Error('Missing branchName or repoName'); } }; diff --git a/libs/util/src/databaseConnection/types.ts b/libs/util/src/databaseConnection/types.ts index 522924835..25d89f0d1 100644 --- a/libs/util/src/databaseConnection/types.ts +++ b/libs/util/src/databaseConnection/types.ts @@ -17,7 +17,7 @@ export type DocsetsDocument = { prefix: EnvironmentConfig; }; -export type clusterZeroConnectionInfo = { +export type ClusterZeroConnectionInfo = { clusterZeroURI: string; databaseName: ClusterZeroDBName; collectionName: string; @@ -99,15 +99,15 @@ export type DocumentsDocument = { source: string; static_assets: Array; github_username: string; - facets?: Array; + facets?: Array; build_id: ObjectId; created_at: Date; }; -export type facet = { +export type Facet = { category: string; value: string; - sub_facets: Array; + sub_facets: Array; display_name: string; }; @@ -131,7 +131,7 @@ type AstHeadings = { depth: number; id: string; title: Array>; - // biome-ignore lint/suspicious/noExplicitAny: + // biome-ignore: selector_ids: unknown; }; diff --git a/libs/util/src/extension.ts b/libs/util/src/extension.ts index b8a71805c..3299cf923 100644 --- a/libs/util/src/extension.ts +++ b/libs/util/src/extension.ts @@ -17,6 +17,7 @@ import type { SearchDBName, SnootyDBName, ProjectsDocument, + OrganizationName, } from './databaseConnection/types'; import { getDbConfig, type StaticEnvVars } from './assertDbEnvVars'; @@ -146,11 +147,13 @@ export class Extension< // }; } -export type GithubOrganizations = 'MongoDB' | '10gen'; export type ConfigEnvironmentVariables = Partial<{ + // The name of the branch in the content repo that is being built BRANCH_NAME: string; + // Usually duplicate of BRANCH_NAME property, this is the git primitve branch that the build is being built on + BRANCH: string; REPO_NAME: string; - ORG: GithubOrganizations; + ORG: OrganizationName; SITE_NAME: string; INCOMING_HOOK_URL: string; INCOMING_HOOK_TITLE: string; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 27455a9ca..34ff20e0b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13365,7 +13365,7 @@ snapshots: '@stackbit/artisanal-names@1.0.1': {} - '@stackbit/cms-contentful@0.4.56': + '@stackbit/cms-contentful@0.4.56(graphql@16.9.0)': dependencies: '@contentful/rich-text-types': 15.15.1 '@stackbit/cms-core': 2.0.1(graphql@16.9.0) @@ -13377,6 +13377,7 @@ snapshots: transitivePeerDependencies: - debug - encoding + - graphql - supports-color '@stackbit/cms-core@2.0.1(graphql@16.9.0)': @@ -13411,7 +13412,7 @@ snapshots: - supports-color - utf-8-validate - '@stackbit/cms-git@0.4.18': + '@stackbit/cms-git@0.4.18(graphql@16.9.0)': dependencies: '@stackbit/artisanal-names': 1.0.1 '@stackbit/cms-core': 2.0.1(graphql@16.9.0) @@ -13425,11 +13426,15 @@ snapshots: micromatch: 4.0.8 slugify: 1.6.6 transitivePeerDependencies: + - bufferutil + - canvas - debug - encoding + - graphql - supports-color + - utf-8-validate - '@stackbit/cms-sanity@0.2.56': + '@stackbit/cms-sanity@0.2.56(graphql@16.9.0)': dependencies: '@sanity/block-tools': 2.36.2 '@sanity/client': 3.4.1 @@ -13449,17 +13454,18 @@ snapshots: - canvas - debug - encoding + - graphql - supports-color - utf-8-validate - '@stackbit/dev-common@0.5.51': + '@stackbit/dev-common@0.5.51(graphql@16.9.0)': dependencies: '@iarna/toml': 2.2.5 '@stackbit/artisanal-names': 1.0.1 - '@stackbit/cms-contentful': 0.4.56 + '@stackbit/cms-contentful': 0.4.56(graphql@16.9.0) '@stackbit/cms-core': 2.0.1(graphql@16.9.0) - '@stackbit/cms-git': 0.4.18 - '@stackbit/cms-sanity': 0.2.56 + '@stackbit/cms-git': 0.4.18(graphql@16.9.0) + '@stackbit/cms-sanity': 0.2.56(graphql@16.9.0) '@stackbit/sdk': 2.0.1 '@stackbit/types': 1.0.1 '@stackbit/utils': 0.4.19 @@ -13485,14 +13491,15 @@ snapshots: - canvas - debug - encoding + - graphql - supports-color - utf-8-validate '@stackbit/dev@1.0.35(graphql@16.9.0)': dependencies: '@stackbit/cms-core': 2.0.1(graphql@16.9.0) - '@stackbit/cms-git': 0.4.18 - '@stackbit/dev-common': 0.5.51 + '@stackbit/cms-git': 0.4.18(graphql@16.9.0) + '@stackbit/dev-common': 0.5.51(graphql@16.9.0) '@stackbit/sdk': 2.0.1 axios: 0.25.0 chalk: 4.1.2