Dop 5214

extensions/populate-metadata/src/getProperties.ts

@@ -8,8 +8,8 @@ import type {
   PoolDBName,
   DocsetsDocument,
   ReposBranchesDocument,
-  clusterZeroConnectionInfo,
-  ProjectMetadataDocument,
+  ClusterZeroConnectionInfo,
+  ProjectsDocument,
   ClusterZeroDBName,
 } from 'util/databaseConnection/types';
 const EXTENSION_NAME = 'populate-metadata-extension';
@@ -23,7 +23,7 @@ const getDocsetEntry = async ({
   projectName,
   environment,
 }: {
-  docsetsConnectionInfo: clusterZeroConnectionInfo;
+  docsetsConnectionInfo: ClusterZeroConnectionInfo;
   projectName: string;
   environment: Environments;
 }): Promise<DocsetsDocument> => {
@@ -56,7 +56,7 @@ const getRepoEntry = async ({
 }: {
   repoName: string;
   branchName: string;
-  connectionInfo: clusterZeroConnectionInfo;
+  connectionInfo: ClusterZeroConnectionInfo;
 }): Promise<ReposBranchesDocument> => {
   const reposBranches = await getReposBranchesCollection(connectionInfo);
 
@@ -88,13 +88,13 @@ const getRepoEntry = async ({
   return repo;
 };
 
-const getMetadataEntry = async ({
+const getProjectsEntry = async ({
   projectName,
   connectionInfo,
 }: {
   projectName: string;
-  connectionInfo: clusterZeroConnectionInfo;
-}): Promise<ProjectMetadataDocument> => {
+  connectionInfo: ClusterZeroConnectionInfo;
+}): Promise<ProjectsDocument> => {
   const projects = await getProjectsCollection(connectionInfo);
   const query = {
     name: projectName,
@@ -109,7 +109,7 @@ const getMetadataEntry = async ({
     },
   };
 
-  const projectMetadata = await projects.findOne<ProjectMetadataDocument>(
+  const projectMetadata = await projects.findOne<ProjectsDocument>(
     query,
     projection,
   );
@@ -139,7 +139,7 @@ export const getProperties = async ({
 }): Promise<{
   repo: ReposBranchesDocument;
   docsetEntry: DocsetsDocument;
-  metadataEntry: ProjectMetadataDocument;
+  projectsEntry: ProjectsDocument;
 }> => {
   const repoBranchesConnectionInfo = {
     clusterZeroURI: dbEnvVars.ATLAS_CLUSTER0_URI,
@@ -174,12 +174,12 @@ export const getProperties = async ({
     extensionName: EXTENSION_NAME,
   };
 
-  const metadataEntry = await getMetadataEntry({
+  const projectsEntry = await getProjectsEntry({
     connectionInfo: projectMetadataConnectionInfo,
     projectName: repo.project,
   });
 
   await closeClusterZeroDb();
 
-  return { repo, docsetEntry, metadataEntry };
+  return { repo, docsetEntry, projectsEntry };
 };

extensions/populate-metadata/src/index.ts

@@ -8,10 +8,11 @@ const extension = new Extension({
 
 extension.addBuildEventHandler(
   'onPreBuild',
-  async ({ netlifyConfig, dbEnvVars }) => {
+  async ({ netlifyConfig, dbEnvVars, utils: { run } }) => {
     await updateConfig({
       configEnvironment: netlifyConfig?.build?.environment,
       dbEnvVars,
+      run,
     });
   },
 );

extensions/populate-metadata/src/updateConfig.ts

@@ -3,10 +3,19 @@ import type {
   PoolDBName,
   SnootyDBName,
   SearchDBName,
+  OrganizationName,
 } from 'util/databaseConnection/types';
 import { getProperties } from './getProperties';
 import type { ConfigEnvironmentVariables } from 'util/extension';
 import type { StaticEnvVars } from 'util/assertDbEnvVars';
+import type { NetlifyPluginUtils } from '@netlify/build';
+import * as fs from 'node:fs';
+
+const FRONTEND_SITES = [
+  'docs-frontend-stg',
+  'docs-frontend-dotcomstg',
+  'docs-frontend-dotcomprd',
+];
 
 const getDbNames = (
   env: Environments,
@@ -47,15 +56,9 @@ const determineEnvironment = ({
   siteName,
 }: { isBuildHookDeploy: boolean; siteName: string }): Environments => {
   // Check if this was an engineer's build or writer's build
-  const frontendSites = [
-    'docs-frontend-stg',
-    'docs-frontend-dotcomstg',
-    'docs-frontend-dotcomprd',
-  ];
-  const isFrontendBuild = frontendSites.includes(siteName);
+  const isFrontendBuild = FRONTEND_SITES.includes(siteName);
 
   //Writer's builds = prd, everything not built on a frontend site (a site with 'Snooty' as git source)
-
   if (!isFrontendBuild) {
     return 'prd';
   }
@@ -69,17 +72,75 @@ const determineEnvironment = ({
   }
   return 'stg';
 };
+
+const cloneContentRepo = async ({
+  run,
+  repoName,
+  branchName,
+  orgName,
+}: {
+  run: NetlifyPluginUtils['run'];
+  repoName: string;
+  branchName: string;
+  orgName: string;
+}) => {
+  if (fs.existsSync(`${process.cwd()}/${repoName}`)) {
+    await run.command(`rm -r ${repoName}`);
+  }
+
+  await run.command(
+    `git clone -b ${branchName} https://${process.env.GITHUB_BOT_USERNAME}:${process.env.GITHUB_BOT_PWD}@github.com/${orgName}/${repoName}.git -s`,
+  );
+
+  if (fs.existsSync(`${repoName}/.git/config`)) {
+    // Remove git config as it stores the connection string in plain text
+    await run.command(`rm -r ${repoName}/.git/config`);
+  }
+  await run.command(`touch ${repoName}/${branchName}`);
+  await run.command(
+    `echo "${repoName}, ${branchName}" > ${repoName}/${branchName}`,
+  );
+  await run.command('ls');
+  console.log(fs.readFileSync(`${repoName}/${branchName}`));
+};
+
 export const updateConfig = async ({
   configEnvironment,
   dbEnvVars,
+  run,
 }: {
   configEnvironment: ConfigEnvironmentVariables;
   dbEnvVars: StaticEnvVars;
+  run: NetlifyPluginUtils['run'];
 }): Promise<void> => {
+  // Check if this was a build triggered by a frontend change or a content repo change
+  const isFrontendBuild = FRONTEND_SITES.includes(
+    configEnvironment.SITE_NAME as string,
+  );
   // Checks if build was triggered by a webhook
   const isBuildHookDeploy = !!(
     configEnvironment.INCOMING_HOOK_URL && configEnvironment.INCOMING_HOOK_TITLE
   );
+
+  // Determine which repository and branch to build
+  // Check if repo name and branch name have been set as environment variables through Netlify UI, allows overwriting of database name values
+  const repoName = isBuildHookDeploy
+    ? JSON.parse(configEnvironment?.INCOMING_HOOK_BODY as string)?.repoName
+    : (process.env.REPO_NAME ??
+      (process.env.REPOSITORY_URL?.split('/')?.pop() as string));
+
+  const branchName: string = isBuildHookDeploy
+    ? JSON.parse(configEnvironment?.INCOMING_HOOK_BODY as string)?.branchName
+    : (process.env.BRANCH_NAME ?? (configEnvironment.BRANCH as string));
+
+  if (!repoName || !branchName) {
+    throw new Error('Repo name or branch name missing from deploy');
+  }
+
+  configEnvironment.BRANCH_NAME = branchName;
+  configEnvironment.REPO_NAME = repoName;
+
+  // Determine which environment the build will run in
   const env = determineEnvironment({
     isBuildHookDeploy,
     siteName: configEnvironment.SITE_NAME as string,
@@ -89,6 +150,7 @@ export const updateConfig = async ({
 
   configEnvironment.ENV = buildEnvironment;
 
+  // Get the names of the databases associated with given build environment
   const { snootyDb, searchDb, poolDb } = getDbNames(buildEnvironment);
 
   // Check if values for the database names have been set as environment variables through Netlify UI
@@ -102,51 +164,34 @@ export const updateConfig = async ({
   configEnvironment.SNOOTY_DB_NAME =
     (process.env.SNOOTY_DB_NAME as SnootyDBName) ?? snootyDb;
 
-  // Check if repo name and branch name have been set as environment variables through Netlify UI
-  // Allows overwriting of database name values for testing
-  let branchName: string;
-  let repoName: string;
-  if (buildEnvironment !== 'dotcomstg' && buildEnvironment !== 'dotcomprd') {
-    branchName =
-      process.env.BRANCH_NAME ?? (configEnvironment.BRANCH as string);
-    repoName =
-      process.env.REPO_NAME ??
-      (process.env.REPOSITORY_URL?.split('/')?.pop() as string);
-  } else {
-    console.log(`Incoming hook body ${configEnvironment?.INCOMING_HOOK_BODY}`);
-    repoName = JSON.parse(
-      configEnvironment?.INCOMING_HOOK_BODY as string,
-    )?.repoName;
-    branchName = JSON.parse(
-      configEnvironment?.INCOMING_HOOK_BODY as string,
-    )?.branchName;
-  }
-
-  if (!repoName) {
-    throw new Error('Repo name missing from deploy');
-  }
-  if (!branchName) {
-    throw new Error('Branch name missing from deploy');
-  }
-  configEnvironment.BRANCH_NAME = branchName;
-  configEnvironment.REPO_NAME = repoName;
-
-  const { repo, docsetEntry, metadataEntry } = await getProperties({
+  const { repo, docsetEntry, projectsEntry } = await getProperties({
     branchName,
     repoName,
     dbEnvVars,
     poolDbName: configEnvironment.POOL_DB_NAME,
     environment: buildEnvironment,
   });
-  configEnvironment.ORG = metadataEntry.github.organization;
-  // Set process.env SNOOTY_ENV and PREFIX_PATH environment variables for frontend to retrieve at build time
-  process.env.SNOOTY_ENV = buildEnvironment;
-  process.env.PATH_PREFIX = docsetEntry.prefix[buildEnvironment];
 
   const { branches: branch, ...repoEntry } = repo;
   configEnvironment.REPO_ENTRY = repoEntry;
   configEnvironment.DOCSET_ENTRY = docsetEntry;
   configEnvironment.BRANCH_ENTRY = branch?.pop();
+  configEnvironment.PROJECTS_ENTRY = projectsEntry;
+
+  const orgName = projectsEntry.github.organization;
+  configEnvironment.ORG = orgName as OrganizationName;
+
+  // Set process.env SNOOTY_ENV and PREFIX_PATH environment variables for frontend to retrieve at build time
+  process.env.SNOOTY_ENV = buildEnvironment;
+  process.env.PATH_PREFIX = docsetEntry.prefix[buildEnvironment];
+
+  // Prep for Snooty frontend build by cloning content repo
+  if (isFrontendBuild) {
+    console.log(
+      `Cloning content repo \n Repo: ${repoName}, branch: ${branchName}, github organization: ${orgName}`,
+    );
+    await cloneContentRepo({ run, repoName, branchName, orgName });
+  }
 
   console.info(
     'BUILD ENVIRONMENT: ',
@@ -158,7 +203,7 @@ export const updateConfig = async ({
     '\n BRANCH ENTRY: ',
     configEnvironment.BRANCH_ENTRY,
     '\n METADATA ENTRY: ',
-    configEnvironment.METADATA_ENTRY,
+    configEnvironment.PROJECTS_ENTRY,
     '\n POOL DB NAME: ',
     configEnvironment.POOL_DB_NAME,
     '\n SEARCH DB NAME: ',

extensions/search-manifest/src/generateAndUpload.ts

@@ -0,0 +1,103 @@
+import type { NetlifyPluginUtils } from '@netlify/build';
+import type { StaticEnvVars } from 'util/assertDbEnvVars';
+import type { ConfigEnvironmentVariables } from 'util/extension';
+import { getSearchProperties } from './uploadToAtlas/getProperties';
+import type {
+  SearchDBName,
+  BranchEntry,
+  DocsetsDocument,
+  ReposBranchesDocument,
+  S3UploadParams,
+} from 'util/databaseConnection/types';
+import { generateManifest } from './generateManifest';
+import { uploadManifest } from './uploadToAtlas/uploadManifest';
+import { uploadManifestToS3 } from './uploadToS3/uploadManifest';
+
+const EXTENSION_NAME = 'search-manifest';
+
+export const generateAndUploadManifests = async ({
+  configEnvironment,
+  run,
+  dbEnvVars,
+}: {
+  configEnvironment: ConfigEnvironmentVariables;
+  run: NetlifyPluginUtils['run'];
+  dbEnvVars: StaticEnvVars;
+}) => {
+  // Get content repo zipfile as AST representation
+  await run.command('unzip -o bundle.zip');
+
+  const branchName = configEnvironment.BRANCH;
+  const repoName = configEnvironment.SITE_NAME;
+  if (!repoName || !branchName) {
+    // Check that an environment variable for repo name was set
+    throw new Error(
+      'Repo or branch name was not found, manifest cannot be uploaded to Atlas or S3 ',
+    );
+  }
+
+  const manifest = await generateManifest();
+
+  console.log('=========== Finished generating manifests ================');
+
+  // TODO:  this should be made into its own type
+  const searchConnectionInfo = {
+    searchURI: dbEnvVars.ATLAS_SEARCH_URI,
+    databaseName: configEnvironment.SEARCH_DB_NAME as SearchDBName,
+    collectionName: dbEnvVars.DOCUMENTS_COLLECTION,
+    extensionName: EXTENSION_NAME,
+  };
+
+  const {
+    url,
+    searchProperty,
+    includeInGlobalSearch,
+  }: {
+    url: string;
+    searchProperty: string;
+    includeInGlobalSearch: boolean;
+  } = await getSearchProperties({
+    branchEntry: configEnvironment.BRANCH_ENTRY as BranchEntry,
+    docsetEntry: configEnvironment.DOCSET_ENTRY as DocsetsDocument,
+    repoEntry: configEnvironment.REPO_ENTRY as ReposBranchesDocument,
+    connectionInfo: searchConnectionInfo,
+  });
+
+  const projectName = configEnvironment.REPO_ENTRY?.project;
+
+  console.log('=========== Uploading Manifests to S3=================');
+  const uploadParams: S3UploadParams = {
+    //TODO: change based on environments
+    bucket:
+      configEnvironment.ENV === 'dotcomstg'
+        ? 'docs-search-indexes-test/preprd'
+        : 'docs-search-indexes-test/prd',
+    prefix: 'search-indexes/',
+    fileName: `${projectName}-${branchName}.json`,
+    manifest: manifest.export(),
+  };
+
+  const s3Status = await uploadManifestToS3({
+    uploadParams,
+    AWS_S3_ACCESS_KEY_ID: dbEnvVars.AWS_S3_ACCESS_KEY_ID,
+    AWS_S3_SECRET_ACCESS_KEY: dbEnvVars.AWS_S3_SECRET_ACCESS_KEY,
+  });
+
+  console.log(`S3 upload status: ${JSON.stringify(s3Status)}`);
+  console.log('=========== Finished Uploading to S3  ================');
+
+  try {
+    manifest.setUrl(url);
+    manifest.setGlobalSearchValue(includeInGlobalSearch);
+    console.log('=========== Uploading Manifests to Atlas =================');
+    const status = await uploadManifest({
+      manifest,
+      searchProperty,
+      connectionInfo: searchConnectionInfo,
+    });
+    console.log(status);
+    console.log('=========== Manifests uploaded to Atlas =================');
+  } catch (e) {
+    console.log('Manifest could not be uploaded', e);
+  }
+};