Remove resource config validation for github-events-to-s3 call (#36)

* Remove resource config validation for github-events-to-s3 call Signed-off-by: Brandon Shien <[email protected]> * Added private repo error log Signed-off-by: Brandon Shien <[email protected]> --------- Signed-off-by: Brandon Shien <[email protected]>
opensearch-project · Nov 7, 2024 · 4a90494 · 4a90494
1 parent b3e35e6
commit 4a90494
Show file tree

Hide file tree

Showing 7 changed files with 62 additions and 40 deletions.
diff --git a/.github/workflows/check-version-bump.yml b/.github/workflows/check-version-bump.yml
@@ -13,7 +13,7 @@ jobs:
         id: check
         uses: EndBug/version-check@v2
         with:
-          file-url: "https://raw.githubusercontent.com/opensearch-project/automation-app/refs/heads/${{ github.event.pull_request.base.ref }}/package.json"
+          file-url: 'https://raw.githubusercontent.com/opensearch-project/automation-app/refs/heads/${{ github.event.pull_request.base.ref }}/package.json'
           static-checking: localIsNew
 
       - name: Log when changed

diff --git a/configs/resources/opensearch-project-only-org.yml b/configs/resources/opensearch-project-only-org.yml
@@ -0,0 +1,3 @@
+---
+organizations:
+  - name: opensearch-project
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "opensearch-automation-app",
-  "version": "0.1.17",
+  "version": "0.1.18",
   "description": "An Automation App that handles all your GitHub Repository Activities",
   "author": "Peter Zhu",
   "homepage": "https://github.com/opensearch-project/automation-app",

diff --git a/src/call/github-events-to-s3.ts b/src/call/github-events-to-s3.ts
@@ -12,30 +12,37 @@
 
 import { Probot } from 'probot';
 import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
-import { Resource } from '../service/resource/resource';
-import { validateResourceConfig } from '../utility/verification/verify-resource';
-
-export default async function githubEventsToS3(app: Probot, context: any, resource: Resource): Promise<void> {
-  if (!(await validateResourceConfig(app, context, resource))) return;
 
+export default async function githubEventsToS3(app: Probot, context: any): Promise<void> {
+  // Removed validateResourceConfig to let this function listen on all repos, and filter for only the repos that are public.
+  // This is done so when a new repo is made public, this app can automatically start processing its events.
+  //
+  // This is only for the s3 data lake specific case, everything else should still specify repos required to be listened in resource config.
+  //
+  // if (!(await validateResourceConfig(app, context, resource))) return;
+  //
   const repoName = context.payload.repository?.name;
-  const eventName = context.payload.action === undefined ? context.name : `${context.name}.${context.payload.action}`;
+  if (context.payload.repository?.private === false) {
+    const eventName = context.payload.action === undefined ? context.name : `${context.name}.${context.payload.action}`;
 
-  context.uploaded_at = new Date().toISOString();
+    context.uploaded_at = new Date().toISOString();
 
-  const now = new Date();
-  const [day, month, year] = [now.getDate(), now.getMonth() + 1, now.getFullYear()].map((num) => String(num).padStart(2, '0'));
+    const now = new Date();
+    const [day, month, year] = [now.getDate(), now.getMonth() + 1, now.getFullYear()].map((num) => String(num).padStart(2, '0'));
 
-  try {
-    const s3Client = new S3Client({ region: String(process.env.REGION) });
-    const putObjectCommand = new PutObjectCommand({
-      Bucket: String(process.env.OPENSEARCH_EVENTS_BUCKET),
-      Body: JSON.stringify(context),
-      Key: `${eventName}/${year}-${month}-${day}/${repoName}-${context.id}`,
-    });
-    await s3Client.send(putObjectCommand);
-    app.log.info('GitHub Event uploaded to S3 successfully.');
-  } catch (error) {
-    app.log.error(`Error uploading GitHub Event to S3 : ${error}`);
+    try {
+      const s3Client = new S3Client({ region: String(process.env.REGION) });
+      const putObjectCommand = new PutObjectCommand({
+        Bucket: String(process.env.OPENSEARCH_EVENTS_BUCKET),
+        Body: JSON.stringify(context),
+        Key: `${eventName}/${year}-${month}-${day}/${repoName}-${context.id}`,
+      });
+      await s3Client.send(putObjectCommand);
+      app.log.info('GitHub Event uploaded to S3 successfully.');
+    } catch (error) {
+      app.log.error(`Error uploading GitHub Event to S3 : ${error}`);
+    }
+  } else {
+    app.log.error(`Event from ${repoName} skipped because it is a private repository.`);
   }
 }
diff --git a/test/call/github-events-to-s3.test.ts b/test/call/github-events-to-s3.test.ts
@@ -16,7 +16,6 @@ jest.mock('@aws-sdk/client-s3');
 describe('githubEventsToS3', () => {
   let app: Probot;
   let context: any;
-  let resource: any;
   let mockS3Client: any;
 
   beforeEach(() => {
@@ -33,19 +32,10 @@ describe('githubEventsToS3', () => {
         repository: {
           name: 'repo',
           owner: { login: 'org' },
+          private: false,
         },
       },
     };
-    resource = {
-      organizations: new Map([
-        [
-          'org',
-          {
-            repositories: new Map([['repo', 'repo object']]),
-          },
-        ],
-      ]),
-    };
 
     mockS3Client = {
       send: jest.fn(),
@@ -60,16 +50,36 @@ describe('githubEventsToS3', () => {
   it('should upload to S3 on event listened', async () => {
     mockS3Client.send.mockResolvedValue({});
 
-    await githubEventsToS3(app, context, resource);
+    await githubEventsToS3(app, context);
 
     expect(mockS3Client.send).toHaveBeenCalledWith(expect.any(PutObjectCommand));
     expect(app.log.info).toHaveBeenCalledWith('GitHub Event uploaded to S3 successfully.');
   });
 
+  it('should not upload to S3 on event listened on private repo', async () => {
+    context = {
+      name: 'name',
+      id: 'id',
+      payload: {
+        repository: {
+          name: 'repo',
+          owner: { login: 'org' },
+          private: true,
+        },
+      },
+    };
+    mockS3Client.send.mockResolvedValue({});
+
+    await githubEventsToS3(app, context);
+
+    expect(mockS3Client.send).not.toHaveBeenCalledWith(expect.any(PutObjectCommand));
+    expect(app.log.error).toHaveBeenCalledWith('Event from repo skipped because it is a private repository.');
+  });
+
   it('should log an error if S3 upload fails', async () => {
     mockS3Client.send.mockRejectedValue(new Error('S3 error'));
 
-    await githubEventsToS3(app, context, resource);
+    await githubEventsToS3(app, context);
 
     expect(app.log.error).toHaveBeenCalledWith('Error uploading GitHub Event to S3 : Error: S3 error');
   });
@@ -82,6 +92,7 @@ describe('githubEventsToS3', () => {
         repository: {
           name: 'repo',
           owner: { login: 'org' },
+          private: false,
         },
         action: 'action',
       },
@@ -92,7 +103,7 @@ describe('githubEventsToS3', () => {
     jest.spyOn(Date.prototype, 'getFullYear').mockReturnValue(2024);
     jest.spyOn(Date.prototype, 'toISOString').mockReturnValue('2024-10-04T21:00:06.875Z');
 
-    await githubEventsToS3(app, context, resource);
+    await githubEventsToS3(app, context);
 
     expect(PutObjectCommand).toHaveBeenCalledWith(
       expect.objectContaining({
@@ -110,6 +121,7 @@ describe('githubEventsToS3', () => {
         repository: {
           name: 'repo',
           owner: { login: 'org' },
+          private: false,
         },
       },
     };
@@ -119,7 +131,7 @@ describe('githubEventsToS3', () => {
     jest.spyOn(Date.prototype, 'getFullYear').mockReturnValue(2024);
     jest.spyOn(Date.prototype, 'toISOString').mockReturnValue('2024-10-04T21:00:06.875Z');
 
-    await githubEventsToS3(app, context, resource);
+    await githubEventsToS3(app, context);
 
     expect(PutObjectCommand).toHaveBeenCalledWith(
       expect.objectContaining({

diff --git a/test/utility/probot/octokit.test.ts b/test/utility/probot/octokit.test.ts
@@ -13,7 +13,7 @@ import { Probot, ProbotOctokit, Logger } from 'probot';
 describe('octokitFunctions', () => {
   let app: Probot;
   let installationId: number;
-  let octokitMock: ProbotOctokit
+  let octokitMock: ProbotOctokit;
 
   beforeEach(() => {
     app = new Probot({ appId: 1, secret: 'test', privateKey: 'test' });