diff --git a/.github/workflows/check-version-bump.yml b/.github/workflows/check-version-bump.yml index ff84f11..9386d35 100644 --- a/.github/workflows/check-version-bump.yml +++ b/.github/workflows/check-version-bump.yml @@ -13,7 +13,7 @@ jobs: id: check uses: EndBug/version-check@v2 with: - file-url: "https://raw.githubusercontent.com/opensearch-project/automation-app/refs/heads/${{ github.event.pull_request.base.ref }}/package.json" + file-url: 'https://raw.githubusercontent.com/opensearch-project/automation-app/refs/heads/${{ github.event.pull_request.base.ref }}/package.json' static-checking: localIsNew - name: Log when changed diff --git a/configs/resources/opensearch-project-only-org.yml b/configs/resources/opensearch-project-only-org.yml new file mode 100644 index 0000000..9459550 --- /dev/null +++ b/configs/resources/opensearch-project-only-org.yml @@ -0,0 +1,3 @@ +--- +organizations: + - name: opensearch-project diff --git a/package-lock.json b/package-lock.json index e9038c4..c6b44a9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "opensearch-automation-app", - "version": "0.1.17", + "version": "0.1.18", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "opensearch-automation-app", - "version": "0.1.17", + "version": "0.1.18", "dependencies": { "@aws-sdk/client-cloudwatch": "^3.664.0", "@aws-sdk/client-opensearch": "^3.658.1", diff --git a/package.json b/package.json index 43e42a9..bb5a82f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "opensearch-automation-app", - "version": "0.1.17", + "version": "0.1.18", "description": "An Automation App that handles all your GitHub Repository Activities", "author": "Peter Zhu", "homepage": "https://github.com/opensearch-project/automation-app", diff --git a/src/call/github-events-to-s3.ts b/src/call/github-events-to-s3.ts index 454bc98..1b3da67 100644 --- a/src/call/github-events-to-s3.ts +++ b/src/call/github-events-to-s3.ts @@ -12,30 +12,37 @@ import { Probot } from 'probot'; import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3'; -import { Resource } from '../service/resource/resource'; -import { validateResourceConfig } from '../utility/verification/verify-resource'; - -export default async function githubEventsToS3(app: Probot, context: any, resource: Resource): Promise { - if (!(await validateResourceConfig(app, context, resource))) return; +export default async function githubEventsToS3(app: Probot, context: any): Promise { + // Removed validateResourceConfig to let this function listen on all repos, and filter for only the repos that are public. + // This is done so when a new repo is made public, this app can automatically start processing its events. + // + // This is only for the s3 data lake specific case, everything else should still specify repos required to be listened in resource config. + // + // if (!(await validateResourceConfig(app, context, resource))) return; + // const repoName = context.payload.repository?.name; - const eventName = context.payload.action === undefined ? context.name : `${context.name}.${context.payload.action}`; + if (context.payload.repository?.private === false) { + const eventName = context.payload.action === undefined ? context.name : `${context.name}.${context.payload.action}`; - context.uploaded_at = new Date().toISOString(); + context.uploaded_at = new Date().toISOString(); - const now = new Date(); - const [day, month, year] = [now.getDate(), now.getMonth() + 1, now.getFullYear()].map((num) => String(num).padStart(2, '0')); + const now = new Date(); + const [day, month, year] = [now.getDate(), now.getMonth() + 1, now.getFullYear()].map((num) => String(num).padStart(2, '0')); - try { - const s3Client = new S3Client({ region: String(process.env.REGION) }); - const putObjectCommand = new PutObjectCommand({ - Bucket: String(process.env.OPENSEARCH_EVENTS_BUCKET), - Body: JSON.stringify(context), - Key: `${eventName}/${year}-${month}-${day}/${repoName}-${context.id}`, - }); - await s3Client.send(putObjectCommand); - app.log.info('GitHub Event uploaded to S3 successfully.'); - } catch (error) { - app.log.error(`Error uploading GitHub Event to S3 : ${error}`); + try { + const s3Client = new S3Client({ region: String(process.env.REGION) }); + const putObjectCommand = new PutObjectCommand({ + Bucket: String(process.env.OPENSEARCH_EVENTS_BUCKET), + Body: JSON.stringify(context), + Key: `${eventName}/${year}-${month}-${day}/${repoName}-${context.id}`, + }); + await s3Client.send(putObjectCommand); + app.log.info('GitHub Event uploaded to S3 successfully.'); + } catch (error) { + app.log.error(`Error uploading GitHub Event to S3 : ${error}`); + } + } else { + app.log.error(`Event from ${repoName} skipped because it is a private repository.`); } } diff --git a/test/call/github-events-to-s3.test.ts b/test/call/github-events-to-s3.test.ts index 0bd12d1..328e8f8 100644 --- a/test/call/github-events-to-s3.test.ts +++ b/test/call/github-events-to-s3.test.ts @@ -16,7 +16,6 @@ jest.mock('@aws-sdk/client-s3'); describe('githubEventsToS3', () => { let app: Probot; let context: any; - let resource: any; let mockS3Client: any; beforeEach(() => { @@ -33,19 +32,10 @@ describe('githubEventsToS3', () => { repository: { name: 'repo', owner: { login: 'org' }, + private: false, }, }, }; - resource = { - organizations: new Map([ - [ - 'org', - { - repositories: new Map([['repo', 'repo object']]), - }, - ], - ]), - }; mockS3Client = { send: jest.fn(), @@ -60,16 +50,36 @@ describe('githubEventsToS3', () => { it('should upload to S3 on event listened', async () => { mockS3Client.send.mockResolvedValue({}); - await githubEventsToS3(app, context, resource); + await githubEventsToS3(app, context); expect(mockS3Client.send).toHaveBeenCalledWith(expect.any(PutObjectCommand)); expect(app.log.info).toHaveBeenCalledWith('GitHub Event uploaded to S3 successfully.'); }); + it('should not upload to S3 on event listened on private repo', async () => { + context = { + name: 'name', + id: 'id', + payload: { + repository: { + name: 'repo', + owner: { login: 'org' }, + private: true, + }, + }, + }; + mockS3Client.send.mockResolvedValue({}); + + await githubEventsToS3(app, context); + + expect(mockS3Client.send).not.toHaveBeenCalledWith(expect.any(PutObjectCommand)); + expect(app.log.error).toHaveBeenCalledWith('Event from repo skipped because it is a private repository.'); + }); + it('should log an error if S3 upload fails', async () => { mockS3Client.send.mockRejectedValue(new Error('S3 error')); - await githubEventsToS3(app, context, resource); + await githubEventsToS3(app, context); expect(app.log.error).toHaveBeenCalledWith('Error uploading GitHub Event to S3 : Error: S3 error'); }); @@ -82,6 +92,7 @@ describe('githubEventsToS3', () => { repository: { name: 'repo', owner: { login: 'org' }, + private: false, }, action: 'action', }, @@ -92,7 +103,7 @@ describe('githubEventsToS3', () => { jest.spyOn(Date.prototype, 'getFullYear').mockReturnValue(2024); jest.spyOn(Date.prototype, 'toISOString').mockReturnValue('2024-10-04T21:00:06.875Z'); - await githubEventsToS3(app, context, resource); + await githubEventsToS3(app, context); expect(PutObjectCommand).toHaveBeenCalledWith( expect.objectContaining({ @@ -110,6 +121,7 @@ describe('githubEventsToS3', () => { repository: { name: 'repo', owner: { login: 'org' }, + private: false, }, }, }; @@ -119,7 +131,7 @@ describe('githubEventsToS3', () => { jest.spyOn(Date.prototype, 'getFullYear').mockReturnValue(2024); jest.spyOn(Date.prototype, 'toISOString').mockReturnValue('2024-10-04T21:00:06.875Z'); - await githubEventsToS3(app, context, resource); + await githubEventsToS3(app, context); expect(PutObjectCommand).toHaveBeenCalledWith( expect.objectContaining({ diff --git a/test/utility/probot/octokit.test.ts b/test/utility/probot/octokit.test.ts index b245ba9..212c5c8 100644 --- a/test/utility/probot/octokit.test.ts +++ b/test/utility/probot/octokit.test.ts @@ -13,7 +13,7 @@ import { Probot, ProbotOctokit, Logger } from 'probot'; describe('octokitFunctions', () => { let app: Probot; let installationId: number; - let octokitMock: ProbotOctokit + let octokitMock: ProbotOctokit; beforeEach(() => { app = new Probot({ appId: 1, secret: 'test', privateKey: 'test' });