Contribution cron job

README.md

@@ -64,6 +64,18 @@
 
 - Generate a Client Secret ID from GitHub and use it to replace the value of `GITHUB_SECRET` within your `.env.local`
 
+## Setting up the Github Access Token
+
+- Login to GitHub, then at the top right click on the icon and go to settings.
+
+- Find the developer settings, then click on Personal access tokens, and create a new access token.
+
+- Name the token something descriptive, no additional options need to be selected.
+
+- Click the Generate token button.
+
+- Next, copy the token and replace the value of `GITHUB_TOKEN` within your `.env.local`
+
 ## Setting up Discord OAuth
 
 - Go to the [Discord Develop Portal](https://discord.com/developers/applications) and log in.

packages/api/package.json

@@ -18,13 +18,18 @@
     "@americanairlines/simple-env": "^1.0.4",
     "@mikro-orm/core": "^4.5.9",
     "@mikro-orm/postgresql": "^4.5.9",
+    "@types/cron": "^1.7.3",
     "@types/express-session": "^1.17.4",
+    "@types/luxon": "^2.0.7",
     "@x/web": "*",
     "cookie-parser": "^1.4.5",
+    "cron": "^1.8.2",
     "dotenv-flow": "^3.2.0",
     "express": "^4.17.1",
     "express-session": "^1.17.2",
+    "fetch-mock": "^9.11.0",
     "fetch-mock-jest": "^1.5.1",
+    "luxon": "^2.1.1",
     "passport": "^0.5.0",
     "passport-github2": "^0.1.12",
     "readline-sync": "^1.4.10",

packages/api/src/__mocks__/env.ts

@@ -3,6 +3,7 @@ export const env = {
   port: '3000',
   githubId: 'mock-client-id',
   githubSecretId: 'Secrete_secret',
+  githubToken: 'mock-token',
   discordClientId: 'mock-discord-id',
   discordSecretId: 'discord-secret',
 };

packages/api/src/cronJobs.ts

@@ -0,0 +1,73 @@
+import { EntityManager } from '@mikro-orm/core';
+import { PostgreSqlDriver } from '@mikro-orm/postgresql';
+import { DateTime } from 'luxon';
+import { Contribution } from './entities/Contribution';
+import { Project } from './entities/Project';
+import { User } from './entities/User';
+import logger from './logger';
+import { searchForContributions } from './utils/github/searchForContributions';
+
+export const contributionPoll = async (entityManager: EntityManager<PostgreSqlDriver>) => {
+  try {
+    const upperBoundDate = DateTime.now();
+
+    const lastCheckedThreshold = upperBoundDate.minus({ hours: 6 });
+
+    const userList = await entityManager.find(User, {
+      contributionsLastCheckedAt: { $lt: lastCheckedThreshold.toJSDate() },
+    });
+
+    if (userList.length === 0) {
+      logger.info('No users need to have their contributions updated');
+      return;
+    }
+
+    const projectList = await entityManager.find(Project, {});
+
+    if (projectList.length === 0) {
+      logger.info('No projects found for contribution polling');
+      return;
+    }
+
+    const lastCheckedTimes = userList.map((u) => DateTime.fromJSDate(u.contributionsLastCheckedAt));
+
+    const lowerBoundDate = DateTime.min(...lastCheckedTimes);
+
+    for (const contribution of await searchForContributions(
+      projectList,
+      lowerBoundDate,
+      upperBoundDate,
+      userList,
+    )) {
+      // Only Add new contributions
+      if (!(await entityManager.count(Contribution, { nodeID: { $eq: contribution.id } }))) {
+        const user = userList.find((u) => u.githubUsername === contribution.author.login);
+
+        if (user) {
+          const newContribution = new Contribution({
+            nodeID: contribution.id,
+            authorGithubId: user.githubId,
+            type: 'Pull Request',
+            description: contribution.title,
+            contributedAt: new Date(Date.parse(contribution.mergedAt)),
+            score: 100,
+          });
+          entityManager.persist(newContribution);
+        }
+      }
+    }
+
+    for (const user of userList) {
+      user.contributionsLastCheckedAt = upperBoundDate.toJSDate();
+      entityManager.persist(user);
+    }
+
+    // Save new info in the db
+    await entityManager.flush();
+    logger.info('Successfully polled and saved new PR contributions');
+    return;
+  } catch (error) {
+    const errorMessage = 'There was an issue saving contribution data to the database';
+    logger.error(errorMessage, error);
+  }
+};

packages/api/src/entities/Contribution.ts

@@ -7,9 +7,12 @@ export type ContributionConstructorValues = ConstructorValues<Contribution>;
 
 @Entity()
 export class Contribution extends Node<Contribution> {
-  @Property({ columnType: 'text' })
+  @Property({ columnType: 'text', unique: true })
   nodeID: string;
 
+  @Property({ columnType: 'text' })
+  authorGithubId: string;
+
   @Property({ columnType: 'text' })
   description: string;
 
@@ -24,6 +27,7 @@ export class Contribution extends Node<Contribution> {
 
   constructor({
     nodeID,
+    authorGithubId,
     description,
     type,
     score,
@@ -32,6 +36,7 @@ export class Contribution extends Node<Contribution> {
   }: ContributionConstructorValues) {
     super(extraValues);
     this.nodeID = nodeID;
+    this.authorGithubId = authorGithubId;
     this.description = description;
     this.type = type;
     this.score = score;

packages/api/src/entities/User.ts

@@ -13,6 +13,9 @@ export class User extends Node<User> {
   @Property({ columnType: 'text' })
   githubId: string;
 
+  @Property({ columnType: 'text' })
+  githubUsername: string;
+
   @Property({ columnType: 'text', nullable: true })
   discordId?: string;
 
@@ -43,16 +46,18 @@ export class User extends Node<User> {
   @Property({ columnType: 'text' })
   email: string;
 
-  @Property({ columnType: 'timestamp', nullable: true })
-  contributionsLastCheckedAt?: Date;
+  @Property({ columnType: 'timestamp' })
+  contributionsLastCheckedAt: Date;
 
   constructor({
     name,
     githubId,
+    githubUsername,
     hireable,
     purpose,
     isAdmin,
     email,
+    contributionsLastCheckedAt,
     ...extraValues
   }: UserConstructorValues) {
     super(extraValues);
@@ -61,7 +66,9 @@ export class User extends Node<User> {
     this.hireable = hireable;
     this.purpose = purpose;
     this.githubId = githubId;
+    this.githubUsername = githubUsername;
     this.isAdmin = isAdmin ?? false;
     this.email = email;
+    this.contributionsLastCheckedAt = contributionsLastCheckedAt;
   }
 }

packages/api/src/env.ts

@@ -13,6 +13,7 @@ export const env = setEnv({
     discordSecretId: 'DISCORD_SECRET_ID',
     githubClientId: 'GITHUB_CLIENT_ID',
     githubSecret: 'GITHUB_SECRET',
+    githubToken: 'GITHUB_TOKEN',
     appUrl: 'APP_URL',
   },
   optional: {

packages/api/src/index.ts

@@ -1,18 +1,21 @@
 /* istanbul ignore file */
 import express, { Handler } from 'express';
 import { web } from '@x/web';
-import { EntityManager } from '@mikro-orm/core';
+import { EntityManager, MikroORM } from '@mikro-orm/core';
 import { PostgreSqlDriver } from '@mikro-orm/postgresql';
 import session from 'express-session';
 import passport from 'passport';
+import { CronJob } from 'cron';
 import { env } from './env';
 import { api } from './api';
 import { initDatabase } from './database';
 import logger from './logger';
 import { User } from './entities/User';
+import { contributionPoll } from './cronJobs';
 
 const app = express();
 const port = Number(env.port ?? '') || 3000;
+let orm: MikroORM<PostgreSqlDriver>;
 const dev = env.nodeEnv === 'development';
 app.use(express.json());
 
@@ -84,9 +87,11 @@ passport.use(
               const newUser = new User({
                 name: profile.username,
                 githubId: profile.id,
+                githubUsername: profile.username,
                 hireable: false,
                 purpose: '',
                 email,
+                contributionsLastCheckedAt: new Date(),
               });
               await authEm?.persistAndFlush(newUser);
               logger.info(`added ${newUser} to the databse`);
@@ -99,6 +104,8 @@ passport.use(
           return done(newUserCreationError, null);
         }
       } else {
+        currentUser.githubUsername = profile.username;
+        await authEm?.flush();
         return done(null, { profile, githubToken: accessToken });
       }
       return done(null, null);
@@ -107,7 +114,7 @@ passport.use(
 );
 
 void (async () => {
-  const orm = await initDatabase();
+  orm = await initDatabase();
   authEm = authEm ?? orm.em.fork();
 
   app.use(
@@ -129,6 +136,14 @@ void (async () => {
   .then(() => {
     app.listen(port, () => {
       logger.info(`🚀 Listening at http://localhost:${port}`);
+
+      const contributionPollCronJob = new CronJob(
+        '0 0,6,12,18 * * *',
+        () => contributionPoll(orm.em.fork()),
+        null,
+        false,
+      ); // every six hours every day
+      contributionPollCronJob.start();
     });
   })
   .catch((err) => {

packages/api/src/migrations/0007-add-github-username-to-user-table.ts

@@ -0,0 +1,11 @@
+import { Migration } from '@mikro-orm/migrations';
+
+export class Migration20211117105116 extends Migration {
+  async up(): Promise<void> {
+    this.addSql('alter table "user" add column "githubUsername" text not null;');
+  }
+
+  async down(): Promise<void> {
+    this.addSql('alter table "user" drop column "githubUsername";');
+  }
+}

packages/api/src/migrations/0008-make-contribution-node-id-unique.ts

@@ -0,0 +1,11 @@
+import { Migration } from '@mikro-orm/migrations';
+
+export class Migration20211118060304 extends Migration {
+  async up(): Promise<void> {
+    this.addSql('alter table "contribution" add constraint "contribution_nodeID_unique" unique ("nodeID");');
+  }
+
+  async down(): Promise<void> {
+    this.addSql('alter table "contribution" drop constraint "contribution_nodeID_unique";');
+  }
+}

packages/api/src/migrations/0009-add-author-github-id-to-contribution-table.ts

@@ -0,0 +1,11 @@
+import { Migration } from '@mikro-orm/migrations';
+
+export class Migration20211118121747 extends Migration {
+  async up(): Promise<void> {
+    this.addSql('alter table "contribution" add column "authorGithubId" text not null;');
+  }
+
+  async down(): Promise<void> {
+    this.addSql('alter table "contribution" drop column "authorGithubId";');
+  }
+}

packages/api/src/utils/github/buildProjectsQuery.ts

@@ -0,0 +1,60 @@
+import { Project } from '../../entities/Project';
+import { env } from '../../env';
+import logger from '../../logger';
+
+interface Repository {
+  id: string;
+  nameWithOwner: string;
+}
+
+interface RepositoryResponse {
+  data: {
+    nodes: Repository[];
+  };
+}
+
+export const buildProjectsQuery = async (projects: Project[]) => {
+  const repoNodeIds = projects.map((project) => project.nodeID);
+
+  const findReposQuery = JSON.stringify({
+    query: `
+      query FindRepositories($repoIds: [ID!]!) { 
+        nodes(ids: $repoIds){
+          ... on Repository {
+            id
+            nameWithOwner
+          }    
+        }
+      }
+    `,
+    variables: {
+      repoIds: repoNodeIds,
+    },
+  });
+
+  try {
+    const fetchRes = await fetch('https://api.github.com/graphql', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        Authorization: `bearer ${env.githubToken}`,
+      },
+      body: findReposQuery,
+    });
+
+    if (!fetchRes.ok) {
+      logger.error('The repository query to GitHub has failed');
+      throw new Error(await fetchRes.text());
+    }
+
+    const { data: responseData }: RepositoryResponse = await fetchRes.json();
+
+    const projectsQuery = responseData.nodes.map((repo) => `repo:${repo.nameWithOwner}`).join(' ');
+
+    return projectsQuery;
+  } catch (error) {
+    const errorMessage = 'There was an issue getting repository info from graphql';
+    logger.error(errorMessage, error);
+    return null;
+  }
+};