refactor: migrate server to typescript, closes #213

- reorganize code files
- adapt file layout of common to be compatible with tsc's quirks
- introduce new script entry `server:dev`

common/package.json

@@ -6,6 +6,8 @@
   "homepage": "",
   "license": "ISC",
   "exports": {
+    "node": "./dist/index.js",
+    "import": "./index.ts",
     "default": "./dist/index.js"
   },
   "type": "module",
@@ -14,7 +16,7 @@
     "test": "__tests__"
   },
   "files": [
-    "dist"
+    "dist/"
   ],
   "publishConfig": {
     "access": "public"

common/tsconfig.json

@@ -1,5 +1,7 @@
 {
-  "include": ["lib/*"],
+  "include": [
+    "index.ts"
+  ],
   "compilerOptions": {
     "declaration": true,
     "module": "commonjs",

graphql.config.yml

@@ -0,0 +1,9 @@
+schema: schema.graphql
+include: schema.graphql
+extensions:
+  endpoints:
+    Default GraphQL Endpoint:
+      url: http://localhost:3001/graphql
+      headers:
+        user-agent: JS GraphQL
+      introspect: false

package.json

@@ -29,6 +29,7 @@
     "client:start:web": "(cd client && npm run start:web)",
     "common:build": "(cd common && npm run build)",
     "server:start": "(cd server && npm run start)",
+    "server:dev": "(cd server && npm run dev)",
     "util:generate-types": "(cd server && npm run generate-schema) && (cd client && npm run generate-graphql-types)",
     "util:create-release": "npx lerna version --conventional-commits --force-publish",
     "version": "npx lerna exec npm i -- --package-lock-only --ignore-scripts && git add **/package-lock.json"

server/.gitignore

@@ -0,0 +1 @@
+/build/

server/lib/cache/cacheModel.ts

@@ -0,0 +1,9 @@
+import mongoose from 'mongoose'
+import { withBaseSchema } from '../util/baseObject'
+import { CacheType } from './cacheType'
+
+const cacheSchema = withBaseSchema<CacheType>({
+  user: { type: String, ref: 'User', index: true },
+  value: {},
+})
+export const Cache = mongoose.model('Cache', cacheSchema)

server/lib/cache/cacheResolvers.ts

@@ -0,0 +1,9 @@
+import { entitiesUpdatedSince } from './methods/entitiesUpdatedSince'
+
+export const cacheResolvers = {
+  Query: {
+    async entitiesUpdatedSince(root, { cacheId }, { user }) {
+      return entitiesUpdatedSince(cacheId, user)
+    },
+  },
+}

server/lib/cache/cacheSchema.ts

@@ -0,0 +1,17 @@
+import gql from 'graphql-tag'
+
+export const cacheSchema = gql`
+  type EntitiesUpdatedSince {
+    addedNotes: [Note!]!
+    updatedNotes: [Note!]!
+    removedNoteIds: [ID!]!
+    addedTags: [Tag!]!
+    updatedTags: [Tag!]!
+    removedTagIds: [ID!]!
+    cacheId: ID!
+  }
+
+  extend type Query {
+    entitiesUpdatedSince(cacheId: ID): EntitiesUpdatedSince!
+  }
+`

server/lib/cache/cacheType.ts

@@ -0,0 +1,5 @@
+import { BaseType } from '../util/baseObject'
+
+export type CacheType = BaseType & {
+  value: any
+}

server/lib/cache/methods/cacheDiff.ts

@@ -0,0 +1,28 @@
+import { BaseType } from '../../util/baseObject'
+import { cacheGetPatch } from './cacheGetPatch'
+
+export const cacheDiff = <T extends BaseType>(
+  entities: Array<T>,
+  cachedIds: Array<string>,
+  { cacheUpdatedAt }: { cacheUpdatedAt: Date },
+) => {
+  const entitiesPatch = cacheGetPatch(entities, cachedIds)
+  const cachePatch = {
+    added: [],
+    removedIds: [],
+    updated: [],
+    patch: entitiesPatch,
+  }
+  entitiesPatch.forEach((patch) => {
+    if (patch.type === 'add') {
+      cachePatch.added = cachePatch.added.concat(patch.items)
+    } else {
+      cachePatch.removedIds = cachePatch.removedIds.concat(patch.items)
+    }
+  })
+  cachePatch.updated = entities.filter(
+    (entity) =>
+      entity.updatedAt > cacheUpdatedAt && !cachePatch.added.includes(entity),
+  )
+  return cachePatch
+}

server/lib/cache/methods/cacheGetPatch.ts

@@ -0,0 +1,28 @@
+import { getPatch } from 'fast-array-diff'
+import { BaseType } from '../../util/baseObject'
+
+type DiffComparator<T> = (cacheEntry: string, dataEntry: T) => boolean
+
+export const cacheGetPatch = <T extends BaseType>(
+  data: Array<T>,
+  cache: Array<string>,
+  {
+    checkAgainstId,
+    customDiffComparator,
+  }: {
+    checkAgainstId: boolean
+    customDiffComparator: DiffComparator<T>
+  } = {
+    checkAgainstId: true,
+    customDiffComparator: undefined,
+  },
+) => {
+  let diffComparator: DiffComparator<T>
+  if (customDiffComparator) {
+    diffComparator = customDiffComparator
+  } else if (checkAgainstId) {
+    // @ts-ignore
+    diffComparator = (cacheEntry, dataEntry) => dataEntry._id.equals(cacheEntry)
+  }
+  return getPatch(cache, data as any, diffComparator as any)
+}

server/lib/cache/methods/entitiesUpdatedSince.ts

@@ -0,0 +1,146 @@
+import { baseNotesQuery, leanTypeEnumFixer } from '../../notes/notesMethods'
+import { Note } from '../../notes/notesModels'
+import { Tag } from '../../tags/tagModel'
+import { baseTagsQuery } from '../../tags/tagsMethods'
+import { Cache } from '../cacheModel'
+import { cacheDiff } from './cacheDiff'
+import { updateCacheFromDiff } from './updateCacheFromDiff'
+
+export const entitiesUpdatedSince = async (cacheId, user) => {
+  if (!user) {
+    throw new Error('Need to be logged in to fetch links.')
+  }
+
+  let cache = (await Cache.findOne({ _id: cacheId, user }).lean()) || {
+    _id: undefined,
+    value: {
+      noteIds: [],
+      tagIds: [],
+    },
+    updatedAt: new Date(0),
+  }
+  const cacheUpdatedAt = cache.updatedAt
+
+  const entityQueryProjection = cache._id
+    ? { _id: 1, updatedAt: 1, createdAt: 1 }
+    : {}
+
+  const fetchNotes = async (transformFilters = (filter) => filter) => {
+    let notesLookup = Note.find(
+      transformFilters({
+        ...(await baseNotesQuery(user, 'read')),
+        deletedAt: null,
+      }),
+      entityQueryProjection,
+    )
+      .sort({ createdAt: -1 })
+      .lean()
+
+    if (!cache._id) {
+      notesLookup = notesLookup.populate('tags')
+    }
+
+    return notesLookup.exec().then(leanTypeEnumFixer)
+  }
+  const notes = await fetchNotes()
+
+  const tags = await Tag.find(
+    {
+      ...baseTagsQuery(user, 'read'),
+    },
+    entityQueryProjection,
+  )
+    .lean()
+    .exec()
+
+  const notesDiff = cacheDiff(notes, cache.value.noteIds, {
+    cacheUpdatedAt,
+  })
+  const tagsDiff = cacheDiff(tags, cache.value.tagIds, {
+    cacheUpdatedAt,
+  })
+
+  if (cache._id) {
+    if (tagsDiff.added.length) {
+      tagsDiff.added = await Tag.find({
+        _id: tagsDiff.added.map(({ _id }) => _id),
+      })
+
+      await Promise.all(
+        tagsDiff.added.map(async (tag) => {
+          // if a shared tag is added (appears for the first time) all notes on which it appears
+          // must be treated as updated for other users (because it could have already been added
+          // and then shared, which does not make notes with the tag 'updatedAt'!
+          const newlySharedTagIds = []
+          if (tag.user !== user._id) {
+            newlySharedTagIds.push(tag._id)
+          }
+          if (newlySharedTagIds.length) {
+            notesDiff.updated = notesDiff.updated.concat(
+              (
+                await fetchNotes((filter) => ({
+                  ...filter,
+                  tags: {
+                    $in: filter.tags.$in.concat(newlySharedTagIds),
+                  },
+                }))
+              ).map(({ _id }) => _id),
+            )
+          }
+        }),
+      )
+    }
+    if (tagsDiff.updated.length) {
+      tagsDiff.updated = await Tag.find({
+        _id: tagsDiff.updated.map(({ _id }) => _id),
+      })
+    }
+    if (notesDiff.added.length) {
+      notesDiff.added = await Note.find({
+        _id: notesDiff.added.map(({ _id }) => _id),
+      })
+        .populate('tags')
+        .lean()
+    }
+    if (notesDiff.updated.length) {
+      notesDiff.updated = await Note.find({
+        _id: notesDiff.updated.map(({ _id }) => _id),
+      })
+        .populate('tags')
+        .lean()
+    }
+  }
+
+  let cacheIdWritten
+  if (cache._id) {
+    await updateCacheFromDiff(user, cache._id, 'noteIds', notesDiff)
+    await updateCacheFromDiff(user, cache._id, 'tagIds', tagsDiff)
+
+    await Cache.collection.updateOne(
+      { _id: cache._id, user },
+      {
+        $set: {
+          updatedAt: new Date(),
+        },
+      },
+    )
+    cacheIdWritten = cache._id
+  } else {
+    const cacheWriteValue = {
+      noteIds: notesDiff.added.map(({ _id }) => _id),
+      tagIds: tagsDiff.added.map(({ _id }) => _id),
+    }
+    cacheIdWritten = (await new Cache({ value: cacheWriteValue, user }).save())
+      ._id
+  }
+
+  return {
+    addedNotes: notesDiff.added,
+    updatedNotes: notesDiff.updated,
+    removedNoteIds: notesDiff.removedIds,
+    addedTags: tagsDiff.added,
+    updatedTags: tagsDiff.updated,
+    removedTagIds: tagsDiff.removedIds,
+    cacheId: cacheIdWritten,
+  }
+}

server/lib/cache/methods/updateCacheFromDiff.ts

@@ -0,0 +1,36 @@
+import { Cache } from '../cacheModel'
+
+export const updateCacheFromDiff = async (
+  user,
+  cacheId,
+  cacheFieldName,
+  diff,
+) => {
+  const cacheValueFieldName = `value.${cacheFieldName}`
+
+  for (const patch of diff.patch) {
+    if (patch.type === 'add') {
+      await Cache.updateOne(
+        { _id: cacheId, user },
+        {
+          $push: {
+            [cacheValueFieldName]: {
+              $each: patch.items.map(({ _id }) => _id),
+              $position: patch.newPos,
+            },
+          },
+        },
+      )
+    } else {
+      // todo: could join all removes!
+      await Cache.updateOne(
+        { _id: cacheId, user },
+        {
+          $pull: {
+            [cacheValueFieldName]: { $in: patch.items.map(({ _id }) => _id) },
+          },
+        },
+      )
+    }
+  }
+}