add higher-level db migration tests

.github/workflows/db-migrations.yml

@@ -0,0 +1,38 @@
+name: Database Migrations
+
+on:
+  push:
+    paths:
+    - .github/workflows/db-migrations.yml
+    - lib/model/migrations/**
+    - test/db-migrations/**
+
+jobs:
+  standard-tests:
+    timeout-minutes: 2
+    # TODO should we use the same container as circle & central?
+    runs-on: ubuntu-latest
+    services:
+      # see: https://docs.github.com/en/[email protected]/actions/using-containerized-services/creating-postgresql-service-containers
+      postgres:
+        image: postgres:14.10
+        env:
+          POSTGRES_PASSWORD: odktest
+        ports:
+          - 5432:5432
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    steps:
+    - uses: actions/checkout@v4
+    - name: Use Node.js 20
+      uses: actions/setup-node@v4
+      with:
+        node-version: 20.17.0
+        cache: 'npm'
+    - run: npm ci
+    - run: node lib/bin/create-docker-databases.js
+    - run: make test-db-migrations

.mocharc.js

@@ -9,6 +9,7 @@
 
 module.exports = {
   ignore: [
+    'test/db-migrations/**',
     'test/e2e/**',
   ],
 };

Makefile

@@ -107,6 +107,12 @@ test: lint
 test-ci: lint
 	BCRYPT=insecure npx mocha --recursive --reporter test/ci-mocha-reporter.js
 
+.PHONY: test-db-migrations
+test-db-migrations:
+	NODE_CONFIG_ENV=db-migration-test npx mocha --bail --sort --timeout=20000 \
+	    --require test/db-migrations/mocha-setup.js \
+	    ./test/db-migrations/**/*.spec.js
+
 .PHONY: test-fast
 test-fast: node_version
 	BCRYPT=insecure npx mocha --recursive --fgrep @slow --invert

config/db-migration-test.json

@@ -0,0 +1,10 @@
+{
+  "default": {
+    "database": {
+      "host": "localhost",
+      "user": "jubilant",
+      "password": "jubilant",
+      "database": "jubilant_test"
+    }
+  }
+}

test/db-migrations/.eslintrc.js

@@ -0,0 +1,19 @@
+module.exports = {
+  extends: '../.eslintrc.js',
+  rules: {
+    'key-spacing': 'off',
+    'keyword-spacing': 'off',
+    'no-console': 'off',
+    'no-multi-spaces': 'off',
+    'no-plusplus': 'off',
+    'no-use-before-define': 'off',
+    'object-curly-newline': 'off',
+    'prefer-arrow-callback': 'off',
+  },
+  globals: {
+    assert: false,
+    db: false,
+    log: false,
+    sql: false,
+  },
+};

test/db-migrations/.gitignore

@@ -0,0 +1 @@
+/.holding-pen/

test/db-migrations/20241008-01-add-user_preferences.spec.js

@@ -0,0 +1,46 @@
+const {
+  assertIndexExists,
+  assertTableDoesNotExist,
+  assertTableSchema,
+  describeMigration,
+} = require('./utils');
+
+describeMigration('20241008-01-add-user_preferences', ({ runMigrationBeingTested }) => {
+  before(async () => {
+    await assertTableDoesNotExist('user_site_preferences');
+    await assertTableDoesNotExist('user_project_preferences');
+
+    await runMigrationBeingTested();
+  });
+
+  it('should create user_site_preferences table', async () => {
+    await assertTableSchema('user_site_preferences',
+      { column_name:'userId',        is_nullable:'NO', data_type:'integer' },
+      { column_name:'propertyName',  is_nullable:'NO', data_type:'text' },
+      { column_name:'propertyValue', is_nullable:'NO', data_type:'jsonb' },
+    );
+  });
+
+  it('should create user_site_preferences userId index', async () => {
+    await assertIndexExists(
+      'user_site_preferences',
+      'CREATE INDEX "user_site_preferences_userId_idx" ON public.user_site_preferences USING btree ("userId")',
+    );
+  });
+
+  it('should create user_project_preferences table', async () => {
+    await assertTableSchema('user_project_preferences',
+      { column_name:'userId',        is_nullable:'NO', data_type:'integer' },
+      { column_name:'projectId',     is_nullable:'NO', data_type:'integer' },
+      { column_name:'propertyName',  is_nullable:'NO', data_type:'text' },
+      { column_name:'propertyValue', is_nullable:'NO', data_type:'jsonb' },
+    );
+  });
+
+  it('should create user_project_preferences userId index', async () => {
+    await assertIndexExists(
+      'user_project_preferences',
+      'CREATE INDEX "user_project_preferences_userId_idx" ON public.user_project_preferences USING btree ("userId")',
+    );
+  });
+});

test/db-migrations/migrator.js

@@ -0,0 +1,101 @@
+module.exports = {
+  exists,
+  hasRun,
+  runBefore,
+  runIncluding,
+  restoreMigrations,
+};
+
+const fs = require('node:fs');
+const { execSync } = require('node:child_process');
+
+// Horrible hacks.  Without this:
+//
+// 1. production migration code needs modifying, and
+// 2. it takes 3 mins+ just to run the migrations
+
+const migrationsDir = './lib/model/migrations';
+const holdingPen = './test/db-migrations/.holding-pen';
+
+fs.mkdirSync(holdingPen, { recursive:true });
+
+restoreMigrations();
+const allMigrations = loadMigrationsList();
+moveMigrationsToHoldingPen();
+
+let lastRunIdx = -1;
+
+function runBefore(migrationName) {
+  const idx = getIndex(migrationName);
+  if(idx === 0) return;
+
+  const previousMigration = allMigrations[idx - 1];
+
+  log('previousMigration:', previousMigration);
+
+  return runIncluding(previousMigration);
+}
+
+function runIncluding(lastMigrationToRun) {
+  const finalIdx = getIndex(lastMigrationToRun);
+
+  for(let restoreIdx=lastRunIdx+1; restoreIdx<=finalIdx; ++restoreIdx) {
+    const f = allMigrations[restoreIdx] + '.js';
+    fs.renameSync(`${holdingPen}/${f}`, `${migrationsDir}/${f}`);
+  }
+
+  log('Running migrations until:', lastMigrationToRun, '...');
+  const res = execSync(`node ./lib/bin/run-migrations.js`, { encoding:'utf8' });
+
+  lastRunIdx = finalIdx;
+
+  log(`Ran migrations up-to-and-including ${lastMigrationToRun}:\n`, res);
+}
+
+function getIndex(migrationName) {
+  const idx = allMigrations.indexOf(migrationName);
+  log('getIndex()', migrationName, 'found at', idx);
+  if(idx === -1) throw new Error(`Unknown migration: ${migrationName}`);
+  return idx;
+}
+
+function restoreMigrations() {
+  moveAll(holdingPen, migrationsDir);
+}
+
+function moveMigrationsToHoldingPen() {
+  moveAll(migrationsDir, holdingPen);
+}
+
+function moveAll(src, tgt) {
+  fs.readdirSync(src)
+    .forEach(f => fs.renameSync(`${src}/${f}`, `${tgt}/${f}`));
+}
+
+function loadMigrationsList() {
+  const migrations = fs.readdirSync(migrationsDir)
+    .filter(f => f.endsWith('.js'))
+    .map(f => f.replace(/\.js$/, ''))
+    .sort(); // TODO check that this is how knex sorts migration files
+  log();
+  log('All migrations:');
+  log();
+  migrations.forEach(m => log('*', m));
+  log();
+  log('Total:', migrations.length);
+  log();
+  return migrations;
+}
+
+function exists(migrationName) {
+  try {
+    getIndex(migrationName);
+    return true;
+  } catch(err) {
+    return false;
+  }
+}
+
+function hasRun(migratioName) {
+  return lastRunIdx >= getIndex(migratioName);
+}

test/db-migrations/mocha-setup.js

@@ -0,0 +1,52 @@
+const _log = level => (...args) => console.log(level, ...args);
+global.log = _log('[INFO]');
+
+global.assert = require('node:assert');
+const fs = require('node:fs');
+const slonik = require('slonik');
+const migrator = require('./migrator');
+
+async function mochaGlobalSetup() {
+  log('mochaGlobalSetup() :: ENTRY');
+
+  global.assert = assert;
+
+  global.sql = slonik.sql;
+
+  const { user, password, host, database } = jsonFile('./config/db-migration-test.json').default.database;
+  const dbUrl = `postgres://${user}:${password}@${host}/${database}`;
+  log('dbUrl:', dbUrl);
+  global.db = slonik.createPool(dbUrl);
+
+  const existingTables = await db.oneFirst(sql`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema='public'`);
+  if(existingTables) {
+    console.log(`
+    Existing tables were found in the public database schema.  Reset the database before running migration tests.
+
+    If you are using odk-postgres14 docker, try:
+
+      docker exec odk-postgres14 psql -U postgres ${database} -c "
+        DROP SCHEMA public CASCADE;
+        CREATE SCHEMA public;
+        GRANT ALL ON SCHEMA public TO postgres;
+        GRANT ALL ON SCHEMA public TO public;
+      "
+    `);
+    process.exit(1);
+  }
+
+  log('mochaGlobalSetup() :: EXIT');
+}
+
+function mochaGlobalTeardown() {
+  log('mochaGlobalTeardown() :: ENTRY');
+  db?.end();
+  migrator.restoreMigrations();
+  log('mochaGlobalTeardown() :: EXIT');
+}
+
+module.exports = { mochaGlobalSetup, mochaGlobalTeardown };
+
+function jsonFile(path) {
+  return JSON.parse(fs.readFileSync(path, { encoding:'utf8' }));
+}