Create an image to load schemas into the ConfigDB (#61)

Create a Dockerfile which builds an image, to be run from service-setup,
which loads the current set of schemas into the ConfigDB.

Schema information is loaded under two different ConfigDB Apps: one
contains the actual schema, the other contains the metadata (name,
version and so on) the Manager needs to display the schema selection
screen.

The `$id` and `$ref` fields of the schemas are rewritten to use URLs
under the `urn:uuid:` scheme. This URL scheme simply names a UUID
without providing any address information. It will be necessary to have
direct access to the schema UUID so that refs can be followed to other
schemas. If necessary this can be changed to some other scheme; there
are various possibilities, from a well-known scheme using
`factoryplus.app.amrc.co.uk` to working out the ConfigDB URL of the
schema config entry itself. But this is simpler and I think will be
sufficient for now.

.dockerignore

@@ -0,0 +1,3 @@
+deploy/node_modules
+deploy/schemas.json
+validate/node_modules

Dockerfile

@@ -0,0 +1,35 @@
+# syntax=docker/dockerfile:1
+
+ARG base_version=v3.0.0
+ARG base_prefix=ghcr.io/amrc-factoryplus/acs-base
+
+FROM ${base_prefix}-js-build:${base_version} AS build
+ARG revision=unknown
+
+USER root
+RUN sh -x <<'SHELL'
+    install -d -o node -g node /home/node/app/deploy
+SHELL
+WORKDIR /home/node/app
+USER node
+COPY deploy/package*.json deploy/
+RUN sh -x <<'SHELL'
+    cd deploy
+    npm install --save=false
+SHELL
+COPY --chown=node . .
+# Finding schemas would be easier if they were in their own subdir. But
+# we can't change that while existing ACS installations rely on the
+# current repo structure.
+RUN sh -x <<'SHELL'
+    cd deploy
+    echo "export const GIT_VERSION=\"$revision\";" > ./lib/git-version.js
+    node bin/find-schemas.js
+SHELL
+
+FROM ${base_prefix}-js-run:${base_version} AS run
+# Copy across from the build container.
+WORKDIR /home/node/app
+COPY --from=build /home/node/app/deploy ./
+USER node
+CMD ["node", "bin/load-schemas.js"]

Service/Service-v1.json

@@ -13,7 +13,7 @@
             "format": "uuid"
         },
         "Device_Information": {
-            "$ref": "../Common/Device_Information-v1.json"
+            "$ref": "https://raw.githubusercontent.com/AMRC-FactoryPlus/schemas/main/Common/Device_Information-v1.json"
         },
 
         "Service_UUID": {

deploy/.gitignore

@@ -0,0 +1,2 @@
+node_modules
+schemas.json

deploy/bin/find-schemas.js

@@ -0,0 +1,112 @@
+import fs from "fs";
+import $fs from "fs/promises";
+import $path from "path";
+
+import Walk from "@root/walk";
+import git from "isomorphic-git";
+
+const ignore = new Set([
+    ".git", ".githooks", ".github",
+    "deploy", "validate",
+]);
+
+const base = "https://raw.githubusercontent.com/AMRC-FactoryPlus/schemas/main";
+const id_rx = RegExp(`^${base}/([\\w/_]+)-v(\\d+).json$`);
+
+const load_json = async f => JSON.parse(await $fs.readFile(f));
+
+/* Walk should be an async iterator really. But meh. */
+const schemas = new Map();
+
+const walker = Walk.create({
+    sort: des => des
+        .filter(d => !(d.parentPath == ".." && ignore.has(d.name))),
+});
+await walker("..", async (err, path, dirent) => {
+    if (err) throw err;
+
+    if (dirent.isDirectory()) return;
+    if (!path.match(/\.json$/)) return;
+
+    const json = await load_json(path);
+
+    const id = json.$id;
+    if (!id) return;
+    if (schemas.has(id))
+        throw `Duplicate $id for ${path}`;
+
+    const matches = id.match(id_rx);
+    if (!matches)
+        throw `Bad $id for ${path}: ${id}`;
+
+    const [, name, version] = matches;
+    const uuid = json.properties?.Schema_UUID?.const;
+    if (!name || !version)
+        throw `Bad name or version for ${path}`;
+
+    schemas.set(id, { uuid, path, name, version, json });
+});
+
+function fixup (obj) {
+    if (obj == null || typeof(obj) != "object")
+        return obj;
+
+    if (Array.isArray(obj))
+        return obj.map(v => fixup(v));
+
+    const fix = Object.fromEntries(
+        Object.entries(obj).map(([k, v]) => [k, fixup(v)]));
+
+    const ref = obj.$ref;
+    if (!ref)
+        return fix;
+
+    const sch = schemas.get(ref);
+    if (!sch)
+        throw `Unknown $ref: ${ref}`;
+
+    if (sch.uuid)
+        return { ...fix, $ref: `urn:uuid:${sch.uuid}` };
+
+    /* The Sparkplug_Types and Eng_Units schemas are sub-schemas of
+     * Metric. As such they cannot include a Schema_UUID metric. For now
+     * just expand them inline (they are only used in Metric). */
+    const expand = { ...sch.json, ...fix };
+    delete expand.$id;
+    delete expand.$ref;
+    delete expand.$schema;
+    return expand;
+}
+
+const configs = {};
+for (const sch of schemas.values()) {
+    if (!sch.uuid)
+        continue;
+
+    const changes = (await git.log({
+        fs, 
+        dir:        "..", 
+        filepath:   $path.relative("..", sch.path),
+    })).map(l => l.commit.author.timestamp);
+
+    configs[sch.uuid] = {
+        metadata: {
+            name:       sch.name,
+            version:    sch.version,
+            created:    changes.at(-1),
+            modified:   changes.at(0),
+        },
+        schema: {
+            ...fixup(sch.json),
+            $id:        `urn:uuid:${sch.uuid}`,
+        },
+    };
+}
+
+await $fs.writeFile("schemas.json", JSON.stringify(configs, null, 2));
+
+const priv = {
+    EdgeAgent:  await load_json("../Edge_Agent_Config.json"),
+    Connection: await load_json("../Device_Connection.json"),
+};
+await $fs.writeFile("private.json", JSON.stringify(priv, null, 2));

deploy/bin/load-schemas.js

@@ -0,0 +1,67 @@
+import $fs from "fs/promises";
+
+import { GIT_VERSION }  from "../lib/git-version.js";
+import { ServiceClient, UUIDs } from "@amrc-factoryplus/service-client";
+
+const App = {
+    Schema:         "b16e85fb-53c2-49f9-8d83-cdf6763304ba",
+    Metadata:       "32093857-9d29-470e-a897-d2b56d5aa978",
+};
+const Class = {
+    Private:        "eda329ca-4e55-4a92-812d-df74993c47e2",
+};
+const Private = {
+    Connection:     "7d08564b-5235-41a4-8acf-bbee7c2e2006",
+    EdgeAgent:      "d4f59d8a-5391-49c3-b591-e74e2468ef43",
+};
+
+console.log(`ACS schemas revision ${GIT_VERSION}`);
+
+const schemas = JSON.parse(await $fs.readFile("schemas.json"));
+
+const fplus = await new ServiceClient({ env: process.env }).init();
+const cdb = fplus.ConfigDB;
+const log = fplus.debug.bound("schemas");
+
+log("Creating required Apps");
+await cdb.create_object(UUIDs.Class.App, App.Schema);
+await cdb.put_config(UUIDs.App.Info, App.Schema,
+    { name: "JSON schema" });
+await cdb.create_object(UUIDs.Class.App, App.Metadata);
+await cdb.put_config(UUIDs.App.Info, App.Metadata,
+    { name: "Metric schema info" });
+
+log("Creating required Classes");
+await cdb.create_object(UUIDs.Class.Class, Class.Private);
+await cdb.put_config(UUIDs.App.Info, Class.Private,
+    { name: "Private configuration" });
+
+for (const [uuid, { metadata, schema }] of Object.entries(schemas)) {
+    log("Updating schema %s v%s (%s)", 
+        metadata.name, metadata.version, uuid);
+
+    await cdb.create_object(UUIDs.Class.Schema, uuid);
+
+    /* XXX It might be better to use the schema title here? But at the
+     * moment those aren't unique. */
+    const name = `${metadata.name} (v${metadata.version})`;
+    await cdb.put_config(UUIDs.App.Info, uuid, { name });
+    await cdb.put_config(App.Metadata, uuid, metadata);
+    await cdb.put_config(App.Schema, uuid, schema);
+}
+
+const priv = JSON.parse(await $fs.readFile("private.json"));
+
+log("Updating Edge Agent Config schema");
+await cdb.create_object(Class.Private, Private.EdgeAgent);
+await cdb.put_config(UUIDs.App.Info, Private.EdgeAgent,
+    { name: "Edge Agent config schema" });
+await cdb.put_config(App.Schema, Private.EdgeAgent, priv.EdgeAgent);
+
+log("Updating Device Connection schema");
+await cdb.create_object(Class.Private, Private.Connection);
+await cdb.put_config(UUIDs.App.Info, Private.Connection,
+    { name: "Device Connection schema" });
+await cdb.put_config(App.Schema, Private.Connection, priv.Connection);
+
+log("Done");

deploy/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "acs-schemas",
+  "version": "0.0.0",
+  "description": "",
+  "main": "index.js",
+  "type": "module",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@amrc-factoryplus/service-client": "^1.3.6",
+    "@root/walk": "^1.1.0",
+    "isomorphic-git": "^1.27.1"
+  }
+}