WIP feat: experimented with db typings using kysely

.gitignore

@@ -11,6 +11,7 @@ release/
 /dev-app-update.yml
 *.sublime*
 sql/
+.env
 
 # React / TypeScript
 ts/**/*.js

TODO.md

@@ -0,0 +1,21 @@
+# Tasks
+
+## kysely integration
+
+1. Add `.env` file.
+
+```
+  DATABASE_URL=[SQLITE_FILE_LOCATION]
+  DATABASE_KEY=[CONFIG.JSON key value] e.g. 3aa09a21008545b5b8ccf9ab9e135e9fa433209d0387e5812167106bf119e7be
+```
+
+2. `yarn install` as normal.
+3. Go into `node_modules/kysely-codegen` and run `yarn install` and then `yarn build`.
+4. Go back to project root `cd ../../` and run `node ./node_modules/kysely-codegen/dist/bin/index.js --out-file ts/types/db.d.ts`.
+5. Run app as normal.
+
+### TODO
+
+- [ ] Auto build kysely-codegen when doing yarn install
+- [ ] Add db type generation into build commandss
+- [ ] Write documentation and add to CONTRIBUTING.md?

package.json

@@ -102,6 +102,7 @@
     "glob": "7.1.2",
     "image-type": "^4.1.0",
     "ip2country": "1.0.1",
+    "kysely": "^0.26.1",
     "libsession_util_nodejs": "https://github.com/oxen-io/libsession-util-nodejs/releases/download/v0.2.2/libsession_util_nodejs-v0.2.2.tar.gz",
     "libsodium-wrappers-sumo": "^0.7.9",
     "linkify-it": "^4.0.1",
@@ -203,6 +204,7 @@
     "events": "^3.3.0",
     "jsdom": "^19.0.0",
     "jsdom-global": "^3.0.2",
+    "kysely-codegen": "github:yougotwill/kysely-codegen#f7732229fca26650bbbf4df57e57ceae2af0f89f",
     "mini-css-extract-plugin": "^2.7.5",
     "mocha": "10.0.0",
     "nan": "^2.17.0",

ts/mains/main_node.ts

@@ -798,7 +798,7 @@ async function showMainWindow(sqlKey: string, passwordAttempt = false) {
     passwordAttempt,
   });
   appStartInitialSpellcheckSetting = await getSpellCheckSetting();
-  sqlChannels.initializeSqlChannel();
+  await sqlChannels.initializeSqlChannel();
 
   await initAttachmentsChannel({
     userDataPath,

ts/node/sql.ts

@@ -66,8 +66,9 @@ import {
 import {
   assertGlobalInstance,
   assertGlobalInstanceOrInstance,
-  closeDbInstance,
-  initDbInstanceWith,
+  assertGlobalKyselyInstance,
+  closeDbInstanceWithKysely,
+  initDbInstanceWithKysely,
   isInstanceInitialized,
 } from './sqlInstance';
 import { configDumpData } from './sql_calls/config_dump';
@@ -182,7 +183,7 @@ async function initializeSql({
     }
 
     // At this point we can allow general access to the database
-    initDbInstanceWith(db);
+    initDbInstanceWithKysely(db);
 
     console.info('total message count before cleaning: ', getMessageCount());
     console.info('total conversation count before cleaning: ', getConversationCount());
@@ -212,7 +213,7 @@ async function initializeSql({
     if (button.response === 0) {
       clipboard.writeText(`Database startup error:\n\n${redactAll(error.stack)}`);
     } else {
-      closeDbInstance();
+      closeDbInstanceWithKysely();
       showFailedToStart();
     }
 
@@ -613,14 +614,16 @@ function getConversationById(id: string, instance?: BetterSqlite3.Database) {
   return formatRowOfConversation(row, 'getConversationById', unreadCount, mentionedUsStillUnread);
 }
 
-function getAllConversations() {
-  const rows = assertGlobalInstance()
-    .prepare(`SELECT * FROM ${CONVERSATIONS_TABLE} ORDER BY id ASC;`)
-    .all();
+async function getAllConversations() {
+  const rows = await assertGlobalKyselyInstance()
+    .selectFrom('conversations')
+    .selectAll()
+    .orderBy('id', 'asc')
+    .execute();
 
   return (rows || []).map(m => {
-    const unreadCount = getUnreadCountByConversation(m.id) || 0;
-    const mentionedUsStillUnread = !!getFirstUnreadMessageWithMention(m.id);
+    const unreadCount = m.id ? getUnreadCountByConversation(m.id) : 0;
+    const mentionedUsStillUnread = m.id ? !!getFirstUnreadMessageWithMention(m.id) : false;
     return formatRowOfConversation(m, 'getAllConversations', unreadCount, mentionedUsStillUnread);
   });
 }
@@ -2326,7 +2329,7 @@ function cleanUpOldOpengroupsOnStart() {
 export type SqlNodeType = typeof sqlNode;
 
 export function close() {
-  closeDbInstance();
+  closeDbInstanceWithKysely();
 }
 
 export const sqlNode = {

ts/node/sqlInstance.ts

@@ -1,4 +1,6 @@
 import * as BetterSqlite3 from 'better-sqlite3';
+import { Kysely, SqliteDialect, sql } from 'kysely';
+import { DB } from '../types/db';
 
 let globalInstance: BetterSqlite3.Database | null = null;
 
@@ -42,3 +44,60 @@ export function closeDbInstance() {
   dbRef.pragma('optimize');
   dbRef.close();
 }
+
+// Kysely functions
+let globalKyselyInstance: Kysely<DB> | null = null;
+
+export function assertGlobalKyselyInstance(): Kysely<DB> {
+  if (!globalKyselyInstance) {
+    throw new Error('globalKyselyInstance is not initialized.');
+  }
+  return globalKyselyInstance;
+}
+
+export function isKyselyInstanceInitialized(): boolean {
+  return !!globalKyselyInstance;
+}
+
+export function assertGlobalKyselyInstanceOrKyselyInstance(
+  instance?: Kysely<DB> | null
+): Kysely<DB> {
+  // if none of them are initialized, throw
+  if (!globalKyselyInstance && !instance) {
+    throw new Error('neither globalKyselyInstance nor initialized is initialized.');
+  }
+  // otherwise, return which ever is true, priority to the global one
+  return globalKyselyInstance || (instance as Kysely<DB>);
+}
+
+export function initDbInstanceWithKysely(instance: BetterSqlite3.Database) {
+  if (globalInstance) {
+    throw new Error('already init');
+  }
+
+  // intiate our typings
+  const dialect = new SqliteDialect({
+    database: instance,
+  });
+
+  const kyselyInstance = new Kysely<DB>({
+    dialect,
+  });
+
+  globalKyselyInstance = kyselyInstance;
+
+  initDbInstanceWith(instance);
+}
+
+export async function closeDbInstanceWithKysely() {
+  if (!globalKyselyInstance) {
+    return;
+  }
+  const dbRef = globalKyselyInstance;
+  globalKyselyInstance = null;
+  // SQLLite documentation suggests that we run `PRAGMA optimize` right before
+  // closing the database connection.
+  await sql`PRAGMA optimize`.execute(dbRef);
+  dbRef.destroy();
+  closeDbInstance();
+}

ts/node/sql_channel.ts

@@ -9,19 +9,19 @@ const SQL_CHANNEL_KEY = 'sql-channel';
 const ERASE_SQL_KEY = 'erase-sql-key';
 // tslint:disable: no-console
 
-export function initializeSqlChannel() {
+export async function initializeSqlChannel() {
   if (initialized) {
     throw new Error('sqlChannels: already initialized!');
   }
 
-  ipcMain.on(SQL_CHANNEL_KEY, (event, jobId, callName, ...args) => {
+  ipcMain.on(SQL_CHANNEL_KEY, async (event, jobId, callName, ...args) => {
     try {
       const fn = (sqlNode as any)[callName];
       if (!fn) {
         throw new Error(`sql channel: ${callName} is not an available function`);
       }
 
-      const result = fn(...args);
+      const result = await fn(...args);
 
       event.sender.send(`${SQL_CHANNEL_KEY}-done`, jobId, null, result);
     } catch (error) {

ts/types/db.d.ts

@@ -0,0 +1,188 @@
+import type { ColumnType } from "kysely";
+
+export type Generated<T> = T extends ColumnType<infer S, infer I, infer U>
+  ? ColumnType<S, I | undefined, U>
+  : ColumnType<T, T | undefined, T>;
+
+export interface AttachmentDownloads {
+  id: string | null;
+  timestamp: number | null;
+  pending: number | null;
+  json: string | null;
+}
+
+export interface ConfigDump {
+  variant: string;
+  publicKey: string;
+  data: Buffer | null;
+}
+
+export interface Conversations {
+  id: string | null;
+  active_at: number | null;
+  type: string | null;
+  members: string | null;
+  zombies: Generated<string | null>;
+  left: number | null;
+  expireTimer: number | null;
+  mentionedUs: number | null;
+  unreadCount: number | null;
+  lastMessageStatus: string | null;
+  lastMessage: string | null;
+  lastJoinedTimestamp: number | null;
+  groupAdmins: Generated<string | null>;
+  isKickedFromGroup: number | null;
+  avatarPointer: string | null;
+  nickname: string | null;
+  profileKey: string | null;
+  triggerNotificationsFor: Generated<string | null>;
+  isTrustedForAttachmentDownload: Generated<number | null>;
+  priority: Generated<number | null>;
+  isApproved: Generated<number | null>;
+  didApproveMe: Generated<number | null>;
+  avatarInProfile: string | null;
+  displayNameInProfile: string | null;
+  conversationIdOrigin: string | null;
+  avatarImageId: number | null;
+  markedAsUnread: number | null;
+}
+
+export interface EncryptionKeyPairsForClosedGroupV2 {
+  id: Generated<number>;
+  groupPublicKey: string | null;
+  timestamp: string | null;
+  json: string | null;
+}
+
+export interface GuardNodes {
+  id: Generated<number>;
+  ed25519PubKey: string | null;
+}
+
+export interface IdentityKeys {
+  id: string | null;
+  json: string | null;
+}
+
+export interface Items {
+  id: string | null;
+  json: string | null;
+}
+
+export interface LastHashes {
+  id: string | null;
+  snode: string | null;
+  hash: string | null;
+  expiresAt: number | null;
+  namespace: Generated<number>;
+}
+
+export interface LokiSchema {
+  id: Generated<number>;
+  version: number | null;
+}
+
+export interface Messages {
+  id: string | null;
+  json: string | null;
+  unread: number | null;
+  expires_at: number | null;
+  sent: number | null;
+  sent_at: number | null;
+  conversationId: string | null;
+  received_at: number | null;
+  source: string | null;
+  hasAttachments: number | null;
+  hasFileAttachments: number | null;
+  hasVisualMediaAttachments: number | null;
+  expireTimer: number | null;
+  expirationStartTimestamp: number | null;
+  type: string | null;
+  body: string | null;
+  serverId: number | null;
+  serverTimestamp: number | null;
+  serverHash: string | null;
+  isDeleted: number | null;
+}
+
+export interface MessagesFts {
+  body: string | null;
+}
+
+export interface MessagesFtsConfig {
+  k: string;
+  v: string | null;
+}
+
+export interface MessagesFtsContent {
+  id: number | null;
+  c0: string | null;
+}
+
+export interface MessagesFtsData {
+  id: number | null;
+  block: Buffer | null;
+}
+
+export interface MessagesFtsDocsize {
+  id: number | null;
+  sz: Buffer | null;
+}
+
+export interface MessagesFtsIdx {
+  segid: string;
+  term: string;
+  pgno: string | null;
+}
+
+export interface NodesForPubkey {
+  pubkey: string | null;
+  json: string | null;
+}
+
+export interface OpenGroupRoomsV2 {
+  serverUrl: string;
+  roomId: string;
+  conversationId: string | null;
+  json: string | null;
+}
+
+export interface SeenMessages {
+  hash: string | null;
+  expiresAt: number | null;
+}
+
+export interface Unprocessed {
+  id: string | null;
+  timestamp: number | null;
+  version: number | null;
+  attempts: number | null;
+  envelope: string | null;
+  decrypted: string | null;
+  source: string | null;
+  senderIdentity: string | null;
+  serverHash: string | null;
+}
+
+export interface DB {
+  attachment_downloads: AttachmentDownloads;
+  configDump: ConfigDump;
+  conversations: Conversations;
+  encryptionKeyPairsForClosedGroupV2: EncryptionKeyPairsForClosedGroupV2;
+  guardNodes: GuardNodes;
+  identityKeys: IdentityKeys;
+  items: Items;
+  lastHashes: LastHashes;
+  loki_schema: LokiSchema;
+  messages: Messages;
+  messages_fts: MessagesFts;
+  messages_fts_config: MessagesFtsConfig;
+  messages_fts_content: MessagesFtsContent;
+  messages_fts_data: MessagesFtsData;
+  messages_fts_docsize: MessagesFtsDocsize;
+  messages_fts_idx: MessagesFtsIdx;
+  nodesForPubkey: NodesForPubkey;
+  openGroupRoomsV2: OpenGroupRoomsV2;
+  seenMessages: SeenMessages;
+  unprocessed: Unprocessed;
+}