feat: revamped the entire VaultsSecretsList handler

package.json

@@ -36,9 +36,6 @@
     },
     {
       "name": "Aditya Varma"
-    },
-    {
-      "name": "Aryan Jassal"
     }
   ],
   "description": "Polykey Core Library",

src/client/callers/vaultsSecretsList.ts

@@ -1,12 +1,5 @@
-import type { HandlerTypes } from '@matrixai/rpc';
-import type VaultsSecretsList from '../handlers/VaultsSecretsList';
-import { UnaryCaller } from '@matrixai/rpc';
+import { RawCaller } from '@matrixai/rpc';
 
-type CallerTypes = HandlerTypes<VaultsSecretsList>;
-
-const vaultsSecretsList = new UnaryCaller<
-  CallerTypes['input'],
-  CallerTypes['output']
->();
+const vaultsSecretsList = new RawCaller();
 
 export default vaultsSecretsList;

src/client/handlers/VaultsSecretsList.ts

@@ -1,73 +1,97 @@
 import type { DB } from '@matrixai/db';
-import type {
-  ClientRPCRequestParams,
-  ClientRPCResponseResult,
-  SecretFilesList,
-  SecretPatternMessage,
-} from '../types';
+import type { JSONObject, JSONRPCRequest } from '@matrixai/rpc';
 import type VaultManager from '../../vaults/VaultManager';
-import type { ContentNode, TreeNode } from '../../vaults/types';
-import { UnaryHandler } from '@matrixai/rpc';
+import type { TreeNode, ContentNode } from '../../vaults/types';
+import { ReadableStream } from 'stream/web';
+import { RawHandler } from '@matrixai/rpc';
 import * as vaultsUtils from '../../vaults/utils';
 import * as vaultsErrors from '../../vaults/errors';
+import * as utils from '../../utils';
 import { fileTree } from '../../vaults';
 
-class VaultsSecretsList extends UnaryHandler<
-  {
-    vaultManager: VaultManager;
-    db: DB;
-  },
-  ClientRPCRequestParams<SecretPatternMessage>,
-  ClientRPCResponseResult<SecretFilesList>
-> {
+class VaultsSecretsGet extends RawHandler<{
+  vaultManager: VaultManager;
+  db: DB;
+}> {
   public handle = async (
-    input: ClientRPCRequestParams<SecretPatternMessage>,
-  ): Promise<ClientRPCResponseResult<SecretFilesList>> => {
+    input: [JSONRPCRequest, ReadableStream<Uint8Array>],
+    _cancel: any,
+    // meta: any,
+    // ctx,
+  ): Promise<[JSONObject, ReadableStream<Uint8Array>]> => {
     const { vaultManager, db } = this.container;
-    return await db.withTransactionF(async (tran) => {
+    const [headerMessage, _] = input;
+
+    const params = headerMessage.params;
+    if (params == null || !utils.isObject(params)) utils.never();
+    utils.checkKeyInJSON('vaultNameOrId', 'string', params);
+    utils.checkKeyInJSON('pattern', 'string', params);
+    utils.checkKeyInJSON('yieldStats', 'boolean', params);
+    utils.checkKeyInJSON('yieldRoot', 'boolean', params);
+    utils.checkKeyInJSON('yieldFiles', 'boolean', params);
+    utils.checkKeyInJSON('yieldParents', 'boolean', params);
+    utils.checkKeyInJSON('yieldDirectories', 'boolean', params);
+    utils.checkKeyInJSON('yieldContents', 'boolean', params);
+
+    const vaultNameOrId = params.vaultNameOrId as string;
+    const vaultId = await db.withTransactionF(async (tran) => {
       const vaultIdFromName = await vaultManager.getVaultId(
-        input.nameOrId,
+        vaultNameOrId,
         tran,
       );
       const vaultId =
-        vaultIdFromName ?? vaultsUtils.decodeVaultId(input.nameOrId);
-      if (vaultId == null) {
-        throw new vaultsErrors.ErrorVaultsVaultUndefined();
+        vaultIdFromName ?? vaultsUtils.decodeVaultId(vaultNameOrId);
+      if (vaultId == null) throw new vaultsErrors.ErrorVaultsVaultUndefined();
+      return vaultId;
+    });
+
+    const filesData = vaultManager.withVaultsG([vaultId], (vault) => {
+      return vault.readG(async function* (fs): AsyncGenerator<
+        TreeNode | ContentNode | Uint8Array
+      > {
+        const fileTreeGen = fileTree.globWalk({
+          fs: fs,
+          basePath: '.',
+          pattern: params.pattern as string,
+          yieldStats: params.yieldStats as boolean,
+          yieldRoot: params.yieldRoot as boolean,
+          yieldFiles: params.yieldFiles as boolean,
+          yieldParents: params.yieldParents as boolean,
+          yieldDirectories: params.yieldDirectories as boolean,
+        });
+        return fileTree.serializerStreamFactory(
+          fs,
+          fileTreeGen,
+          params.yieldContents as boolean,
+        );
+      });
+    });
+
+    async function* filterTransform(
+      source: AsyncGenerator<TreeNode | ContentNode | Uint8Array>,
+    ): AsyncGenerator<Uint8Array> {
+      for await (const chunk of source) {
+        if (chunk instanceof Uint8Array) {
+          // @ts-ignore while the types don't fully match, they match enough
+          yield Buffer.from(chunk).toString();
+        }
+      }
+    }
+    const filteredFilesData = filterTransform(filesData);
+
+    const filesDataStream = new ReadableStream<Uint8Array>({
+      pull: async (controller) => {
+        const next = await filteredFilesData.next();
+        if (next.done === true) return controller.close();
+        controller.enqueue(next.value);
+      },
+      cancel: async() => {
+        filteredFilesData.return('Closed');
       }
-      return await vaultManager.withVaults(
-        [vaultId],
-        async (vault) => {
-          const data = await vault.readF(async (fs) => {
-            const fileTreeGen = fileTree.globWalk({
-              fs: fs,
-              basePath: '.',
-              pattern: input.pattern,
-              yieldStats: true,
-              yieldRoot: false,
-              yieldFiles: true,
-              yieldParents: true,
-              yieldDirectories: true,
-            });
-            const data: Array<TreeNode | ContentNode | Uint8Array> = [];
-            const serializedStream = fileTree.serializerStreamFactory(
-              fs,
-              fileTreeGen,
-              true,
-            );
-            for await (const output of serializedStream) {
-              data.push(output);
-            }
-            return data;
-          });
-          const contents = data
-            .filter((v) => v instanceof Uint8Array)
-            .map((v) => Buffer.from(v as Uint8Array).toString());
-          return { secretFilesList: contents };
-        },
-        tran,
-      );
     });
+
+    return [{}, filesDataStream];
   };
 }
 
-export default VaultsSecretsList;
+export default VaultsSecretsGet;

src/client/types.ts

@@ -301,12 +301,18 @@ type VaultsLatestVersionMessage = {
 
 // Secrets
 
+// NOTE: see if we still need this. probably dont, but just be sure.
 type SecretFilesList = {
   secretFilesList: Array<string>;
 };
 
-type SecretPatternMessage = VaultIdentifierMessage & {
+type SecretFilesMessage = VaultIdentifierMessage & {
   pattern: string;
+  yieldStats: boolean;
+  yieldRoot: boolean;
+  yieldFiles: boolean;
+  yieldParents: boolean;
+  yieldDirectories: boolean;
 };
 
 // NOTE: we used to use SecretNameMessage before. do we need to keep it? look into this.
@@ -423,7 +429,7 @@ export type {
   SecretFilesList,
   SecretNameMessage,
   SecretIdentifierMessage,
-  SecretPatternMessage,
+  SecretFilesMessage,
   ContentMessage,
   SecretContentMessage,
   SecretMkdirMessage,

src/utils/utils.ts

@@ -14,6 +14,7 @@ import lexi from 'lexicographic-integer';
 import { PromiseCancellable } from '@matrixai/async-cancellable';
 import { timedCancellable } from '@matrixai/contexts/dist/functions';
 import * as utilsErrors from './errors';
+import { JSONObject } from '@matrixai/rpc';
 
 const AsyncFunction = (async () => {}).constructor;
 const GeneratorFunction = function* () {}.constructor;
@@ -541,6 +542,10 @@ function setMaxListeners(
   nodesEvents.setMaxListeners(limit, target);
 }
 
+function checkKeyInJSON(key: string, type: string, params: JSONObject) {
+  if (!(key in params) || typeof params[key] != type) never();
+}
+
 export {
   AsyncFunction,
   GeneratorFunction,
@@ -581,4 +586,5 @@ export {
   isBufferSource,
   yieldMicro,
   setMaxListeners,
+  checkKeyInJSON,
 };

tests/client/handlers/vaults.test.ts

@@ -71,6 +71,7 @@ import * as vaultsErrors from '@/vaults/errors';
 import * as networkUtils from '@/network/utils';
 import * as testsUtils from '../../utils';
 import { fileTree } from '@/vaults';
+import { ContentNode, TreeNode } from '@/vaults/types';
 
 describe('vaultsClone', () => {
   const logger = new Logger('vaultsClone test', LogLevel.WARN, [
@@ -1590,17 +1591,31 @@ describe('vaultsSecretsNewDir and vaultsSecretsList', () => {
     expect(addResponse.success).toBeTruthy();
     // List secrets
     const secrets = await rpcClient.methods.vaultsSecretsList({
-      nameOrId: vaultsIdEncoded,
+      vaultNameOrId: vaultsIdEncoded,
       pattern: '**/*',
+      yieldStats: false,
+      yieldRoot: false,
+      yieldFiles: true,
+      yieldParents: false,
+      yieldDirectories: false,
+      yieldContents: false,
     });
 
     console.log(secrets);
+    console.log(secrets.readable);
 
-    const secretFiles = secrets.secretFilesList;
+    const secretDataStream = secrets.readable;
     const parserTransform = fileTree.parserTransformStreamFactory();
-    // const parsedFiles = dataStream.pipeThrough(parserTransform);
+    const parsedFilesStream = secretDataStream.pipeThrough(parserTransform);
+    const parsedFiles: Array<Uint8Array | TreeNode | ContentNode> = []
+
+    for await (const file of parsedFilesStream) {
+      parsedFiles.push(file);
+    }
+
+    console.log(parsedFiles);
 
-    expect(secretFiles).toIncludeAllMembers([
+    expect(parsedFiles).toIncludeAllMembers([
       ...secretList.map((secret) => path.join('secretDir', secret)),
       'secretDir'
     ]);