From 39972fc1ade6462918e2072b78aa172a384da95b Mon Sep 17 00:00:00 2001 From: Aryan Jassal Date: Fri, 20 Sep 2024 17:17:28 +1000 Subject: [PATCH] feat: updated RPC handlers taking multiple secret paths to use duplex streams --- src/client/callers/vaultsSecretsGet.ts | 4 +- src/client/callers/vaultsSecretsRemove.ts | 4 +- src/client/handlers/VaultsSecretsGet.ts | 44 ++-- src/client/handlers/VaultsSecretsRemove.ts | 31 ++- src/client/types.ts | 14 -- tests/client/handlers/vaults.test.ts | 265 +++++++++++++++------ 6 files changed, 246 insertions(+), 116 deletions(-) diff --git a/src/client/callers/vaultsSecretsGet.ts b/src/client/callers/vaultsSecretsGet.ts index 7e7172255..4c00b0c4b 100644 --- a/src/client/callers/vaultsSecretsGet.ts +++ b/src/client/callers/vaultsSecretsGet.ts @@ -1,10 +1,10 @@ import type { HandlerTypes } from '@matrixai/rpc'; import type VaultsSecretsGet from '../handlers/VaultsSecretsGet'; -import { ServerCaller } from '@matrixai/rpc'; +import { DuplexCaller } from '@matrixai/rpc'; type CallerTypes = HandlerTypes; -const vaultsSecretsGet = new ServerCaller< +const vaultsSecretsGet = new DuplexCaller< CallerTypes['input'], CallerTypes['output'] >(); diff --git a/src/client/callers/vaultsSecretsRemove.ts b/src/client/callers/vaultsSecretsRemove.ts index e546b43ac..e2bda1e28 100644 --- a/src/client/callers/vaultsSecretsRemove.ts +++ b/src/client/callers/vaultsSecretsRemove.ts @@ -1,10 +1,10 @@ import type { HandlerTypes } from '@matrixai/rpc'; import type VaultsSecretsRemove from '../handlers/VaultsSecretsRemove'; -import { UnaryCaller } from '@matrixai/rpc'; +import { ClientCaller } from '@matrixai/rpc'; type CallerTypes = HandlerTypes; -const vaultsSecretsRemove = new UnaryCaller< +const vaultsSecretsRemove = new ClientCaller< CallerTypes['input'], CallerTypes['output'] >(); diff --git a/src/client/handlers/VaultsSecretsGet.ts b/src/client/handlers/VaultsSecretsGet.ts index 5a9d5c6bc..ead41f0e9 100644 --- a/src/client/handlers/VaultsSecretsGet.ts +++ b/src/client/handlers/VaultsSecretsGet.ts @@ -3,51 +3,53 @@ import type { ClientRPCRequestParams, ClientRPCResponseResult, ContentMessage, - SecretManyPathMessage, + SecretIdentifierMessage, } from '../types'; import type VaultManager from '../../vaults/VaultManager'; -import { ServerHandler } from '@matrixai/rpc'; +import { DuplexHandler } from '@matrixai/rpc'; import * as vaultsUtils from '../../vaults/utils'; import * as vaultsErrors from '../../vaults/errors'; import * as vaultOps from '../../vaults/VaultOps'; -class VaultsSecretsGet extends ServerHandler< +class VaultsSecretsGet extends DuplexHandler< { vaultManager: VaultManager; db: DB; }, - ClientRPCRequestParams, + ClientRPCRequestParams, ClientRPCResponseResult > { - public async *handle( - input: ClientRPCRequestParams, - ): AsyncGenerator, void, void> { + public handle = async function* ( + input: AsyncIterable>, + _cancel, + _meta, + ctx, + ): AsyncGenerator> { + if (ctx.signal.aborted) throw ctx.signal.reason; const { vaultManager, db } = this.container; yield* db.withTransactionG(async function* (tran): AsyncGenerator< - ContentMessage, - void, - void + ClientRPCResponseResult > { + if (ctx.signal.aborted) throw ctx.signal.reason; // As we need to preserve the order of parameters, we need to loop over // them individually, as grouping them would make them go out of order. - for (const [vaultName, secretName] of input.secretNames) { - const vaultIdFromName = await vaultManager.getVaultId(vaultName, tran); - const vaultId = vaultIdFromName ?? vaultsUtils.decodeVaultId(vaultName); + for await (const secretIdentiferMessage of input) { + const { nameOrId, secretName } = secretIdentiferMessage; + const vaultIdFromName = await vaultManager.getVaultId(nameOrId, tran); + const vaultId = vaultIdFromName ?? vaultsUtils.decodeVaultId(nameOrId); if (vaultId == null) throw new vaultsErrors.ErrorVaultsVaultUndefined(); - yield* vaultManager.withVaultsG( + const content: Buffer = await vaultManager.withVaults( [vaultId], - async function* (vault): AsyncGenerator { - yield { - secretContent: ( - await vaultOps.getSecret(vault, secretName) - ).toString('binary'), - }; + async (vault) => { + return await vaultOps.getSecret(vault, secretName); }, tran, ); + + yield { secretContent: content.toString('binary') }; } }); - } + }; } export default VaultsSecretsGet; diff --git a/src/client/handlers/VaultsSecretsRemove.ts b/src/client/handlers/VaultsSecretsRemove.ts index e7a0c8ce9..0395acf8e 100644 --- a/src/client/handlers/VaultsSecretsRemove.ts +++ b/src/client/handlers/VaultsSecretsRemove.ts @@ -3,34 +3,46 @@ import type { ClientRPCRequestParams, ClientRPCResponseResult, SuccessMessage, - SecretRemoveMessage, + SecretIdentifierMessage, } from '../types'; import type VaultManager from '../../vaults/VaultManager'; -import { UnaryHandler } from '@matrixai/rpc'; +import { ClientHandler } from '@matrixai/rpc'; import * as vaultsUtils from '../../vaults/utils'; import * as vaultsErrors from '../../vaults/errors'; import * as vaultOps from '../../vaults/VaultOps'; -class VaultsSecretsRemove extends UnaryHandler< +class VaultsSecretsRemove extends ClientHandler< { vaultManager: VaultManager; db: DB; }, - ClientRPCRequestParams, + ClientRPCRequestParams, ClientRPCResponseResult > { public handle = async ( - input: ClientRPCRequestParams, + input: AsyncIterable>, ): Promise> => { const { vaultManager, db } = this.container; // Create a record of secrets to be removed, grouped by vault names - const vaultGroups: Record = {}; - input.secretNames.forEach(([vaultName, secretName]) => { + const vaultGroups: Record> = {}; + const secretNames: Array<[string, string]> = []; + let metadata: any = {}; + let first = true; + for await (const secretRemoveMessage of input) { + if (first) metadata = secretRemoveMessage.metadata; + secretNames.push([ + secretRemoveMessage.nameOrId, + secretRemoveMessage.secretName, + ]); + first = false; + } + secretNames.forEach(([vaultName, secretName]) => { if (vaultGroups[vaultName] == null) { vaultGroups[vaultName] = []; } vaultGroups[vaultName].push(secretName); }); + await db.withTransactionF(async (tran) => { for (const [vaultName, secretNames] of Object.entries(vaultGroups)) { const vaultIdFromName = await vaultManager.getVaultId(vaultName, tran); @@ -39,8 +51,11 @@ class VaultsSecretsRemove extends UnaryHandler< await vaultManager.withVaults( [vaultId], async (vault) => { + // console.log('in here', metadata); + // console.log('options', metadata?.options); + // console.log('recursive', metadata?.options?.recursive); await vaultOps.deleteSecret(vault, secretNames, { - recursive: input.options?.recursive, + recursive: metadata?.options?.recursive, }); }, tran, diff --git a/src/client/types.ts b/src/client/types.ts index 45def3382..0b125b18d 100644 --- a/src/client/types.ts +++ b/src/client/types.ts @@ -304,20 +304,8 @@ type SecretPathMessage = { secretName: string; }; -// The secrets must be in a format like: -// [ [vaultName, secretPath], [vaultName, secretPath ] ] -type SecretManyPathMessage = { - secretNames: Array>; -}; - type SecretIdentifierMessage = VaultIdentifierMessage & SecretPathMessage; -type SecretRemoveMessage = SecretManyPathMessage & { - options?: { - recursive?: boolean; - }; -}; - // Contains binary content as a binary string 'toString('binary')' type ContentMessage = { secretContent: string; @@ -427,9 +415,7 @@ export type { VaultsVersionMessage, VaultsLatestVersionMessage, SecretPathMessage, - SecretManyPathMessage, SecretIdentifierMessage, - SecretRemoveMessage, ContentMessage, SecretContentMessage, SecretMkdirMessage, diff --git a/tests/client/handlers/vaults.test.ts b/tests/client/handlers/vaults.test.ts index ab425f8f1..d6779fdbc 100644 --- a/tests/client/handlers/vaults.test.ts +++ b/tests/client/handlers/vaults.test.ts @@ -5,6 +5,7 @@ import type NodeManager from '@/nodes/NodeManager'; import type { LogEntryMessage, SecretContentMessage, + ContentMessage, VaultListMessage, VaultPermissionMessage, } from '@/client/types'; @@ -1462,27 +1463,48 @@ describe('vaultsSecretsNew and vaultsSecretsDelete, vaultsSecretsGet', () => { }); expect(createResponse.success).toBeTruthy(); // Get secret - const getResponse = await rpcClient.methods.vaultsSecretsGet({ - secretNames: [[vaultIdEncoded, secret]], + const getStream = await rpcClient.methods.vaultsSecretsGet(); + await new Promise(async (resolve) => { + const writer = getStream.writable.getWriter(); + await writer.write({ + nameOrId: vaultIdEncoded, + secretName: secret, + }); + await writer.close(); + resolve(0); }); const secretContent: Array = []; - for await (const data of getResponse) { + for await (const data of getStream.readable) { secretContent.push(data.secretContent); } const concatenatedContent = secretContent.join(''); expect(concatenatedContent).toStrictEqual(secret); // Delete secret - const deleteResponse = await rpcClient.methods.vaultsSecretsRemove({ - secretNames: [[vaultIdEncoded, secret]], + const deleteStream = await rpcClient.methods.vaultsSecretsRemove(); + await new Promise(async (resolve) => { + const writer = deleteStream.writable.getWriter(); + await writer.write({ + nameOrId: vaultIdEncoded, + secretName: secret, + }); + await writer.close(); + resolve(0); }); - expect(deleteResponse.success).toBeTruthy(); + expect((await deleteStream.output).success).toBeTruthy(); // Check secret was deleted - const deleteGetResponse = await rpcClient.methods.vaultsSecretsGet({ - secretNames: [[vaultIdEncoded, secret]], + const deleteGetStream = await rpcClient.methods.vaultsSecretsGet(); + await new Promise(async (resolve) => { + const writer = deleteGetStream.writable.getWriter(); + await writer.write({ + nameOrId: vaultIdEncoded, + secretName: secret, + }); + await writer.close(); + resolve(0); }); await expect(async () => { try { - for await (const _ of deleteGetResponse); + for await (const _ of deleteGetStream.readable); } catch (e) { throw e.cause; } @@ -1502,11 +1524,20 @@ describe('vaultsSecretsNew and vaultsSecretsDelete, vaultsSecretsGet', () => { expect(createResponse.success).toBeTruthy(); } // Get secret - const getResponse = await rpcClient.methods.vaultsSecretsGet({ - secretNames: secretNames.map((v) => [vaultIdEncoded, v]), + const getStream = await rpcClient.methods.vaultsSecretsGet(); + await new Promise(async (resolve) => { + const writer = getStream.writable.getWriter(); + for (const name of secretNames) { + await writer.write({ + nameOrId: vaultIdEncoded, + secretName: name, + }); + } + await writer.close(); + resolve(0); }); const secretContent: Array = []; - for await (const data of getResponse) { + for await (const data of getStream.readable) { secretContent.push(data.secretContent); } expect(secretContent.join('')).toStrictEqual(secretNames.join('')); @@ -1524,27 +1555,52 @@ describe('vaultsSecretsNew and vaultsSecretsDelete, vaultsSecretsGet', () => { }); } // Get secret - const getResponse = await rpcClient.methods.vaultsSecretsGet({ - secretNames: secretNames.map((v) => [vaultIdEncoded, v]), + const getStream = await rpcClient.methods.vaultsSecretsGet(); + await new Promise(async (resolve) => { + const writer = getStream.writable.getWriter(); + for (const name of secretNames) { + await writer.write({ + nameOrId: vaultIdEncoded, + secretName: name, + }); + } + await writer.close(); + resolve(0); }); const secretContent: Array = []; - for await (const data of getResponse) { + for await (const data of getStream.readable) { secretContent.push(data.secretContent); } expect(secretContent.join('')).toStrictEqual(secretNames.join('')); // Delete secret - const deleteResponse = await rpcClient.methods.vaultsSecretsRemove({ - secretNames: secretNames.map((v) => [vaultIdEncoded, v]), + const deleteStream = await rpcClient.methods.vaultsSecretsRemove(); + await new Promise(async (resolve) => { + const writer = deleteStream.writable.getWriter(); + for (const secretName of secretNames) { + await writer.write({ + nameOrId: vaultIdEncoded, + secretName: secretName, + }); + } + await writer.close(); + resolve(0); }); - expect(deleteResponse.success).toBeTruthy(); + expect((await deleteStream.output).success).toBeTruthy(); // Check each secret was deleted for (const secretName of secretNames) { - const response = await rpcClient.methods.vaultsSecretsGet({ - secretNames: [[vaultIdEncoded, secretName]], + const getStream = await rpcClient.methods.vaultsSecretsGet(); + await new Promise(async (resolve) => { + const writer = getStream.writable.getWriter(); + await writer.write({ + nameOrId: vaultIdEncoded, + secretName: secretName, + }); + await writer.close(); + resolve(0); }); await expect(async () => { try { - for await (const _ of response); // Consume + for await (const _ of getStream.readable); // Consume } catch (e) { throw e.cause; } @@ -1572,27 +1628,52 @@ describe('vaultsSecretsNew and vaultsSecretsDelete, vaultsSecretsGet', () => { }); } // Get secret - const getResponse = await rpcClient.methods.vaultsSecretsGet({ - secretNames: secretVaultNames, + const getStream = await rpcClient.methods.vaultsSecretsGet(); + await new Promise(async (resolve) => { + const writer = getStream.writable.getWriter(); + for (const [vault, name] of secretVaultNames) { + await writer.write({ + nameOrId: vault, + secretName: name, + }); + } + await writer.close(); + resolve(0); }); const secretContent: Array = []; - for await (const data of getResponse) { + for await (const data of getStream.readable) { secretContent.push(data.secretContent); } expect(secretContent.join('')).toStrictEqual(secretNames.join('')); // Delete secret - const deleteResponse = await rpcClient.methods.vaultsSecretsRemove({ - secretNames: secretVaultNames, + const deleteStream = await rpcClient.methods.vaultsSecretsRemove(); + await new Promise(async (resolve) => { + const writer = deleteStream.writable.getWriter(); + for (const [vault, name] of secretVaultNames) { + await writer.write({ + nameOrId: vault, + secretName: name, + }); + } + await writer.close(); + resolve(0); }); - expect(deleteResponse.success).toBeTruthy(); + expect((await deleteStream.output).success).toBeTruthy(); // Check each secret was deleted - for (const secretPair of secretVaultNames) { - const response = await rpcClient.methods.vaultsSecretsGet({ - secretNames: [secretPair], + for (const [vault, name] of secretVaultNames) { + const getStream = await rpcClient.methods.vaultsSecretsGet(); + await new Promise(async (resolve) => { + const writer = getStream.writable.getWriter(); + await writer.write({ + nameOrId: vault, + secretName: name, + }); + await writer.close(); + resolve(0); }); await expect(async () => { try { - for await (const _ of response); // Consume + for await (const _ of getStream.readable); // Consume } catch (e) { throw e.cause; } @@ -1620,48 +1701,69 @@ describe('vaultsSecretsNew and vaultsSecretsDelete, vaultsSecretsGet', () => { }); } // Get secret - const getResponse = await rpcClient.methods.vaultsSecretsGet({ - secretNames: secretVaultNames, + const getStream = await rpcClient.methods.vaultsSecretsGet(); + await new Promise(async (resolve) => { + const writer = getStream.writable.getWriter(); + for (const [vault, name] of secretVaultNames) { + await writer.write({ + nameOrId: vault, + secretName: name, + }); + } + await writer.close(); + resolve(0); }); const secretContent: Array = []; - for await (const data of getResponse) { + for await (const data of getStream.readable) { secretContent.push(data.secretContent); } expect(secretContent.join('')).toStrictEqual(secretNames.join('')); // Get log size - let logLength1: number; - let logLength2: number; - await vaultManager.withVaults([vaultIds[0]], async (vault) => { - logLength1 = (await vault.log()).length; - }); - await vaultManager.withVaults([vaultIds[1]], async (vault) => { - logLength2 = (await vault.log()).length; + let logLength: [number, number] = [0, 0]; + await vaultManager.withVaults(vaultIds, async (...vaults) => { + for (let i = 0; i < vaults.length; i++) { + logLength[i] = (await vaults[i].log()).length; + } }); // Delete secret - const deleteResponse = await rpcClient.methods.vaultsSecretsRemove({ - secretNames: secretVaultNames, + const deleteStream = await rpcClient.methods.vaultsSecretsRemove(); + await new Promise(async (resolve) => { + const writer = deleteStream.writable.getWriter(); + for (const [vault, name] of secretVaultNames) { + await writer.write({ + nameOrId: vault, + secretName: name, + }); + } + await writer.close(); + resolve(0); }); - expect(deleteResponse.success).toBeTruthy(); + expect((await deleteStream.output).success).toBeTruthy(); // Check each secret was deleted - for (const secretPair of secretVaultNames) { - const response = await rpcClient.methods.vaultsSecretsGet({ - secretNames: [secretPair], + for (const [vault, name] of secretVaultNames) { + const getStream = await rpcClient.methods.vaultsSecretsGet(); + await new Promise(async (resolve) => { + const writer = getStream.writable.getWriter(); + await writer.write({ + nameOrId: vault, + secretName: name, + }); + await writer.close(); + resolve(0); }); await expect(async () => { try { - for await (const _ of response); // Consume + for await (const _ of getStream.readable); // Consume } catch (e) { throw e.cause; } }).rejects.toThrow(vaultsErrors.ErrorSecretsSecretUndefined); } // Ensure single log message for deleting the secrets - // TODO: find a way to collapse this condition into one withVaults - await vaultManager.withVaults([vaultIds[0]], async (vault) => { - expect((await vault.log()).length).toEqual(logLength1 + 1); - }); - await vaultManager.withVaults([vaultIds[1]], async (vault) => { - expect((await vault.log()).length).toEqual(logLength2 + 1); + await vaultManager.withVaults(vaultIds, async (...vaults) => { + for (let i = 0; i < vaults.length; i++) { + expect((await vaults[i].log()).length).toEqual(logLength[i] + 1); + } }); }); test('deletes directory recursively', async () => { @@ -1685,25 +1787,50 @@ describe('vaultsSecretsNew and vaultsSecretsDelete, vaultsSecretsGet', () => { }); expect(addResponse.success).toBeTruthy(); // Delete secret - await testsUtils.expectRemoteError( - rpcClient.methods.vaultsSecretsRemove({ - secretNames: [[vaultIdEncoded, secretDirName]], - }), - vaultsErrors.ErrorVaultsRecursive, - ); - const deleteResponse = await rpcClient.methods.vaultsSecretsRemove({ - secretNames: [[vaultIdEncoded, secretDirName]], - options: { recursive: true }, + const failDeleteStream = await rpcClient.methods.vaultsSecretsRemove(); + await new Promise(async (resolve) => { + const writer = failDeleteStream.writable.getWriter(); + await writer.write({ + nameOrId: vaultIdEncoded, + secretName: secretDirName, + }); + await writer.close(); + resolve(0); }); - expect(deleteResponse.success).toBeTruthy(); - // Check secret was deleted - for (const secretName of secretNames) { - const response = await rpcClient.methods.vaultsSecretsGet({ - secretNames: [[vaultIdEncoded, secretName]], + await expect(async () => { + try { + (await failDeleteStream.output).success; + } catch (e) { + throw e.cause; + } + }).rejects.toThrow(vaultsErrors.ErrorVaultsRecursive); + const deleteStream = await rpcClient.methods.vaultsSecretsRemove(); + await new Promise(async (resolve) => { + const writer = deleteStream.writable.getWriter(); + await writer.write({ + nameOrId: vaultIdEncoded, + secretName: secretDirName, + metadata: { options: { recursive: true } }, + }); + await writer.close(); + resolve(0); + }); + expect((await deleteStream.output).success).toBeTruthy(); + // Check each secret and the secret directory were deleted + for (const name of secretNames) { + const getStream = await rpcClient.methods.vaultsSecretsGet(); + await new Promise(async (resolve) => { + const writer = getStream.writable.getWriter(); + await writer.write({ + nameOrId: vaultIdEncoded, + secretName: name, + }); + await writer.close(); + resolve(0); }); await expect(async () => { try { - for await (const _ of response); // Consume + for await (const _ of getStream.readable); // Consume } catch (e) { throw e.cause; }