Limited sqs size

CHANGELOG.md

@@ -48,6 +48,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
 
 ### Changed
 
+- **CUMULUS-3678**
+  - sqs messages truncated to max sqs length if necessary
 - **CUMULUS-3928**
   - updated publish scripting to use [email protected] for user email
   - updated publish scripting to use esm over common import of latest-version

packages/aws-client/src/SQS.ts

@@ -18,7 +18,9 @@ import {
   Message,
   QueueAttributeName,
   ReceiveMessageCommand,
-  SendMessageCommand } from '@aws-sdk/client-sqs';
+  SendMessageCommand,
+  SendMessageCommandOutput
+} from '@aws-sdk/client-sqs';
 
 import { StepFunctionEventBridgeEvent } from './Lambda';
 import { sqs } from './services';
@@ -94,12 +96,17 @@ export const sendSQSMessage = (
   queueUrl: string,
   message: string | object,
   logOverride: Logger | undefined = undefined
-) => {
+): Promise<SendMessageCommandOutput> => {
   const logger = logOverride || log;
   let messageBody;
   if (isString(message)) messageBody = message;
   else if (isObject(message)) messageBody = JSON.stringify(message);
   else throw new Error('body type is not accepted');
+  const maxLength = 262144;
+  if (messageBody.length > maxLength) {
+    const warningString = '...TruncatedForLength';
+    messageBody = `${messageBody.substring(0, maxLength - warningString.length)}${warningString}`;
+  }
 
   const command = new SendMessageCommand({
     MessageBody: messageBody,

packages/aws-client/tests/test-SQS.js

@@ -12,6 +12,7 @@ const {
   sqsQueueExists,
   sendSQSMessage,
   isSQSRecordLike,
+  receiveSQSMessages,
 } = require('../SQS');
 
 const randomString = () => cryptoRandomString({ length: 10 });
@@ -100,3 +101,29 @@ test('isSQSRecordLike filters correctly for sqs record shape', (t) => {
   body *should* be a string form json object,
   but strictly checking is not compatible with the current use-case*/
 });
+
+test('sendSQSMessage truncates oversized messages safely', async (t) => {
+  const queueName = randomString();
+  const queueUrl = await createQueue(queueName);
+  const maxLength = 262144;
+  const overflowMessage = '0'.repeat(maxLength + 2);
+  await sendSQSMessage(queueUrl, overflowMessage);
+
+  let recievedMessage = await receiveSQSMessages(queueUrl, {});
+  let messageBody = recievedMessage[0].Body;
+  t.true(messageBody.endsWith('...TruncatedForLength'));
+  t.is(messageBody.length, maxLength);
+});
+
+test('sendSQSMessage does not alter objects of normal size', async (t) => {
+  const queueName = randomString();
+  const queueUrl = await createQueue(queueName);
+  const maxLength = 262144;
+  const underflowMessage = '0'.repeat(maxLength);
+  await sendSQSMessage(queueUrl, underflowMessage);
+  recievedMessage = await receiveSQSMessages(queueUrl, {});
+  messageBody = recievedMessage[0].Body;
+
+  t.deepEqual(messageBody, underflowMessage);
+  t.false(messageBody.endsWith('...TruncatedForLength'));
+});