Add validation to check that notifications dont exceed sqs msg limit

app/notifications/validators.py

@@ -52,6 +52,7 @@
     increment_todays_requested_sms_count,
 )
 from app.utils import (
+    flatten_dct,
     get_document_url,
     get_limit_reset_time_et,
     get_public_notify_type_text,
@@ -594,3 +595,18 @@ def decode_personalisation_files(json_personalisation):
                 }
             )
     return json_personalisation, errors
+
+
+def validate_notification_does_not_exceed_sqs_limit(notification):
+    # SQS max payload size is 256KB
+    if len(str(notification)) >= 262144:
+        # find the largest value in the payload for logging and return message
+        max_key, max_length = max(
+            ((key, len(str(value))) for key, value in flatten_dct(dict(notification)).items()), key=lambda x: x[1]
+        )
+        current_app.logger.debug(
+            f"Unable to send notification {notification['id']}. Payload size exceeds SQS limit of 262144 bytes. Largest key: {max_key} is {max_length} bytes."
+        )
+        raise BadRequestError(
+            message=f"Notification size cannot exceed 256Kb. Consider reducing the size of: {max_key}.", status_code=413
+        )

app/utils.py

@@ -1,6 +1,7 @@
 import os
+from collections.abc import MutableMapping
 from datetime import datetime, timedelta
-from typing import Any
+from typing import Any, List, Tuple
 
 import pytz
 from flask import current_app, url_for
@@ -142,6 +143,30 @@ def email_address_is_nhs(email_address):
     )
 
 
+def flatten_dct(dictionary: MutableMapping[Any, Any], parent_key: str = "", separator: str = ".") -> dict:
+    """Recursively flatten a nested dict structure into a single level dict structure.
+    Depth is represented by composite keys separated by the separator arg.
+
+    Args:
+        dictionary `MutableMapping`: The dictionary to flatten
+        parent_key `str`: Top level key used when compositing keys, typically an object
+          identifier (notification, service, template, etc.)
+          Defaults to the top level key of the dict being flattened.
+        separator `str` : Separator used to delimit composited keys.
+
+    Returns:
+        dict `dict`: The flattened dict with composite keys
+    """
+    items: List[Tuple[str, Any]] = []
+    for key, value in dictionary.items():
+        new_key = parent_key + separator + key if parent_key else key
+        if isinstance(value, MutableMapping):
+            items.extend(flatten_dct(value, new_key, separator=separator).items())
+        else:
+            items.append((new_key, value))
+    return dict(items)
+
+
 def update_dct_to_str(update_dct, lang):
     if lang not in ["EN", "FR"]:
         raise NotImplementedError

app/v2/notifications/post_notifications.py

@@ -83,6 +83,7 @@
     increment_email_daily_count_send_warnings_if_needed,
     increment_sms_daily_count_send_warnings_if_needed,
     validate_and_format_recipient,
+    validate_notification_does_not_exceed_sqs_limit,
     validate_template,
     validate_template_exists,
 )
@@ -415,6 +416,8 @@ def process_sms_or_email_notification(
         "reply_to_text": reply_to_text,
     }
 
+    validate_notification_does_not_exceed_sqs_limit(_notification)
+
     signed_notification_data = signer_notification.sign(_notification)
     notification = {**_notification}
     scheduled_for = form.get("scheduled_for", None)

tests/app/notifications/test_validators.py

@@ -1,3 +1,4 @@
+import uuid
 from unittest.mock import call
 
 import pytest
@@ -31,6 +32,7 @@
     increment_sms_daily_count_send_warnings_if_needed,
     service_can_send_to_recipient,
     validate_and_format_recipient,
+    validate_notification_does_not_exceed_sqs_limit,
 )
 from app.utils import get_document_url
 from app.v2.errors import (
@@ -698,3 +700,37 @@ def test_check_reply_to_sms_type(sample_service):
 def test_check_reply_to_letter_type(sample_service):
     letter_contact = create_letter_contact(service=sample_service, contact_block="123456")
     assert check_reply_to(sample_service.id, letter_contact.id, LETTER_TYPE) == "123456"
+
+
+def test_validate_notification_does_not_exceed_sqs_limit_exceeds_limit(mocker):
+    notification = {
+        "id": str(uuid.uuid4()),
+        "payload": {
+            "key1": "value1",
+            "key2": "value2" * 100000,  # Exceeds the SQS limit
+            "key3": "value3",
+        },
+    }
+    logger = mocker.patch("app.notifications.validators.current_app.logger.debug")
+
+    with pytest.raises(BadRequestError) as e:
+        validate_notification_does_not_exceed_sqs_limit(notification)
+
+    # We flatten the dict here so the key is composite
+    assert e.value.message == "Notification size cannot exceed 256Kb. Consider reducing the size of: payload.key2."
+    assert e.value.status_code == 413
+    logger.assert_called_once_with(
+        f"Unable to send notification {notification['id']}. Payload size exceeds SQS limit of 262144 bytes. Largest key: payload.key2 is {len(notification['payload']['key2'])} bytes."
+    )
+
+
+def test_validate_notification_does_not_exceed_sqs_limit_within_limit(mocker):
+    notification = {
+        "id": "123",
+        "payload": {
+            "key1": "value1",
+            "key2": "value2",
+            "key3": "value3",
+        },
+    }
+    assert validate_notification_does_not_exceed_sqs_limit(notification) is None

tests/app/service/test_utils.py

@@ -3,6 +3,7 @@
 
 from app.dao.date_util import get_current_financial_year_start_year
 from app.service.utils import get_organisation_id_from_crm_org_notes
+from app.utils import flatten_dct
 
 
 # see get_financial_year for conversion of financial years.
@@ -39,3 +40,69 @@ def test_get_organisation_id_from_crm_org_notes(mocker, org_notes, expected_id):
     ]
     mocker.patch("app.service.utils.get_gc_organisation_data", return_value=mock_gc_org_data)
     assert get_organisation_id_from_crm_org_notes(org_notes) == expected_id
+
+
+@pytest.mark.parametrize(
+    "dictionary, expected_result",
+    [
+        (
+            {
+                "key1": {
+                    "nested_key1": {"deeply_nested_key1": "deeply_nested_value1", "deeply_nested_key2": "deeply_nested_value2"},
+                    "nested_key2": "nested_value2",
+                },
+                "key2": "value2",
+                "key3": {"nested_key3": "nested_value3"},
+            },
+            {
+                "key1.nested_key1.deeply_nested_key1": "deeply_nested_value1",
+                "key1.nested_key1.deeply_nested_key2": "deeply_nested_value2",
+                "key1.nested_key2": "nested_value2",
+                "key2": "value2",
+                "key3.nested_key3": "nested_value3",
+            },
+        ),
+        (
+            {
+                "key1": {
+                    "nested_key1": {
+                        "deeply_nested_key1": "deeply_nested_value1",
+                        "nested_key2": {
+                            "deeply_nested_key2": "deeply_nested_value2",
+                            "deeply_nested_key3": "deeply_nested_value3",
+                        },
+                    },
+                    "nested_key2": "nested_value2",
+                },
+                "key2": "value2",
+                "key3": {
+                    "nested_key1": {
+                        "deeply_nested_key1": "deeply_nested_value1",
+                        "nested_key2": {
+                            "deeply_nested_key2": "deeply_nested_value2",
+                            "deeply_nested_key3": "deeply_nested_value3",
+                        },
+                    },
+                },
+            },
+            {
+                "key1.nested_key1.deeply_nested_key1": "deeply_nested_value1",
+                "key1.nested_key1.nested_key2.deeply_nested_key2": "deeply_nested_value2",
+                "key1.nested_key1.nested_key2.deeply_nested_key3": "deeply_nested_value3",
+                "key1.nested_key2": "nested_value2",
+                "key2": "value2",
+                "key3.nested_key1.deeply_nested_key1": "deeply_nested_value1",
+                "key3.nested_key1.nested_key2.deeply_nested_key2": "deeply_nested_value2",
+                "key3.nested_key1.nested_key2.deeply_nested_key3": "deeply_nested_value3",
+            },
+        ),
+        (
+            {"key1": "value1", "key2": {"nested_key1": "nested_value1", "nested_key2": "nested_value2"}, "key3": "value3"},
+            {"key1": "value1", "key2.nested_key1": "nested_value1", "key2.nested_key2": "nested_value2", "key3": "value3"},
+        ),
+        ({"key1": "value1", "key2": "value2", "key3": "value3"}, {"key1": "value1", "key2": "value2", "key3": "value3"}),
+        ({}, {}),
+    ],
+)
+def test_flatten_dct_deeply_nested_dict(dictionary, expected_result):
+    assert flatten_dct(dictionary) == expected_result