ti_misp: add request rate limit (#11641)

Adding the possibility to set the request rate limit for the TI MISP integration. The nature of API responses that require nested splits to decode events may lead to high resource utilization in the agent, which cannot be avoided by the filters that MISP currently provides. Setting a request rate limit may help in the processing of the responses to avoid the input being overwhelmed.
elastic · Nov 7, 2024 · 6926115 · 6926115
1 parent e368bfe
commit 6926115
Show file tree

Hide file tree

Showing 6 changed files with 28 additions and 1 deletion.
diff --git a/packages/ti_misp/changelog.yml b/packages/ti_misp/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.35.7"
+  changes:
+    - description: Set request rate limits.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/11641
 - version: "1.35.6"
   changes:
     - description: Fix the processing of ISO8601 dates.

diff --git a/packages/ti_misp/data_stream/threat/agent/stream/httpjson.yml.hbs b/packages/ti_misp/data_stream/threat/agent/stream/httpjson.yml.hbs
@@ -14,6 +14,9 @@ request.ssl: {{ssl}}
 {{#if http_client_timeout}}
 request.timeout: {{http_client_timeout}}
 {{/if}}
+{{#if http_request_rate_limit}}
+request.rate_limit.limit: "{{http_request_rate_limit}}"
+{{/if}}
 {{#if proxy_url}}
 request.proxy_url: {{proxy_url}}
 {{/if}}

diff --git a/packages/ti_misp/data_stream/threat/manifest.yml b/packages/ti_misp/data_stream/threat/manifest.yml
@@ -43,6 +43,14 @@ streams:
         required: false
         show_user: false
         default: 30s
+      - name: http_request_rate_limit
+        type: text
+        title: HTTP Request Rate limit
+        description: "The maximum per endpoint request rate, in requests per second (e.g. 0.5 reqs/sec for 30 reqs/min). Controlling the rate limit may help with the processing of large responses from the MISP API."
+        default: 1
+        multi: false
+        required: false
+        show_user: false
       - name: filters
         type: yaml
         title: MISP API Filters

diff --git a/packages/ti_misp/data_stream/threat_attributes/agent/stream/httpjson.yml.hbs b/packages/ti_misp/data_stream/threat_attributes/agent/stream/httpjson.yml.hbs
@@ -14,6 +14,9 @@ request.ssl: {{ssl}}
 {{#if http_client_timeout}}
 request.timeout: {{http_client_timeout}}
 {{/if}}
+{{#if http_request_rate_limit}}
+request.rate_limit.limit: "{{http_request_rate_limit}}"
+{{/if}}
 {{#if proxy_url}}
 request.proxy_url: {{proxy_url}}
 {{/if}}

diff --git a/packages/ti_misp/data_stream/threat_attributes/manifest.yml b/packages/ti_misp/data_stream/threat_attributes/manifest.yml
@@ -52,6 +52,14 @@ streams:
         required: false
         show_user: false
         default: 30s
+      - name: http_request_rate_limit
+        type: text
+        title: HTTP Request Rate limit
+        description: "The maximum per endpoint request rate, in requests per second (e.g. 0.5 reqs/sec for 30 reqs/min). Controlling the rate limit may help with the processing of large responses from the MISP API."
+        default: 1
+        multi: false
+        required: false
+        show_user: false
       - name: filters
         type: yaml
         title: MISP API Filters

diff --git a/packages/ti_misp/manifest.yml b/packages/ti_misp/manifest.yml
@@ -1,6 +1,6 @@
 name: ti_misp
 title: MISP
-version: "1.35.6"
+version: "1.35.7"
 description: Ingest threat intelligence indicators from MISP platform with Elastic Agent.
 type: integration
 format_version: "3.0.2"