ti_custom: Improve pagination options for TAXII servers (#12084)

Changes added: - Add a limit parameter, that can be used to control the size of responses from TAXII servers (see https://docs.oasis-open.org/cti/taxii/v2.1/os/taxii-v2.1-os.html#_Toc31107517) - To avoid fetching duplicate indicators every interval, now the response header X-Taxii-Date-Added-Last is stored in the cursor and used to populate the added_after parameter every iteration (see https://docs.oasis-open.org/cti/taxii/v2.1/os/taxii-v2.1-os.html#_Toc31107519)
elastic · Dec 17, 2024 · 97ab6aa · 97ab6aa
1 parent 52ddf71
commit 97ab6aa
Show file tree

Hide file tree

Showing 4 changed files with 31 additions and 2 deletions.
diff --git a/packages/ti_custom/changelog.yml b/packages/ti_custom/changelog.yml
@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "0.6.0"
+  changes:
+    - description: Support for limit parameter in API requests.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/12084
+    - description: Use the header X-Taxii-Date-Added-Last to avoid duplicates in successive requests.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/12084
 - version: "0.5.0"
   changes:
     - description: Improve documentation on how to proceed with ISAC feeds.

diff --git a/packages/ti_custom/data_stream/indicator/agent/stream/cel.yml.hbs b/packages/ti_custom/data_stream/indicator/agent/stream/cel.yml.hbs
@@ -37,7 +37,9 @@ resource.tracer.maxsize: 5
 {{#if enable_taxii}}
 state:
   url: {{url}}
+{{#if initial_interval}}
   initial_interval: {{initial_interval}}
+{{/if}}
   want_more: false
 {{#if api_key}}
   api_key: {{api_key}}
@@ -48,6 +50,9 @@ state:
   password: {{password}}
 {{/if}}
 {{/if}}
+{{#if limit}}
+  limit: {{limit}}
+{{/if}}
 redact:
   fields:
     - api_key
@@ -64,7 +69,12 @@ program: |
             : (has(state.initial_interval) && state.initial_interval != "") ?
                 (
                     state.url.trim_right("/") + "/?" + {
-                        "added_after": [(now() - duration(state.initial_interval)).format(time_layout.RFC3339)],
+                        ?"added_after": state.?cursor.last_timestamp.optMap(ts,
+                            [ts]
+                        ).or(
+                            state.?initial_interval.optMap(i, [(now() - duration(i)).format(time_layout.RFC3339)])
+                        ),
+                        ?"limit": state.?limit.optMap(l, [string(int(l))]),
                     }.format_query()
                 )
                 :
@@ -99,10 +109,14 @@ program: |
                         (
                             state.url.trim_right("/") + "/?" + {
                                 "next": [string(body.next)],
+                                ?"limit": state.?limit.optMap(l, [string(int(l))]),
                             }.format_query()
                         )
                     :
                         state.url,
+                    "cursor": {
+                        ?"last_timestamp": resp.Header[?"X-Taxii-Date-Added-Last"][0],
+                    },
                 }
             )
         :

diff --git a/packages/ti_custom/data_stream/indicator/manifest.yml b/packages/ti_custom/data_stream/indicator/manifest.yml
@@ -173,6 +173,13 @@ streams:
         multi: false
         required: false
         show_user: true
+      - name: limit
+        type: integer
+        title: Limit
+        description: The maximum number of objects to return in each API response. It must be a positive number.
+        multi: false
+        required: false
+        show_user: false
       - name: feed_name
         type: text
         title: Feed name

diff --git a/packages/ti_custom/manifest.yml b/packages/ti_custom/manifest.yml
@@ -3,7 +3,7 @@ name: ti_custom
 title: Custom Threat Intelligence
 description: Ingest threat intelligence data in STIX 2.1 format with Elastic Agent
 type: integration
-version: 0.5.0
+version: 0.6.0
 categories:
   - custom
   - security