From 97ab6aa393835c00467bfed3ac98f6df84d02dc4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Chema=20Mart=C3=ADnez?= <chema.martinez@elastic.co>
Date: Tue, 17 Dec 2024 13:09:11 +0100
Subject: [PATCH] ti_custom: Improve pagination options for TAXII servers
 (#12084)

Changes added:
- Add a limit parameter, that can be used to control the size of responses from TAXII servers (see https://docs.oasis-open.org/cti/taxii/v2.1/os/taxii-v2.1-os.html#_Toc31107517)
- To avoid fetching duplicate indicators every interval, now the response header X-Taxii-Date-Added-Last is stored in the cursor and used to populate the added_after parameter every iteration (see https://docs.oasis-open.org/cti/taxii/v2.1/os/taxii-v2.1-os.html#_Toc31107519)
---
 packages/ti_custom/changelog.yml                 |  8 ++++++++
 .../indicator/agent/stream/cel.yml.hbs           | 16 +++++++++++++++-
 .../ti_custom/data_stream/indicator/manifest.yml |  7 +++++++
 packages/ti_custom/manifest.yml                  |  2 +-
 4 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/packages/ti_custom/changelog.yml b/packages/ti_custom/changelog.yml
index 38fcb3c41f1..b5be4733337 100644
--- a/packages/ti_custom/changelog.yml
+++ b/packages/ti_custom/changelog.yml
@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "0.6.0"
+  changes:
+    - description: Support for limit parameter in API requests.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/12084
+    - description: Use the header X-Taxii-Date-Added-Last to avoid duplicates in successive requests.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/12084
 - version: "0.5.0"
   changes:
     - description: Improve documentation on how to proceed with ISAC feeds.
diff --git a/packages/ti_custom/data_stream/indicator/agent/stream/cel.yml.hbs b/packages/ti_custom/data_stream/indicator/agent/stream/cel.yml.hbs
index dc4d059339a..1311ecd0e84 100644
--- a/packages/ti_custom/data_stream/indicator/agent/stream/cel.yml.hbs
+++ b/packages/ti_custom/data_stream/indicator/agent/stream/cel.yml.hbs
@@ -37,7 +37,9 @@ resource.tracer.maxsize: 5
 {{#if enable_taxii}}
 state:
   url: {{url}}
+{{#if initial_interval}}
   initial_interval: {{initial_interval}}
+{{/if}}
   want_more: false
 {{#if api_key}}
   api_key: {{api_key}}
@@ -48,6 +50,9 @@ state:
   password: {{password}}
 {{/if}}
 {{/if}}
+{{#if limit}}
+  limit: {{limit}}
+{{/if}}
 redact:
   fields:
     - api_key
@@ -64,7 +69,12 @@ program: |
             : (has(state.initial_interval) && state.initial_interval != "") ?
                 (
                     state.url.trim_right("/") + "/?" + {
-                        "added_after": [(now() - duration(state.initial_interval)).format(time_layout.RFC3339)],
+                        ?"added_after": state.?cursor.last_timestamp.optMap(ts,
+                            [ts]
+                        ).or(
+                            state.?initial_interval.optMap(i, [(now() - duration(i)).format(time_layout.RFC3339)])
+                        ),
+                        ?"limit": state.?limit.optMap(l, [string(int(l))]),
                     }.format_query()
                 )
                 :
@@ -99,10 +109,14 @@ program: |
                         (
                             state.url.trim_right("/") + "/?" + {
                                 "next": [string(body.next)],
+                                ?"limit": state.?limit.optMap(l, [string(int(l))]),
                             }.format_query()
                         )
                     :
                         state.url,
+                    "cursor": {
+                        ?"last_timestamp": resp.Header[?"X-Taxii-Date-Added-Last"][0],
+                    },
                 }
             )
         :
diff --git a/packages/ti_custom/data_stream/indicator/manifest.yml b/packages/ti_custom/data_stream/indicator/manifest.yml
index 79e374adca9..317f0977f1e 100644
--- a/packages/ti_custom/data_stream/indicator/manifest.yml
+++ b/packages/ti_custom/data_stream/indicator/manifest.yml
@@ -173,6 +173,13 @@ streams:
         multi: false
         required: false
         show_user: true
+      - name: limit
+        type: integer
+        title: Limit
+        description: The maximum number of objects to return in each API response. It must be a positive number.
+        multi: false
+        required: false
+        show_user: false
       - name: feed_name
         type: text
         title: Feed name
diff --git a/packages/ti_custom/manifest.yml b/packages/ti_custom/manifest.yml
index 13d247f1069..9895bbf4db3 100644
--- a/packages/ti_custom/manifest.yml
+++ b/packages/ti_custom/manifest.yml
@@ -3,7 +3,7 @@ name: ti_custom
 title: Custom Threat Intelligence
 description: Ingest threat intelligence data in STIX 2.1 format with Elastic Agent
 type: integration
-version: 0.5.0
+version: 0.6.0
 categories:
   - custom
   - security