From f4550b7468a7b559ed3e18a3834652ad14d92492 Mon Sep 17 00:00:00 2001 From: ShourieG Date: Sat, 9 Nov 2024 00:03:21 +0530 Subject: [PATCH] [Cisco Duo] - Added fix for 400 and 401 errors in telephony v2 data stream (#11640) * added fix in telephony_v2 data stream for 400 & 401 errors --- packages/cisco_duo/changelog.yml | 5 + .../telephony_v2/agent/stream/cel.yml.hbs | 196 +++++++++--------- packages/cisco_duo/manifest.yml | 2 +- 3 files changed, 100 insertions(+), 103 deletions(-) diff --git a/packages/cisco_duo/changelog.yml b/packages/cisco_duo/changelog.yml index 456ef26304c..3ca4f1a2fbf 100644 --- a/packages/cisco_duo/changelog.yml +++ b/packages/cisco_duo/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.2.2" + changes: + - description: Fix for 40103 & 400 errors in the telephony_v2 data stream. + type: bugfix + link: https://github.com/elastic/integrations/pull/11640 - version: "2.2.1" changes: - description: Fix errors in Trust Monitor CEL. diff --git a/packages/cisco_duo/data_stream/telephony_v2/agent/stream/cel.yml.hbs b/packages/cisco_duo/data_stream/telephony_v2/agent/stream/cel.yml.hbs index fb8f07067f1..fe7ac5c8a6c 100644 --- a/packages/cisco_duo/data_stream/telephony_v2/agent/stream/cel.yml.hbs +++ b/packages/cisco_duo/data_stream/telephony_v2/agent/stream/cel.yml.hbs @@ -10,9 +10,6 @@ resource.tracer.maxbackups: 5 resource.tracer.maxsize: 5 {{/if}} -regexp: - "next_offset_timestamp": '^\d+' - state: url: {{hostname}} integration_key: {{integration_key}} @@ -31,114 +28,109 @@ program: | state : state.with({ - "mintime": state.?cursor.last_published.orValue(int(now - duration(state.initial_interval)) * 1000), - "maxtime": int(now - duration("2m")) * 1000, - "date": now.format(time_layout.RFC1123Z), + "mintime": string(int(state.?cursor.last_published.orValue(int(now - duration(state.initial_interval)) * 1000))), }) - ).as(state, state.with( - request( - "GET", - state.?want_more.orValue(false) ? - state.next_url - : - state.url.trim_right("/") + "/admin/v2/logs/telephony?" + { - "limit": [string(int(state.limit))], - "maxtime": [string(int(state.maxtime))], - "mintime": [string(int(state.mintime))], - "sort": ["ts:asc"], - }.format_query() - ).with( - { + ).as(state, state.with({ + // calculate maxtime to be the current time minus a buffer (2 minutes) to avoid potential synchronization issues. + "maxtime": state.want_more ? + string(state.maxtime) + : + string(int([ + int(timestamp(int(state.mintime)/1000) + duration("4320h")), + int(now - duration("2m")) + ].min()) * 1000) + })).as(state, state.with( + { + // prepare request data dynamically for each request to prevent 40103 error. + "date": now().format(time_layout.RFC1123Z), + "method": "GET", + "url_base": state.url.trim_right("/"), + "url_path": "/admin/v2/logs/telephony", + "query_string": { + "limit": [string(dyn(state).limit)], + "maxtime": [string(state.maxtime)], + "mintime": [string(state.mintime)], + ?"next_offset": state.?next_offset.optMap(v, [string(v)]), + "sort": ["ts:asc"], + }.format_query(), + }.as(r, r.with({ + "authorization": "Basic " + ( + state.integration_key + ":" + ( + [ + r.date, + r.method, + r.url_base.trim_prefix("https://"), + r.url_path, + r.query_string, + ].join("\n") + .hmac("sha1", bytes(state.secret_key)) + .hex() + ) + ).base64(), + })).as(r, + request( + r.method, + [r.url_base, r.url_path, "?", r.query_string].join("") + ).with({ "Header": { "Content-Type": ["application/x-www-form-urlencoded"], - "Date": [state.date], - "Authorization": ["Basic " + ( - state.integration_key + ":" + ( - [ - state.date, - "GET", - state.url.trim_prefix("https://"), - "/admin/v2/logs/telephony", - { - "limit": [string(int(state.limit))], - "maxtime": [string(int(state.maxtime))], - "mintime": [string(int(state.mintime))], - ?"next_offset": has(state.next_offset) ? - optional.of([string(state.next_offset)]) - : - optional.none(), - "sort": ["ts:asc"], - }.format_query() - ].join("\n") - .hmac("sha1", bytes(state.secret_key)) - .hex() - ) - ).base64()], + "Date": [r.date], + "Authorization": [r.authorization], }, - } - ).do_request().as(resp, (resp.StatusCode == 200) ? - bytes(resp.Body).decode_json().as(body, has(body.?response.items) && size(body.response.items) > 0 ? - { - "events": body.response.items.map(item, - { - "message": item.encode_json(), - } - ), - "url": state.url, - "integration_key": state.integration_key, - "secret_key": state.secret_key, - "limit": state.limit, - "mintime": state.mintime, - "maxtime": state.maxtime, - "date": now.format(time_layout.RFC1123Z), - "want_more": has(body.?response.?metadata.next_offset), - ?"next_offset": (body.?response.metadata.next_offset.orValue(null) != null) ? - optional.of(string(body.response.metadata.next_offset)) - : - optional.none(), - "next_url": (body.?response.metadata.next_offset.orValue(null) != null) ? - ( - state.url.trim_right("/") + "/admin/v2/logs/telephony?" + { - "limit": [string(int(state.limit))], - "maxtime": [string(int(state.maxtime))], - "mintime": [string(int(state.mintime))], - "next_offset": [string(body.response.metadata.next_offset)], - "sort": ["ts:asc"], - }.format_query() - ) - : - state.url, - "cursor": { - ?"last_published": (body.?response.metadata.next_offset.orValue(null) != null) ? - optional.of(body.response.metadata.next_offset.re_find("next_offset_timestamp")) + }).do_request().as(resp, (resp.StatusCode == 200) ? + bytes(resp.Body).decode_json().as(body, has(body.?response.items) && size(body.response.items) > 0 ? + { + "events": body.response.items.map(item, + { + "message": item.encode_json(), + } + ), + "want_more": has(body.?response.?metadata.next_offset), + ?"next_offset": (body.?response.metadata.next_offset.orValue(null) != null) ? + optional.of(string(body.response.metadata.next_offset)) : optional.none(), + "cursor": { + ?"last_published": (has(body.?response.items) && size(body.response.items) > 0) ? + optional.of( + body.response.items.map(i, i.ts).max().as(last_timestamp, + !has(dyn(state).?cursor.last_published) ? + last_timestamp + : (last_timestamp < dyn(state).cursor.last_published) ? + dyn(state).cursor.last_published + : + last_timestamp + ) + ) + : + dyn(state).?cursor.last_published, + } + } + : + { + "events": [], + "want_more": false, } - } - : - { - "events":[], - "want_more": false, - } - ) - : - bytes(resp.Body).decode_json().as(body, - { - "events": { - "error": { - "code": has(body.code) ? string(body.code) : string(resp.StatusCode), - "id": string(resp.Status), - "message": "GET:"+( - size(resp.Body) != 0 ? - string(resp.Body) - : - string(resp.Status) + ' (' + string(resp.StatusCode) + ')' - ), + ) + : + bytes(resp.Body).decode_json().as(body, + { + "events": { + "error": { + "code": has(body.code) ? string(body.code) : string(resp.StatusCode), + "id": string(resp.Status), + "message": "GET:"+( + size(resp.Body) != 0 ? + string(resp.Body) + : + string(resp.Status) + ' (' + string(resp.StatusCode) + ')' + ), + }, }, - }, - "want_more": false, - } + "want_more": false, + } + ) ) ) )) diff --git a/packages/cisco_duo/manifest.yml b/packages/cisco_duo/manifest.yml index c6f67331ebe..4666747c1b7 100644 --- a/packages/cisco_duo/manifest.yml +++ b/packages/cisco_duo/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: cisco_duo title: Cisco Duo -version: "2.2.1" +version: "2.2.2" description: Collect logs from Cisco Duo with Elastic Agent. type: integration categories: