From 19bd853359a11d1aa4c8d347f42e4194fbe2bf31 Mon Sep 17 00:00:00 2001 From: Chris Berkhout Date: Fri, 18 Oct 2024 08:19:57 +1100 Subject: [PATCH] [cisco_duo] Fix auth CEL cursor handling (#11456) In the auth CEL program, the error `type conversion error from 'string' to 'int'` seems to have been happening because of `cursor.last_published` being set to a value of the form `1532951895000,af0ba235-0b33-23c8-bc23-a31aa0231de8`, which can't be parsed as an int. Now `cursor.last_published` is replaced with `cursor.last_timestamp_ms`, which is taken from the last result, and so will be available if the last page of sequence has results but no value in `response.metadata.next_offset`. Also, the date is no longer shared across requests, the request building is simplified, and redundant overrides of state are removed. Related documentation: https://duo.com/docs/adminapi#authentication-logs --- .../_dev/deploy/docker/files/config.yml | 6 +- packages/cisco_duo/changelog.yml | 5 + .../data_stream/auth/agent/stream/cel.yml.hbs | 171 +++++++----------- packages/cisco_duo/manifest.yml | 2 +- 4 files changed, 76 insertions(+), 108 deletions(-) diff --git a/packages/cisco_duo/_dev/deploy/docker/files/config.yml b/packages/cisco_duo/_dev/deploy/docker/files/config.yml index 4d1c392750a..26a76616b2c 100644 --- a/packages/cisco_duo/_dev/deploy/docker/files/config.yml +++ b/packages/cisco_duo/_dev/deploy/docker/files/config.yml @@ -35,7 +35,7 @@ rules: ], "metadata": { "next_offset": ["1666714065305","5bf1a860-fe39-49e3-be29-217659663a74"], - "total_objects": 5 + "total_objects": 4 }}, "stat":"OK"} - path: /admin/v2/logs/authentication @@ -50,7 +50,7 @@ rules: ], "metadata": { "next_offset": ["1666714065306","5bf1a860-fe39-49e3-be29-217659663a74"], - "total_objects": 5 + "total_objects": 3 }}, "stat":"OK"} - path: /admin/v2/logs/authentication @@ -65,7 +65,7 @@ rules: {"access_device":{"browser":"Chrome","browser_version":"67.0.3396.99","flash_version":"uninstalled","hostname":null,"ip":"89.160.20.156","is_encryption_enabled":true,"is_firewall_enabled":true,"is_password_set":true,"java_version":"uninstalled","location":{"city":"Ann Arbor","country":"United States","state":"Michigan"},"os":"Mac OS X","os_version":"10.14.1","security_agents":[]},"alias":"","application":{"key":"DIY231J8BR23QK4UKBY8","name":"Microsoft Azure Active Directory"},"auth_device":{"ip":"192.168.225.254","location":{"city":"Ann Arbor","country":"United States","state":"Michigan"},"name":"My iPhone X (734-555-2342)"},"email":"narroway@example.com","event_type":"authentication","factor":"duo_push","isotimestamp":"2020-02-13T18:56:20.351346+00:00","ood_software":null,"reason":"user_approved","result":"success","timestamp":1581620180,"trusted_endpoint_status":"not trusted","txid":"340a23e3-23f3-23c1-87dc-1491a23dfdbe","user":{"groups":["Duo Users","CorpHQ Users"],"key":"DU3KC77WJ06Y5HIV7XKQ","name":"narroway@example.com"}} ], "metadata": { - "total_objects": 5 + "total_objects": 2 }}, "stat":"OK"} - path: /admin/v1/logs/offline_enrollment diff --git a/packages/cisco_duo/changelog.yml b/packages/cisco_duo/changelog.yml index 1d0ac29e882..630be6db204 100644 --- a/packages/cisco_duo/changelog.yml +++ b/packages/cisco_duo/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.0.4" + changes: + - description: Fix auth CEL cursor handling. + type: bugfix + link: https://github.com/elastic/integrations/pull/11456 - version: "2.0.3" changes: - description: Set request rate limits. diff --git a/packages/cisco_duo/data_stream/auth/agent/stream/cel.yml.hbs b/packages/cisco_duo/data_stream/auth/agent/stream/cel.yml.hbs index 7d5de1c8f65..50bdf2cb837 100644 --- a/packages/cisco_duo/data_stream/auth/agent/stream/cel.yml.hbs +++ b/packages/cisco_duo/data_stream/auth/agent/stream/cel.yml.hbs @@ -27,121 +27,84 @@ program: | state : state.with({ - "mintime": state.?cursor.last_published.orValue(int(now - duration(state.initial_interval)) * 1000), - "maxtime": int(now - duration("2m")) * 1000, - "date": now.format(time_layout.RFC1123Z), + "mintime": state.?cursor.last_timestamp_ms.orValue(string(int(now - duration(state.initial_interval)) * 1000)), + "maxtime": string(int(now - duration("2m")) * 1000), }) ).as(state, state.with( - request( - "GET", - state.?want_more.orValue(false) ? - state.next_url - : - state.url.trim_right("/") + "/admin/v2/logs/authentication?" + { - "limit": [string(int(state.limit))], - "maxtime": [string(int(state.maxtime))], - "mintime": [string(int(state.mintime))], - "sort": ["ts:asc"], - }.format_query() - ).with( - { + { + // prepare request data + "date": now.format(time_layout.RFC1123Z), + "method": "GET", + "url_base": state.url.trim_right("/"), + "url_path": "/admin/v2/logs/authentication", + "query_string": { + "limit": [string(int(state.limit))], + "maxtime": [state.maxtime], + "mintime": [state.mintime], + ?"next_offset": state.?next_offset_joined.optMap(v, [v]), + "sort": ["ts:asc"], + }.format_query(), + }.as(r, r.with({ + // add an authorization header value + "authorization": "Basic " + ( + state.integration_key + ":" + ( + [ + r.date, + r.method, + r.url_base.trim_prefix("https://"), + r.url_path, + r.query_string, + ].join("\n") + .hmac("sha1", bytes(state.secret_key)) + .hex() + ) + ).base64(), + })).as(r, + // now do the request using the prepared data + request( + r.method, + [r.url_base, r.url_path, "?", r.query_string].join("") + ).with({ "Header": { "Content-Type": ["application/x-www-form-urlencoded"], - "Date": [state.date], - "Authorization": ["Basic " + ( - state.integration_key + ":" + ( - [ - state.date, - "GET", - state.url.trim_prefix("https://"), - "/admin/v2/logs/authentication", - { - "limit": [string(int(state.limit))], - "maxtime": [string(int(state.maxtime))], - "mintime": [string(int(state.mintime))], - ?"next_offset": has(state.next_offset) ? - optional.of([string(state.next_offset)]) - : - optional.none(), - "sort": ["ts:asc"], - }.format_query() - ].join("\n") - .hmac("sha1", bytes(state.secret_key)) - .hex() - ) - ).base64()], + "Date": [r.date], + "Authorization": [r.authorization], }, - } - ).do_request().as(resp, (resp.StatusCode == 200) ? - bytes(resp.Body).decode_json().as(body, has(body.?response.authlogs) && size(body.response.authlogs) > 0 ? - ( - body.?response.metadata.next_offset.orValue(null) == null ? - optional.none() - : type(body.response.metadata.next_offset) == type([]) && size(body.response.metadata.next_offset) == 2 ? - optional.of(string(body.response.metadata.next_offset[0])+","+string(body.response.metadata.next_offset[1])) - : - // Fall back to the actual value. This will result in a - // failure on the next iteration, but will expose the - // value to logging to aid identification of a change - // in the format of this field should that happen. - body.?response.metadata.next_offset - ).as(next_offset, - { - "events": body.response.authlogs.map(item, - { - "message": item.encode_json(), - } - ), - "url": state.url, - "integration_key": state.integration_key, - "secret_key": state.secret_key, - "limit": state.limit, - "mintime": state.mintime, - "maxtime": state.maxtime, - "date": now.format(time_layout.RFC1123Z), - "want_more": next_offset.hasValue(), - ?"next_offset": next_offset, - "next_url": next_offset.hasValue() ? - ( - state.url.trim_right("/") + "/admin/v2/logs/authentication?" + { - "limit": [string(int(state.limit))], - "maxtime": [string(int(state.maxtime))], - "mintime": [string(int(state.mintime))], - "next_offset": [next_offset.value()], - "sort": ["ts:asc"], - }.format_query() - ) - : - state.url, + }).do_request().as(resp, (resp.StatusCode == 200) ? + bytes(resp.Body).decode_json().as(body, has(body.?response.authlogs) && size(body.response.authlogs) > 0 ? + body.response.as(r, { + "events": r.authlogs.map(item, { "message": item.encode_json() }), + "want_more": r.?metadata.next_offset.hasValue(), + ?"next_offset_joined": r.?metadata.next_offset.optMap(v, [v].flatten().join(",")), "cursor": { - ?"last_published": next_offset, + "last_timestamp_ms": string(int(r.authlogs[size(r.authlogs) - 1].timestamp) * 1000), } + }) + : + { + "events":[], + "want_more": false, } + ) : - { - "events":[], - "want_more": false, - } - - ) - : - bytes(resp.Body).decode_json().as(body, - { - "events": { - "error": { - "code": has(body.code) ? string(body.code) : string(resp.StatusCode), - "id": string(resp.Status), - "message": "GET:"+( - size(resp.Body) != 0 ? - string(resp.Body) - : - string(resp.Status) + ' (' + string(resp.StatusCode) + ')' - ), + bytes(resp.Body).decode_json().as(body, + { + "events": { + "error": { + "code": has(body.code) ? string(body.code) : string(resp.StatusCode), + "id": string(resp.Status), + "message": "GET:"+( + size(resp.Body) != 0 ? + string(resp.Body) + : + string(resp.Status) + ' (' + string(resp.StatusCode) + ')' + ), + }, }, - }, - "want_more": false, - } + "want_more": false, + } + ) ) ) )) diff --git a/packages/cisco_duo/manifest.yml b/packages/cisco_duo/manifest.yml index 8e4d01c2853..5c9ae07cb1d 100644 --- a/packages/cisco_duo/manifest.yml +++ b/packages/cisco_duo/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: cisco_duo title: Cisco Duo -version: "2.0.3" +version: "2.0.4" description: Collect logs from Cisco Duo with Elastic Agent. type: integration categories: