Skip to content

Commit

Permalink
proofpoint_tap: improve clarity of agent config and fix pagination lo…
Browse files Browse the repository at this point in the history 
…gic (#11361)

There was no check for the pagination termination state, which for this API is
an empty array of events. Add that check after clarifying the HTTPJSON
configuration, including ensuring that all time ranges are valid queries.

Extend system test conditions to exercise pagination logic and termination,
and fix configuration of request tracing in system tests.
  • Loading branch information
@efd6
efd6 authored Oct 9, 2024
1 parent f088617 commit 1c07c84
Show file tree
Hide file tree
Showing 11 changed files with 700 additions and 25 deletions.
511 changes: 503 additions & 8 deletions packages/proofpoint_tap/_dev/deploy/docker/files/config.yml
View file

Large diffs are not rendered by default.

8 changes: 8 additions & 0 deletions packages/proofpoint_tap/changelog.yml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,12 @@
# newer versions go on top
- version: "1.24.0"
changes:
- description: Improve clarity of agent behavior configuration.
type: enhancement
link: https://github.com/elastic/integrations/pull/11361
- description: Fix pagination termination condition check.
type: bugfix
link: https://github.com/elastic/integrations/pull/11361
- version: "1.23.0"
changes:
- description: Set default search period to one day.
Expand Down
4 changes: 3 additions & 1 deletion packages/proofpoint_tap/data_stream/clicks_blocked/_dev/test/system/test-default-config.yml
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
input: httpjson
service: proofpoint_tap
vars:
enable_request_tracer: true
url: http://{{Hostname}}:{{Port}}
principal: xxxx
secret: xxxx
data_stream:
vars:
preserve_original_event: true
enable_request_tracer: true
assert:
hit_count: 2
47 changes: 44 additions & 3 deletions packages/proofpoint_tap/data_stream/clicks_blocked/agent/stream/httpjson.yml.hbs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,53 @@ request.transforms:
value: json
- set:
target: url.params.interval
value: '[[if (le (formatDate ((parseDate .cursor.last_received_time "RFC3339").Add (parseDuration "1h"))) (formatDate (now)))]][[formatDate (parseDate .cursor.last_received_time "RFC3339")]]/[[formatDate ((parseDate .cursor.last_received_time "RFC3339").Add (parseDuration "1h"))]][[else]][[formatDate (parseDate .cursor.last_received_time "RFC3339")]]/[[formatDate now]][[end]]'
default: '[[formatDate (now (parseDuration "-{{initial_interval}}"))]]/[[formatDate ((now (parseDuration "-{{initial_interval}}")).Add (parseDuration "1h"))]]'
value: |-
[[- $last := (parseDate .cursor.last_received_time "RFC3339") -]]
[[- $hour := (parseDuration "1h") -]]
[[- $end := 0 -]][[- /* Predeclare $end. */ -]]
[[- with $last -]]
[[- $end = .Add $hour -]]
[[- end -]]
[[- with $end -]]
[[- if .Before now -]]
[[- formatDate $last ]]/[[ formatDate $end -]]
[[- else -]]
[[- formatDate $last ]]/[[ formatDate now -]]
[[- end -]]
[[- end -]]
default: |-
[[- $start := (now (parseDuration "-{{initial_interval}}")) -]]
[[- $hour := (parseDuration "1h") -]]
[[- $end := 0 -]][[- /* Predeclare $end. */ -]]
[[- with $start -]]
[[- $end = .Add $hour -]]
[[- end -]]
[[- with $end -]]
[[- if .Before now -]]
[[- formatDate $start ]]/[[ formatDate $end -]]
[[- else -]]
[[- formatDate $start ]]/[[ formatDate now -]]
[[- end -]]
[[- end -]]
response.pagination:
- set:
target: url.params.interval
value: '[[if (le (formatDate ((parseDate .last_response.body.queryEndTime "RFC3339").Add (parseDuration "1h"))) (formatDate (now)))]][[formatDate (parseDate .last_response.body.queryEndTime "RFC3339")]]/[[formatDate ((parseDate .last_response.body.queryEndTime "RFC3339").Add (parseDuration "1h"))]][[end]]'
value: |-
[[- if ne (len .last_response.body.clicksBlocked) 0 -]]
[[- $last := (parseDate .last_response.body.queryEndTime "RFC3339") -]]
[[- $hour := (parseDuration "1h") -]]
[[- $end := 0 -]][[- /* Predeclare $end. */ -]]
[[- with $last -]]
[[- $end = .Add $hour -]]
[[- end -]]
[[- with $end -]]
[[- if .Before now -]]
[[- formatDate $last ]]/[[ formatDate $end -]]
[[- else -]]
[[- formatDate $last ]]/[[ formatDate now -]]
[[- end -]]
[[- end -]]
[[- end -]]
fail_on_template_error: true
cursor:
last_received_time:
Expand Down
4 changes: 3 additions & 1 deletion ...ages/proofpoint_tap/data_stream/clicks_permitted/_dev/test/system/test-default-config.yml
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
input: httpjson
service: proofpoint_tap
vars:
enable_request_tracer: true
url: http://{{Hostname}}:{{Port}}
principal: xxxx
secret: xxxx
data_stream:
vars:
preserve_original_event: true
enable_request_tracer: true
assert:
hit_count: 2
47 changes: 44 additions & 3 deletions packages/proofpoint_tap/data_stream/clicks_permitted/agent/stream/httpjson.yml.hbs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,53 @@ request.transforms:
value: json
- set:
target: url.params.interval
value: '[[if (le (formatDate ((parseDate .cursor.last_received_time "RFC3339").Add (parseDuration "1h"))) (formatDate (now)))]][[formatDate (parseDate .cursor.last_received_time "RFC3339")]]/[[formatDate ((parseDate .cursor.last_received_time "RFC3339").Add (parseDuration "1h"))]][[else]][[formatDate (parseDate .cursor.last_received_time "RFC3339")]]/[[formatDate now]][[end]]'
default: '[[formatDate (now (parseDuration "-{{initial_interval}}"))]]/[[formatDate ((now (parseDuration "-{{initial_interval}}")).Add (parseDuration "1h"))]]'
value: |-
[[- $last := (parseDate .cursor.last_received_time "RFC3339") -]]
[[- $hour := (parseDuration "1h") -]]
[[- $end := 0 -]][[- /* Predeclare $end. */ -]]
[[- with $last -]]
[[- $end = .Add $hour -]]
[[- end -]]
[[- with $end -]]
[[- if .Before now -]]
[[- formatDate $last ]]/[[ formatDate $end -]]
[[- else -]]
[[- formatDate $last ]]/[[ formatDate now -]]
[[- end -]]
[[- end -]]
default: |-
[[- $start := (now (parseDuration "-{{initial_interval}}")) -]]
[[- $hour := (parseDuration "1h") -]]
[[- $end := 0 -]][[- /* Predeclare $end. */ -]]
[[- with $start -]]
[[- $end = .Add $hour -]]
[[- end -]]
[[- with $end -]]
[[- if .Before now -]]
[[- formatDate $start ]]/[[ formatDate $end -]]
[[- else -]]
[[- formatDate $start ]]/[[ formatDate now -]]
[[- end -]]
[[- end -]]
response.pagination:
- set:
target: url.params.interval
value: '[[if (le (formatDate ((parseDate .last_response.body.queryEndTime "RFC3339").Add (parseDuration "1h"))) (formatDate (now)))]][[formatDate (parseDate .last_response.body.queryEndTime "RFC3339")]]/[[formatDate ((parseDate .last_response.body.queryEndTime "RFC3339").Add (parseDuration "1h"))]][[end]]'
value: |-
[[- if ne (len .last_response.body.clicksPermitted) 0 -]]
[[- $last := (parseDate .last_response.body.queryEndTime "RFC3339") -]]
[[- $hour := (parseDuration "1h") -]]
[[- $end := 0 -]][[- /* Predeclare $end. */ -]]
[[- with $last -]]
[[- $end = .Add $hour -]]
[[- end -]]
[[- with $end -]]
[[- if .Before now -]]
[[- formatDate $last ]]/[[ formatDate $end -]]
[[- else -]]
[[- formatDate $last ]]/[[ formatDate now -]]
[[- end -]]
[[- end -]]
[[- end -]]
fail_on_template_error: true
cursor:
last_received_time:
Expand Down
4 changes: 3 additions & 1 deletion packages/proofpoint_tap/data_stream/message_blocked/_dev/test/system/test-default-config.yml
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
input: httpjson
service: proofpoint_tap
vars:
enable_request_tracer: true
url: http://{{Hostname}}:{{Port}}
principal: xxxx
secret: xxxx
data_stream:
vars:
preserve_original_event: true
enable_request_tracer: true
assert:
hit_count: 2
47 changes: 44 additions & 3 deletions packages/proofpoint_tap/data_stream/message_blocked/agent/stream/httpjson.yml.hbs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,53 @@ request.transforms:
value: json
- set:
target: url.params.interval
value: '[[if (le (formatDate ((parseDate .cursor.last_received_time "RFC3339").Add (parseDuration "1h"))) (formatDate (now)))]][[formatDate (parseDate .cursor.last_received_time "RFC3339")]]/[[formatDate ((parseDate .cursor.last_received_time "RFC3339").Add (parseDuration "1h"))]][[else]][[formatDate (parseDate .cursor.last_received_time "RFC3339")]]/[[formatDate now]][[end]]'
default: '[[formatDate (now (parseDuration "-{{initial_interval}}"))]]/[[formatDate ((now (parseDuration "-{{initial_interval}}")).Add (parseDuration "1h"))]]'
value: |-
[[- $last := (parseDate .cursor.last_received_time "RFC3339") -]]
[[- $hour := (parseDuration "1h") -]]
[[- $end := 0 -]][[- /* Predeclare $end. */ -]]
[[- with $last -]]
[[- $end = .Add $hour -]]
[[- end -]]
[[- with $end -]]
[[- if .Before now -]]
[[- formatDate $last ]]/[[ formatDate $end -]]
[[- else -]]
[[- formatDate $last ]]/[[ formatDate now -]]
[[- end -]]
[[- end -]]
default: |-
[[- $start := (now (parseDuration "-{{initial_interval}}")) -]]
[[- $hour := (parseDuration "1h") -]]
[[- $end := 0 -]][[- /* Predeclare $end. */ -]]
[[- with $start -]]
[[- $end = .Add $hour -]]
[[- end -]]
[[- with $end -]]
[[- if .Before now -]]
[[- formatDate $start ]]/[[ formatDate $end -]]
[[- else -]]
[[- formatDate $start ]]/[[ formatDate now -]]
[[- end -]]
[[- end -]]
response.pagination:
- set:
target: url.params.interval
value: '[[if (le (formatDate ((parseDate .last_response.body.queryEndTime "RFC3339").Add (parseDuration "1h"))) (formatDate (now)))]][[formatDate (parseDate .last_response.body.queryEndTime "RFC3339")]]/[[formatDate ((parseDate .last_response.body.queryEndTime "RFC3339").Add (parseDuration "1h"))]][[end]]'
value: |-
[[- if ne (len .last_response.body.messagesBlocked) 0 -]]
[[- $last := (parseDate .last_response.body.queryEndTime "RFC3339") -]]
[[- $hour := (parseDuration "1h") -]]
[[- $end := 0 -]][[- /* Predeclare $end. */ -]]
[[- with $last -]]
[[- $end = .Add $hour -]]
[[- end -]]
[[- with $end -]]
[[- if .Before now -]]
[[- formatDate $last ]]/[[ formatDate $end -]]
[[- else -]]
[[- formatDate $last ]]/[[ formatDate now -]]
[[- end -]]
[[- end -]]
[[- end -]]
fail_on_template_error: true
cursor:
last_received_time:
Expand Down
4 changes: 3 additions & 1 deletion ...ges/proofpoint_tap/data_stream/message_delivered/_dev/test/system/test-default-config.yml
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
input: httpjson
service: proofpoint_tap
vars:
enable_request_tracer: true
url: http://{{Hostname}}:{{Port}}
principal: xxxx
secret: xxxx
data_stream:
vars:
preserve_original_event: true
enable_request_tracer: true
assert:
hit_count: 2
47 changes: 44 additions & 3 deletions packages/proofpoint_tap/data_stream/message_delivered/agent/stream/httpjson.yml.hbs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,53 @@ request.transforms:
value: json
- set:
target: url.params.interval
value: '[[if (le (formatDate ((parseDate .cursor.last_received_time "RFC3339").Add (parseDuration "1h"))) (formatDate (now)))]][[formatDate (parseDate .cursor.last_received_time "RFC3339")]]/[[formatDate ((parseDate .cursor.last_received_time "RFC3339").Add (parseDuration "1h"))]][[else]][[formatDate (parseDate .cursor.last_received_time "RFC3339")]]/[[formatDate now]][[end]]'
default: '[[formatDate (now (parseDuration "-{{initial_interval}}"))]]/[[formatDate ((now (parseDuration "-{{initial_interval}}")).Add (parseDuration "1h"))]]'
value: |-
[[- $last := (parseDate .cursor.last_received_time "RFC3339") -]]
[[- $hour := (parseDuration "1h") -]]
[[- $end := 0 -]][[- /* Predeclare $end. */ -]]
[[- with $last -]]
[[- $end = .Add $hour -]]
[[- end -]]
[[- with $end -]]
[[- if .Before now -]]
[[- formatDate $last ]]/[[ formatDate $end -]]
[[- else -]]
[[- formatDate $last ]]/[[ formatDate now -]]
[[- end -]]
[[- end -]]
default: |-
[[- $start := (now (parseDuration "-{{initial_interval}}")) -]]
[[- $hour := (parseDuration "1h") -]]
[[- $end := 0 -]][[- /* Predeclare $end. */ -]]
[[- with $start -]]
[[- $end = .Add $hour -]]
[[- end -]]
[[- with $end -]]
[[- if .Before now -]]
[[- formatDate $start ]]/[[ formatDate $end -]]
[[- else -]]
[[- formatDate $start ]]/[[ formatDate now -]]
[[- end -]]
[[- end -]]
response.pagination:
- set:
target: url.params.interval
value: '[[if (le (formatDate ((parseDate .last_response.body.queryEndTime "RFC3339").Add (parseDuration "1h"))) (formatDate (now)))]][[formatDate (parseDate .last_response.body.queryEndTime "RFC3339")]]/[[formatDate ((parseDate .last_response.body.queryEndTime "RFC3339").Add (parseDuration "1h"))]][[end]]'
value: |-
[[- if ne (len .last_response.body.messagesDelivered) 0 -]]
[[- $last := (parseDate .last_response.body.queryEndTime "RFC3339") -]]
[[- $hour := (parseDuration "1h") -]]
[[- $end := 0 -]][[- /* Predeclare $end. */ -]]
[[- with $last -]]
[[- $end = .Add $hour -]]
[[- end -]]
[[- with $end -]]
[[- if .Before now -]]
[[- formatDate $last ]]/[[ formatDate $end -]]
[[- else -]]
[[- formatDate $last ]]/[[ formatDate now -]]
[[- end -]]
[[- end -]]
[[- end -]]
fail_on_template_error: true
cursor:
last_received_time:
Expand Down
2 changes: 1 addition & 1 deletion packages/proofpoint_tap/manifest.yml
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
format_version: "3.0.3"
name: proofpoint_tap
title: Proofpoint TAP
version: "1.23.0"
version: "1.24.0"
description: Collect logs from Proofpoint TAP with Elastic Agent.
type: integration
categories:
Expand Down

0 comments on commit 1c07c84

Please sign in to comment.