okta: allow user configuration of debug_data flattened use (#9868)

packages/okta/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.12.0"
+  changes:
+    - description: Allow user configuration of debug_data flattened use.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/9868
 - version: "2.11.0"
   changes:
     - description: Modified the field definitions to remove ECS fields made redundant by the ecs@mappings component template.

packages/okta/data_stream/system/_dev/test/pipeline/test-okta-system-no-flattened-events.log-config.yml

@@ -0,0 +1,6 @@
+fields:
+  "@timestamp": "2020-04-28T11:07:58.223Z"
+  tags:
+    - preserve_original_event
+  _conf:
+    remove_flattened_debug: true

packages/okta/data_stream/system/_dev/test/pipeline/test-okta-system-yes-flattened-events.log-config.yml

@@ -0,0 +1,6 @@
+fields:
+  "@timestamp": "2020-04-28T11:07:58.223Z"
+  tags:
+    - preserve_original_event
+  _conf:
+    remove_flattened_debug: false

packages/okta/data_stream/system/_dev/test/system/test-api-key-no-flattened-config.yml

@@ -0,0 +1,14 @@
+input: httpjson
+service: okta
+vars:
+  url: http://{{Hostname}}:{{Port}}/api/v1/logs
+  interval: 10s
+  api_key: testing
+  enable_request_tracer: true
+policy_template: okta
+data_stream:
+  vars:
+    preserve_original_event: true
+    remove_flattened_debug: true
+assert:
+  hit_count: 4

packages/okta/data_stream/system/_dev/test/system/test-oauth2-config.yml

@@ -13,5 +13,6 @@ policy_template: okta
 data_stream:
   vars:
     preserve_original_event: true
+    remove_flattened_debug: false
 assert:
   hit_count: 4

packages/okta/data_stream/system/agent/stream/httpjson.yml.hbs

@@ -105,6 +105,10 @@ tags:
 {{#contains "forwarded" tags}}
 publisher_pipeline.disable_host: true
 {{/contains}}
+fields_under_root: true
+fields:
+  _conf:
+    remove_flattened_debug: {{remove_flattened_debug}}
 {{#if processors}}
 processors:
 {{processors}}

packages/okta/data_stream/system/elasticsearch/ingest_pipeline/default.yml

@@ -282,145 +282,12 @@ processors:
       target_field: okta.transaction.detail.request_api_token_id
       ignore_missing: true
       ignore_failure: true
-  - set:
-      field: okta.debug_context.debug_data.flattened
-      copy_from: json.debugContext.debugData
-      ignore_failure: true
-  - json:
-      field: okta.debug_context.debug_data.flattened.logOnlySecurityData
-      ignore_failure: true
-  - dissect:
-      field: okta.debug_context.debug_data.flattened.behaviors
-      pattern: "{%{okta.debug_context.debug_data.flattened.behaviors}}"
-      ignore_missing: true
-      ignore_failure: true
-  - kv:
-      field: okta.debug_context.debug_data.flattened.behaviors
-      field_split: ", "
-      value_split: "="
-      target_field: _behaviors_object
-      if: ctx.okta?.debug_context?.debug_data?.flattened?.behaviors != null
-  - remove:
-      field: okta.debug_context.debug_data.flattened.behaviors
-      if: ctx._behaviors_object != null
-  - rename:
-      field: _behaviors_object
-      target_field: okta.debug_context.debug_data.flattened.behaviors
-      ignore_missing: true
-      ignore_failure: true
-  - set:
-      field: okta.debug_context.debug_data.flattened.risk_object
-      copy_from: okta.debug_context.debug_data.flattened.risk
-      if: ctx.okta?.debug_context?.debug_data?.flattened?.risk != null
-  - dissect:
-      field: okta.debug_context.debug_data.flattened.risk
-      pattern: "{%{okta.debug_context.debug_data.flattened.risk}}"
-      ignore_missing: true
-      ignore_failure: true
-  - kv:
-      field: okta.debug_context.debug_data.flattened.risk
-      field_split: ", "
-      value_split: "="
-      target_field: _risk_object
-      if: ctx.okta?.debug_context?.debug_data?.flattened?.risk != null
-      on_failure:
-        - remove:
-            field: _risk_object
-  - remove:
-      field: okta.debug_context.debug_data.flattened.risk_object
-      if: ctx._risk_object != null
-  # Make heroic efforts to capture risk level and reason if kv could not get them.
-  - grok:
-      field: okta.debug_context.debug_data.flattened.risk
-      patterns:
-        - level=%{NOTSPACE:_risk_object.level}
-      if: ctx.okta?.debug_context?.debug_data?.flattened?.risk_object != null && ctx.okta?.debug_context?.debug_data?.flattened?.risk != null
-      ignore_failure: true
-  - grok:
-      field: okta.debug_context.debug_data.flattened.risk
-      patterns:
-        - reasons=%{DATA:_risk_object.reasons}, %{KEY}
-        - reasons=%{DATA:_risk_object.reasons}$
-      pattern_definitions:
-        KEY: '%{NOTSPACE}='
-      if: ctx.okta?.debug_context?.debug_data?.flattened?.risk_object != null && ctx.okta?.debug_context?.debug_data?.flattened?.risk != null
-      ignore_failure: true
-  - remove:
-      field: okta.debug_context.debug_data.flattened.risk
-      if: ctx._risk_object != null
-  - rename:
-      field: _risk_object
-      target_field: okta.debug_context.debug_data.flattened.risk
-      ignore_missing: true
-      ignore_failure: true
-  - rename:
-      field: json.debugContext.debugData.deviceFingerprint
-      target_field: okta.debug_context.debug_data.device_fingerprint
-      ignore_missing: true
-      ignore_failure: true
-  - rename:
-      field: json.debugContext.debugData.requestId
-      target_field: okta.debug_context.debug_data.request_id
-      ignore_missing: true
-      ignore_failure: true
-  - rename:
-      field: json.debugContext.debugData.requestUri
-      target_field: okta.debug_context.debug_data.request_uri
-      ignore_missing: true
-      ignore_failure: true
-  - rename:
-      field: json.debugContext.debugData.threatSuspected
-      target_field: okta.debug_context.debug_data.threat_suspected
-      ignore_missing: true
-      ignore_failure: true
-  - rename:
-      field: json.debugContext.debugData.url
-      target_field: okta.debug_context.debug_data.url
-      ignore_missing: true
-      ignore_failure: true
-  - rename:
-      field: json.debugContext.debugData.dtHash
-      target_field: okta.debug_context.debug_data.dt_hash
-      ignore_missing: true
-      ignore_failure: true
-  - set:
-      field: okta.debug_context.debug_data.risk_level
-      value: "{{{okta.debug_context.debug_data.flattened.logOnlySecurityData.risk.level}}}"
-      if: 'ctx.okta?.debug_context?.debug_data?.flattened?.logOnlySecurityData?.risk?.level != null && ctx.okta?.debug_context?.debug_data?.flattened?.logOnlySecurityData?.risk?.level != ""'
-  - split:
-      field: okta.debug_context.debug_data.flattened.logOnlySecurityData.risk.reasons
-      target_field: okta.debug_context.debug_data.risk_reasons
-      separator: ',\s*'
-      if: 'ctx.okta?.debug_context?.debug_data?.flattened?.logOnlySecurityData?.risk?.reasons != null && ctx.okta?.debug_context?.debug_data?.flattened?.logOnlySecurityData?.risk?.reasons != ""'
-  - set:
-      field: okta.debug_context.debug_data.risk_level
-      value: "{{{okta.debug_context.debug_data.flattened.risk.level}}}"
-      if: 'ctx.okta?.debug_context?.debug_data?.risk_level == null && ctx.okta?.debug_context?.debug_data?.flattened?.risk?.level != null && ctx.okta?.debug_context?.debug_data?.flattened?.risk?.level != ""'
-  - set:
-      field: okta.debug_context.debug_data.factor
-      value: "{{{okta.debug_context.debug_data.flattened.factor}}}"
-      if: 'ctx.okta?.debug_context?.debug_data?.factor == null && ctx.okta?.debug_context?.debug_data?.flattened?.factor != null && ctx.okta?.debug_context?.debug_data?.flattened?.factor != ""'
-  - split:
-      field: okta.debug_context.debug_data.flattened.risk.reasons
-      target_field: okta.debug_context.debug_data.risk_reasons
-      separator: ',\s*'
-      if: 'ctx.okta?.debug_context?.debug_data?.risk_reasons == null && ctx.okta?.debug_context?.debug_data?.flattened?.risk?.reasons != null && ctx.okta?.debug_context?.debug_data?.flattened?.risk?.reasons != ""'
-  - script:
-      lang: painless
-      source: |
-        def src = ctx.okta?.debug_context?.debug_data?.flattened?.behaviors;
-        if (src == null) {
-          return;
-        }
-        def dst = new ArrayList();
-        for (e in src.entrySet()) {
-          if (e != null && e.getValue() == "POSITIVE") {
-            dst.add(e.getKey());
-          }
-        }
-        if (dst.length != 0) {
-          ctx.okta.debug_context.debug_data['risk_behaviors'] = dst;
-        }
+  - pipeline:
+      if: ctx._conf?.remove_flattened_debug != true
+      name: '{{ IngestPipeline "use_flattened_debug" }}'
+  - pipeline:
+      if: ctx._conf?.remove_flattened_debug == true
+      name: '{{ IngestPipeline "no_use_flattened_debug" }}'
   - rename:
       field: json.authenticationContext.authenticationProvider
       target_field: okta.authentication_context.authentication_provider
@@ -723,6 +590,10 @@ processors:
       if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
       ignore_failure: true
       ignore_missing: true
+  - remove:
+      field: _conf
+      ignore_missing: true
+      ignore_failure: true
 on_failure:
   - set:
       field: event.kind