zscaler_zia: Remove department field and fix parsing errors in web lo…

…gs (#10874)

Remove `department` field and fix parsing errors in web logs.

The `department` value is already being ingested from `dept` field. This PR removes the 
hex-encoded `department` field in favour of `dept` due to noticed improper hex-encoding in 
fields prefixed with `e` [here](#10855 (comment)). 

Remove `urldecode` processor on parsing urls causing parsing errors. The `b64url` 
field is already being decoded by `script` processor. 

Also adds `on_failure` clause on `urldecode` processors to continue pipeline execution 
to prevent parsing errors on hex-encoded values to stop running rest of the pipeline.

packages/zscaler_zia/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "3.2.3"
+  changes:
+    - description: Remove department field and add on_failure clauses.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/10874
 - version: "3.2.2"
   changes:
     - description: Sanitize unwanted characters in firewall.

packages/zscaler_zia/data_stream/web/elasticsearch/ingest_pipeline/default.yml

@@ -121,6 +121,10 @@ processors:
       tag: urldecode_apprulelabel
       target_field: zscaler_zia.web.app.rule_label
       ignore_missing: true
+      on_failure:
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - append:
       field: rule.name
       tag: append_zscaler_zia_web_app_rule_label_into_rule_name
@@ -361,6 +365,10 @@ processors:
       tag: urldecode_devicename
       target_field: zscaler_zia.web.device.name
       ignore_missing: true
+      on_failure:
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - set:
       field: host.hostname
       tag: set_host_hostname_from_web_device_name
@@ -531,6 +539,10 @@ processors:
       tag: urldecode_filename
       target_field: zscaler_zia.web.file.name
       ignore_missing: true
+      on_failure:
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - append:
       field: file.name
       tag: append_zscaler_zia_web_file_name_into_file_name
@@ -593,6 +605,10 @@ processors:
       tag: urldecode_host
       target_field: zscaler_zia.web.host
       ignore_missing: true
+      on_failure:
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - set:
       field: destination.domain
       tag: set_destination_domain_from_web_host
@@ -634,16 +650,19 @@ processors:
       tag: urldecode_location
       target_field: zscaler_zia.web.location
       ignore_missing: true
-  - urldecode:
-      field: json.department
-      tag: urldecode_department
-      target_field: zscaler_zia.web.department
-      ignore_missing: true
+      on_failure:
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - urldecode:
       field: json.login
       tag: urldecode_login
       target_field: zscaler_zia.web.login
       ignore_missing: true
+      on_failure:
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - set:
       field: user.email
       tag: set_user_email_from_web_login
@@ -723,6 +742,10 @@ processors:
       tag: urldecode_mobappname
       target_field: zscaler_zia.web.mobile.application.name
       ignore_missing: true
+      on_failure:
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - rename:
       field: json.mobdevtype
       tag: rename_mobdevtype
@@ -902,6 +925,10 @@ processors:
       tag: urldecode_refererhost
       target_field: zscaler_zia.web.referer.host
       ignore_missing: true
+      on_failure:
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - script:
       description: Decode referer
       tag: decode_b64referer
@@ -1046,6 +1073,10 @@ processors:
       tag: urldecode_rulelabel
       target_field: zscaler_zia.web.rule.name
       ignore_missing: true
+      on_failure:
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - append:
       field: rule.name
       tag: append_zscaler_zia_web_rule_name_into_rule_name
@@ -1280,6 +1311,10 @@ processors:
       tag: urldecode_upload_filename
       target_field: zscaler_zia.web.upload.file.name
       ignore_missing: true
+      on_failure:
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - append:
       field: file.name
       tag: append_zscaler_zia_web_upload_file_name_into_file_name
@@ -1327,6 +1362,10 @@ processors:
       tag: urldecode_urlfilterrulelabel
       target_field: zscaler_zia.web.url.filter_rule_label
       ignore_missing: true
+      on_failure:
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - append:
       field: rule.name
       tag: append_zscaler_zia_web_url_filter_rule_label_into_rule_name
@@ -1373,15 +1412,6 @@ processors:
       field: url.full
       value: '{{{url.original}}}'
       if: ctx.url?.original != null && ctx.url.original != ''
-  - urldecode:
-      field: url.original
-      ignore_missing: true
-      tag: urldecode_url_original
-      if: ctx.url?.original != null && ctx.url.original != ''
-      on_failure:
-        - append:
-            field: error.message
-            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - rename:
       field: json.useragentclass
       tag: rename_useragentclass
@@ -1417,6 +1447,10 @@ processors:
       tag: urldecode_userlocationname
       target_field: zscaler_zia.web.user_location_name
       ignore_missing: true
+      on_failure:
+        - append:
+            field: error.message
+            value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}'
   - convert:
       field: json.year
       tag: convert_year_to_long