imperva_cloud_waf: improve error reporting for API requests (#12648)

Author: efd6

packages/imperva_cloud_waf/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.6.0"
+  changes:
+    - description: Improve error reporting for API requests.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/12648
 - version: "1.5.1"
   changes:
     - description: Tolerate no separator in log files.

packages/imperva_cloud_waf/data_stream/event/agent/stream/cel.yml.hbs

@@ -17,55 +17,100 @@ state:
   want_more: false
   user: "{{api_id}}"
   password: "{{api_key}}"
+redact:
+  fields:
+    - password
 program: |
   (
     has(state.worklist) && size(state.worklist) > 0 ?
       state
     :
-      request("GET", ( state.url + "logs.index")).with({
+      request("GET", (state.url + "logs.index")).with({
         "Header":{
           "Authorization": ["Basic "+string(base64(state.user+":"+state.password))],
         }
-      }).do_request().as(resp, bytes(resp.Body).as(body, {
-        "worklist": (
-          has(state.cursor) && has(state.cursor.log_file) && state.cursor.log_file != null
-          ?
-            string(body).split(state.cursor.log_file)[1].split("\n").filter(x,x!="").map(x,{"filename":x})
-          :
-            string(body).split("\n").map(x,{"filename":x})
-        ),
-        "next": 0,
-      }))
-  ).as(v, v.next < size(v.worklist) ?
+      }).do_request().as(resp, resp.StatusCode == 200 ?
+        bytes(resp.Body).as(body, {
+          "worklist": (
+            has(state.cursor) && has(state.cursor.log_file) && state.cursor.log_file != null
+            ?
+              string(body).split(state.cursor.log_file)[1].split("\n").filter(x,x!="").map(x,{"filename":x})
+            :
+              string(body).split("\n").map(x,{"filename":x})
+          ),
+          "next": 0,
+        })
+      :
+        {
+          "events": {
+            "error": {
+              "code": string(resp.StatusCode),
+              "id": string(resp.Status),
+              "message": "GET " + state.url + "logs.index: " + "+(
+                size(resp.Body) != 0 ?
+                  string(resp.Body)
+                :
+                  string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
+              ),
+            },
+          },
+          "want_more": false,
+          "user": state.user,
+          "password": state.password,
+         }
+      )
+  ).as(v, has(v.?events.error) ?
+    v
+  : v.next < size(v.worklist) ?
     (
       request("GET",
         state.url+v.worklist[v.next].filename
       ).with({
         "Header":{
           "Authorization": ["Basic "+string(base64(state.user + ":" + state.password))],
         }
-      }).do_request().as(resp, bytes(resp.Body).as(body, {
-        "events": (string(body)+"|==|").split("|==|")[1].split("\n").filter(x,x!="").map(x,{"message":x}),
-        "cursor": {
-          "log_file": (
-            has(state.cursor) && has(state.cursor.log_file) && state.cursor.log_file != null
-            ?
-              (
-                (v.worklist[v.next].filename).split(".")[0] != (state.cursor.log_file).split(".")[0] ?
-                  v.worklist[v.next].filename
+      }).do_request().as(resp, resp.StatusCode == 200 ?
+        bytes(resp.Body).as(body, {
+          "events": (string(body)+"|==|").split("|==|")[1].split("\n").filter(x,x!="").map(x,{"message":x}),
+          "cursor": {
+            "log_file": (
+              has(state.cursor) && has(state.cursor.log_file) && state.cursor.log_file != null
+              ?
+                (
+                  (v.worklist[v.next].filename).split(".")[0] != (state.cursor.log_file).split(".")[0] ?
+                    v.worklist[v.next].filename
+                  :
+                    state.cursor.log_file
+                )
+              :
+                v.worklist[v.next].filename
+            )
+          },
+          "worklist": (int(v.next)+1) < size(v.worklist) ? v.worklist : [],
+          "next": (int(v.next)+1) < size(v.worklist) ? (int(v.next)+1) : 0,
+          "want_more": (int(v.next)+1) < size(v.worklist),
+          "user": state.user,
+          "password": state.password,
+        })
+      :
+        {
+          "events": {
+            "error": {
+              "code": string(resp.StatusCode),
+              "id": string(resp.Status),
+              "message": "GET " + state.url+v.worklist[v.next].filename + ": " + "+(
+                size(resp.Body) != 0 ?
+                  string(resp.Body)
                 :
-                  state.cursor.log_file
-              )
-            :
-              v.worklist[v.next].filename
-          )
-        },
-        "worklist": (int(v.next)+1) < size(v.worklist) ? v.worklist : [],
-        "next": (int(v.next)+1) < size(v.worklist) ? (int(v.next)+1) : 0,
-        "want_more": (int(v.next)+1) < size(v.worklist),
-        "user": state.user,
-        "password": state.password
-      }))
+                  string(resp.Status) + ' (' + string(resp.StatusCode) + ')'
+              ),
+            },
+          },
+          "want_more": false,
+          "user": state.user,
+          "password": state.password,
+        }
+      )
     )
   :
     {

packages/imperva_cloud_waf/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 3.0.3
 name: imperva_cloud_waf
 title: Imperva Cloud WAF
-version: "1.5.1"
+version: "1.6.0"
 description: Collect logs from Imperva Cloud WAF with Elastic Agent.
 type: integration
 categories: