test/integration.sh: retry query

Currently, querying for the time series produced by the ruler is racey
because the ruler may not yet have had a chance to generate the time
series specified by the recording rules. This raciness can cause the
integration test to occasionally fail. This commit adds retries to the
testing of the rules. Eventually, these integration tests could be
ported to the same e2e test framework used in observatorium/api.

Signed-off-by: Lucas Servén Marín <[email protected]>

Author: squat

test/integration.sh

@@ -117,18 +117,28 @@ echo "-------------------------------------------"
 echo "- Rules File tests                        -"
 echo "-------------------------------------------"
 
-query="$(curl \
-    --fail \
-    'http://127.0.0.1:8443/api/metrics/v1/test-oidc/api/v1/query?query=trs' \
-    -H "Authorization: bearer $token")"
+i=0
+while [ "$i" -lt 10 ]; do
+  query="$(curl \
+      --fail \
+      'http://127.0.0.1:8443/api/metrics/v1/test-oidc/api/v1/query?query=trs' \
+      -H "Authorization: bearer $token")"
+  
+  if [[ "$query" == *'"result":[{"metric":{"__name__":"trs","tenant_id":"1610b0c3-c509-4592-a256-a1871353dbfa"}'* ]]; then
+    result=0
+    break
+  else
+    result=1
+    ((i=i+1))
+    sleep 5
+  fi
+done
 
-if [[ "$query" == *'"result":[{"metric":{"__name__":"trs","tenant_id":"1610b0c3-c509-4592-a256-a1871353dbfa"}'* ]]; then
-  result=0
+if [ "$result" -eq 0 ]; then
   echo "-------------------------------------------"
   echo "- Rules File: OK                          -"
   echo "-------------------------------------------"
 else
-  result=1
   echo "-------------------------------------------"
   echo "- Rules File: FAILED                      -"
   echo "-------------------------------------------"