Adjust metadata (#20)

* Remove version and tool

* Use agent_uuid as raw.Metadata.UUID

* Set raw.Metadata.TracerouteCallerVersion to NULL

* Use hash as raw.Metadata.CachedUUID

* Set parse.Time to NULL

* Set raw.Tracelb.type to tracelb

* Testing & Validation
- Verified pipeline integrity on the server.
- Verified correctness of field values.

Author: Elenetta17

db/iris_to_mlab.sql

@@ -15,12 +15,10 @@ DECLARE
   measurement_agent STRING DEFAULT @measurement_agent_param;
 DECLARE
   hostname STRING DEFAULT @host_param;
-DECLARE
-  version STRING DEFAULT @version_param;
-DECLARE
-  tool STRING DEFAULT @tool_param;
 DECLARE
   start_time STRING DEFAULT @start_time_param;
+DECLARE
+  agent_uuid STRING DEFAULT @agent_uuid_param;
 DECLARE
   min_ttl STRING DEFAULT @min_ttl_param;
 DECLARE
@@ -122,7 +120,7 @@ links AS (
 	    scamper1.id = '%s' AND
 	    scamper1.raw.CycleStart.Hostname = '%s' AND
 	    -- Check for duplicate rows
-	    scamper1.raw.Metadata.UUID = near.sha256
+	    scamper1.raw.Metadata.CachedUUID = near.sha256
     )
 ),
 
@@ -175,7 +173,7 @@ SELECT
     '%s' AS id, -- measurement_uuid AS id
     STRUCT(
         CAST(NULL AS STRING) AS Version,
-        CURRENT_TIMESTAMP() AS Time,
+        CAST(NULL AS TIMESTAMP) AS Time,
         CAST(NULL AS STRING) AS ArchiveURL,
         CAST(NULL AS STRING) AS Filename,
         CAST(NULL AS INT64) AS Priority,
@@ -184,10 +182,10 @@ SELECT
     DATE(TIMESTAMP('%s')) AS date,
     STRUCT(
         STRUCT(
-            ANY_VALUE(sha256) AS UUID,  -- Propagate SHA256 from the previous CTE
-            '%s' AS TracerouteCallerVersion,
+            '%s' AS UUID, -- agent UUID
+            CAST(NULL AS STRING)  AS TracerouteCallerVersion,
             CAST(NULL AS BOOLEAN) AS CachedResult,
-            CAST(NULL AS STRING) AS CachedUUID
+            ANY_VALUE(sha256)     AS CachedUUID -- Propagate SHA256 from the previous CTE
         ) AS Metadata,
         STRUCT(
             'cycle-start' AS Type,
@@ -197,10 +195,7 @@ SELECT
             UNIX_SECONDS(MIN(first_timestamp)) AS start_time
         ) AS CycleStart,
         STRUCT(
-            CASE
-               WHEN '%s'='diamond-miner' THEN 'MDA'
-               ELSE CAST(NULL AS STRING)
-	    END AS type, -- TODO: case when tool is yarrp
+	    'tracelb' AS type,
             CAST(NULL AS STRING) AS version,
             CAST(NULL AS INT64) AS userid,
             'icmp-echo' AS method,
@@ -226,10 +221,10 @@ SELECT
                   AND probe_dst_addr = lbn.probe_dst_addr
             ) AS linkc,
             ARRAY_AGG(STRUCT(
-                GENERATE_UUID() AS hop_id,
+                GENERATE_UUID()          AS hop_id,
                 COALESCE(near_addr, '*') AS addr,
-                CAST(NULL AS STRING) AS name,  -- Not applicable
-                NULL AS q_ttl,  -- Not applicable
+                CAST(NULL AS STRING)     AS name,  -- Not applicable
+                CAST(NULL AS INT64)      AS q_ttl,  -- Not applicable
                 (
                     SELECT COUNT(DISTINCT CONCAT(near_addr, '|', far_addr))
                     FROM links
@@ -251,6 +246,6 @@ SELECT
     ) AS raw
 FROM links_by_node lbn
 GROUP BY probe_protocol, probe_src_addr, probe_dst_addr
-""", scamper1_table, measurement_agent, table_name, scamper1_table, measurement_uuid, hostname, measurement_uuid, start_time, version, hostname, tool, min_ttl, failure_probability, hostname);
+""", scamper1_table, measurement_agent, table_name, scamper1_table, measurement_uuid, hostname, measurement_uuid, start_time, agent_uuid, hostname, min_ttl, failure_probability, hostname);
 EXECUTE IMMEDIATE
   convert_iris_to_scamper1;

tools/upload_tables.sh

@@ -146,7 +146,6 @@ convert_and_insert_values() {
 	local bq_tmp_table="$3"
 	local agent
 	local index
-	local tool
 	local start_time
 	local md_fields
 	local MD_FIELDS=()
@@ -162,17 +161,11 @@ convert_and_insert_values() {
 		fi
 	done
 
-	tool="$(jq -r .tool "${meas_md_tmpfile}")"
 	start_time="$(jq -r .start_time "${meas_md_tmpfile}")"
-	# sanity check
-	if [[ "${tool}" != "diamond-miner" && "${tool}" != "yarrp" ]]; then
-		echo "error: invalid tool: ${tool}"
-		return 1
-	fi
 	# order of these fields should match --parameter lines in the bq command below
 	MD_FIELDS=(
+	    .agent_uuid
 	    .agent_parameters.hostname
-	    .agent_parameters.version
 	    .agent_parameters.min_ttl
 	    .tool_parameters.failure_probability
 	)
@@ -190,10 +183,9 @@ convert_and_insert_values() {
 		--parameter="measurement_uuid_param:STRING:${meas_uuid}" \
 		--parameter="table_name_param:STRING:${bq_tmp_table}" \
 		--parameter="measurement_agent_param:STRING:${bq_tmp_table#*__}" \
-		--parameter="tool_param:STRING:${tool}" \
 		--parameter="start_time_param:STRING:${start_time}" \
-		--parameter="host_param:STRING:${MD_VALUES[0]}" \
-		--parameter="version_param:STRING:${MD_VALUES[1]}" \
+		--parameter="agent_uuid_param:STRING:${MD_VALUES[0]}" \
+		--parameter="host_param:STRING:${MD_VALUES[1]}" \
 		--parameter="min_ttl_param:STRING:${MD_VALUES[2]}" \
 		--parameter="failure_probability_param:STRING:${MD_VALUES[3]}" \
 		< "${TABLE_CONVERSION_QUERY}"