From b4fc48e3606558a3383908c6b2823ac69af49829 Mon Sep 17 00:00:00 2001 From: Thibault Martinez Date: Mon, 9 Feb 2026 13:43:58 +0100 Subject: [PATCH] feat(indexer): add influxdb --- .../iota_names_influxdb_dashboard.json | 766 ++++++++++++++++++ .../docker/assets/grafana/datasources.yaml | 16 + indexer/docker/docker-compose.yml | 32 + indexer/src/influxdb.rs | 273 +++++++ indexer/src/main.rs | 26 + indexer/src/worker.rs | 38 +- 6 files changed, 1147 insertions(+), 4 deletions(-) create mode 100644 indexer/docker/assets/grafana/dashboards/iota_names_influxdb_dashboard.json create mode 100644 indexer/src/influxdb.rs diff --git a/indexer/docker/assets/grafana/dashboards/iota_names_influxdb_dashboard.json b/indexer/docker/assets/grafana/dashboards/iota_names_influxdb_dashboard.json new file mode 100644 index 000000000..765595507 --- /dev/null +++ b/indexer/docker/assets/grafana/dashboards/iota_names_influxdb_dashboard.json @@ -0,0 +1,766 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { "type": "grafana", "uid": "-- Grafana --" }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }, { "color": "red", "value": 80 }] }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 3, "w": 5, "x": 0, "y": 0 }, + "id": 7, + "options": { + "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "editorMode": "builder", "expr": "data_ingestion_checkpoint", + "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" + } + ], + "title": "Processed Checkpoint", + "type": "stat" + }, + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }, { "color": "red", "value": 80 }] } + }, + "overrides": [] + }, + "gridPos": { "h": 3, "w": 6, "x": 5, "y": 0 }, + "id": 11, + "options": { + "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true + }, + "targets": [ + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "query": "from(bucket: \"iota-names\")\n |> range(start: 0)\n |> filter(fn: (r) => r._measurement == \"name_record_added\" and r._field == \"length\")\n |> group()\n |> count()\n |> rename(columns: {_value: \"Total\"})", + "refId": "A" + } + ], + "title": "Total Name Records ever created", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 }, + "id": 10, + "panels": [], + "title": "General", + "type": "row" + }, + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "description": "The IOTA balance of the IotaNames object.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, + "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, + "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }, { "color": "red", "value": 80 }] } + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 8, "x": 0, "y": 4 }, + "id": 2, + "options": { + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "query": "from(bucket: \"iota-names\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"balance\" and r.source == \"iota_names\" and r._field == \"value\")\n |> map(fn: (r) => ({r with _value: float(v: r._value) / 1000000000.0}))\n |> set(key: \"_field\", value: \"Balance in IOTA\")", + "refId": "A" + } + ], + "title": "IOTA-Names Balance", + "type": "timeseries" + }, + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, + "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, + "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }, { "color": "red", "value": 80 }] } + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 8, "x": 8, "y": 4 }, + "id": 1, + "options": { + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "query": "added = from(bucket: \"iota-names\")\n |> range(start: 0, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"name_record_added\" and r._field == \"length\")\n |> map(fn: (r) => ({_time: r._time, _value: 1, _field: \"Record Count\", _measurement: \"records\"}))\n\nremoved = from(bucket: \"iota-names\")\n |> range(start: 0, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"name_record_removed\" and r._field == \"length\")\n |> map(fn: (r) => ({_time: r._time, _value: -1, _field: \"Record Count\", _measurement: \"records\"}))\n\nunion(tables: [added, removed])\n |> group(columns: [\"_field\"])\n |> sort(columns: [\"_time\"])\n |> cumulativeSum()\n |> filter(fn: (r) => r._time >= v.timeRangeStart)", + "refId": "A" + } + ], + "title": "Total Name Records", + "type": "timeseries" + }, + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, + "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, + "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }, { "color": "red", "value": 80 }] } + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 8, "x": 16, "y": 4 }, + "id": 5, + "options": { + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "query": "created = from(bucket: \"iota-names\")\n |> range(start: 0, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"node_subname_created\" and r._field == \"name\")\n |> map(fn: (r) => ({_time: r._time, _value: 1, _field: \"Node Subnames\", _measurement: \"subnames\"}))\n\nburned = from(bucket: \"iota-names\")\n |> range(start: 0, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"node_subname_burned\" and r._field == \"name\")\n |> map(fn: (r) => ({_time: r._time, _value: -1, _field: \"Node Subnames\", _measurement: \"subnames\"}))\n\nunion(tables: [created, burned])\n |> group(columns: [\"_field\"])\n |> sort(columns: [\"_time\"])\n |> cumulativeSum()\n |> filter(fn: (r) => r._time >= v.timeRangeStart)", + "refId": "A" + }, + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "query": "created = from(bucket: \"iota-names\")\n |> range(start: 0, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"leaf_subname_created\" and r._field == \"name\")\n |> map(fn: (r) => ({_time: r._time, _value: 1, _field: \"Leaf Subnames\", _measurement: \"subnames\"}))\n\nremoved = from(bucket: \"iota-names\")\n |> range(start: 0, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"leaf_subname_removed\" and r._field == \"name\")\n |> map(fn: (r) => ({_time: r._time, _value: -1, _field: \"Leaf Subnames\", _measurement: \"subnames\"}))\n\nunion(tables: [created, removed])\n |> group(columns: [\"_field\"])\n |> sort(columns: [\"_time\"])\n |> cumulativeSum()\n |> filter(fn: (r) => r._time >= v.timeRangeStart)", + "refId": "B", + "hide": false + } + ], + "title": "Subname Records", + "type": "timeseries" + }, + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, + "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, + "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }, { "color": "red", "value": 80 }] } + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 8, "x": 0, "y": 10 }, + "id": 12, + "options": { + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "query": "set = from(bucket: \"iota-names\")\n |> range(start: 0, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"target_address_set\" and r._field == \"set\")\n |> map(fn: (r) => ({_time: r._time, _value: if r._value then 1 else -1, _field: \"Target Addresses\", _measurement: \"target_addresses\"}))\n |> group(columns: [\"_field\"])\n |> sort(columns: [\"_time\"])\n |> cumulativeSum()\n |> filter(fn: (r) => r._time >= v.timeRangeStart)", + "refId": "A" + } + ], + "title": "Target addresses", + "type": "timeseries" + }, + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, + "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, + "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }, { "color": "red", "value": 80 }] } + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 8, "x": 8, "y": 10 }, + "id": 13, + "options": { + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "query": "setEvt = from(bucket: \"iota-names\")\n |> range(start: 0, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"reverse_lookup_set\" and r._field == \"name\")\n |> map(fn: (r) => ({_time: r._time, _value: 1, _field: \"Default Names\", _measurement: \"defaults\"}))\n\nunsetEvt = from(bucket: \"iota-names\")\n |> range(start: 0, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"reverse_lookup_unset\" and r._field == \"name\")\n |> map(fn: (r) => ({_time: r._time, _value: -1, _field: \"Default Names\", _measurement: \"defaults\"}))\n\nunion(tables: [setEvt, unsetEvt])\n |> group(columns: [\"_field\"])\n |> sort(columns: [\"_time\"])\n |> cumulativeSum()\n |> filter(fn: (r) => r._time >= v.timeRangeStart)", + "refId": "A" + } + ], + "title": "Default names", + "type": "timeseries" + }, + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 8, "x": 16, "y": 10 }, + "id": 16, + "options": { + "displayLabels": ["percent"], + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "pieType": "pie", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "query": "created = from(bucket: \"iota-names\")\n |> range(start: 0)\n |> filter(fn: (r) => r._measurement == \"node_subname_created\" and r._field == \"name\")\n |> group()\n |> count()\n |> set(key: \"_field\", value: \"Node\")\n\nburned = from(bucket: \"iota-names\")\n |> range(start: 0)\n |> filter(fn: (r) => r._measurement == \"node_subname_burned\" and r._field == \"name\")\n |> group()\n |> count()\n |> set(key: \"_field\", value: \"node_burned\")\n\nnodeTotal = join(tables: {created: created, burned: burned}, on: [\"_field\"], method: \"inner\")\n\nfrom(bucket: \"iota-names\")\n |> range(start: 0)\n |> filter(fn: (r) => r._measurement == \"node_subname_created\" and r._field == \"name\")\n |> group()\n |> count()\n |> set(key: \"_field\", value: \"Node\")", + "refId": "A" + }, + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "query": "from(bucket: \"iota-names\")\n |> range(start: 0)\n |> filter(fn: (r) => r._measurement == \"leaf_subname_created\" and r._field == \"name\")\n |> group()\n |> count()\n |> set(key: \"_field\", value: \"Leaf\")", + "refId": "B", + "hide": false + } + ], + "title": "Proportion of node/leaf subnames", + "type": "piechart" + }, + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { "h": 7, "w": 8, "x": 0, "y": 16 }, + "id": 20, + "options": { + "displayLabels": ["name", "percent"], + "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, + "pieType": "pie", + "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "query": "from(bucket: \"iota-names\")\n |> range(start: 0)\n |> filter(fn: (r) => r._measurement == \"coupon_applied\" and r._field == \"discount\" and r.kind == \"percentage\")\n |> group()\n |> sum()\n |> set(key: \"_field\", value: \"Percentage\")", + "refId": "A" + }, + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "query": "from(bucket: \"iota-names\")\n |> range(start: 0)\n |> filter(fn: (r) => r._measurement == \"coupon_applied\" and r._field == \"discount\" and r.kind == \"fixed\")\n |> group()\n |> sum()\n |> set(key: \"_field\", value: \"Fixed\")", + "refId": "B", + "hide": false + } + ], + "title": "Coupon discounts", + "type": "piechart" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, + "id": 9, + "panels": [], + "title": "Distributions", + "type": "row" + }, + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", + "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, + "fillOpacity": 80, "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineWidth": 1, "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } + }, + "decimals": 0, "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }, { "color": "red", "value": 80 }] }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 8, "x": 0, "y": 24 }, + "id": 6, + "options": { + "barRadius": 0, "barWidth": 0.97, "fullHighlight": false, "groupWidth": 0.7, + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, + "orientation": "auto", "showValue": "auto", "stacking": "none", + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" }, + "xField": "years", "xTickLabelRotation": 0, "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "query": "from(bucket: \"iota-names\")\n |> range(start: 0)\n |> filter(fn: (r) => r._measurement == \"transaction\" and r._field == \"years\" and r.is_renewal == \"true\")\n |> group()\n |> histogram(bins: [1.0, 2.0, 3.0, 4.0, 5.0])", + "refId": "A" + } + ], + "title": "Number of years per renewal", + "transformations": [ + { "id": "convertFieldType", "options": { "conversions": [{ "destinationType": "number", "targetField": "years" }], "fields": {} } }, + { "id": "sortBy", "options": { "fields": {}, "sort": [{ "desc": false, "field": "years" }] } } + ], + "type": "barchart" + }, + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", + "axisLabel": "", "axisPlacement": "auto", "axisSoftMin": 0, + "fillOpacity": 80, "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineWidth": 1, "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } + }, + "decimals": 0, "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }, { "color": "red", "value": 80 }] }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 8, "x": 8, "y": 24 }, + "id": 3, + "options": { + "barRadius": 0, "barWidth": 0.97, "fullHighlight": false, "groupWidth": 0.7, + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, + "orientation": "auto", "showValue": "auto", "stacking": "none", + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" }, + "xField": "length", "xTickLabelRotation": 0, "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "query": "import \"strings\"\n\nadded = from(bucket: \"iota-names\")\n |> range(start: 0)\n |> filter(fn: (r) => r._measurement == \"name_record_added\" and r._field == \"length\")\n |> map(fn: (r) => ({_time: r._time, _value: 1, _field: \"count\", length: string(v: r._value)}))\n\nremoved = from(bucket: \"iota-names\")\n |> range(start: 0)\n |> filter(fn: (r) => r._measurement == \"name_record_removed\" and r._field == \"length\")\n |> map(fn: (r) => ({_time: r._time, _value: -1, _field: \"count\", length: string(v: r._value)}))\n\nunion(tables: [added, removed])\n |> group(columns: [\"length\"])\n |> sum()\n |> group()", + "refId": "A" + } + ], + "title": "Count per second level name length", + "transformations": [ + { "id": "convertFieldType", "options": { "conversions": [{ "destinationType": "number", "targetField": "length" }], "fields": {} } }, + { "id": "sortBy", "options": { "fields": {}, "sort": [{ "desc": false, "field": "length" }] } } + ], + "type": "barchart" + }, + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", + "axisLabel": "", "axisPlacement": "auto", + "fillOpacity": 80, "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineWidth": 1, "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }, { "color": "red", "value": 80 }] } + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 8, "x": 16, "y": 24 }, + "id": 17, + "options": { + "barRadius": 0, "barWidth": 0.97, "fullHighlight": false, "groupWidth": 0.7, + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, + "orientation": "auto", "showValue": "auto", "stacking": "none", + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" }, + "xField": "depth", "xTickLabelRotation": 0, "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "query": "added = from(bucket: \"iota-names\")\n |> range(start: 0)\n |> filter(fn: (r) => r._measurement == \"name_record_added\" and r._field == \"depth\")\n |> map(fn: (r) => ({_time: r._time, _value: 1, _field: \"count\", depth: string(v: r._value)}))\n\nremoved = from(bucket: \"iota-names\")\n |> range(start: 0)\n |> filter(fn: (r) => r._measurement == \"name_record_removed\" and r._field == \"depth\")\n |> map(fn: (r) => ({_time: r._time, _value: -1, _field: \"count\", depth: string(v: r._value)}))\n\nunion(tables: [added, removed])\n |> group(columns: [\"depth\"])\n |> sum()\n |> group()", + "refId": "A" + } + ], + "title": "Name depth", + "type": "barchart" + }, + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", + "axisLabel": "", "axisPlacement": "auto", + "fillOpacity": 80, "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineWidth": 1, "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }, { "color": "red", "value": 80 }] } + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 8, "x": 0, "y": 30 }, + "id": 24, + "options": { + "barRadius": 0, "barWidth": 0.97, "fullHighlight": false, "groupWidth": 0.7, + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, + "orientation": "auto", "showValue": "auto", "stacking": "none", + "text": { "valueSize": 30 }, + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" }, + "xField": "key", "xTickLabelRotation": 0, "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "query": "setEvt = from(bucket: \"iota-names\")\n |> range(start: 0)\n |> filter(fn: (r) => r._measurement == \"user_data_set\" and r._field == \"new\" and r._value == true)\n |> map(fn: (r) => ({_time: r._time, _value: 1, _field: \"count\", key: r.key}))\n\nunsetEvt = from(bucket: \"iota-names\")\n |> range(start: 0)\n |> filter(fn: (r) => r._measurement == \"user_data_unset\" and r._field == \"count\")\n |> map(fn: (r) => ({_time: r._time, _value: -1, _field: \"count\", key: r.key}))\n\nunion(tables: [setEvt, unsetEvt])\n |> group(columns: [\"key\"])\n |> sum()\n |> group()", + "refId": "A" + } + ], + "title": "User data keys", + "type": "barchart" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 36 }, + "id": 8, + "panels": [], + "title": "Auctions", + "type": "row" + }, + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "description": "The IOTA balance of the AuctionHouse object.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, + "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, + "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }, { "color": "red", "value": 80 }] } + }, + "overrides": [] + }, + "gridPos": { "h": 7, "w": 8, "x": 0, "y": 37 }, + "id": 15, + "options": { + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "query": "from(bucket: \"iota-names\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"balance\" and r.source == \"auction_house\" and r._field == \"value\")\n |> map(fn: (r) => ({r with _value: float(v: r._value) / 1000000000.0}))\n |> set(key: \"_field\", value: \"Balance in IOTA\")", + "refId": "A" + } + ], + "title": "Auction House Balance", + "type": "timeseries" + }, + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "fillOpacity": 80, "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineWidth": 1, + "stacking": { "group": "A", "mode": "none" } + }, + "displayName": "Duration", + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }, { "color": "red", "value": 80 }] }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { "h": 7, "w": 8, "x": 8, "y": 37 }, + "id": 18, + "options": { + "bucketSize": 60000, + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "query": "from(bucket: \"iota-names\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"auction_finalized\" and r._field == \"duration_ms\")\n |> set(key: \"_field\", value: \"Duration (ms)\")", + "refId": "A" + } + ], + "title": "Auction durations", + "type": "histogram" + }, + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }, { "color": "red", "value": 80 }] }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { "h": 7, "w": 8, "x": 16, "y": 37 }, + "id": 19, + "options": { + "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { "calcs": ["mean"], "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true + }, + "targets": [ + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "query": "from(bucket: \"iota-names\")\n |> range(start: 0)\n |> filter(fn: (r) => r._measurement == \"auction_finalized\" and r._field == \"duration_ms\")\n |> group()\n |> mean()", + "refId": "A" + } + ], + "title": "Average auction duration", + "type": "stat" + }, + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, + "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, + "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }, { "color": "red", "value": 80 }] } + }, + "overrides": [] + }, + "gridPos": { "h": 7, "w": 8, "x": 0, "y": 44 }, + "id": 4, + "options": { + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "query": "started = from(bucket: \"iota-names\")\n |> range(start: 0, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"auction_started\" and r._field == \"starting_bid\")\n |> map(fn: (r) => ({_time: r._time, _value: 1, _field: \"Auctions Started\", _measurement: \"auctions\"}))\n |> group(columns: [\"_field\"])\n |> sort(columns: [\"_time\"])\n |> cumulativeSum()\n |> filter(fn: (r) => r._time >= v.timeRangeStart)\n\nstartedCount = from(bucket: \"iota-names\")\n |> range(start: 0, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"auction_started\" and r._field == \"starting_bid\")\n |> map(fn: (r) => ({_time: r._time, _value: 1, _field: \"_tmp\", _measurement: \"auctions\"}))\n |> group(columns: [\"_field\"])\n |> sort(columns: [\"_time\"])\n |> cumulativeSum()\n |> filter(fn: (r) => r._time >= v.timeRangeStart)\n\nstarted", + "refId": "A" + }, + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "query": "from(bucket: \"iota-names\")\n |> range(start: 0, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"auction_finalized\" and r._field == \"winning_bid\")\n |> map(fn: (r) => ({_time: r._time, _value: 1, _field: \"Auctions Finalized\", _measurement: \"auctions\"}))\n |> group(columns: [\"_field\"])\n |> sort(columns: [\"_time\"])\n |> cumulativeSum()\n |> filter(fn: (r) => r._time >= v.timeRangeStart)", + "refId": "B", + "hide": false + } + ], + "title": "Number of auctions started and finalized", + "type": "timeseries" + }, + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "description": "Bucket size is 1 IOTA", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "fillOpacity": 80, "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineWidth": 1, + "stacking": { "group": "A", "mode": "none" } + }, + "displayName": "Price", + "fieldMinMax": false, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }, { "color": "red", "value": 80 }] }, + "unit": "NANOs" + }, + "overrides": [] + }, + "gridPos": { "h": 7, "w": 8, "x": 8, "y": 44 }, + "id": 14, + "options": { + "bucketSize": 1000000000, + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "query": "from(bucket: \"iota-names\")\n |> range(start: v.timeRangeStart, stop: v.timeRangeStop)\n |> filter(fn: (r) => r._measurement == \"auction_finalized\" and r._field == \"winning_bid\")\n |> set(key: \"_field\", value: \"Price\")", + "refId": "A" + } + ], + "title": "Final auction prices", + "type": "histogram" + }, + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }, { "color": "red", "value": 80 }] }, + "unit": "IOTA" + }, + "overrides": [] + }, + "gridPos": { "h": 7, "w": 8, "x": 16, "y": 44 }, + "id": 22, + "options": { + "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { "calcs": ["mean"], "fields": "", "values": false }, + "showPercentChange": false, "textMode": "auto", "wideLayout": true + }, + "targets": [ + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "query": "from(bucket: \"iota-names\")\n |> range(start: 0)\n |> filter(fn: (r) => r._measurement == \"auction_finalized\" and r._field == \"winning_bid\")\n |> group()\n |> map(fn: (r) => ({r with _value: float(v: r._value) / 1000000000.0}))\n |> mean()", + "refId": "A" + } + ], + "title": "Average auction price", + "type": "stat" + }, + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "description": "Shows how many auctions received a certain number of bids.", + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", + "axisLabel": "", "axisPlacement": "auto", + "fillOpacity": 80, "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineWidth": 1, "scaleDistribution": { "type": "linear" }, "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green" }, { "color": "red", "value": 80 }] } + }, + "overrides": [] + }, + "gridPos": { "h": 7, "w": 8, "x": 0, "y": 51 }, + "id": 23, + "options": { + "barRadius": 0, "barWidth": 0.97, "fullHighlight": false, "groupWidth": 0.7, + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, + "orientation": "auto", "showValue": "auto", "stacking": "none", + "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" }, + "xField": "bid_count", "xTickLabelRotation": 0, "xTickLabelSpacing": 0 + }, + "targets": [ + { + "datasource": { "type": "influxdb", "uid": "influxdb" }, + "query": "import \"experimental\"\n\nfrom(bucket: \"iota-names\")\n |> range(start: 0)\n |> filter(fn: (r) => r._measurement == \"auction_started\" and r._field == \"name\")\n |> group(columns: [\"_value\"])\n |> count()\n |> rename(columns: {_value: \"bid_count\"})\n |> group()", + "refId": "A" + } + ], + "title": "Auctions per number of bids", + "transformations": [ + { "id": "convertFieldType", "options": { "conversions": [{ "destinationType": "number", "targetField": "bid_count" }], "fields": {} } }, + { "id": "sortBy", "options": { "fields": {}, "sort": [{ "field": "bid_count" }] } } + ], + "type": "barchart" + } + ], + "preload": false, + "schemaVersion": 41, + "tags": [], + "templating": { "list": [] }, + "time": { "from": "now-6h", "to": "now" }, + "timepicker": {}, + "timezone": "browser", + "title": "IOTA-Names (InfluxDB)", + "uid": "iota-names-influxdb", + "version": 1 +} diff --git a/indexer/docker/assets/grafana/datasources.yaml b/indexer/docker/assets/grafana/datasources.yaml index 22b668e8d..76decc077 100755 --- a/indexer/docker/assets/grafana/datasources.yaml +++ b/indexer/docker/assets/grafana/datasources.yaml @@ -13,3 +13,19 @@ datasources: editable: false jsonData: httpMethod: GET + + - name: InfluxDB + type: influxdb + uid: influxdb + access: proxy + orgId: 1 + url: http://influxdb:8086 + isDefault: true + version: 1 + editable: false + jsonData: + version: Flux + organization: iota + defaultBucket: iota-names + secureJsonData: + token: ${INFLUXDB_TOKEN:-iota-names-token} diff --git a/indexer/docker/docker-compose.yml b/indexer/docker/docker-compose.yml index 71515c5ed..4e13db800 100644 --- a/indexer/docker/docker-compose.yml +++ b/indexer/docker/docker-compose.yml @@ -21,6 +21,8 @@ services: depends_on: prometheus: condition: service_healthy + influxdb: + condition: service_healthy networks: - indexer-metrics-net volumes: @@ -38,11 +40,15 @@ services: - IOTA_NAMES_TEMP_SUBNAME_PROXY_PACKAGE_ADDRESS=${IOTA_NAMES_TEMP_SUBNAME_PROXY_PACKAGE_ADDRESS} - IOTA_NAMES_AUCTION_HOUSE_OBJECT_ID=${IOTA_NAMES_AUCTION_HOUSE_OBJECT_ID} - EVENT_PACKAGE_IDS=${EVENT_PACKAGE_IDS} + - INFLUXDB_TOKEN=${INFLUXDB_TOKEN:-iota-names-token} + - INFLUXDB_ORG=${INFLUXDB_ORG:-iota} + - INFLUXDB_BUCKET=${INFLUXDB_BUCKET:-iota-names} extra_hosts: - "host.docker.internal:host-gateway" command: - "start" - "--prometheus-url=http://prometheus:9090" + - "--influxdb-url=http://influxdb:8086" - "--num-workers=10" # Localnet values: # - "--node-url=http://host.docker.internal:9000" @@ -83,6 +89,31 @@ services: retries: 8 start_period: 30s + influxdb: + image: influxdb:2 + container_name: influxdb + restart: unless-stopped + ports: + - 8086:8086 + environment: + - DOCKER_INFLUXDB_INIT_MODE=setup + - DOCKER_INFLUXDB_INIT_USERNAME=${INFLUXDB_USER:-admin} + - DOCKER_INFLUXDB_INIT_PASSWORD=${INFLUXDB_PASSWORD:-adminadmin} + - DOCKER_INFLUXDB_INIT_ORG=${INFLUXDB_ORG:-iota} + - DOCKER_INFLUXDB_INIT_BUCKET=${INFLUXDB_BUCKET:-iota-names} + - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=${INFLUXDB_TOKEN:-iota-names-token} + - DOCKER_INFLUXDB_INIT_RETENTION=0s + volumes: + - influxdb_data:/var/lib/influxdb2 + networks: + - indexer-metrics-net + healthcheck: + test: ["CMD", "influx", "ping"] + interval: 15s + timeout: 5s + retries: 8 + start_period: 30s + grafana: image: grafana/grafana:latest container_name: grafana @@ -102,6 +133,7 @@ services: volumes: prometheus_data: grafana_data: + influxdb_data: indexer_progress: networks: diff --git a/indexer/src/influxdb.rs b/indexer/src/influxdb.rs new file mode 100644 index 000000000..c5e134547 --- /dev/null +++ b/indexer/src/influxdb.rs @@ -0,0 +1,273 @@ +// Copyright (c) 2025 IOTA Stiftung +// SPDX-License-Identifier: Apache-2.0 + +use std::fmt::Write; + +use reqwest::Client; +use tracing::{debug, warn}; + +use crate::events::{CouponKind, IotaNamesEvent}; + +/// InfluxDB v2 client for writing event data with actual on-chain timestamps. +pub(crate) struct InfluxDb { + client: Client, + write_url: String, + token: String, +} + +impl InfluxDb { + pub fn new(url: &str, token: &str, org: &str, bucket: &str) -> Self { + let url = url.trim_end_matches('/'); + let write_url = format!( + "{url}/api/v2/write?org={org}&bucket={bucket}&precision=ms", + ); + Self { + client: Client::new(), + write_url, + token: token.to_string(), + } + } + + /// Write all events from a checkpoint to InfluxDB. + /// + /// `timestamp_ms` is the checkpoint's on-chain timestamp in milliseconds. + pub async fn write_events( + &self, + events: &[(IotaNamesEvent, u64)], + balance_updates: &[BalanceUpdate], + ) { + let mut body = String::new(); + + for (event, timestamp_ms) in events { + if let Err(e) = write_event_line(&mut body, event, *timestamp_ms) { + warn!("Failed to format InfluxDB line for event {event:?}: {e}"); + } + } + + for update in balance_updates { + let _ = writeln!( + body, + "balance,source={} value={}i {}", + update.source, update.value, update.timestamp_ms, + ); + } + + if body.is_empty() { + return; + } + + debug!("Writing {} bytes to InfluxDB", body.len()); + + if let Err(e) = self + .client + .post(&self.write_url) + .header("Authorization", format!("Token {}", self.token)) + .header("Content-Type", "text/plain; charset=utf-8") + .body(body) + .send() + .await + .and_then(|r| r.error_for_status()) + { + warn!("Failed to write to InfluxDB: {e}"); + } + } +} + +pub(crate) struct BalanceUpdate { + pub source: &'static str, + pub value: i64, + pub timestamp_ms: u64, +} + +/// Escape a string value for InfluxDB line protocol tag values. +/// Tags: escape commas, equals, spaces. +fn escape_tag(s: &str) -> String { + s.replace(',', r"\,") + .replace('=', r"\=") + .replace(' ', r"\ ") +} + +/// Escape a string value for InfluxDB line protocol field values (quoted strings). +/// Inside double quotes: escape double quotes and backslashes. +fn escape_field_str(s: &str) -> String { + s.replace('\\', r"\\").replace('"', r#"\""#) +} + +fn write_event_line( + buf: &mut String, + event: &IotaNamesEvent, + timestamp_ms: u64, +) -> std::fmt::Result { + match event { + IotaNamesEvent::AuctionStarted(e) => { + writeln!( + buf, + "auction_started name=\"{}\",starting_bid={}i,bidder=\"{}\",start_timestamp_ms={}i,end_timestamp_ms={}i {}", + escape_field_str(&e.name.to_string()), + e.starting_bid, + e.bidder, + e.start_timestamp_ms, + e.end_timestamp_ms, + timestamp_ms, + ) + } + IotaNamesEvent::AuctionBid(e) => { + writeln!( + buf, + "auction_bid name=\"{}\",bid={}i,bidder=\"{}\" {}", + escape_field_str(&e.name.to_string()), + e.bid, + e.bidder, + timestamp_ms, + ) + } + IotaNamesEvent::AuctionExtended(e) => { + writeln!( + buf, + "auction_extended name=\"{}\",end_timestamp_ms={}i {}", + escape_field_str(&e.name.to_string()), + e.end_timestamp_ms, + timestamp_ms, + ) + } + IotaNamesEvent::AuctionFinalized(e) => { + writeln!( + buf, + "auction_finalized name=\"{}\",winning_bid={}i,winner=\"{}\",duration_ms={}i {}", + escape_field_str(&e.name.to_string()), + e.winning_bid, + e.winner, + e.end_timestamp_ms.saturating_sub(e.start_timestamp_ms), + timestamp_ms, + ) + } + IotaNamesEvent::CouponApplied(e) => { + let kind = match e.kind { + CouponKind::Percentage => "percentage", + CouponKind::Fixed => "fixed", + }; + writeln!( + buf, + "coupon_applied,kind={kind} discount={}i {}", + e.discount, timestamp_ms, + ) + } + IotaNamesEvent::NameRecordAdded(e) => { + let sln_length = e.name.label(1).expect("missing SLN").len(); + let depth = e.name.num_labels(); + writeln!( + buf, + "name_record_added name=\"{}\",length={}i,depth={}i {}", + escape_field_str(&e.name.to_string()), + sln_length, + depth, + timestamp_ms, + ) + } + IotaNamesEvent::NameRecordRemoved(e) => { + let sln_length = e.name.label(1).expect("missing SLN").len(); + let depth = e.name.num_labels(); + writeln!( + buf, + "name_record_removed name=\"{}\",length={}i,depth={}i {}", + escape_field_str(&e.name.to_string()), + sln_length, + depth, + timestamp_ms, + ) + } + IotaNamesEvent::TargetAddressSet(e) => { + let set = e.target_address.is_some(); + writeln!( + buf, + "target_address_set name=\"{}\",set={set} {}", + escape_field_str(&e.name.to_string()), + timestamp_ms, + ) + } + IotaNamesEvent::ReverseLookupSet(e) => { + writeln!( + buf, + "reverse_lookup_set address=\"{}\",name=\"{}\" {}", + e.default_address, + escape_field_str(&e.default_name.to_string()), + timestamp_ms, + ) + } + IotaNamesEvent::ReverseLookupUnset(e) => { + writeln!( + buf, + "reverse_lookup_unset address=\"{}\",name=\"{}\" {}", + e.default_address, + escape_field_str(&e.default_name.to_string()), + timestamp_ms, + ) + } + IotaNamesEvent::UserDataSet(e) => { + writeln!( + buf, + "user_data_set,key={} value=\"{}\",new={} {}", + escape_tag(&e.key), + escape_field_str(&e.value), + e.new, + timestamp_ms, + ) + } + IotaNamesEvent::UserDataUnset(e) => { + writeln!( + buf, + "user_data_unset,key={} count=1i {}", + escape_tag(&e.key), + timestamp_ms, + ) + } + IotaNamesEvent::Transaction(e) => { + writeln!( + buf, + "transaction,app={},is_renewal={} name=\"{}\",years={}i,base_amount={}i,currency=\"{}\",currency_amount={}i {}", + escape_tag(&e.app), + e.is_renewal, + escape_field_str(&e.name.to_string()), + e.years, + e.base_amount, + escape_field_str(&e.currency), + e.currency_amount, + timestamp_ms, + ) + } + IotaNamesEvent::NodeSubnameCreated(e) => { + writeln!( + buf, + "node_subname_created name=\"{}\",expiration_timestamp_ms={}i {}", + escape_field_str(&e.name.to_string()), + e.expiration_timestamp_ms, + timestamp_ms, + ) + } + IotaNamesEvent::NodeSubnameBurned(e) => { + writeln!( + buf, + "node_subname_burned name=\"{}\" {}", + escape_field_str(&e.name.to_string()), + timestamp_ms, + ) + } + IotaNamesEvent::LeafSubnameCreated(e) => { + writeln!( + buf, + "leaf_subname_created name=\"{}\",target=\"{}\" {}", + escape_field_str(&e.name.to_string()), + e.target, + timestamp_ms, + ) + } + IotaNamesEvent::LeafSubnameRemoved(e) => { + writeln!( + buf, + "leaf_subname_removed name=\"{}\" {}", + escape_field_str(&e.name.to_string()), + timestamp_ms, + ) + } + } +} diff --git a/indexer/src/main.rs b/indexer/src/main.rs index 18d99716b..2c9a8fc99 100644 --- a/indexer/src/main.rs +++ b/indexer/src/main.rs @@ -5,6 +5,7 @@ mod api; mod config; mod db; mod events; +mod influxdb; mod metrics; mod worker; @@ -26,6 +27,7 @@ use crate::{ api::start_api_server, config::IotaNamesExtendedConfig, db::pool::{DbConnectionPool, DbConnectionPoolConfig}, + influxdb::InfluxDb, metrics::{IotaNamesMetrics, PrometheusServer}, worker::{IotaNamesWorker, run_iota_names_reader}, }; @@ -63,6 +65,20 @@ enum Command { /// Resets metrics in case of a Prometheus error. #[arg(long, default_value_t = false)] reset_metrics: bool, + /// The URL of the InfluxDB v2 instance (e.g. http://localhost:8086). + /// When set, events are written to InfluxDB with actual on-chain + /// timestamps. + #[arg(long, env = "INFLUXDB_URL")] + influxdb_url: Option, + /// The InfluxDB authentication token. + #[arg(long, env = "INFLUXDB_TOKEN", default_value = "")] + influxdb_token: String, + /// The InfluxDB organization. + #[arg(long, env = "INFLUXDB_ORG", default_value = "iota")] + influxdb_org: String, + /// The InfluxDB bucket to write to. + #[arg(long, env = "INFLUXDB_BUCKET", default_value = "iota-names")] + influxdb_bucket: String, }, } @@ -77,6 +93,10 @@ impl Command { api_port, prometheus_url, reset_metrics, + influxdb_url, + influxdb_token, + influxdb_org, + influxdb_bucket, } => { info!("Starting IOTA Names Indexer"); @@ -146,6 +166,11 @@ impl Command { if iota_names_config.event_package_ids.is_empty() { panic!("No EVENT_PACKAGE_IDS provided in the environment variables"); } + let influxdb = influxdb_url.map(|url| { + info!("InfluxDB enabled at {url}"); + Arc::new(InfluxDb::new(&url, &influxdb_token, &influxdb_org, &influxdb_bucket)) + }); + info!("Starting with IOTA-Names config: {iota_names_config:#?}"); tasks.spawn(async move { let worker = IotaNamesWorker::new( @@ -153,6 +178,7 @@ impl Command { iota_names_config, metrics, handle.clone(), + influxdb, )?; tokio::select! { diff --git a/indexer/src/worker.rs b/indexer/src/worker.rs index 2cc0af553..a907d173d 100644 --- a/indexer/src/worker.rs +++ b/indexer/src/worker.rs @@ -39,6 +39,7 @@ use crate::{ config::IotaNamesExtendedConfig, db::{pool::DbConnectionPool, queries}, events::{CouponKind, IotaNamesEvent}, + influxdb::{BalanceUpdate, InfluxDb}, }; pub(crate) async fn run_iota_names_reader( @@ -104,6 +105,7 @@ pub(crate) struct IotaNamesWorker { metrics: Arc, token: CancellationToken, balance_object_id: ObjectID, + influxdb: Option>, } impl IotaNamesWorker { @@ -112,6 +114,7 @@ impl IotaNamesWorker { extended_config: IotaNamesExtendedConfig, metrics: Arc, token: CancellationToken, + influxdb: Option>, ) -> anyhow::Result { let config_type = StructTag::from_str(&format!( "{}::iota_names::BalanceKey<0x2::iota::IOTA>", @@ -137,10 +140,11 @@ impl IotaNamesWorker { metrics, token, balance_object_id, + influxdb, }) } - fn process_event(&self, event: IotaNamesEvent) -> anyhow::Result<()> { + fn process_event(&self, event: &IotaNamesEvent) -> anyhow::Result<()> { match event { // `auctions` IotaNamesEvent::AuctionStarted(event) => { @@ -292,16 +296,31 @@ impl IotaNamesWorker { ); } + let timestamp_ms = checkpoint.checkpoint_summary.timestamp_ms; + + let mut balance_updates = Vec::new(); let mut iota_names_balance = false; let mut auction_house_balance = false; for object in checkpoint.latest_live_output_objects() { if object.id() == self.balance_object_id { let balance = get_iota_names_balance(object)?; - self.metrics.balance.set(balance.value() as _); + let value = balance.value() as i64; + self.metrics.balance.set(value); + balance_updates.push(BalanceUpdate { + source: "iota_names", + value, + timestamp_ms, + }); iota_names_balance = true; } else if object.id() == self.extended_config.auction_house_id { let balance = get_auction_house_balance(object)?; - self.metrics.auction_house_balance.set(balance.value() as _); + let value = balance.value() as i64; + self.metrics.auction_house_balance.set(value); + balance_updates.push(BalanceUpdate { + source: "auction_house", + value, + timestamp_ms, + }); auction_house_balance = true; } else { continue; @@ -311,6 +330,8 @@ impl IotaNamesWorker { } } + let mut influxdb_events = Vec::new(); + for transaction in &checkpoint.transactions { let TransactionEffects::V1(effects) = &transaction.effects; @@ -327,7 +348,8 @@ impl IotaNamesWorker { checkpoint.checkpoint_summary.sequence_number ); - self.process_event(event)? + self.process_event(&event)?; + influxdb_events.push((event, timestamp_ms)); } Err(e) => warn!("parsing event failed: {e}"), _ => {} @@ -336,6 +358,14 @@ impl IotaNamesWorker { } } + if let Some(influxdb) = &self.influxdb { + if !influxdb_events.is_empty() || !balance_updates.is_empty() { + influxdb + .write_events(&influxdb_events, &balance_updates) + .await; + } + } + Ok(()) } }