feat(stats): reset charts (#1226)

- add authorization w/ api keys
- on-demand chart update
- chart reupdate logic
- api-key protected endpoint for requesting the chart reupdates
- readability / code structure improvements
- tests performance improvements
- fix last_accurate_point for counters

Author: bragov4ik

.github/workflows/stats.yml

@@ -66,7 +66,7 @@ jobs:
         if: success() || failure()
 
       - name: DB tests
-        run: RUST_BACKTRACE=1 RUST_LOG=info cargo test --locked --workspace -- --nocapture --ignored
+        run: RUST_BACKTRACE=1 cargo test --locked --workspace -- --nocapture --ignored
         if: success() || failure()
         env:
           DATABASE_URL: postgres://postgres:admin@localhost:5432/

stats/Cargo.lock

@@ -10,6 +10,8 @@ RUN cargo chef cook --release --recipe-path recipe.json
 
 FROM chef AS build
 
+# Include proto common definitions (`proto` must be passed as build context)
+COPY --from=proto . /proto
 COPY . .
 COPY --from=cache /app/target target
 COPY --from=cache $CARGO_HOME $CARGO_HOME

stats/Dockerfile

@@ -11,6 +11,9 @@ export DATABASE_URL := "postgres://" + db-user + ":" + db-password + "@" + db-ho
 docker-name := env_var_or_default('DOCKER_NAME', "stats-postgres")
 test-db-port := env_var_or_default('TEST_DB_PORT', "9433")
 
+docker-build *args:
+    docker build --build-context proto=../proto . {{args}}
+
 start-postgres:
     # we run it in --rm mode, so all data will be deleted after stopping
     docker run -p {{db-port}}:5432 --name {{docker-name}} -e POSTGRES_PASSWORD={{db-password}} -e POSTGRES_USER={{db-user}} --rm -d postgres -N 500

stats/justfile

@@ -41,7 +41,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
     ]));
     compile(
         &["proto/stats.proto", "proto/health.proto"],
-        &["proto"],
+        &["proto", "../../proto"],
         gens,
     )?;
     Ok(())

stats/stats-proto/build.rs

@@ -15,6 +15,9 @@ http:
       get: /api/v1/pages/transactions
     - selector: blockscout.stats.v1.StatsService.GetContractsPageStats
       get: /api/v1/pages/contracts
+    - selector: blockscout.stats.v1.StatsService.BatchUpdateCharts
+      post: /api/v1/charts/batch-update
+      body: "*"
 
     - selector: blockscout.stats.v1.Health.Check
       get: /health

stats/stats-proto/proto/api_config_http.yaml

@@ -2,15 +2,53 @@ syntax = "proto3";
 
 package blockscout.stats.v1;
 
+import "protoc-gen-openapiv2/options/annotations.proto";
+
+
 option go_package = "github.com/blockscout/blockscout-rs/stats";
 
+option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_swagger) = {
+  info: {
+    title: "Stats service";
+    contact: {
+      name: "Blockscout";
+      url: "https://blockscout.com";
+      email: "[email protected]";
+    };
+  };
+  schemes: [HTTPS],
+  external_docs: {
+    url: "https://github.com/blockscout/blockscout-rs";
+    description: "More about blockscout microservices";
+  }
+  security_definitions: {
+    security: {
+      key: "ApiKeyAuth";
+      value: {
+        type: TYPE_API_KEY;
+        in: IN_HEADER;
+        name: "x-api-key";
+      }
+    }
+  }
+};
+
+
 service StatsService {
   rpc GetCounters(GetCountersRequest) returns (Counters);
   rpc GetLineCharts(GetLineChartsRequest) returns (LineCharts);
   rpc GetLineChart(GetLineChartRequest) returns (LineChart);
   rpc GetMainPageStats(GetMainPageStatsRequest) returns (MainPageStats);
   rpc GetTransactionsPageStats(GetTransactionsPageStatsRequest) returns (TransactionsPageStats);
   rpc GetContractsPageStats(GetContractsPageStatsRequest) returns (ContractsPageStats);
+  rpc BatchUpdateCharts(BatchUpdateChartsRequest) returns (BatchUpdateChartsResult) {
+    option (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_operation) = {
+      security: {
+        security_requirement: {key: "ApiKeyAuth"}
+      }
+      tags: "Instances"
+    };
+  };
 }
 
 message GetCountersRequest {}
@@ -109,3 +147,22 @@ message ContractsPageStats {
   optional Counter total_verified_contracts = 3;
   optional Counter new_verified_contracts_24h = 4;
 }
+
+message BatchUpdateChartsRequest {
+  repeated string chart_names = 1;
+  // Default is today
+  optional string from = 2;
+  optional bool update_later = 3;
+}
+
+message BatchUpdateChartRejection {
+  string name = 1;
+  string reason = 2;
+}
+
+message BatchUpdateChartsResult {
+  uint32 total = 1;
+  uint32 total_rejected = 2;
+  repeated string accepted = 3;
+  repeated BatchUpdateChartRejection rejected = 4;
+}

stats/stats-proto/proto/stats.proto

@@ -1,15 +1,43 @@
 swagger: "2.0"
 info:
-  title: stats.proto
+  title: Stats service
   version: version not set
+  contact:
+    name: Blockscout
+    url: https://blockscout.com
+    email: [email protected]
 tags:
   - name: StatsService
   - name: Health
+schemes:
+  - https
 consumes:
   - application/json
 produces:
   - application/json
 paths:
+  /api/v1/charts/batch-update:
+    post:
+      operationId: StatsService_BatchUpdateCharts
+      responses:
+        "200":
+          description: A successful response.
+          schema:
+            $ref: '#/definitions/v1BatchUpdateChartsResult'
+        default:
+          description: An unexpected error response.
+          schema:
+            $ref: '#/definitions/rpcStatus'
+      parameters:
+        - name: body
+          in: body
+          required: true
+          schema:
+            $ref: '#/definitions/v1BatchUpdateChartsRequest'
+      tags:
+        - Instances
+      security:
+        - ApiKeyAuth: []
   /api/v1/counters:
     get:
       operationId: StatsService_GetCounters
@@ -171,6 +199,43 @@ definitions:
         items:
           type: object
           $ref: '#/definitions/protobufAny'
+  v1BatchUpdateChartRejection:
+    type: object
+    properties:
+      name:
+        type: string
+      reason:
+        type: string
+  v1BatchUpdateChartsRequest:
+    type: object
+    properties:
+      chart_names:
+        type: array
+        items:
+          type: string
+      from:
+        type: string
+        title: Default is today
+      update_later:
+        type: boolean
+  v1BatchUpdateChartsResult:
+    type: object
+    properties:
+      total:
+        type: integer
+        format: int64
+      total_rejected:
+        type: integer
+        format: int64
+      accepted:
+        type: array
+        items:
+          type: string
+      rejected:
+        type: array
+        items:
+          type: object
+          $ref: '#/definitions/v1BatchUpdateChartRejection'
   v1ContractsPageStats:
     type: object
     properties:
@@ -308,3 +373,11 @@ definitions:
         $ref: '#/definitions/v1Counter'
       operational_transactions_24h:
         $ref: '#/definitions/v1Counter'
+securityDefinitions:
+  ApiKeyAuth:
+    type: apiKey
+    name: x-api-key
+    in: header
+externalDocs:
+  description: More about blockscout microservices
+  url: https://github.com/blockscout/blockscout-rs

stats/stats-proto/swagger/stats.swagger.yaml

@@ -18,6 +18,7 @@ config = "0.13"
 tracing = "0.1"
 futures = "0.3"
 anyhow = "1.0"
+thiserror = "1.0"
 chrono = "0.4"
 sea-orm = { workspace = true, features = [
     "sqlx-postgres",

stats/stats-server/Cargo.toml

@@ -0,0 +1,58 @@
+use std::collections::HashMap;
+
+use serde::{Deserialize, Serialize};
+use tonic::Status;
+
+pub struct AuthorizationProvider {
+    keys: HashMap<String, ApiKey>,
+}
+
+pub const API_KEY_NAME: &str = "x-api-key";
+
+impl AuthorizationProvider {
+    pub fn new(keys: HashMap<String, ApiKey>) -> Self {
+        Self { keys }
+    }
+
+    pub fn is_request_authorized<T>(&self, request: &tonic::Request<T>) -> bool {
+        let Some(key) = request.metadata().get(API_KEY_NAME) else {
+            return false;
+        };
+        let Ok(api_key) = key
+            .to_str()
+            .inspect_err(|e| tracing::warn!("could not read http header as ascii: {}", e))
+        else {
+            return false;
+        };
+        self.is_key_authorized(api_key)
+    }
+
+    pub fn is_key_authorized(&self, api_key: &str) -> bool {
+        self.keys.values().any(|key| key.key.eq(api_key))
+    }
+
+    /// Unified error message
+    pub fn unauthorized(&self) -> Status {
+        Status::unauthenticated(format!(
+            "Request not authorized: Invalid or missing API key in {API_KEY_NAME} header"
+        ))
+    }
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(deny_unknown_fields)]
+pub struct ApiKey {
+    pub key: String,
+}
+
+impl ApiKey {
+    pub fn new(key: String) -> Self {
+        Self { key }
+    }
+
+    pub fn from_str_infallible(key: &str) -> Self {
+        Self {
+            key: key.to_string(),
+        }
+    }
+}