(BIDS-2872) wip

Author: guybrush

cmd/explorer/main.go

@@ -607,7 +607,7 @@ func main() {
 			router.Use(metrics.HttpMiddleware)
 		}
 
-		ratelimit.Init(utils.Config.RedisSessionStoreEndpoint, ratelimit.DefaultRequestCollector)
+		ratelimit.Init()
 		router.Use(ratelimit.HttpMiddleware)
 
 		n := negroni.New(negroni.NewRecovery())

cmd/misc/main.go

@@ -393,7 +393,7 @@ func main() {
 	case "fix-ens-addresses":
 		err = fixEnsAddresses(erigonClient)
 	case "update-ratelimits":
-		err = updateRatelimits()
+		ratelimit.DBUpdater()
 	default:
 		utils.LogFatal(nil, fmt.Sprintf("unknown command %s", opts.Command), 0)
 	}
@@ -1936,14 +1936,3 @@ func reExportSyncCommittee(rpcClient rpc.Client, p uint64, dryRun bool) error {
 		return tx.Commit()
 	}
 }
-
-func updateRatelimits() error {
-	for {
-		err := ratelimit.DBUpdate()
-		if err != nil {
-			logrus.WithError(err).Errorf("error in updateRatelimits")
-		}
-		time.Sleep(time.Second * 10)
-	}
-	return nil
-}

db/migrations/20240125120000_add_ratelimits.sql

@@ -33,7 +33,7 @@ CREATE TABLE IF NOT EXISTS
         endpoint TEXT NOT NULL,
         method TEXT NOT NULL,
         params TEXT NOT NULL,
-        weight INT NOT NULL DEFAULT 0,
+        weight INT NOT NULL DEFAULT 1,
         valid_from TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT TO_TIMESTAMP(0),
         PRIMARY KEY (endpoint, valid_from)
     );
@@ -49,6 +49,19 @@ CREATE TABLE IF NOT EXISTS
         valid_from TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT TO_TIMESTAMP(0),
         PRIMARY KEY (name, valid_from)
     ); 
+
+SELECT 'up SQL query - add table api_products';
+CREATE TABLE IF NOT EXISTS
+    api_stats (
+        ts TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT NOW(),
+        user_id INT NOT NULL,
+        api_key VARCHAR(256) NOT NULL,
+        endpoint TEXT NOT NULL,
+        count INT NOT NULL,
+        PRIMARY KEY (ts, user_id, api_key, endpoint)
+    ); 
+
+CREATE INDEX IF NOT EXISTS idx_api_stats_ts_user_id ON api_stats (ts, user_id);
 -- +goose StatementEnd
 
 -- +goose Down
@@ -65,4 +78,8 @@ SELECT 'down SQL query - drop table api_weights';
 DROP TABLE IF EXISTS api_weights;
 SELECT 'down SQL query - drop table api_products';
 DROP TABLE IF EXISTS api_products;
+SELECT 'down SQL query - drop table api_stats';
+DROP TABLE IF EXISTS api_stats;
+SELECT 'down SQL query - drop index idx_api_stats_ts_user_id';
+DROP INDEX IF EXISTS idx_api_stats_ts_user_id;
 -- +goose StatementEnd

local-deployment/network-params.json

@@ -26,7 +26,7 @@
       "electra_fork_epoch": 999999999
     },
     "global_client_log_level": "info",
-    "start_tx_spammer": true,
+    "start_tx_spammer": false,
     "start_blob_spammer": false
   }
 

local-deployment/provision-explorer-config.sh

@@ -101,8 +101,6 @@ frontend:
     sapphire: price_sapphire
     emerald: price_emerald
     diamond: price_diamond
-    custom1: price_custom1
-    custom2: price_custom2
 
 indexer:
   # fullIndexOnStartup: false # Perform a one time full db index on startup

local-deployment/run.sh

@@ -43,7 +43,11 @@ fn_sql() {
 }
 
 fn_redis() {
-    docker compose exec redis-sessions redis-cli
+    if [ -z "${1}" ]; then
+        docker compose exec redis-sessions redis-cli
+    else
+        docker compose exec redis-sessions redis-cli "$@"
+    fi
     #redis-cli -h localhost -p $REDIS_PORT
 }