diff --git a/go.mod b/go.mod index 65783d46d1b..9955299b7d2 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/shopspring/decimal v1.3.1 github.com/stretchr/testify v1.9.0 github.com/swaggo/swag v1.16.3 - github.com/testcontainers/testcontainers-go v0.28.0 + github.com/testcontainers/testcontainers-go v0.29.1 github.com/urfave/cli/v2 v2.27.1 golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa golang.org/x/sync v0.6.0 @@ -75,7 +75,7 @@ require ( github.com/deckarep/golang-set/v2 v2.1.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect github.com/distribution/reference v0.5.0 // indirect - github.com/docker/docker v25.0.2+incompatible // indirect + github.com/docker/docker v25.0.3+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect @@ -137,7 +137,7 @@ require ( github.com/neko-neko/echo-logrus/v2 v2.0.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0-rc5 // indirect + github.com/opencontainers/image-spec v1.1.0 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect @@ -170,7 +170,7 @@ require ( go.opentelemetry.io/otel/trace v1.20.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.18.0 // indirect - golang.org/x/mod v0.14.0 // indirect + golang.org/x/mod v0.16.0 // indirect golang.org/x/net v0.20.0 // indirect golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/go.sum b/go.sum index 5f38aca4112..ee62345fba0 100644 --- a/go.sum +++ b/go.sum @@ -143,8 +143,8 @@ github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/cli v24.0.7+incompatible h1:wa/nIwYFW7BVTGa7SWPVyyXU9lgORqUb1xfI36MSkFg= github.com/docker/cli v24.0.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/docker v25.0.2+incompatible h1:/OaKeauroa10K4Nqavw4zlhcDq/WBcPMc5DbjOGgozY= -github.com/docker/docker v25.0.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v25.0.3+incompatible h1:D5fy/lYmY7bvZa0XTZ5/UJPljor41F+vdyJG5luQLfQ= +github.com/docker/docker v25.0.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -513,8 +513,8 @@ github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI= -github.com/opencontainers/image-spec v1.1.0-rc5/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/opencontainers/runc v1.1.12 h1:BOIssBaW1La0/qbNZHXOOa71dZfZEQOzW7dqQf3phss= github.com/opencontainers/runc v1.1.12/go.mod h1:S+lQwSfncpBha7XTy/5lBwWgm5+y5Ma/O44Ekby9FK8= github.com/ory/dockertest/v3 v3.10.0 h1:4K3z2VMe8Woe++invjaTB7VRyQXQy5UY+loujO4aNE4= @@ -626,8 +626,8 @@ github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/taikoxyz/taiko-geth v0.0.0-20240228040257-a00cd0963e60 h1:kJpiKwsHtNF8z8KZVEjLgwPWtI2AbMGNwrrr8VQc1EY= github.com/taikoxyz/taiko-geth v0.0.0-20240228040257-a00cd0963e60/go.mod h1:TN8ZiHrdJwSe8Cb6x+p0hs5CxhJZPbqB7hHkaUXcmIU= -github.com/testcontainers/testcontainers-go v0.28.0 h1:1HLm9qm+J5VikzFDYhOd+Zw12NtOl+8drH2E8nTY1r8= -github.com/testcontainers/testcontainers-go v0.28.0/go.mod h1:COlDpUXbwW3owtpMkEB1zo9gwb1CoKVKlyrVPejF4AU= +github.com/testcontainers/testcontainers-go v0.29.1 h1:z8kxdFlovA2y97RWx98v/TQ+tR+SXZm6p35M+xB92zk= +github.com/testcontainers/testcontainers-go v0.29.1/go.mod h1:SnKnKQav8UcgtKqjp/AD8bE1MqZm+3TDb/B8crE3XnI= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= @@ -737,8 +737,8 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/packages/blobstorage/.default.indexer.env b/packages/blobstorage/.default.indexer.env index a2fddf76ede..13073c65b35 100644 --- a/packages/blobstorage/.default.indexer.env +++ b/packages/blobstorage/.default.indexer.env @@ -2,8 +2,8 @@ RPC_URL=wss://l1ws.internal.taiko.xyz BEACON_URL=https://l1beacon.internal.taiko.xyz/eth/v1/beacon/blob_sidecars/ TAIKO_L1_CONTRACT_ADDRESS=0xC069c3d2a9f2479F559AD34485698ad5199C555f DATABASE_HOST=localhost -DATABASE_PORT=27017 +DATABASE_PORT=3306 DATABASE_USER=root -DATABASE_PASSWORD=root +DATABASE_PASSWORD=passw00d DATABASE_NAME=blobs -METRICS_HTTP_PORT=7472 \ No newline at end of file +METRICS_HTTP_PORT=7472 diff --git a/packages/blobstorage/.default.server.env b/packages/blobstorage/.default.server.env index 7c469c7f072..d7c46611e1d 100644 --- a/packages/blobstorage/.default.server.env +++ b/packages/blobstorage/.default.server.env @@ -1,7 +1,7 @@ HTTP_PORT=3282 DATABASE_HOST=localhost -DATABASE_PORT=27017 +DATABASE_PORT=3306 DATABASE_USER=root -DATABASE_PASSWORD=root +DATABASE_PASSWORD=passw00d DATABASE_NAME=blobs -METRICS_HTTP_PORT=7471 \ No newline at end of file +METRICS_HTTP_PORT=7471 diff --git a/packages/blobstorage/README.md b/packages/blobstorage/README.md index 5c2165de59d..e7fe0e15ce9 100644 --- a/packages/blobstorage/README.md +++ b/packages/blobstorage/README.md @@ -1,55 +1,93 @@ # blob-storage -Repo for BLOB storage (archive and serve data) +Repository for BLOB storage (archive and serve data) -## how to run ? +## Prerequisites -Prerequisite is to have docker engine up and running. +- Docker engine up and running. +- Go installed on your system. -1. Start the mongoDB +## Configuration and Setup -```bash -cd local_docker && docker-compose up -d -``` +### Setting up MySQL -2. Start the `indexer` +1. **Start MySQL**: -```bash -ENV_FILE=.default.indexer.env run cmd/main.go indexer -``` + Navigate to the `docker-compose` directory and start the MySQL service: -By default the above command starts the app from the latest block height. If we want to specify a previous blockheight, we can run change it from the `.default.indexer.env` file, by adding a `STARTING_BLOCK_ID` variable. + ```bash + cd ./docker-compose + docker-compose up -d + ``` -2. Start the `server`. + This command starts your MySQL instance as defined in your `docker-compose.yml` file. -```bash -ENV_FILE=.default.server.env run cmd/main.go server -``` +2. **Migrate Database Schema**: -## how to test / use ? + Navigate to the `migrations` directory to apply database migrations: -When the `DB`, `blob-catcher` and `server` is running, the `blob-catcher` is outputting the `blobHash` to the terminal (with the `networkName` variable too, tho it is not written into the DB). Use that `blobHash` (including the 0x) in + ```bash + cd ./migrations + goose mysql "root:passw00d@tcp(localhost:3306)/blobs" status + goose mysql "root:passw00d@tcp(localhost:3306)/blobs" up + ``` -1. Either in a curl command like this (you can query multiple blobHashes - comma separated - with one go and the result will be a respective array): + These commands apply migrations to the `blobs` database. -```bash -curl -X GET "http://localhost:27001/getBlob?blobHash=0x01a2a1cdc7ad221934061642a79a760776a013d0e6fa1a1c6b642ace009c372a,0xWRONG_HASH" -``` +### Environment Configuration -The result will be something like this: +Ensure your `.default.indexer.env` and `.default.server.env` files are configured with the correct database credentials, host, and any other necessary environment variables. -```bash -{"data":[{"blob":"0x123...00","kzg_commitment":"0xabd68b406920aa74b83cf19655f1179d373b5a8cba21b126b2c18baf2096c8eb9ab7116a89b375546a3c30038485939e"}, {"blob":"NOT_FOUND","kzg_commitment":"NOT_FOUND"}]} -``` +## Running the Application -2. Or to backtest, use the simple python script below, after overwriting the `blob_hash` variable: +1. **Start the Indexer**: -```bash -python3 python_query.py -``` + With the environment file configured, start the indexer: -## todos + ```bash + ENV_FILE=.default.indexer.env go run cmd/main.go indexer + ``` + + This starts the app from the latest block height by default. Adjust the `STARTING_BLOCK_ID` in the environment file if needed. + +2. **Start the Server**: + + Similarly, start the server: + + ```bash + ENV_FILE=.default.server.env go run cmd/main.go server + ``` + +## Testing and Usage + +When the `DB`, `blob-catcher` and `server` are running, the `blob-catcher` is outputting the `blobHash` to the terminal (with the `networkName` variable too, though it is not written into the DB). Use that `blobHash` (including the 0x) in + +1. **Querying Blob Data via HTTP Request**: + + To retrieve blob data, you can execute a `curl` command. This allows for querying multiple `blobHashes` simultaneously, separated by commas. A single request can yield an array of results: + + ```bash + curl -X GET "http://localhost:3282/getBlob?blobHash=0x01a2a1cdc7ad221934061642a79a760776a013d0e6fa1a1c6b642ace009c372a,0xWRONG_HASH" + ``` + + **Expected Output**: + + ```bash + {"data":[{"blob":"0x123...00","kzg_commitment":"0xabd68b406920aa74b83cf19655f1179d373b5a8cba21b126b2c18baf2096c8eb9ab7116a89b375546a3c30038485939e"}, {"blob":"NOT_FOUND","kzg_commitment":"NOT_FOUND"}]} + ``` + +2. **Backtesting with a Python Script**: + + This script facilitates querying the database directly based on a specified `blob_hash`. Modify the `blob_hash` variable in the script to match the hash you wish to query. + + To run the script: + + ```bash + python3 python_query.py + ``` + +## Todos What is still missing is: -- small refinements and DevOps (prod-grade DB with creditentials, proper containerization) +- small refinements and DevOps (prod-grade DB with credentials, proper containerization) diff --git a/packages/blobstorage/docker-compose/docker-compose.yml b/packages/blobstorage/docker-compose/docker-compose.yml new file mode 100644 index 00000000000..09976b8f4ca --- /dev/null +++ b/packages/blobstorage/docker-compose/docker-compose.yml @@ -0,0 +1,14 @@ +version: "3.8" +services: + db: + image: mysql:8.0 + cap_add: + - SYS_NICE + restart: always + environment: + - MYSQL_DATABASE=blobs + - MYSQL_ROOT_PASSWORD=passw00d + ports: + - "3306:3306" + volumes: + - ~/.docker-conf/mysql/data/:/var/lib/mysql diff --git a/packages/blobstorage/docs/docs.go b/packages/blobstorage/docs/docs.go new file mode 100644 index 00000000000..3e482f062ce --- /dev/null +++ b/packages/blobstorage/docs/docs.go @@ -0,0 +1,98 @@ +// Package docs Code generated by swaggo/swag. DO NOT EDIT +package docs + +import "github.com/swaggo/swag" + +const docTemplate = `{ + "schemes": {{ marshal .Schemes }}, + "swagger": "2.0", + "info": { + "description": "{{escape .Description}}", + "title": "{{.Title}}", + "termsOfService": "http://swagger.io/terms/", + "contact": { + "name": "API Support", + "url": "https://community.taiko.xyz/", + "email": "info@taiko.xyz" + }, + "license": { + "name": "MIT" + }, + "version": "{{.Version}}" + }, + "host": "{{.Host}}", + "basePath": "{{.BasePath}}", + "paths": { + "/getBlob": { + "get": { + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "summary": "Get blob(s) and KZG commitment(s)", + "operationId": "get-blob", + "parameters": [ + { + "type": "string", + "description": "blobHash to query", + "name": "blobHash", + "in": "query", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/http.getBlobResponse" + } + } + } + } + } + }, + "definitions": { + "http.blobData": { + "type": "object", + "properties": { + "blob_hash": { + "type": "string" + }, + "kzg_commitment": { + "type": "string" + } + } + }, + "http.getBlobResponse": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/definitions/http.blobData" + } + } + } + } + } +}` + +// SwaggerInfo holds exported Swagger Info so clients can modify it +var SwaggerInfo = &swag.Spec{ + Version: "1.0", + Host: "blobs.internal.taiko.xyz", + BasePath: "", + Schemes: []string{}, + Title: "Taiko Blobstorage API", + Description: "", + InfoInstanceName: "swagger", + SwaggerTemplate: docTemplate, + LeftDelim: "{{", + RightDelim: "}}", +} + +func init() { + swag.Register(SwaggerInfo.InstanceName(), SwaggerInfo) +} diff --git a/packages/blobstorage/docs/index.html b/packages/blobstorage/docs/index.html new file mode 100644 index 00000000000..f4dec8355c7 --- /dev/null +++ b/packages/blobstorage/docs/index.html @@ -0,0 +1,29 @@ + +
+ + + +