Feat importer offline e2e (#2014)

* e2e test for importer offline

* ignore database path

* configure test to run on github actions

* improve gitignore

* stop using \t for f strings in python

* adding python requirements

* fail justfile in case of failure

* building rpc downloader and importer offline

* second parameter is not target type

* waiting for stratus 3001 to be ready

* second stratus executing must be with release

* test don't run importer

* must run importer offline in the task

* dont build binaries at the beginning

* missing just

Author: arthurmm-cloudwalk

.github/workflows/_setup-e2e.yml

@@ -51,6 +51,9 @@ jobs:
         with:
           node-version-file: .tool-versions
 
+      - name: Install docker-compose
+        run: sudo apt-get install docker-compose -y
+        
       - name: Install jq
         run: sudo apt-get install jq -y
 

.github/workflows/e2e-test.yml

@@ -116,3 +116,13 @@ jobs:
     concurrency:
       group: ${{ github.workflow }}-rpc-downloader-${{ github.ref || github.run_id }}
       cancel-in-progress: true
+  
+  e2e-importer-offline:
+    name: E2E Importer Offline
+    uses: ./.github/workflows/_setup-e2e.yml
+    with:
+      justfile_recipe: "e2e-importer-offline"
+    
+    concurrency:
+      group: ${{ github.workflow }}-importer-offline-${{ github.ref || github.run_id }}
+      cancel-in-progress: true

.gitignore

@@ -32,6 +32,7 @@ node_modules/
 # E2E (Run With Importer)
 e2e_logs/
 temp_*-rocksdb/
+tests/importer-offline-database/*
 
 # Build
 crates/*/target

justfile

@@ -461,6 +461,45 @@ e2e-rpc-downloader:
     just _log "Killing PostgreSQL"
     docker-compose down postgres
 
+# E2E Importer Offline
+e2e-importer-offline:
+    mkdir -p e2e_logs
+
+    rm -rf data/importer-offline-database-rocksdb
+
+    just _log "Starting Stratus"
+    just stratus -a 0.0.0.0:3000 > e2e_logs/e2e-importer-offline-stratus.log &
+    just _wait_for_stratus
+
+    just _log "Running TestContractBalances tests"
+    just e2e stratus automine
+
+    just _log "Starting PostgreSQL"
+    docker-compose up -d postgres
+    
+    just _log "Running rpc downloader"
+    just rpc-downloader --external-rpc http://localhost:3000/ --external-rpc-storage postgres://postgres:123@localhost:5432/stratus --metrics-exporter-address 0.0.0.0:9001 --initial-accounts 0x70997970c51812dc3a010c7d01b50e0d17dc79c8,0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266
+
+    just _log "Run importer-offline"
+    just importer-offline --external-rpc-storage postgres://postgres:123@localhost:5432/stratus --rocks-path-prefix=data/importer-offline-database --metrics-exporter-address 0.0.0.0:9002
+
+    just _log "Stratus for importer-offline"
+    cargo run --release --bin stratus -- --leader -a 0.0.0.0:3001 --perm-storage=rocks --rocks-path-prefix=data/importer-offline-database --metrics-exporter-address 0.0.0.0:9002 > e2e_logs/e2e-importer-offline-stratus-3001.log &
+    just _wait_for_stratus 3001
+
+    just _log "Compare blocks of stratus and importer-offline"
+    pip install -r utils/compare_block/requirements.txt
+    python utils/compare_block/main.py http://localhost:3000 http://localhost:3001 1 --ignore timestamp --ignore type
+    
+    just _log "Killing Stratus"
+    killport 3000 -s sigterm
+
+    just _log "Killing importer-offline"
+    killport 3001 -s sigterm
+    
+    just _log "Killing PostgreSQL"
+    docker-compose down postgres
+
 # ------------------------------------------------------------------------------
 # Hive tests
 # ------------------------------------------------------------------------------

utils/compare_block/main.py

@@ -10,7 +10,7 @@ def print_items(changes: list, ignore: list[str], indent: int = 0):
     for change in changes:
         field = change.path(root="", output_format="list")[0]
         if field not in ignore:
-            print(f"{'\t'*indent}[bold red]{field}[/bold red]")
+            print(f"{'    ' * indent}[bold red]{field}[/bold red]")
     print()
 
 
@@ -20,11 +20,11 @@ def print_changes(changes: list, ignore: list[str], left, right, indent: int = 0
         if field not in ignore:
             if isinstance(left[field], HexBytes) and isinstance(right[field], HexBytes):
                 print(
-                    f"{'\t'*indent}{field}:\n{'\t'*indent}\t[bold red]Left: {left[field].hex()}[/bold red]\n{'\t'*indent}\t[bold green]Right: {right[field].hex()}[/bold green]"
+                    f"{'    ' * indent}{field}:\n{'    ' * indent}    [bold red]Left: {left[field].hex()}[/bold red]\n{'    ' * indent}    [bold green]Right: {right[field].hex()}[/bold green]"
                 )
             else:
                 print(
-                    f"{'\t'*indent}{field}:\n{'\t'*indent}\t[bold red]Left: {left[field]}[/bold red]\n{'\t'*indent}\t[bold green]Right: {right[field]}[/bold green]"
+                    f"{'    ' * indent}{field}:\n{'    ' * indent}    [bold red]Left: {left[field]}[/bold red]\n{'    ' * indent}    [bold green]Right: {right[field]}[/bold green]"
                 )
     print()
 
@@ -38,25 +38,25 @@ def print_diff(
     ignore_in_item: list[str],
     indent: int = 0,
 ):
-    print(f"{'\t'*indent}[bold blue]{name} Diff:[/bold blue]")
+    print(f"{'    ' * indent}[bold blue]{name} Diff:[/bold blue]")
     if len(diff.get("dictionary_item_added", [])):
         print(
-            f"{'\t'*indent}\t[red]The left {name} is missing the following fields:[/red]"
+            f"{'    ' * indent}    [red]The left {name} is missing the following fields:[/red]"
         )
         print_items(
             diff["dictionary_item_added"], [*ignore, *ignore_in_item], indent + 2
         )
 
     if len(diff.get("dictionary_item_removed", [])):
         print(
-            f"{'\t'*indent}\t[red]The right {name} is missing the following fields:[/red]"
+            f"{'    ' * indent}    [red]The right {name} is missing the following fields:[/red]"
         )
         print_items(
             diff["dictionary_item_removed"], [*ignore, *ignore_in_item], indent + 2
         )
 
     if len(diff.get("values_changed", [])):
-        print(f"{'\t'*indent}\t[red]The follwing values don't match:[/red]")
+        print(f"{'    ' * indent}    [red]The follwing values don't match:[/red]")
         print_changes(diff["values_changed"], ignore, left, right, indent + 2)
 
 

utils/compare_block/requirements.txt

@@ -0,0 +1,5 @@
+deepdiff
+web3
+hexbytes
+typer
+rich