NDEV-342: Platform synthetic performance test (do not merge)

.vscode/settings.json

@@ -0,0 +1,8 @@
+{
+    "python.defaultInterpreterPath": "/Users/gigimon/workspaces/neon/venv/bin/python",
+    "python.testing.pytestArgs": [
+        "."
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true
+}

clickfile.py

@@ -10,10 +10,14 @@
 import subprocess
 import sys
 import typing as tp
+import platform
 from collections import defaultdict
 from multiprocessing.dummy import Pool
 from urllib.parse import urlparse
 
+from deploy.infra.utils import env
+from deploy.infra.utils import docker as docker_utils
+
 import requests
 
 from deploy.infra.utils import docker as docker_utils

deploy/infra/Dockerfile

@@ -33,12 +33,19 @@ RUN apt-get update && apt-get install -qy --no-install-recommends ${PIP_BUILD_DE
 
 # Install PIP requirements
 RUN python3 -m ensurepip \
+<<<<<<< HEAD
+  && pip install -U --upgrade-strategy=only-if-needed wheel pip==21.1 pip-tools==6.2.0 \
+=======
   && pip install -U --upgrade-strategy=only-if-needed wheel pip pip-tools \
+>>>>>>> b0e94e36f104a0bd63a5d11c9b3b9c45772e8081
   && pip install --upgrade \
   -r ${HOME_DIR}/deploy/requirements/prod.txt \
   -r ${HOME_DIR}/deploy/requirements/devel.txt \
   -r ${HOME_DIR}/deploy/requirements/ui.txt \
+<<<<<<< HEAD
+=======
   #-r ${HOME_DIR}/deploy/requirements/nodeps.txt \
+>>>>>>> b0e94e36f104a0bd63a5d11c9b3b9c45772e8081
   --src=/usr/local/lib/python3.8/site-packages ;
 
 # Fixes for ctypes.cdll.LoadLibrary:
@@ -47,11 +54,14 @@ RUN python3 -m ensurepip \
 RUN [ ! -L "/usr/lib/x86_64-linux-gnu/libssh2.so" ] || ln -sf libssh2.so.1 /usr/lib/x86_64-linux-gnu/libssh2.so
 
 
+<<<<<<< HEAD
+=======
 # Install tfenv fo terraform using in devbox
 RUN git clone --depth=1 https://github.com/tfutils/tfenv.git ~/.tfenv \
   && echo 'export PATH="$HOME/.tfenv/bin:$PATH"' >> ~/.bash_profile \
   && ln -s ~/.tfenv/bin/* /usr/local/bin
 
+>>>>>>> b0e94e36f104a0bd63a5d11c9b3b9c45772e8081
 # Cleanup:
 ## Git cache saves ~ 1min on pip-sync
 RUN find /usr/local/lib/python3.8/site-packages -name .git | xargs rm -rf \

deploy/infra/devbox/.gitignore

@@ -1 +1,5 @@
-*.bash_history
+<<<<<<< HEAD
+*bash_history
+=======
+*.bash_history
+>>>>>>> b0e94e36f104a0bd63a5d11c9b3b9c45772e8081

deploy/requirements/nodeps.txt

@@ -1,2 +1,2 @@
 # this file for packages which has conflicts with
-web3==5.26.0
+web3==5.26.0

integration/tests/basic/test_rpc_calls.py

@@ -1,4 +1,3 @@
-import time
 import typing as tp
 from enum import Enum
 
@@ -7,13 +6,11 @@
 import sha3
 import web3
 
+from integration.tests.basic.helpers import rpc_checks
 from integration.tests.basic.helpers.assert_message import AssertMessage
 from integration.tests.basic.helpers.basic import BaseMixin
-from integration.tests.basic.helpers import rpc_checks
 from utils import helpers
-from utils.consts import Unit
 from utils.helpers import gen_hash_of_block
-from ui.libs import try_until
 
 """
 12.	Verify implemented rpc calls work
@@ -162,7 +159,7 @@ def test_eth_gas_price(self, params: tp.Union[str, None], raises: bool):
             (Tag.EARLIEST, Tag.EARLIEST),
             (Tag.EARLIEST, Tag.PENDING),
             (Tag.EARLIEST, Tag.LATEST),
-            (Tag.EARLIEST, None)
+            (Tag.EARLIEST, None),
         ],
     )
     def test_eth_get_logs_via_tags(self, tag1: Tag, tag2: Tag, send_erc20: tp.Any):
@@ -232,7 +229,7 @@ def test_net_version(self, params: tp.Union[str, None], raises: bool):
         else:
             assert "error" not in response
             assert (
-                    int(response["result"]) == self.web3_client._chain_id
+                int(response["result"]) == self.web3_client._chain_id
             ), f"Invalid response result {response['result']}"
 
     @pytest.mark.parametrize(
@@ -283,22 +280,30 @@ def test_rpc_call_eth_get_transaction_receipt(self):
         assert "error" not in response
         assert "result" in response, AssertMessage.DOES_NOT_CONTAIN_RESULT
         result = response["result"]
-        expected_hex_fields = ['transactionHash', 'transactionIndex', 'blockNumber', 'blockHash', 'cumulativeGasUsed',
-                               'gasUsed', 'logsBloom', 'status']
+        expected_hex_fields = [
+            "transactionHash",
+            "transactionIndex",
+            "blockNumber",
+            "blockHash",
+            "cumulativeGasUsed",
+            "gasUsed",
+            "logsBloom",
+            "status",
+        ]
         for field in expected_hex_fields:
             assert rpc_checks.is_hex(result[field])
-        assert result["status"] == '0x1', "Transaction status must be 0x1"
+        assert result["status"] == "0x1", "Transaction status must be 0x1"
         assert result["transactionHash"] == transaction_hash
         assert result["blockHash"] == tx_receipt.blockHash.hex()
-        assert result["from"].upper() == tx_receipt['from'].upper()
-        assert result["to"].upper() == tx_receipt['to'].upper()
+        assert result["from"].upper() == tx_receipt["from"].upper()
+        assert result["to"].upper() == tx_receipt["to"].upper()
         assert result["contractAddress"] is None
         assert result["logs"] == []
 
     def test_rpc_call_eth_get_transaction_receipt_when_hash_doesnt_exist(self):
         """Verify implemented rpc calls work eth_getTransactionReceipt when transaction hash doesn't exist"""
         response = self.proxy_api.send_rpc(method="eth_getTransactionReceipt", params=gen_hash_of_block(32))
-        assert response['result'] is None, "Result should be None"
+        assert response["result"] is None, "Result should be None"
 
     def test_rpc_call_eth_get_transaction_receipt_with_incorrect_hash(self):
         """Verify implemented rpc calls work eth_getTransactionReceipt when transaction hash is not correct"""
@@ -316,10 +321,14 @@ def test_eth_get_block_by_hash(self, full_trx: bool):
         rpc_checks.assert_block_fields(response, full_trx, tx_receipt)
 
     @pytest.mark.xfail(reason="NDEV-605")
-    @pytest.mark.parametrize("hash, full_trx, msg",
-                             [(gen_hash_of_block(31), False, "Key not found in database"),
-                              (gen_hash_of_block(32), True, "Key not found in database"),
-                              ("bad_hash", True, "Invalid `blockNumber`: \"bad_hash\"")])
+    @pytest.mark.parametrize(
+        "hash, full_trx, msg",
+        [
+            (gen_hash_of_block(31), False, "Key not found in database"),
+            (gen_hash_of_block(32), True, "Key not found in database"),
+            ("bad_hash", True, 'Invalid `blockNumber`: "bad_hash"'),
+        ],
+    )
     def test_eth_get_block_by_hash_with_incorrect_hash(self, hash, full_trx, msg):
         """Verify implemented rpc calls work eth_getBlockByHash with incorrect hash"""
         params = [hash, full_trx]
@@ -333,21 +342,23 @@ def test_eth_get_block_by_hash_with_incorrect_hash(self, hash, full_trx, msg):
     def test_eth_get_block_by_number_via_numbers(self, full_trx):
         """Verify implemented rpc calls work eth_getBlockByNumber"""
         tx_receipt = self.send_neon(self.sender_account, self.recipient_account, 10)
-        response = self.proxy_api.send_rpc(method="eth_getBlockByNumber",
-                                           params=[tx_receipt.blockNumber, full_trx])
+        response = self.proxy_api.send_rpc(method="eth_getBlockByNumber", params=[tx_receipt.blockNumber, full_trx])
         rpc_checks.assert_block_fields(response, full_trx, tx_receipt)
 
     @pytest.mark.xfail(reason="NDEV-605")
-    @pytest.mark.parametrize("number, full_trx, msg",
-                             [(gen_hash_of_block(31), False, "Key not found in database"),
-                              (1, True, "Key not found in database"),
-                              (1, True, "Invalid `blockNumber`: \"1\""),
-                              ("bad_tag", True, "Invalid `blockNumber`: \"bad_hash\"")])
+    @pytest.mark.parametrize(
+        "number, full_trx, msg",
+        [
+            (gen_hash_of_block(31), False, "Key not found in database"),
+            (1, True, "Key not found in database"),
+            (1, True, 'Invalid `blockNumber`: "1"'),
+            ("bad_tag", True, 'Invalid `blockNumber`: "bad_hash"'),
+        ],
+    )
     def test_eth_get_block_by_number_with_incorrect_data(self, number, full_trx, msg):
         """Verify implemented rpc calls work eth_getBlockByNumber"""
         tx_receipt = self.send_neon(self.sender_account, self.recipient_account, 10)
-        response = self.proxy_api.send_rpc(method="eth_getBlockByNumber",
-                                           params=[tx_receipt.blockNumber, full_trx])
+        response = self.proxy_api.send_rpc(method="eth_getBlockByNumber", params=[tx_receipt.blockNumber, full_trx])
         rpc_checks.assert_block_fields(response, full_trx, tx_receipt)
         assert "error" in response, "Error not in response"
         assert response["error"]["code"] == -32000
@@ -544,7 +555,7 @@ def constructor_args(self, request: tp.Any) -> tp.List[int]:
 
     @pytest.fixture(params=["BigGasFactory1", "BigGasFactory2"])
     def deploy_big_gas_requirements_contract(
-            self, request: tp.Any, constructor_args: tp.List[int]
+        self, request: tp.Any, constructor_args: tp.List[int]
     ) -> "web3._utils.datatypes.Contract":
         """Deploy contracts"""
         self.account = self.sender_account

loadtesting/proxy/__init__.py

@@ -1,5 +1,9 @@
 # coding: utf-8
 """
+<<<<<<< HEAD
+Created on 2022-08-18 
+=======
 Created on 2022-09-06 
+>>>>>>> b0e94e36f104a0bd63a5d11c9b3b9c45772e8081
 @author: Eugeny Kurkovich
 """

loadtesting/synthetic/__init__.py

@@ -0,0 +1,5 @@
+# coding: utf-8
+"""
+Created on 2022-08-18 
+@author: Eugeny Kurkovich
+"""

loadtesting/synthetic/get_cu.py

@@ -0,0 +1,22 @@
+import os
+import re
+from solana.rpc.api import Client as SolanaClient
+
+
+client = SolanaClient(os.environ.get("SOLANA_URL"))
+
+block_height = client.get_slot()["result"]
+
+for num in range(block_height-32, block_height):
+    block = client.get_block(num)["result"]
+
+    cu = 0
+
+    for tr in block["transactions"]:
+        res = re.findall(r"consumed (\d+) of (:?\d+) compute units", "".join(tr["meta"]["logMessages"]))
+        if not res:
+            continue
+        cu += int(res[0][0])
+    print(f"Transactions in block {num}: ", len(block["transactions"]))
+    print("CU: ", cu)
+

loadtesting/synthetic/helpers.py

@@ -0,0 +1,114 @@
+# coding: utf-8
+"""
+Created on 2022-08-29
+@author: Eugeny Kurkovich
+"""
+import typing as tp
+from dataclasses import dataclass
+from hashlib import sha256
+
+from construct import Bytes, Int8ul, Struct, Int32ul
+from eth_keys import keys as eth_keys
+from solana.publickey import PublicKey
+from solana.rpc import commitment
+from solana.transaction import Transaction, TransactionInstruction
+
+ACCOUNT_INFO_LAYOUT = Struct(
+    "type" / Int8ul,
+    "ether" / Bytes(20),
+    "nonce" / Int8ul,
+    "tx_count" / Bytes(8),
+    "balance" / Bytes(32),
+    "generation" / Int32ul,
+    "code_size" / Int32ul,
+    "is_rw_blocked" / Int8ul,
+)
+
+
+DEFAULT_UNITS = 500 * 1000
+DEFAULT_HEAP_FRAME = 256 * 1024
+DEFAULT_ADDITIONAL_FEE = 0
+COMPUTE_BUDGET_ID: PublicKey = PublicKey("ComputeBudget111111111111111111111111111111")
+SYSTEM_ADDRESS = "11111111111111111111111111111111"
+
+
+@dataclass
+class AccountInfo:
+    ether: eth_keys.PublicKey
+    trx_count: int
+
+    @staticmethod
+    def from_bytes(data: bytes):
+        cont = ACCOUNT_INFO_LAYOUT.parse(data)
+        return AccountInfo(cont.ether, cont.tx_count)
+
+
+@dataclass
+class SOLCommitmentState:
+    """Bank states to solana query"""
+
+    CONFIRMED: str = commitment.Confirmed
+    FINALIZED: str = commitment.Finalized
+    PROCESSED: str = commitment.Processed
+
+
+@dataclass
+class TreasuryPoolBases:
+    """Collateral pool base address"""
+
+    DEVNET: str = "7SBdHNeF9FFYySEoszpjZXXQsAiwa5Lzpsz6nUJWusEx"
+    NIGHT_STAND: str = "4sW3SZDJB7qXUyCYKA7pFL8eCTfm3REr8oSiKkww7MaT"
+
+    @classmethod
+    def get_by_network(cls, network: str) -> str:
+        """Get collateral pool base address"""
+        return getattr(cls, network.replace("-", "_").upper(), cls.NIGHT_STAND)
+
+
+@dataclass
+class TreasuryPool:
+    index: int
+    account: PublicKey
+    buffer: bytes
+
+
+class ComputeBudget:
+    @staticmethod
+    def request_units(units, additional_fee):
+        return TransactionInstruction(
+            program_id=COMPUTE_BUDGET_ID,
+            keys=[],
+            data=bytes.fromhex("00") + units.to_bytes(4, "little") + additional_fee.to_bytes(4, "little"),
+        )
+
+    @staticmethod
+    def request_heap_frame(heap_frame):
+        return TransactionInstruction(
+            program_id=COMPUTE_BUDGET_ID, keys=[], data=bytes.fromhex("01") + heap_frame.to_bytes(4, "little")
+        )
+
+
+class TransactionWithComputeBudget(Transaction):
+    def __init__(
+        self, units=DEFAULT_UNITS, additional_fee=DEFAULT_ADDITIONAL_FEE, heap_frame=DEFAULT_HEAP_FRAME, *args, **kwargs
+    ):
+        super().__init__(*args, **kwargs)
+        if units:
+            self.instructions.append(ComputeBudget.request_units(units, additional_fee))
+        if heap_frame:
+            self.instructions.append(ComputeBudget.request_heap_frame(heap_frame))
+
+
+def gen_account_with_seed(base, seed, program) -> PublicKey:
+    return PublicKey(sha256(bytes(base) + bytes(seed, "utf8") + bytes(program)).digest())
+
+
+def create_treasury_pool_address(
+    network: str, loader_id: str, collateral_pool_index: tp.Optional[int] = 2
+) -> TreasuryPool:
+    """Create treasury pool address"""
+    collateral_seed_prefix = "treasury_pool"
+    index_buf = collateral_pool_index.to_bytes(4, "little")
+    address, bump = PublicKey.find_program_address([bytes(collateral_seed_prefix, "utf8"), index_buf], PublicKey(loader_id))
+    # address = gen_account_with_seed(PublicKey(TreasuryPoolBases.get_by_network(network)), seed, PublicKey(loader_id))
+    return TreasuryPool(collateral_pool_index, address, index_buf)