Merge pull request #2845 from CounterpartyXCP/develop

v10.8.0

Author: ouziel-slama

.github/workflows/test_compose.sh

@@ -3,7 +3,7 @@
 set -e
 set -x
 
-#exit 0
+exit 0
 
 export PATH="/snap/bin:$PATH"
 

.github/workflows/test_compose.yml

@@ -1456,7 +1456,7 @@ Returns server information and the list of documented routes in JSON format.
         "result": {
             "server_ready": true,
             "network": "mainnet",
-            "version": "10.8.0-rc.2",
+            "version": "10.8.0",
             "backend_height": 850214,
             "counterparty_height": 850214,
             "documentation": "https://counterpartycore.docs.apiary.io/",

apiary.apib

@@ -363,6 +363,8 @@ def handle_route(**kwargs):
     except Exception as e:
         capture_exception(e)
         logger.error("Error in API: %s", e)
+        # import traceback
+        # print(traceback.format_exc())
         return return_result(
             503, error="Unknown error", start_time=start_time, query_args=query_args
         )

counterparty-core/counterpartycore/lib/api/api_server.py

@@ -692,6 +692,10 @@ def get_oldest_tx(self, address, block_index, timeout=ADDRINDEXRS_CLIENT_TIMEOUT
         "block_index": 820321,
         "tx_hash": "b61ac3ab1ba9d63d484e8f83e8b9607bd932c8f4b742095445c3527ab575d972",
     },  # {}
+    "842375-bc1q2rn0c89mylzj6c26exda5y2nkezmzh4lh5rkkl": {
+        "block_index": 842374,
+        "tx_hash": "3043db292b7f64cbe1cafa32bc733316b162d8a1f41f31a1e6cd224fd5b72415",
+    },
 }
 ADDRINDEXRS_CLIENT = None
 

counterparty-core/counterpartycore/lib/backend/addrindexrs.py

@@ -87,7 +87,7 @@ def rpc_call(payload, retry=0):
             f"{response_json['error']} Is `txindex` enabled in {config.BTC_NAME} Core?"
         )
     elif response_json["error"]["code"] in [-28, -8, -2]:
-        # "Verifying blocks..." or "Block height out of range" or "The network does not appear to fully agree!"
+        # "Verifying blocks..." or "Block height out of range" or "The network does not appear to fully agree!""
         logger.debug(f"Backend not ready. Sleeping for ten seconds. ({response_json['error']})")
         logger.debug(f"Payload: {payload}")
         if retry >= 10:
@@ -101,8 +101,15 @@ def rpc_call(payload, retry=0):
     else:
         raise exceptions.BitcoindRPCError(response_json["error"]["message"])
 
+    if isinstance(payload, dict):
+        method = payload["method"]
+    elif isinstance(payload, list):
+        method = payload[0]["method"]
+    else:
+        method = "unknown"
     elapsed = time.time() - start_time
-    logger.trace(f"Bitcoin Core RPC call {payload['method']} took {elapsed:.3f}s")
+    logger.trace(f"Bitcoin Core RPC call {method} took {elapsed:.3f}s")
+
     return result
 
 

counterparty-core/counterpartycore/lib/backend/bitcoind.py

@@ -691,6 +691,10 @@
         "ledger_hash": "8e55b64d0dfd85a58e3a9dd27ce49efd98559d96f16f53beb66a10b7671ea857",
         "txlist_hash": "b3f549168f56702287c7b06c0348c4ac0adffcd219bab386d2f19326c0cd491c",
     },
+    874883: {
+        "ledger_hash": "4c4d6b660af23bb03a04bbf93ddd0a4b8e615dd7b883ecf827274cabe658bfc2",
+        "txlist_hash": "f6a99d60337c33c1822c048f56e241455cd7e45bb5a9515096f1ac609d50f669",
+    },
 }
 
 CONSENSUS_HASH_VERSION_TESTNET = 7
@@ -1114,5 +1118,6 @@ def database_version(db):
                 message = "`VERSION_STRING` not found in dataase. "
             else:
                 message = f"Client pre-release version number mismatch: {version_pre_release} ≠ {config.VERSION_PRE_RELEASE}. "
-            message += "Checking if a reparse is needed..."
+            message += "Checking if a rollback or a reparse is needed..."
+            check_need_rollback(version_minor, message)
             check_need_reparse(version_minor, message)

counterparty-core/counterpartycore/lib/check.py

@@ -5,7 +5,7 @@
 
 
 # Semantic Version
-__version__ = "10.8.0-rc.2"  # for hatch
+__version__ = "10.8.0"  # for hatch
 VERSION_STRING = __version__
 version = VERSION_STRING.split("-")[0].split(".")
 VERSION_MAJOR = int(version[0])
@@ -155,7 +155,7 @@
 # Custom exit codes
 EXITCODE_UPDATE_REQUIRED = 5
 
-BACKEND_RAW_TRANSACTIONS_CACHE_SIZE = 20000
+BACKEND_RAW_TRANSACTIONS_CACHE_SIZE = 1000
 BACKEND_RPC_BATCH_NUM_WORKERS = 6
 
 DEFAULT_UTXO_LOCKS_MAX_ADDRESSES = 1000

counterparty-core/counterpartycore/lib/config.py

@@ -741,7 +741,8 @@ def get_op_return_vout(decoded_tx):
 
 
 KNOWN_SOURCES = {
-    "92ad58f5aa35c503489efbdd2a466e942baa9ac5cd67cb7544adf03e47a457d0": "a71da7169db3672408c7b25f84be425839548e63fa480c0478f91e3c2aa3ec67:0"
+    "92ad58f5aa35c503489efbdd2a466e942baa9ac5cd67cb7544adf03e47a457d0": "a71da7169db3672408c7b25f84be425839548e63fa480c0478f91e3c2aa3ec67:0",
+    "c80143886181ebbc782d23a50acca0f5ea7ac005d3164d7c76fc5e14f72d47c8": "",
 }
 
 

counterparty-core/counterpartycore/lib/gettxinfo.py

@@ -131,22 +131,22 @@ def parse_raw_mempool(db):
     timestamps = {}
     cursor = db.cursor()
     logger.debug(f"Found {len(raw_mempool)} transaction(s) in the mempool...")
+    txhash_list = []
     for txid, tx_info in raw_mempool.items():
         existing_tx_in_mempool = cursor.execute(
             "SELECT * FROM mempool WHERE tx_hash = ? LIMIT 1", (txid,)
         ).fetchone()
         if existing_tx_in_mempool:
             continue
-        try:
-            logger.trace(f"Getting raw transaction `{txid}` from the mempool...")
-            raw_tx = backend.bitcoind.getrawtransaction(txid)
-            raw_tx_list.append(raw_tx)
-            timestamps[txid] = tx_info["time"]
-        except exceptions.BitcoindRPCError as e:
-            if "No such mempool or blockchain transaction" in str(e):
-                pass
-            else:
-                raise e
+        txhash_list.append(txid)
+        timestamps[txid] = tx_info["time"]
+
+    logger.debug(f"Getting {len(txhash_list)} raw transactions by batch from the mempool...")
+    raw_transactions_by_hash = backend.addrindexrs.getrawtransaction_batch(
+        txhash_list, skip_missing=True
+    )
+    raw_tx_list = [raw_hex for raw_hex in raw_transactions_by_hash.values() if raw_hex is not None]
+
     logger.debug(f"Parsing {len(raw_tx_list)} transaction(s) from the mempool...")
     parse_mempool_transactions(db, raw_tx_list, timestamps)
     logger.debug("Raw mempool parsed successfully.")

counterparty-core/counterpartycore/lib/mempool.py

@@ -988,13 +988,18 @@ def bootstrap(no_confirm=False, snapshot_url=None):
     if config.TESTNET:
         ledger_database_path += ".testnet"
     ledger_database_path += ".db"
-    api_database_path = ledger_database_path.replace(".db", ".api.db")
+
+    old_api_database_path = ledger_database_path.replace(".db", ".api.db")
+    if config.TESTNET:
+        api_database_path = os.path.join(config.DATA_DIR, "state.testnet.db")
+    else:
+        api_database_path = os.path.join(config.DATA_DIR, "state.db")
 
     # Prepare Directory.
     if not os.path.exists(config.DATA_DIR):
         os.makedirs(config.DATA_DIR, mode=0o755)
 
-    for database_path in [ledger_database_path, api_database_path]:
+    for database_path in [ledger_database_path, api_database_path, old_api_database_path]:
         if os.path.exists(database_path):
             os.remove(database_path)
         # Delete SQLite Write-Ahead-Log
@@ -1037,10 +1042,13 @@ def bootstrap_progress(blocknum, blocksize, totalsize):
             tar_file.extractall(path=config.DATA_DIR)  # nosec B202  # noqa: S202
 
     assert os.path.exists(ledger_database_path)
-    assert os.path.exists(api_database_path)
+    assert os.path.exists(api_database_path) or os.path.exists(old_api_database_path)
     # user and group have "rw" access
     os.chmod(ledger_database_path, 0o660)  # nosec B103
-    os.chmod(api_database_path, 0o660)  # nosec B103
+    if os.path.exists(api_database_path):
+        os.chmod(api_database_path, 0o660)  # nosec B103
+    if os.path.exists(old_api_database_path):
+        os.chmod(old_api_database_path, 0o660)  # nosec B103
 
     with log.Spinner("Cleaning up..."):
         os.remove(tarball_path)

counterparty-core/counterpartycore/server.py

@@ -165,7 +165,7 @@ Returns server information and the list of documented routes in JSON format.
         "result": {
             "server_ready": true,
             "network": "mainnet",
-            "version": "10.8.0-rc.2",
+            "version": "10.8.0",
             "backend_height": 850214,
             "counterparty_height": 850214,
             "documentation": "https://counterpartycore.docs.apiary.io/",

counterparty-core/counterpartycore/test/regtest/apidoc/blueprint-template.md

@@ -36,4 +36,4 @@ gunicorn==23.0.0
 waitress==3.0.1
 hypothesis==6.116.0
 bitcoin-utils==0.7.1
-counterparty-rs==10.8.0-rc.2
+counterparty-rs==10.8.0

counterparty-core/requirements.txt

@@ -1,6 +1,6 @@
 [package]
 name = "counterparty-rs"
-version = "10.8.0-rc.2"
+version = "10.8.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

counterparty-rs/Cargo.lock

@@ -5,4 +5,4 @@ colorlog==6.8.0
 python-dateutil==2.8.2
 requests==2.32.0
 termcolor==2.4.0
-counterparty-core==10.8.0-rc.2
+counterparty-core==10.8.0

counterparty-rs/Cargo.toml

@@ -15,7 +15,7 @@ x-addrindexrs-common: &addrindexrs-common
   restart: unless-stopped
 
 x-counterparty-common: &counterparty-common
-  image: counterparty/counterparty:v10.8.0-rc.2
+  image: counterparty/counterparty:v10.8.0
   stop_grace_period: 1m
   volumes:
     - data:/root/.bitcoin

counterparty-wallet/requirements.txt

@@ -5,7 +5,7 @@ This release includes some significant architectural changes to the codebase to
 
 # Upgrading
 
-This upgrade requires a mandatory, automatic reparse from block 871780.
+This upgrade requires a mandatory, automatic rollback from block 871780.
 
 
 # ChangeLog
@@ -47,6 +47,8 @@ This upgrade requires a mandatory, automatic reparse from block 871780.
 - Exclude transactions by `SIGHASH`
 - Be able to trigger a rollback on a minor version change
 - Add several new checkpoints
+- Parse raw mempool by batch before following
+- Set `BACKEND_RAW_TRANSACTIONS_CACHE_SIZE` to 1000
 
 ## API