diff --git a/contrib/query.py b/contrib/query.py index 6cf8d58f..c960cf34 100755 --- a/contrib/query.py +++ b/contrib/query.py @@ -7,33 +7,34 @@ # See the file "LICENCE" for information about the copyright # and warranty status of this software. -'''Script to query the database for debugging purposes. +"""Script to query the database for debugging purposes. Not currently documented; might become easier to use in future. -''' +""" import argparse import asyncio -from electrumx import Env -from electrumx.server.db import DB from electrumx.lib.hash import hash_to_hex_str, Base58Error +from electrumx.server.db import DB +from electrumx.server.env import Env +from electrumx.server.history import History -async def print_stats(hist_db, utxo_db): - count = 0 - for key in utxo_db.iterator(prefix=b'u', include_value=False): - count += 1 - print(f'UTXO count: {utxos}') +async def print_stats(hist_db: History, utxo_db): + utxo_count = 0 + for _ in utxo_db.iterator(prefix=b'u', include_value=False): + utxo_count += 1 + print(f'UTXO count: {utxo_count}') - count = 0 - for key in utxo_db.iterator(prefix=b'h', include_value=False): - count += 1 - print(f'HashX count: {count}') + hash_count = 0 + for _ in utxo_db.iterator(prefix=b'h', include_value=False): + hash_count += 1 + print(f'HashX count: {hash_count}') hist = 0 hist_len = 0 - for key, value in hist_db.iterator(prefix=b'H'): + for key, value in hist_db.db.iterator(prefix=b'H'): hist += 1 hist_len += len(value) // 4 print(f'History rows {hist:,d} entries {hist_len:,d}') @@ -64,7 +65,7 @@ async def query(args): await db.open_for_serving() if not args.scripts: - await print_stats(db.hist_db, db.utxo_db) + await print_stats(db.history, db.utxo_db) return limit = args.limit for arg in args.scripts: @@ -97,15 +98,16 @@ def main(): parser = argparse.ArgumentParser( 'query.py', description='Invoke with COIN and DB_DIRECTORY set in the ' - 'environment as they would be invoking electrumx_server' + 'environment as they would be invoking electrumx_server' ) parser.add_argument('-l', '--limit', metavar='limit', type=int, default=10, help=f'maximum number of entries to ' - f'return (default: {default_limit})') + f'return (default: {default_limit})') parser.add_argument('scripts', nargs='*', default=[], type=str, help='hex scripts to query') args = parser.parse_args() asyncio.run(query(args)) + if __name__ == '__main__': main() diff --git a/electrumx/__init__.py b/electrumx/__init__.py index 88f1ae85..e69de29b 100644 --- a/electrumx/__init__.py +++ b/electrumx/__init__.py @@ -1,6 +0,0 @@ -__version__ = "1.16.0" -version = f'ElectrumX {__version__}' -version_short = __version__ - -from electrumx.server.controller import Controller -from electrumx.server.env import Env diff --git a/electrumx/lib/atomicals_blueprint_builder.py b/electrumx/lib/atomicals_blueprint_builder.py index 7fc73b1d..2095d037 100644 --- a/electrumx/lib/atomicals_blueprint_builder.py +++ b/electrumx/lib/atomicals_blueprint_builder.py @@ -595,7 +595,15 @@ def validate_ft_transfer_has_no_inflation(self, atomical_id_to_expected_outs_map input_value = ft_info['value'] if sum_out_value and sum_out_value > input_value: atomical_id_compact = location_id_bytes_to_compact(atomical_id) - raise AtomicalsTransferBlueprintBuilderError(f'validate_ft_transfer_has_no_inflation: Fatal error the output sum of outputs is greater than input sum for Atomical: atomical_id={atomical_id_compact} input_value={input_value} sum_out_value={sum_out_value} {hash_to_hex_str(tx_hash)} ft_atomicals={ft_atomicals}') + raise AtomicalsTransferBlueprintBuilderError( + 'validate_ft_transfer_has_no_inflation: ' + 'Fatal error the output sum of outputs is greater than input sum for Atomical: ' + f'atomical_id={atomical_id_compact} ' + f'input_value={input_value} ' + f'sum_out_value={sum_out_value} ' + f'{hash_to_hex_str(self.tx_hash)} ' + f'ft_atomicals={ft_atomicals}' + ) def is_split_operation(self): return is_split_operation(self.operations_found_at_inputs) diff --git a/electrumx/lib/coins.py b/electrumx/lib/coins.py index 95d9b1e0..742d58e9 100644 --- a/electrumx/lib/coins.py +++ b/electrumx/lib/coins.py @@ -48,9 +48,9 @@ import electrumx.lib.tx_axe as lib_tx_axe import electrumx.server.block_processor as block_proc import electrumx.server.daemon as daemon -from electrumx.server.session import (ElectrumX, DashElectrumX, - SmartCashElectrumX, AuxPoWElectrumX, - NameIndexElectrumX, NameIndexAuxPoWElectrumX) +from electrumx.server.session.electrumx_session import (ElectrumX, DashElectrumX, + SmartCashElectrumX, AuxPoWElectrumX, + NameIndexElectrumX, NameIndexAuxPoWElectrumX) @dataclass @@ -65,8 +65,19 @@ class CoinError(Exception): '''Exception raised for coin-related errors.''' -class Coin: - '''Base class of coin hierarchy.''' +class CoinHeaderHashMixin: + @classmethod + def header_hash(cls, header): + """Given a header return hash""" + return double_sha256(header) + + +class CoinShortNameMixin: + SHORTNAME: str + + +class Coin(CoinHeaderHashMixin, CoinShortNameMixin): + """Base class of coin hierarchy.""" REORG_LIMIT = 200 # Not sure if these are coin-specific @@ -225,11 +236,6 @@ def privkey_WIF(cls, privkey_bytes, compressed): payload.append(0x01) return cls.ENCODE_CHECK(payload) - @classmethod - def header_hash(cls, header): - '''Given a header return hash''' - return double_sha256(header) - @classmethod def header_prevhash(cls, header): '''Given a header return previous hash''' @@ -328,7 +334,7 @@ def block_header(cls, block, height): return deserializer.read_header(cls.BASIC_HEADER_SIZE) -class ScryptMixin: +class ScryptMixin(CoinHeaderHashMixin): DESERIALIZER = lib_tx.DeserializerTxTime HEADER_HASH = None @@ -357,7 +363,7 @@ class KomodoMixin: DESERIALIZER = lib_tx.DeserializerZcash -class BitcoinMixin: +class BitcoinMixin(CoinShortNameMixin): SHORTNAME = "BTC" NET = "mainnet" XPUB_VERBYTES = bytes.fromhex("0488b21e") @@ -845,7 +851,7 @@ def hashX_from_script(cls, script): return super().hashX_from_script(address_script) -class BitcoinTestnetMixin: +class BitcoinTestnetMixin(CoinShortNameMixin): SHORTNAME = "XTN" NET = "testnet" XPUB_VERBYTES = bytes.fromhex("043587cf") diff --git a/electrumx/server/block_processor.py b/electrumx/server/block_processor.py index bf8e9e4b..098cc468 100644 --- a/electrumx/server/block_processor.py +++ b/electrumx/server/block_processor.py @@ -9,23 +9,19 @@ '''Block prefetcher and chain processor.''' import asyncio -import os import time from typing import Sequence, Tuple, List, Callable, Optional, TYPE_CHECKING, Type, Union from aiorpcx import run_in_thread, CancelledError -import electrumx -from electrumx.server.daemon import DaemonError, Daemon +from electrumx.lib.atomicals_blueprint_builder import AtomicalsTransferBlueprintBuilder from electrumx.lib.hash import hash_to_hex_str, HASHX_LEN, double_sha256 from electrumx.lib.script import SCRIPTHASH_LEN, is_unspendable_legacy, is_unspendable_genesis +from electrumx.lib.tx import Tx from electrumx.lib.util import ( - chunks, class_logger, pack_le_uint32, unpack_le_uint32, pack_le_uint64, unpack_le_uint64, pack_be_uint64, unpack_be_uint64, OldTaskGroup, pack_byte, pack_le_uint16, unpack_le_uint16_from + chunks, class_logger, pack_le_uint32, unpack_le_uint32, pack_le_uint64, unpack_le_uint64, pack_be_uint64, + OldTaskGroup, pack_le_uint16 ) -import math -from electrumx.lib.tx import Tx -from electrumx.server.db import FlushData, COMP_TXID_LEN, DB -from electrumx.server.history import TXNUM_LEN from electrumx.lib.util_atomicals import ( is_within_acceptable_blocks_for_general_reveal, auto_encode_bytes_elements, @@ -35,29 +31,27 @@ get_subname_request_candidate_status, is_within_acceptable_blocks_for_name_reveal, compact_to_location_id_bytes, - is_proof_of_work_prefix_match, format_name_type_candidates_to_rpc_for_subname, format_name_type_candidates_to_rpc, - pad_bytes_n, - has_requested_proof_of_work, - is_valid_container_string_name, + pad_bytes_n, + has_requested_proof_of_work, + is_valid_container_string_name, calculate_expected_bitwork, expand_spend_utxo_data, - encode_tx_hash_hex, SUBREALM_MINT_PATH, DMINT_PATH, MINT_SUBNAME_RULES_BECOME_EFFECTIVE_IN_BLOCKS, - MINT_REALM_CONTAINER_TICKER_COMMIT_REVEAL_DELAY_BLOCKS, - MINT_SUBNAME_COMMIT_PAYMENT_DELAY_BLOCKS, - is_valid_dmt_op_format, - is_compact_atomical_id, + MINT_REALM_CONTAINER_TICKER_COMMIT_REVEAL_DELAY_BLOCKS, + MINT_SUBNAME_COMMIT_PAYMENT_DELAY_BLOCKS, + is_valid_dmt_op_format, + is_compact_atomical_id, is_valid_regex, - unpack_mint_info, - parse_protocols_operations_from_witness_array, - location_id_bytes_to_compact, - is_valid_subrealm_string_name, - is_valid_realm_string_name, - is_valid_ticker_string, + unpack_mint_info, + parse_protocols_operations_from_witness_array, + location_id_bytes_to_compact, + is_valid_subrealm_string_name, + is_valid_realm_string_name, + is_valid_ticker_string, get_mint_info_op_factory, convert_db_mint_info_to_rpc_mint_info_format, calculate_latest_state_from_mod_history, @@ -72,21 +66,19 @@ is_txid_valid_for_perpetual_bitwork, auto_encode_bytes_items ) - -from electrumx.lib.atomicals_blueprint_builder import AtomicalsTransferBlueprintBuilder - -import copy +from electrumx.server.daemon import DaemonError, Daemon +from electrumx.server.db import FlushData, COMP_TXID_LEN, DB +from electrumx.server.history import TXNUM_LEN +from electrumx.version import electrumx_version if TYPE_CHECKING: from electrumx.lib.coins import Coin, AtomicalsCoinMixin from electrumx.server.env import Env from electrumx.server.controller import Notifications -from cbor2 import dumps, loads, CBORDecodeError -import pickle +from cbor2 import dumps, loads import pylru -import regex -import sys +import sys import re TX_HASH_LEN = 32 @@ -184,7 +176,7 @@ async def _prefetch_blocks(self): first = self.fetched_height + 1 # Try and catch up all blocks but limit to room in cache. cache_room = max(self.min_cache_size // self.ave_size, 1) - count = min(daemon_height - self.fetched_height, cache_room) + count: int = min(daemon_height - self.fetched_height, cache_room) # Don't make too large a request count = min(self.coin.max_fetch_blocks(first), max(count, 0)) if not count: @@ -193,8 +185,7 @@ async def _prefetch_blocks(self): hex_hashes = await daemon.block_hex_hashes(first, count) if self.caught_up: - self.logger.info(f'new block height {first + count-1:,d} ' - f'hash {hex_hashes[-1]}') + self.logger.info(f'new block height {first + count - 1:,d} hash {hex_hashes[-1]}') blocks = await daemon.raw_blocks(hex_hashes) assert count == len(blocks) @@ -934,9 +925,14 @@ def spend_atomicals_utxo(self, tx_hash: bytes, tx_idx: int, live_run) -> bytes: found_at_least_one = False for atomical_a_db_key, atomical_a_db_value in self.db.utxo_db.iterator(prefix=prefix): found_at_least_one = True - # For live_run == True we must throw an exception since the b'a' record should always be there when we are spending - if live_run and found_at_least_one == False: - raise IndexError(f'Did not find expected at least one entry for atomicals table for atomical: {location_id_bytes_to_compact(atomical_id)} at location {location_id_bytes_to_compact(location_id)}') + # For live_run == True we must throw an exception since the b'a' record + # should always be there when we are spending + if live_run and not found_at_least_one: + raise IndexError( + 'Did not find expected at least one entry for atomicals table for atomical: ' + f'{location_id_bytes_to_compact(atomical_id)} at location ' + f'{location_id_bytes_to_compact(location_id)}' + ) # Only do the db delete if this was a live run if live_run: self.delete_general_data(b'a' + atomical_id + location_id) @@ -965,14 +961,20 @@ def delete_state_data(self, db_key_prefix, db_key_suffix, expected_entry_value): if state_map: cached_value = state_map.pop(db_key_suffix, None) if cached_value != expected_entry_value: - raise IndexError(f'IndexError: delete_state_data cache data does not match expected value {expected_entry_value} {db_value}') + raise IndexError( + 'IndexError: delete_state_data cache data does not match expected value' + f'{expected_entry_value} {cached_value}' + ) # return intentionally fall through to catch in db just in case db_delete_key = db_key_prefix + db_key_suffix db_value = self.db.utxo_db.get(db_delete_key) if db_value: if db_value != expected_entry_value: - raise IndexError(f'IndexError: delete_state_data db data does not match expected atomical id {expected_entry_value} {db_value}') + raise IndexError( + 'IndexError: delete_state_data db data does not match expected atomical id' + f'{expected_entry_value} {db_value}' + ) self.delete_general_data(db_delete_key) return cached_value or db_value @@ -1055,17 +1057,25 @@ def delete_pay_record(self, atomical_id, tx_num, expected_entry_value, db_prefix return cached_value or db_value # Delete the distributed mint data that is used to track how many mints were made - def delete_decentralized_mint_data(self, atomical_id, location_id) -> bytes: + def delete_decentralized_mint_data(self, atomical_id, location_id): cache_map = self.distmint_data_cache.get(atomical_id, None) - if cache_map != None: + if cache_map is not None: cache_map.pop(location_id, None) - self.logger.info(f'delete_decentralized_mint_data: distmint_data_cache. location_id={location_id_bytes_to_compact(location_id)}, atomical_id={location_id_bytes_to_compact(atomical_id)}') + self.logger.info( + 'delete_decentralized_mint_data: distmint_data_cache. ' + f'location_id={location_id_bytes_to_compact(location_id)}, ' + f'atomical_id={location_id_bytes_to_compact(atomical_id)}' + ) gi_key = b'gi' + atomical_id + location_id gi_value = self.db.utxo_db.get(gi_key) if gi_value: # not do the i entry beuse it's deleted elsewhere self.delete_general_data(gi_key) - self.logger.info(f'delete_decentralized_mint_data: db_deletes:. location_id={location_id_bytes_to_compact(location_id)}, atomical_id={location_id_bytes_to_compact(atomical_id)}') + self.logger.info( + 'delete_decentralized_mint_data: db_deletes:. ' + f'location_id={location_id_bytes_to_compact(location_id)}, ' + f'atomical_id={location_id_bytes_to_compact(atomical_id)}' + ) def log_subrealm_request(self, method, msg, status, subrealm, parent_realm_atomical_id, height): self.logger.info(f'{method} - {msg}, status={status} subrealm={subrealm}, parent_realm_atomical_id={parent_realm_atomical_id.hex()}, height={height}') @@ -3561,7 +3571,7 @@ def spend_utxo(self, tx_hash: bytes, tx_idx: int) -> bytes: ''' # Fast track is it being in the cache idx_packed = pack_le_uint32(tx_idx) - cache_value = self.utxo_cache.pop(tx_hash + idx_packed, None) + cache_value: bytes | None = self.utxo_cache.pop(tx_hash + idx_packed, None) if cache_value: return cache_value @@ -3621,8 +3631,7 @@ async def _first_caught_up(self): self.db.first_sync = False await self.flush(True) if first_sync: - self.logger.info(f'{electrumx.version} synced to ' - f'height {self.height:,d}') + self.logger.info(f'{electrumx_version} synced to height {self.height:,d}') # Reopen for serving await self.db.open_for_serving() @@ -3683,8 +3692,14 @@ async def calc_reorg_range(self, count): class NameIndexBlockProcessor(BlockProcessor): - def advance_txs(self, txs, is_unspendable): - result = super().advance_txs(txs, is_unspendable) + def advance_txs( + self, + txs: Sequence[Tuple[Tx, bytes]], + is_unspendable: Callable[[bytes], bool], + header, + height + ): + result = super().advance_txs(txs, is_unspendable, header, height) tx_num = self.tx_count - len(txs) script_name_hashX = self.coin.name_hashX_from_script @@ -3714,7 +3729,13 @@ def advance_txs(self, txs, is_unspendable): class LTORBlockProcessor(BlockProcessor): - def advance_txs(self, txs, is_unspendable): + def advance_txs( + self, + txs: Sequence[Tuple[Tx, bytes]], + is_unspendable: Callable[[bytes], bool], + header, + height + ): self.tx_hashes.append(b''.join(tx_hash for tx, tx_hash in txs)) # Use local vars for speed in the loops diff --git a/electrumx/server/controller.py b/electrumx/server/controller.py index 12013bf4..ac1c8aef 100644 --- a/electrumx/server/controller.py +++ b/electrumx/server/controller.py @@ -9,12 +9,12 @@ from aiorpcx import _version as aiorpcx_version -import electrumx from electrumx.lib.server_base import ServerBase from electrumx.lib.util import version_string, OldTaskGroup from electrumx.server.db import DB from electrumx.server.session.session_manager import SessionManager from electrumx.server.mempool import MemPool, MemPoolAPI +from electrumx.version import electrumx_version class Notifications: @@ -87,7 +87,7 @@ async def serve(self, shutdown_event): env = self.env min_str, max_str = env.coin.SESSIONCLS.protocol_min_max_strings() - self.logger.info(f'software version: {electrumx.version}') + self.logger.info(f'software version: {electrumx_version}') self.logger.info(f'aiorpcX version: {version_string(aiorpcx_version)}') self.logger.info(f'supported protocol versions: {min_str}-{max_str}') self.logger.info(f'event loop policy: {env.loop_policy}') diff --git a/electrumx/server/db.py b/electrumx/server/db.py index efcfc30c..6f8dc5b9 100644 --- a/electrumx/server/db.py +++ b/electrumx/server/db.py @@ -41,6 +41,7 @@ ATOMICAL_ID_LEN = 36 TX_HASH_LEN = 32 + @dataclass(order=True) class UTXO: __slots__ = 'tx_num', 'tx_pos', 'tx_hash', 'height', 'value' @@ -50,8 +51,8 @@ class UTXO: height: int # block height value: int # in satoshis -@attr.s(slots=True) +@attr.s(slots=True) class FlushData: height = attr.ib() tx_count = attr.ib() @@ -72,12 +73,12 @@ class FlushData: # atomicals_adds is used to track atomicals locations and unspent utxos with the b'i' and b'a' indexes # It uses a field 'deleted' to indicate whether to write the b'a' (active unspent utxo) or not - because it may have been spent before the cache flushed # Maps location_id to atomical_ids and the value/deleted entry - atomicals_adds = attr.ib() # type: Dict[bytes, Dict[bytes, { value: bytes, deleted: Boolean}] ] + atomicals_adds = attr.ib() # type: Dict[bytes, Dict[bytes, { value: bytes, deleted: bool }] ] # general_adds is a general purpose storage for key-value, used for the majority of atomicals data general_adds = attr.ib() # type: List[Tuple[Sequence[bytes], Sequence[bytes]]] # realm_adds map realm names to tx_num ints, which then map onto an atomical_id # The purpose is to track the earliest appearance of a realm name claim request in the order of the commit tx number - realm_adds = attr.ib() # type: Dict[bytes, Dict[int, bytes] + realm_adds = attr.ib() # type: Dict[bytes, Dict[int, bytes]] # container_adds map container names to tx_num ints, which then map onto an atomical_id # The purpose is to track the earliest appearance of a container name claim request in the order of the commit tx number container_adds = attr.ib() # type: List[Tuple[Sequence[bytes], Sequence[bytes]]] @@ -85,24 +86,26 @@ class FlushData: # The purpose is to track the earliest appearance of a ticker name claim request in the order of the commit tx number ticker_adds = attr.ib() # type: List[Tuple[Sequence[bytes], Sequence[bytes]]] # subrealm_adds maps parent_realm_id + subrealm name to tx_num ints, which then map onto an atomical_id - subrealm_adds = attr.ib() # type: Dict[bytes, Dict[int, bytes] + subrealm_adds = attr.ib() # type: Dict[bytes, Dict[int, bytes]] # subrealmpay_adds maps atomical_id to tx_num ints, which then map onto payment_outpoints - subrealmpay_adds = attr.ib() # type: Dict[bytes, Dict[int, bytes] + subrealmpay_adds = attr.ib() # type: Dict[bytes, Dict[int, bytes]] # dmitem_adds maps parent_realm_id + dmitem name to tx_num ints, which then map onto an atomical_id - dmitem_adds = attr.ib() # type: Dict[bytes, Dict[int, bytes] + dmitem_adds = attr.ib() # type: Dict[bytes, Dict[int, bytes]] # dmpay_adds maps atomical_id to tx_num ints, which then map onto payment_outpoints - dmpay_adds = attr.ib() # type: Dict[bytes, Dict[int, bytes] + dmpay_adds = attr.ib() # type: Dict[bytes, Dict[int, bytes]] # distmint_adds tracks the b'gi' which is the initial distributed mint location tracked to determine if any more mints are allowed # It maps atomical_id (of the dft deploy token mint) to location_ids and then the details of the scripthash+value_sats of the mint - distmint_adds = attr.ib() # type: Dict[bytes, Dict[bytes, bytes] + distmint_adds = attr.ib() # type: Dict[bytes, Dict[bytes, bytes]] # state_adds is for evt, mod state updates # It maps atomical_id to the data of the state update - state_adds = attr.ib() # type: Dict[bytes, Dict[bytes, bytes] + state_adds = attr.ib() # type: Dict[bytes, Dict[bytes, bytes]] # op_adds is for record tx operation of one tx - op_adds = attr.ib() # type: Dict[bytes, Dict[bytes] - + op_adds = attr.ib() # type: Dict[bytes, Dict[bytes]] + + COMP_TXID_LEN = 4 + class DB: '''Simple wrapper of the backend database for querying. @@ -262,7 +265,7 @@ def __init__(self, env: 'Env'): # Value: paylaod_bytes of the operation found # "maps pow reveal prefix to height to non atomicals operation data" - self.utxo_db = None + self.utxo_db: Optional[Storage] = None self.utxo_flush_count = 0 self.fs_height = -1 self.fs_tx_count = 0 @@ -270,7 +273,7 @@ def __init__(self, env: 'Env'): self.db_height = -1 self.db_tx_count = 0 self.db_atomical_count = 0 - self.db_tip = None # type: Optional[bytes] + self.db_tip: Optional[bytes] = None self.tx_counts = None self.atomical_counts = None self.last_flush = time.time() diff --git a/electrumx/server/session/__init__.py b/electrumx/server/session/__init__.py index e69de29b..1d63197b 100644 --- a/electrumx/server/session/__init__.py +++ b/electrumx/server/session/__init__.py @@ -0,0 +1,4 @@ +BAD_REQUEST = 1 +DAEMON_ERROR = 2 +MAX_TX_QUERY = 50 +ATOMICALS_INVALID_TX = 800422 diff --git a/electrumx/server/session/electrumx_session.py b/electrumx/server/session/electrumx_session.py index 0d7acafd..c5a47ef9 100644 --- a/electrumx/server/session/electrumx_session.py +++ b/electrumx/server/session/electrumx_session.py @@ -4,11 +4,12 @@ from aiorpcx import timeout_after, TaskTimeout -import electrumx +from electrumx.lib import util from electrumx.lib.script2addr import get_address_from_output_script from electrumx.lib.util_atomicals import * from electrumx.server.daemon import DaemonError from electrumx.server.session.session_base import * +from electrumx.version import electrumx_version, electrumx_version_short class ElectrumX(SessionBase): @@ -46,7 +47,7 @@ def server_features(cls, env): return { 'hosts': hosts_dict, 'pruning': None, - 'server_version': electrumx.version, + 'server_version': electrumx_version, 'protocol_min': min_str, 'protocol_max': max_str, 'genesis_hash': env.coin.GENESIS_HASH, @@ -55,13 +56,12 @@ def server_features(cls, env): } async def server_features_async(self): - self.bump_cost(0.2) return self.server_features(self.env) @classmethod def server_version_args(cls): """The arguments to a server.version RPC call to a peer.""" - return [electrumx.version, cls.protocol_min_max_strings()] + return [electrumx_version, cls.protocol_min_max_strings()] def protocol_version_string(self): return util.version_string(self.protocol_tuple) @@ -1506,8 +1506,8 @@ async def replaced_banner(self, banner): revision //= 100 daemon_version = f'{major:d}.{minor:d}.{revision:d}' for pair in [ - ('$SERVER_VERSION', electrumx.version_short), - ('$SERVER_SUBVERSION', electrumx.version), + ('$SERVER_VERSION', electrumx_version_short), + ('$SERVER_SUBVERSION', electrumx_version), ('$DAEMON_VERSION', daemon_version), ('$DAEMON_SUBVERSION', network_info['subversion']), ('$DONATION_ADDRESS', self.env.donation_address), @@ -1522,7 +1522,7 @@ async def donation_address(self): async def banner(self): """Return the server banner text.""" - banner = f'You are connected to an {electrumx.version} server.' + banner = f'You are connected to an {electrumx_version} server.' self.bump_cost(0.5) if self.is_tor(): @@ -1630,7 +1630,7 @@ async def server_version(self, client_name='', protocol_version=None): BAD_REQUEST, f'unsupported protocol version: {protocol_version}')) self.set_request_handlers(ptuple) - return electrumx.version, self.protocol_version_string() + return electrumx_version, self.protocol_version_string() async def crash_old_client(self, ptuple, crash_client_ver): if crash_client_ver: @@ -1645,50 +1645,15 @@ async def crash_old_client(self, ptuple, crash_client_ver): await self.send_notification('blockchain.estimatefee', ()) async def transaction_broadcast_validate(self, raw_tx): - """Simulate a Broadcast a raw transaction to the network. - - raw_tx: the raw transaction as a hexadecimal string to validate for Atomicals FT rules""" self.bump_cost(0.25 + len(raw_tx) / 5000) - # This returns errors as JSON RPC errors, as is natural - try: - hex_hash = await self.session_mgr.broadcast_transaction_validated(raw_tx, False) - return hex_hash - except AtomicalsValidationError as e: - self.logger.info(f'error validating atomicals transaction: {e}') - raise RPCError(ATOMICALS_INVALID_TX, 'the transaction was rejected by ' - f'atomicals rules.\n\n{e}\n[{raw_tx}]') + return await self.ss.transaction_broadcast_validate() async def transaction_broadcast(self, raw_tx): """Broadcast a raw transaction to the network. raw_tx: the raw transaction as a hexadecimal string""" self.bump_cost(0.25 + len(raw_tx) / 5000) - # This returns errors as JSON RPC errors, as is natural - try: - hex_hash = await self.session_mgr.broadcast_transaction_validated(raw_tx, True) - except DaemonError as e: - error, = e.args - message = error['message'] - self.logger.info(f'error sending transaction: {message}') - raise RPCError(BAD_REQUEST, 'the transaction was rejected by ' - f'network rules.\n\n{message}\n[{raw_tx}]') - except AtomicalsValidationError as e: - self.logger.info(f'error validating atomicals transaction: {e}') - raise RPCError(ATOMICALS_INVALID_TX, 'the transaction was rejected by ' - f'atomicals rules.\n\n{e}\n[{raw_tx}]') - - else: - self.txs_sent += 1 - client_ver = util.protocol_tuple(self.client) - if client_ver != (0,): - msg = self.coin.warn_old_client_on_tx_broadcast(client_ver) - if msg: - self.logger.info(f'sent tx: {hex_hash}. and warned user to upgrade their ' - f'client from {self.client}') - return msg - - self.logger.info(f'sent tx: {hex_hash}') - return hex_hash + return await self.ss.transaction_broadcast(raw_tx) async def transaction_broadcast_force(self, raw_tx): """Broadcast a raw transaction to the network. Force even if invalid FT transfer diff --git a/electrumx/server/session/http_session.py b/electrumx/server/session/http_session.py index 82699b51..b6c60c97 100644 --- a/electrumx/server/session/http_session.py +++ b/electrumx/server/session/http_session.py @@ -10,13 +10,14 @@ from aiohttp import web from aiorpcx import RPCError -import electrumx import electrumx.lib.util as util from electrumx.lib.script2addr import get_address_from_output_script from electrumx.lib.util_atomicals import * -from electrumx.server.daemon import DaemonError +from electrumx.server.session import BAD_REQUEST from electrumx.server.session.session_base import assert_tx_hash, scripthash_to_hashX, non_negative_integer, \ - assert_atomical_id, BAD_REQUEST, ATOMICALS_INVALID_TX + assert_atomical_id +from electrumx.server.session.shared_session import SharedSession +from electrumx.version import electrumx_version class DecimalEncoder(json.JSONEncoder): @@ -55,23 +56,20 @@ def __init__(self, session_mgr, db, mempool, peer_mgr, kind): self.coin = self.env.coin self.client = 'unknown' self.anon_logs = self.env.anon_logs - self.txs_sent = 0 self.log_me = False self.daemon_request = self.session_mgr.daemon_request self.mempool_statuses = {} self.sv_seen = False self.MAX_CHUNK_SIZE = 2016 self.hashX_subs = {} + # Use the sharing session to manage handlers. + self.ss = SharedSession(self.session_mgr, self.logger) async def get_rpc_server(self): for service in self.env.services: if service.protocol == 'tcp': return service - def remote_address(self): - """Returns a NetAddress or None if not connected.""" - return self.transport.remote_address() - @classmethod def protocol_min_max_strings(cls): return [util.version_string(ver) @@ -90,7 +88,7 @@ def server_features(cls, env): return { 'hosts': hosts_dict, 'pruning': None, - 'server_version': electrumx.version, + 'server_version': electrumx_version, 'protocol_min': min_str, 'protocol_max': max_str, 'genesis_hash': env.coin.GENESIS_HASH, @@ -98,17 +96,6 @@ def server_features(cls, env): 'services': [str(service) for service in env.report_services], } - def is_tor(self): - """Try to detect if the connection is to a tor hidden service we are - running.""" - proxy_address = self.peer_mgr.proxy_address() - if not proxy_address: - return False - remote_addr = self.remote_address() - if not remote_addr: - return False - return remote_addr.host == proxy_address.host - async def _merkle_proof(self, cp_height, height): max_height = self.db.db_height if not height <= cp_height <= max_height: @@ -815,34 +802,7 @@ async def transaction_broadcast(self, request): raw_tx: the raw transaction as a hexadecimal string""" params = await format_params(request) raw_tx = params.get(0, "") - - # self.bump_cost(0.25 + len(raw_tx) / 5000) - # This returns errors as JSON RPC errors, as is natural - try: - hex_hash = await self.session_mgr.broadcast_transaction_validated(raw_tx, True) - except DaemonError as e: - error, = e.args - message = error['message'] - self.logger.info(f'error sending transaction: {message}') - raise RPCError(BAD_REQUEST, 'the transaction was rejected by ' - f'network rules.\n\n{message}\n[{raw_tx}]') - except AtomicalsValidationError as e: - self.logger.info(f'error validating atomicals transaction: {e}') - raise RPCError(ATOMICALS_INVALID_TX, 'the transaction was rejected by ' - f'atomicals rules.\n\n{e}\n[{raw_tx}]') - - else: - self.txs_sent += 1 - client_ver = util.protocol_tuple(self.client) - if client_ver != (0,): - msg = self.coin.warn_old_client_on_tx_broadcast(client_ver) - if msg: - self.logger.info(f'sent tx: {hex_hash}. and warned user to upgrade their ' - f'client from {self.client}') - return msg - - self.logger.info(f'sent tx: {hex_hash}') - return hex_hash + return await self.ss.transaction_broadcast(raw_tx) # verified async def scripthash_get_history(self, request): @@ -1087,13 +1047,6 @@ async def rpc_add_peer(self, request): res = f"peer '{real_name}' added" return res - async def add_peer(self, request): - """Add a peer (but only if the peer resolves to the source).""" - params = await format_params(request) - features = params.get(0, None) - self.is_peer = True - return await self.peer_mgr.on_add_peer(features, self.remote_address()) - async def donation_address(self, request): """Return the donation address as a string, empty if there is none.""" return self.env.donation_address @@ -1103,7 +1056,7 @@ async def server_features_async(self, request): async def peers_subscribe(self, request): """Return the server peers as a list of (ip, host, details) tuples.""" - return self.peer_mgr.on_peers_subscribe(self.is_tor()) + return self.peer_mgr.on_peers_subscribe(False) async def ping(self, request): """Serves as a connection keep-alive mechanism and for the client to @@ -1117,14 +1070,7 @@ async def transaction_broadcast_validate(self, request): raw_tx: the raw transaction as a hexadecimal string to validate for Atomicals FT rules""" params = await format_params(request) raw_tx = params.get(0, "") - # This returns errors as JSON RPC errors, as is natural - try: - hex_hash = await self.session_mgr.broadcast_transaction_validated(raw_tx, False) - return hex_hash - except AtomicalsValidationError as e: - self.logger.info(f'error validating atomicals transaction: {e}') - raise RPCError(ATOMICALS_INVALID_TX, 'the transaction was rejected by ' - f'atomicals rules.\n\n{e}\n[{raw_tx}]') + return await self.ss.transaction_broadcast_validate(raw_tx) async def atomicals_get_ft_balances(self, request): """Return the FT balances for a scripthash address""" diff --git a/electrumx/server/session/session_base.py b/electrumx/server/session/session_base.py index c6671a46..f9c3319e 100644 --- a/electrumx/server/session/session_base.py +++ b/electrumx/server/session/session_base.py @@ -10,12 +10,8 @@ from electrumx.server.db import DB from electrumx.server.mempool import MemPool from electrumx.server.peers import PeerManager - - -BAD_REQUEST = 1 -DAEMON_ERROR = 2 -MAX_TX_QUERY = 50 -ATOMICALS_INVALID_TX = 800422 +from electrumx.server.session import BAD_REQUEST +from electrumx.server.session.shared_session import SharedSession def scripthash_to_hashX(scripthash): @@ -116,6 +112,8 @@ def __init__( self.recalc_concurrency() # must be called after session_mgr.add_session self.protocol_tuple: Optional[Tuple[int, ...]] = None self.request_handlers: Optional[Dict[str, Callable]] = None + # Use the sharing session to manage handlers. + self.ss = SharedSession(self.session_mgr, self.logger) async def notify(self, touched, height_changed): pass diff --git a/electrumx/server/session/session_manager.py b/electrumx/server/session/session_manager.py index eea6d565..43b72f80 100644 --- a/electrumx/server/session/session_manager.py +++ b/electrumx/server/session/session_manager.py @@ -9,8 +9,6 @@ import pylru from aiorpcx import serve_ws, serve_rs, RPCError, run_in_thread -import electrumx -from electrumx import Env from electrumx.lib import util from electrumx.lib.atomicals_blueprint_builder import AtomicalsTransferBlueprintBuilder from electrumx.lib.hash import Base58Error @@ -22,15 +20,19 @@ from electrumx.server.block_processor import BlockProcessor from electrumx.server.daemon import DaemonError, Daemon from electrumx.server.db import DB +from electrumx.server.env import Env from electrumx.server.history import TXNUM_LEN from electrumx.server.http_middleware import * from electrumx.server.mempool import MemPool +from electrumx.server.session import BAD_REQUEST, DAEMON_ERROR from electrumx.server.session.http_session import HttpHandler -from electrumx.server.session.session_base import LocalRPC, SessionBase, non_negative_integer, BAD_REQUEST, DAEMON_ERROR +from electrumx.server.session.session_base import LocalRPC, SessionBase, non_negative_integer from electrumx.server.peers import PeerManager from typing import TYPE_CHECKING +from electrumx.version import electrumx_version + if TYPE_CHECKING: pass @@ -452,7 +454,7 @@ def _get_info(self): self._tx_hashes_lookups, self._tx_hashes_hits, len(self._tx_hashes_cache)), 'txs sent': self.txs_sent, 'uptime': util.formatted_time(time.time() - self.start_time), - 'version': electrumx.version, + 'version': electrumx_version, } def _session_data(self, for_log): diff --git a/electrumx/server/session/shared_session.py b/electrumx/server/session/shared_session.py new file mode 100644 index 00000000..1c928ff8 --- /dev/null +++ b/electrumx/server/session/shared_session.py @@ -0,0 +1,64 @@ +from aiorpcx import RPCError +from logging import LoggerAdapter + +from electrumx.lib import util +from electrumx.lib.util_atomicals import AtomicalsValidationError +from electrumx.server.daemon import DaemonError +from electrumx.server.session import ATOMICALS_INVALID_TX, BAD_REQUEST + + +class SharedSession: + def __init__(self, session_mgr: 'SessionManager', logger: LoggerAdapter): + self.session_mgr = session_mgr + self.logger = logger + self.txs_sent = 0 + + async def transaction_broadcast_validate(self, raw_tx: str = ""): + """Simulate a Broadcast a raw transaction to the network. + + raw_tx: the raw transaction as a hexadecimal string to validate for Atomicals FT rules""" + # This returns errors as JSON RPC errors, as is natural + try: + hex_hash = await self.session_mgr.broadcast_transaction_validated(raw_tx, False) + return hex_hash + except AtomicalsValidationError as e: + self.logger.info(f'error validating atomicals transaction: {e}') + raise RPCError( + ATOMICALS_INVALID_TX, + f'the transaction was rejected by atomicals rules.\n\n{e}\n[{raw_tx}]' + ) + + async def transaction_broadcast(self, raw_tx): + """Broadcast a raw transaction to the network. + + raw_tx: the raw transaction as a hexadecimal string""" + # This returns errors as JSON RPC errors, as is natural. + try: + hex_hash = await self.session_mgr.broadcast_transaction_validated(raw_tx, True) + hex_hash = await self.session_mgr.broadcast_transaction(raw_tx) + except DaemonError as e: + error, = e.args + message = error['message'] + self.logger.info(f'error sending transaction: {message}') + raise RPCError( + BAD_REQUEST, + f'the transaction was rejected by network rules.\n\n{message}\n[{raw_tx}]' + ) + except AtomicalsValidationError as e: + self.logger.info(f'error validating atomicals transaction: {e}') + raise RPCError( + ATOMICALS_INVALID_TX, + f'the transaction was rejected by atomicals rules.\n\n{e}\n[{raw_tx}]' + ) + else: + self.txs_sent += 1 + client_ver = util.protocol_tuple(self.client) + if client_ver != (0,): + msg = self.coin.warn_old_client_on_tx_broadcast(client_ver) + if msg: + self.logger.info(f'sent tx: {hex_hash}. and warned user to upgrade their ' + f'client from {self.client}') + return msg + + self.logger.info(f'sent tx: {hex_hash}') + return hex_hash diff --git a/electrumx/version.py b/electrumx/version.py new file mode 100644 index 00000000..77cdc88b --- /dev/null +++ b/electrumx/version.py @@ -0,0 +1,3 @@ +__version__ = "1.4.2.0" +electrumx_version = f'ElectrumX {__version__}' +electrumx_version_short = __version__ diff --git a/electrumx_compact_history b/electrumx_compact_history index 1449b53c..c9752c57 100755 --- a/electrumx_compact_history +++ b/electrumx_compact_history @@ -38,8 +38,8 @@ import traceback from os import environ from dotenv import load_dotenv -from electrumx import Env from electrumx.server.db import DB +from electrumx.server.env import Env load_dotenv() diff --git a/electrumx_server b/electrumx_server index 1098cc61..4420ca19 100755 --- a/electrumx_server +++ b/electrumx_server @@ -16,8 +16,9 @@ import sys import logging.handlers from dotenv import load_dotenv -from electrumx import Controller, Env from electrumx.lib.util import CompactFormatter, make_logger +from electrumx.server.controller import Controller +from electrumx.server.env import Env load_dotenv() diff --git a/tests/server/test_api.py b/tests/server/test_api.py index 7cad87e1..c60f336d 100644 --- a/tests/server/test_api.py +++ b/tests/server/test_api.py @@ -1,8 +1,9 @@ import asyncio +from aiorpcx import RPCError from unittest import mock -from aiorpcx import RPCError -from electrumx import Controller, Env +from electrumx.server.controller import Controller +from electrumx.server.env import Env loop = asyncio.get_event_loop()