Merge pull request #2502 from CounterpartyXCP/develop

v10.5.0-rc.1

counterparty-core/counterpartycore/cli.py

@@ -2,6 +2,7 @@
 
 import argparse
 import logging
+import os
 from urllib.parse import quote_plus as urlencode
 
 from termcolor import cprint
@@ -306,6 +307,7 @@ def float_range_checker(arg):
         ("--db-connection-pool-size",),
         {
             "type": int,
+            "default": 20,
             "help": "size of the database connection pool",
         },
     ],
@@ -325,12 +327,28 @@ def float_range_checker(arg):
             "choices": ["waitress", "gunicorn", "werkzeug"],
         },
     ],
+    [
+        ("--waitress-threads",),
+        {
+            "type": int,
+            "default": 10,
+            "help": "number of threads for the Waitress WSGI server (if enabled)",
+        },
+    ],
     [
         ("--gunicorn-workers",),
+        {
+            "type": int,
+            "default": 2 * os.cpu_count() + 1,
+            "help": "number of worker processes for gunicorn (if enabled)",
+        },
+    ],
+    [
+        ("--gunicorn-threads-per-worker",),
         {
             "type": int,
             "default": 2,
-            "help": "number of worker processes for gunicorn",
+            "help": "number of threads per worker for the Gunicorn WSGI server (if enabled)",
         },
     ],
 ]

counterparty-core/counterpartycore/lib/api/api_server.py

@@ -392,7 +392,7 @@ def init_flask_app():
     return app
 
 
-def run_api_server(args, interruped_value, server_ready_value):
+def run_api_server(args, interrupted_value, server_ready_value):
     sentry.init()
     # Initialise log and config
     server.initialise_log_and_config(argparse.Namespace(**args))
@@ -404,24 +404,46 @@ def run_api_server(args, interruped_value, server_ready_value):
     app = init_flask_app()
 
     wsgi_server = None
+    parent_checker = None
 
     try:
         # Init the HTTP Server.
         wsgi_server = wsgi.WSGIApplication(app, args=args)
-        ParentProcessChecker(interruped_value, wsgi_server).start()
+        parent_checker = ParentProcessChecker(interrupted_value, wsgi_server)
+        parent_checker.start()
         app.app_context().push()
         # Run app server (blocking)
         server_ready_value.value = 1
         wsgi_server.run()
     except KeyboardInterrupt:
         logger.trace("Keyboard Interrupt!")
+    except Exception as e:
+        capture_exception(e)
+        logger.error("Error in API Server: %s", e)
     finally:
         logger.trace("Shutting down API Server...")
-        watcher.stop()
-        watcher.join()
+        try:
+            watcher.stop()
+            watcher.join()
+        except Exception as e:
+            logger.error("Error stopping API Watcher: %s", e)
+
         if wsgi_server is not None:
-            wsgi_server.stop()
-        APIDBConnectionPool().close()
+            try:
+                wsgi_server.stop()
+            except Exception as e:
+                logger.error("Error stopping WSGI Server: %s", e)
+
+        if parent_checker is not None:
+            try:
+                parent_checker.join()
+            except Exception as e:
+                logger.error("Error joining ParentProcessChecker: %s", e)
+
+        try:
+            APIDBConnectionPool().close()
+        except Exception as e:
+            logger.error("Error closing DB connection pool: %s", e)
 
 
 # This thread is used for the following two reasons:
@@ -466,12 +488,12 @@ def is_ready(self):
 
     def stop(self):
         logger.info("Stopping API Server...")
-        self.interrupted.value = 1
+        self.interrupted.value = 1  # stop the thread
         waiting_start_time = time.time()
         while self.process.is_alive():
-            time.sleep(0.5)
-            logger.trace("Waiting for API Server to stop...")
-            if time.time() - waiting_start_time > 2:
+            time.sleep(1)
+            logger.trace("Waiting 10 seconds for API Server to stop...")
+            if time.time() - waiting_start_time > 10:
                 logger.error("API Server did not stop in time. Terminating...")
                 self.process.kill()
                 break

counterparty-core/counterpartycore/lib/api/api_v1.py

@@ -621,7 +621,7 @@ def create_method(**kwargs):
                 ) as error:
                     # TypeError happens when unexpected keyword arguments are passed in
                     error_msg = f"Error composing {tx} transaction via API: {str(error)}"
-                    logging.warning(error_msg)
+                    logger.trace(error_msg)
                     raise JSONRPCDispatchException(  # noqa: B904
                         code=JSON_RPC_ERROR_API_COMPOSE, message=error_msg
                     )
@@ -1227,7 +1227,7 @@ def handle_rest(path_args, flask_request):
                     exceptions.TransactionError,
                     exceptions.BalanceError,
                 ) as error:
-                    error_msg = logging.warning(
+                    error_msg = logger.trace(
                         f"{error.__class__.__name__} -- error composing {query_type} transaction via API: {error}"
                     )
                     return flask.Response(error_msg, 400, mimetype="application/json")

counterparty-core/counterpartycore/lib/api/api_watcher.py

@@ -802,6 +802,8 @@ def gen_random_tx_index(event):
 
 
 def synchronize_mempool(api_db, ledger_db):
+    if config.NO_MEMPOOL:
+        return
     logger.trace("API Watcher - Synchronizing mempool...")
     global MEMPOOL_SKIP_EVENT_HASHES  # noqa: PLW0602
     try:

counterparty-core/counterpartycore/lib/api/compose.py

@@ -563,7 +563,7 @@ def compose_fairminter(
     :param lock_description: If True, the description of the asset is locked
     :param lock_quantity: If True, the quantity of the asset cannot be changed after the minting
     :param divisible: If True, the asset is divisible
-    :param description: The description of the asset
+    :param description: The description of the asset. Overrides the current description if the asset already exists.
     """
     params = {
         "source": address,

counterparty-core/counterpartycore/lib/api/queries.py

@@ -2714,3 +2714,25 @@ def get_fairmints_by_address_and_asset(
         limit=limit,
         offset=offset,
     )
+
+
+def get_all_fairmints(db, cursor: str = None, limit: int = 100, offset: int = None):
+    """
+    Returns all fairmints
+    :param str cursor: The last index of the fairmint to return
+    :param int limit: The maximum number of fairmint to return (e.g. 5)
+    :param int offset: The number of lines to skip before returning results (overrides the `cursor` parameter)
+    """
+    return select_rows(db, "fairmints", last_cursor=cursor, limit=limit, offset=offset)
+
+
+def get_fairmint(db, tx_hash: str):
+    """
+    Returns the fairmint by its hash
+    :param str tx_hash: The hash of the fairmint to return (e.g. $LAST_FAIRMINT_TX_HASH)
+    """
+    return select_row(
+        db,
+        "fairmints",
+        where={"tx_hash": tx_hash},
+    )

counterparty-core/counterpartycore/lib/api/routes.py

@@ -159,7 +159,10 @@ def get_routes():
         ### /fairminters ###
         "/v2/fairminters": queries.get_all_fairminters,
         "/v2/fairminters/<tx_hash>": queries.get_fairminter,
-        "/v2/fairminters/<tx_hash>/mints": queries.get_fairmints_by_fairminter,
+        "/v2/fairminters/<tx_hash>/fairmints": queries.get_fairmints_by_fairminter,
+        ### /fairmints ###
+        "/v2/fairmints": queries.get_all_fairmints,
+        "/v2/fairmints/<tx_hash>": queries.get_fairmint,
         ### /bitcoin ###
         "/v2/bitcoin/addresses/utxos": addrindexrs.get_unspent_txouts_by_addresses,
         "/v2/bitcoin/addresses/<address>/transactions": addrindexrs.get_transactions_by_address,

counterparty-core/counterpartycore/lib/api/wsgi.py

@@ -209,7 +209,7 @@ def __init__(self, app, args=None):
             "workers": config.GUNICORN_WORKERS,
             "worker_class": "gthread",
             "daemon": True,
-            "threads": 2,
+            "threads": config.GUNICORN_THREADS_PER_WORKER,
             "loglevel": "debug",
             # "access-logfile": "-",
         }
@@ -274,7 +274,7 @@ def __init__(self, app, args=None):
         self.args = args
         self.timer_db = get_db_connection(config.API_DATABASE, read_only=True, check_wal=False)
         self.server = waitress.server.create_server(
-            self.app, host=config.API_HOST, port=config.API_PORT
+            self.app, host=config.API_HOST, port=config.API_PORT, threads=config.WAITRESS_THREADS
         )
 
     def run(self):

counterparty-core/counterpartycore/lib/blocks.py

@@ -972,6 +972,10 @@ def list_tx(db, block_hash, block_index, block_time, tx_hash, tx_index, decoded_
     if block_hash is None or block_hash == config.MEMPOOL_BLOCK_HASH:
         block_hash = config.MEMPOOL_BLOCK_HASH
         block_index = config.MEMPOOL_BLOCK_INDEX
+        existing_tx = ledger.get_transaction(db, tx_hash)
+        if existing_tx:
+            util.CURRENT_TX_HASH = None
+            return tx_index
     else:
         assert block_index == util.CURRENT_BLOCK_INDEX
 

counterparty-core/counterpartycore/lib/check.py

@@ -651,6 +651,14 @@
         "ledger_hash": "76ca4415a24ae04579b05b517f41441b7cb45eb881db71d1b59de5d373a4bc28",
         "txlist_hash": "b05b906391cf96d5b4b5893c5fda13fb64635d26785ee3ad1330fe701eb41cb4",
     },
+    866000: {
+        "ledger_hash": "5bfa1fef4356b1326c8edfe1af582911461d86132dd768028511d20ed2d9e3f5",
+        "txlist_hash": "19d1621ea05abd741e05e361ce96708e8dc1b442fb89079af370abd2698549db",
+    },
+    866330: {
+        "ledger_hash": "1c5f82ee5009fbc3d94bb8ff88d644aa0b2ec42f21409b315f555f1006bca6fc",
+        "txlist_hash": "1f5db508a80205eaaa6d915402c7833a0851bb369bea54a600e2fdda7e1d7ff5",
+    },
 }
 
 CONSENSUS_HASH_VERSION_TESTNET = 7
@@ -1001,6 +1009,22 @@ def __init__(self, message, required_action, from_block_index=None):
         self.from_block_index = from_block_index
 
 
+def check_need_reparse(version_minor, message):
+    need_reparse_from = (
+        config.NEED_REPARSE_IF_MINOR_IS_LESS_THAN_TESTNET
+        if config.TESTNET
+        else config.NEED_REPARSE_IF_MINOR_IS_LESS_THAN
+    )
+    if need_reparse_from is not None:
+        for min_version_minor, min_version_block_index in need_reparse_from:
+            if version_minor < min_version_minor:
+                raise DatabaseVersionError(
+                    message=message,
+                    required_action="reparse",
+                    from_block_index=min_version_block_index,
+                )
+
+
 def database_version(db):
     if config.FORCE:
         return
@@ -1016,18 +1040,22 @@ def database_version(db):
         )
     elif version_minor != config.VERSION_MINOR:
         # Reparse transactions from the vesion block if minor version has changed.
-        message = f"Client minor version number mismatch. Triggering a reparse... ({version_minor} ≠ {config.VERSION_MINOR})"
-        need_reparse_from = (
-            config.NEED_REPARSE_IF_MINOR_IS_LESS_THAN_TESTNET
-            if config.TESTNET
-            else config.NEED_REPARSE_IF_MINOR_IS_LESS_THAN
+        message = (
+            f"Client minor version number mismatch: {version_minor} ≠ {config.VERSION_MINOR}. "
         )
-        if need_reparse_from is not None:
-            for min_version_minor, min_version_block_index in need_reparse_from:
-                if version_minor < min_version_minor:
-                    raise DatabaseVersionError(
-                        message=message,
-                        required_action="reparse",
-                        from_block_index=min_version_block_index,
-                    )
+        message += "Checking if a reparse is needed..."
+        check_need_reparse(version_minor, message)
         raise DatabaseVersionError(message=message, required_action=None)
+    else:
+        version_string = database.get_config_value(db, "VERSION_STRING")
+        if version_string:
+            version_pre_release = "-".join(version_string.split("-")[1:])
+        else:
+            # if version_string is not set, that mean we are on a version before 10.5.0 and after 10.4.8
+            # let's assume it's a pre-release version
+            # and set an arbitrary value different from config.VERSION_PRE_RELEASE
+            version_pre_release = "xxxx"
+        if version_pre_release != config.VERSION_PRE_RELEASE:
+            message = f"Client pre-release version number mismatch: {version_pre_release} ≠ {config.VERSION_PRE_RELEASE}. "
+            message += "Checking if a reparse is needed..."
+            check_need_reparse(version_minor, message)

counterparty-core/counterpartycore/lib/config.py

@@ -5,7 +5,7 @@
 
 
 # Semantic Version
-__version__ = "10.5.0-alpha.3"  # for hatch
+__version__ = "10.5.0-rc.1"  # for hatch
 VERSION_STRING = __version__
 version = VERSION_STRING.split("-")[0].split(".")
 VERSION_MAJOR = int(version[0])
@@ -184,5 +184,3 @@
 INFLUX_DB_BUCKET = "node-telemetry"
 
 LOG_IN_CONSOLE = False
-
-DEFAULT_DB_CONNECTION_POOL_SIZE = 10

counterparty-core/counterpartycore/lib/database.py

@@ -203,10 +203,39 @@ def version(db):
     return version_major, version_minor
 
 
+def init_config_table(db):
+    sql = """
+        CREATE TABLE IF NOT EXISTS config (
+            name TEXT PRIMARY KEY,
+            value TEXT
+        )
+    """
+    cursor = db.cursor()
+    cursor.execute(sql)
+    cursor.execute("CREATE INDEX IF NOT EXISTS config_config_name_idx ON config (name)")
+
+
+def set_config_value(db, name, value):
+    init_config_table(db)
+    cursor = db.cursor()
+    cursor.execute("INSERT OR REPLACE INTO config (name, value) VALUES (?, ?)", (name, value))
+
+
+def get_config_value(db, name):
+    init_config_table(db)
+    cursor = db.cursor()
+    cursor.execute("SELECT value FROM config WHERE name = ?", (name,))
+    rows = cursor.fetchall()
+    if rows:
+        return rows[0]["value"]
+    return None
+
+
 def update_version(db):
     cursor = db.cursor()
     user_version = (config.VERSION_MAJOR * 1000) + config.VERSION_MINOR
     cursor.execute(f"PRAGMA user_version = {user_version}")  # Syntax?!
+    set_config_value(db, "VERSION_STRING", config.VERSION_STRING)
     logger.info("Database version number updated.")
 
 

counterparty-core/counterpartycore/lib/follow.py

@@ -89,8 +89,9 @@ def __init__(self, db):
         self.last_block_check_time = 0
         self.last_software_version_check_time = 0
         # catch up and clean mempool before starting
-        mempool.parse_raw_mempool(self.db)
-        mempool.clean_mempool(self.db)
+        if not config.NO_MEMPOOL:
+            mempool.parse_raw_mempool(self.db)
+            mempool.clean_mempool(self.db)
 
     def connect_to_zmq(self):
         self.zmq_context = zmq.asyncio.Context()
@@ -99,7 +100,8 @@ def connect_to_zmq(self):
         self.zmq_sub_socket_sequence.setsockopt(zmq.RCVTIMEO, ZMQ_TIMEOUT)
         self.zmq_sub_socket_sequence.setsockopt_string(zmq.SUBSCRIBE, "rawtx")
         self.zmq_sub_socket_sequence.setsockopt_string(zmq.SUBSCRIBE, "hashtx")
-        self.zmq_sub_socket_sequence.setsockopt_string(zmq.SUBSCRIBE, "sequence")
+        if not config.NO_MEMPOOL:
+            self.zmq_sub_socket_sequence.setsockopt_string(zmq.SUBSCRIBE, "sequence")
         self.zmq_sub_socket_sequence.connect(self.zmq_sequence_address)
         self.zmq_sub_socket_rawblock = self.zmq_context.socket(zmq.SUB)
         self.zmq_sub_socket_rawblock.setsockopt(zmq.RCVHWM, 0)
@@ -222,7 +224,8 @@ async def handle(self):
         util.BLOCK_PARSER_STATUS = "following"
 
         # sequence topic
-        await self.receive_multipart(self.zmq_sub_socket_sequence, "sequence")
+        if not config.NO_MEMPOOL:
+            await self.receive_multipart(self.zmq_sub_socket_sequence, "sequence")
         # check rawblock topic
         check_block_delay = 10 if config.NETWORK_NAME == "mainnet" else 0.5
         if time.time() - self.last_block_check_time > check_block_delay:

counterparty-core/counterpartycore/lib/messages/fairmint.py

@@ -290,7 +290,7 @@ def parse(db, tx, message):
     # we check if the hard cap is reached and in this case...
     if fairminter["hard_cap"] > 0:
         asset_supply = ledger.asset_supply(db, fairminter["asset"])
-        if asset_supply + quantity == fairminter["hard_cap"]:
+        if asset_supply + earn_quantity == fairminter["hard_cap"]:
             # ...we unlock the issuances for this assets
             bindings["fair_minting"] = False
             # we check if we need to lock the assets