From 164ddaa051bf21d0e70d6c682d76d376259f4ab8 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Fri, 10 Jan 2025 08:41:56 +0000 Subject: [PATCH 01/31] Fix reorg --- .../counterpartycore/lib/backend/bitcoind.py | 13 ++++--------- counterparty-core/counterpartycore/lib/blocks.py | 7 ++++++- .../counterpartycore/lib/exceptions.py | 4 ++++ release-notes/release-notes-v10.9.0.md | 2 ++ 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/counterparty-core/counterpartycore/lib/backend/bitcoind.py b/counterparty-core/counterpartycore/lib/backend/bitcoind.py index 329e2927d1..2d991307e4 100644 --- a/counterparty-core/counterpartycore/lib/backend/bitcoind.py +++ b/counterparty-core/counterpartycore/lib/backend/bitcoind.py @@ -41,16 +41,8 @@ def rpc_call(payload, retry=0): ) if response is None: # noqa: E711 - if config.TESTNET: - network = "testnet" - elif config.TESTNET4: - network = "testnet4" - elif config.REGTEST: - network = "regtest" - else: - network = "mainnet" raise exceptions.BitcoindRPCError( - f"Cannot communicate with Bitcoin Core at `{util.clean_url_for_log(url)}`. (server is set to run on {network}, is backend?)" + f"Cannot communicate with Bitcoin Core at `{util.clean_url_for_log(url)}`. (server is set to run on {config.NETWORK_NAME}, is backend?)" ) if response.status_code in (401,): raise exceptions.BitcoindRPCError( @@ -93,6 +85,9 @@ def rpc_call(payload, retry=0): ) elif response_json["error"]["code"] in [-28, -8, -2]: # "Verifying blocks..." or "Block height out of range" or "The network does not appear to fully agree!"" + if "Block height out of range" in response_json["error"]["message"]: + # this error should be managed by the caller + raise exceptions.BlockOutOfRange(response_json["error"]["message"]) logger.debug(f"Backend not ready. Sleeping for ten seconds. ({response_json['error']})") logger.debug(f"Payload: {payload}") if retry >= 10: diff --git a/counterparty-core/counterpartycore/lib/blocks.py b/counterparty-core/counterpartycore/lib/blocks.py index c3918c7154..4977c18a07 100644 --- a/counterparty-core/counterpartycore/lib/blocks.py +++ b/counterparty-core/counterpartycore/lib/blocks.py @@ -1296,7 +1296,12 @@ def parse_new_block(db, decoded_block, tx_index=None): # search last block with the correct hash previous_block_index = util.CURRENT_BLOCK_INDEX - 1 while True: - bitcoin_block_hash = backend.bitcoind.getblockhash(previous_block_index) + try: + bitcoin_block_hash = backend.bitcoind.getblockhash(previous_block_index) + except exceptions.BlockOutOfRange: + # previous block and current block are not in the blockchain + previous_block_index -= 2 + continue counterparty_block_hash = ledger.get_block_hash(db, previous_block_index) if bitcoin_block_hash != counterparty_block_hash: previous_block_index -= 1 diff --git a/counterparty-core/counterpartycore/lib/exceptions.py b/counterparty-core/counterpartycore/lib/exceptions.py index 67764c7e99..38e2dc1bd5 100644 --- a/counterparty-core/counterpartycore/lib/exceptions.py +++ b/counterparty-core/counterpartycore/lib/exceptions.py @@ -145,3 +145,7 @@ class RSFetchError(Exception): class ElectrsError(Exception): pass + + +class BlockOutOfRange(Exception): + pass diff --git a/release-notes/release-notes-v10.9.0.md b/release-notes/release-notes-v10.9.0.md index 6e61bcf532..b6be1511f4 100644 --- a/release-notes/release-notes-v10.9.0.md +++ b/release-notes/release-notes-v10.9.0.md @@ -37,6 +37,8 @@ The following transaction construction parameters have been deprecated (but rema - Fix the `dispensers` table in State DB: include dispensers with same the `source` and `asset` but a different `tx_hash` - Fix endpoint to get info from raw transaction when block index is not provided - Fix issue where composed transactions contained `script_pubkey` (lock script) where the `script_sig` (unlock script) should be +- Fix bootstrap when using `--bootstrap-url` flag +- Fix Blockchain reorg of several blocks ## Codebase From 03cab6ed64e49986bb9e92f02df08d2f1cb82c1e Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Fri, 10 Jan 2025 08:45:05 +0000 Subject: [PATCH 02/31] Use Bitcoin Core for decoderawtransaction --- counterparty-core/counterpartycore/lib/backend/bitcoind.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/counterparty-core/counterpartycore/lib/backend/bitcoind.py b/counterparty-core/counterpartycore/lib/backend/bitcoind.py index 2d991307e4..7ac759f311 100644 --- a/counterparty-core/counterpartycore/lib/backend/bitcoind.py +++ b/counterparty-core/counterpartycore/lib/backend/bitcoind.py @@ -383,7 +383,7 @@ def decoderawtransaction(rawtx: str): Proxy to `decoderawtransaction` RPC call. :param rawtx: The raw transaction hex. (e.g. 0200000000010199c94580cbea44aead18f429be20552e640804dc3b4808e39115197f1312954d000000001600147c6b1112ed7bc76fd03af8b91d02fd6942c5a8d0ffffffff0280f0fa02000000001976a914a11b66a67b3ff69671c8f82254099faf374b800e88ac70da0a27010000001600147c6b1112ed7bc76fd03af8b91d02fd6942c5a8d002000000000000) """ - return deserialize.deserialize_tx(rawtx) + return rpc("decoderawtransaction", [rawtx]) def search_pubkey_in_transactions(pubkeyhash, tx_hashes): From 0a5102aef549f79536a8bb82a257754c68363428 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Fri, 10 Jan 2025 08:50:25 +0000 Subject: [PATCH 03/31] tweak --- counterparty-core/counterpartycore/lib/backend/bitcoind.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/counterparty-core/counterpartycore/lib/backend/bitcoind.py b/counterparty-core/counterpartycore/lib/backend/bitcoind.py index 7ac759f311..0295c88118 100644 --- a/counterparty-core/counterpartycore/lib/backend/bitcoind.py +++ b/counterparty-core/counterpartycore/lib/backend/bitcoind.py @@ -83,11 +83,11 @@ def rpc_call(payload, retry=0): raise exceptions.BitcoindRPCError( f"{response_json['error']} Is `txindex` enabled in {config.BTC_NAME} Core?" ) + elif "Block height out of range" in response_json["error"]["message"]: + # this error should be managed by the caller + raise exceptions.BlockOutOfRange(response_json["error"]["message"]) elif response_json["error"]["code"] in [-28, -8, -2]: # "Verifying blocks..." or "Block height out of range" or "The network does not appear to fully agree!"" - if "Block height out of range" in response_json["error"]["message"]: - # this error should be managed by the caller - raise exceptions.BlockOutOfRange(response_json["error"]["message"]) logger.debug(f"Backend not ready. Sleeping for ten seconds. ({response_json['error']})") logger.debug(f"Payload: {payload}") if retry >= 10: From cf7fc277bc2e8da9f19415a518c717babaf7524a Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Fri, 10 Jan 2025 09:22:02 +0000 Subject: [PATCH 04/31] Stop async loop on error --- counterparty-core/counterpartycore/lib/follow.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/counterparty-core/counterpartycore/lib/follow.py b/counterparty-core/counterpartycore/lib/follow.py index 461bbe3f45..ba9646e0c6 100644 --- a/counterparty-core/counterpartycore/lib/follow.py +++ b/counterparty-core/counterpartycore/lib/follow.py @@ -238,9 +238,8 @@ async def receive_multipart(self, socket, topic_name): self.receive_message(topic, body, seq) except Exception as e: logger.error("Error processing message: %s", e) - import traceback - - print(traceback.format_exc()) # for debugging + # import traceback + # print(traceback.format_exc()) # for debugging capture_exception(e) raise e @@ -297,6 +296,7 @@ async def handle(self): except Exception as e: logger.error("Error in handle loop: %s", e) capture_exception(e) + self.stop() break # Optionally break the loop on other exceptions def start(self): From 400a95a18b947ff74242b7b29526b6f029d558ba Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Fri, 10 Jan 2025 09:43:49 +0000 Subject: [PATCH 05/31] split function --- .../counterpartycore/lib/blocks.py | 61 ++++++++++--------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/counterparty-core/counterpartycore/lib/blocks.py b/counterparty-core/counterpartycore/lib/blocks.py index 4977c18a07..c14e36d76a 100644 --- a/counterparty-core/counterpartycore/lib/blocks.py +++ b/counterparty-core/counterpartycore/lib/blocks.py @@ -1270,6 +1270,38 @@ def get_next_tx_index(db): return tx_index +def handle_reorg(db): + # search last block with the correct hash + previous_block_index = util.CURRENT_BLOCK_INDEX - 1 + while True: + try: + bitcoin_block_hash = backend.bitcoind.getblockhash(previous_block_index) + except exceptions.BlockOutOfRange: + # previous block and current block are not in the blockchain + previous_block_index -= 2 + continue + counterparty_block_hash = ledger.get_block_hash(db, previous_block_index) + if bitcoin_block_hash != counterparty_block_hash: + previous_block_index -= 1 + else: + break + current_block_hash = backend.bitcoind.getblockhash(previous_block_index + 1) + raw_current_block = backend.bitcoind.getblock(current_block_hash) + decoded_block = deserialize.deserialize_block( + raw_current_block, + parse_vouts=True, + block_index=previous_block_index + 1, + ) + logger.warning("Blockchain reorganization detected at block %s.", previous_block_index) + # rollback to the previous block + rollback(db, block_index=previous_block_index + 1) + previous_block = ledger.get_block(db, previous_block_index) + util.CURRENT_BLOCK_INDEX = previous_block_index + 1 + tx_index = get_next_tx_index(db) + + return decoded_block, previous_block, tx_index + + def parse_new_block(db, decoded_block, tx_index=None): start_time = time.time() @@ -1290,36 +1322,9 @@ def parse_new_block(db, decoded_block, tx_index=None): else: # get previous block previous_block = ledger.get_block(db, util.CURRENT_BLOCK_INDEX - 1) - # check if reorg is needed if decoded_block["hash_prev"] != previous_block["block_hash"]: - # search last block with the correct hash - previous_block_index = util.CURRENT_BLOCK_INDEX - 1 - while True: - try: - bitcoin_block_hash = backend.bitcoind.getblockhash(previous_block_index) - except exceptions.BlockOutOfRange: - # previous block and current block are not in the blockchain - previous_block_index -= 2 - continue - counterparty_block_hash = ledger.get_block_hash(db, previous_block_index) - if bitcoin_block_hash != counterparty_block_hash: - previous_block_index -= 1 - else: - break - current_block_hash = backend.bitcoind.getblockhash(previous_block_index + 1) - raw_current_block = backend.bitcoind.getblock(current_block_hash) - decoded_block = deserialize.deserialize_block( - raw_current_block, - parse_vouts=True, - block_index=previous_block_index + 1, - ) - logger.warning("Blockchain reorganization detected at block %s.", previous_block_index) - # rollback to the previous block - rollback(db, block_index=previous_block_index + 1) - previous_block = ledger.get_block(db, previous_block_index) - util.CURRENT_BLOCK_INDEX = previous_block_index + 1 - tx_index = get_next_tx_index(db) + decoded_block, previous_block, tx_index = handle_reorg(db) if "height" not in decoded_block: decoded_block["block_index"] = util.CURRENT_BLOCK_INDEX From 8628aaca3629fc0495b5c4652afa043e22f442e4 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Fri, 10 Jan 2025 09:44:29 +0000 Subject: [PATCH 06/31] Repeat indefinitely, even on error -5 --- .../counterpartycore/lib/backend/bitcoind.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/counterparty-core/counterpartycore/lib/backend/bitcoind.py b/counterparty-core/counterpartycore/lib/backend/bitcoind.py index 0295c88118..c558e8f3ed 100644 --- a/counterparty-core/counterpartycore/lib/backend/bitcoind.py +++ b/counterparty-core/counterpartycore/lib/backend/bitcoind.py @@ -79,21 +79,15 @@ def rpc_call(payload, retry=0): result = response_json elif "error" not in response_json.keys() or response_json["error"] is None: # noqa: E711 result = response_json["result"] - elif response_json["error"]["code"] == -5: # RPC_INVALID_ADDRESS_OR_KEY - raise exceptions.BitcoindRPCError( - f"{response_json['error']} Is `txindex` enabled in {config.BTC_NAME} Core?" - ) elif "Block height out of range" in response_json["error"]["message"]: # this error should be managed by the caller raise exceptions.BlockOutOfRange(response_json["error"]["message"]) - elif response_json["error"]["code"] in [-28, -8, -2]: + elif response_json["error"]["code"] in [-28, -8, -5, -2]: # "Verifying blocks..." or "Block height out of range" or "The network does not appear to fully agree!"" - logger.debug(f"Backend not ready. Sleeping for ten seconds. ({response_json['error']})") - logger.debug(f"Payload: {payload}") - if retry >= 10: - raise exceptions.BitcoindRPCError( - f"Backend not ready after {retry} retries. ({response_json['error']})" - ) + logger.debug(f"Error calling {payload}: {response_json['error']}") + logger.debug("Sleeping for ten seconds and retrying...") + if response_json["error"]["code"] == -5: # RPC_INVALID_ADDRESS_OR_KEY + logger.warning(f"Is `txindex` enabled in {config.BTC_NAME} Core?") # If Bitcoin Core takes more than `sys.getrecursionlimit() * 10 = 9970` # seconds to start, this'll hit the maximum recursion depth limit. time.sleep(10) From 3d074c59e14dc8b08f710e3473d532fcb32ce153 Mon Sep 17 00:00:00 2001 From: Adam Krellenstein Date: Fri, 10 Jan 2025 05:02:49 -0500 Subject: [PATCH 07/31] Tweak Release Notes for v10.9.0 --- release-notes/release-notes-v10.9.0.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/release-notes/release-notes-v10.9.0.md b/release-notes/release-notes-v10.9.0.md index b6be1511f4..0e2be950f6 100644 --- a/release-notes/release-notes-v10.9.0.md +++ b/release-notes/release-notes-v10.9.0.md @@ -38,12 +38,15 @@ The following transaction construction parameters have been deprecated (but rema - Fix endpoint to get info from raw transaction when block index is not provided - Fix issue where composed transactions contained `script_pubkey` (lock script) where the `script_sig` (unlock script) should be - Fix bootstrap when using `--bootstrap-url` flag -- Fix Blockchain reorg of several blocks +- Fix logic for blockchain reorgs of several blocks +- Have the node terminate when the `follow` loop raises an error +- Don't stop the server on "No such mempool or blockchain" error + ## Codebase - Remove the AddrIndexRs dependency -- Replacement of `transaction.py` and `transaction_helper/*` with `composer.py` +- Replace `transaction.py` and `transaction_helper/*` with `composer.py` - Use the `bitcoin-utils` library for generating transactions - No longer block the follow process on mempool parsing - Add a timeout when parsing mempool transaction from ZMQ @@ -52,6 +55,8 @@ The following transaction construction parameters have been deprecated (but rema - Trigger State DB refreshes automatically on version bumps - Use only Rust to deserialize blocks and transactions - Add `testnet4` support +- Repeat the RPC call to Bitcoin Core indefinitely until it succeeds +- Raise a specific `BlockOutOfRange` error when querying an unknown block ## API From 58f24fc7d51f2687d32052da9e361fb4ae78cd1e Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Fri, 10 Jan 2025 10:22:02 +0000 Subject: [PATCH 08/31] clean reorg code --- .../counterpartycore/lib/blocks.py | 51 ++++++++++--------- .../counterpartycore/lib/deserialize.py | 4 +- 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/counterparty-core/counterpartycore/lib/blocks.py b/counterparty-core/counterpartycore/lib/blocks.py index c14e36d76a..e5c2969b0a 100644 --- a/counterparty-core/counterpartycore/lib/blocks.py +++ b/counterparty-core/counterpartycore/lib/blocks.py @@ -1275,31 +1275,40 @@ def handle_reorg(db): previous_block_index = util.CURRENT_BLOCK_INDEX - 1 while True: try: - bitcoin_block_hash = backend.bitcoind.getblockhash(previous_block_index) + previous_block_hash = backend.bitcoind.getblockhash(previous_block_index) except exceptions.BlockOutOfRange: # previous block and current block are not in the blockchain previous_block_index -= 2 continue - counterparty_block_hash = ledger.get_block_hash(db, previous_block_index) - if bitcoin_block_hash != counterparty_block_hash: + + try: + current_block_hash = backend.bitcoind.getblockhash(previous_block_index + 1) + except exceptions.BlockOutOfRange: + # current block is not in the blockchain previous_block_index -= 1 - else: - break - current_block_hash = backend.bitcoind.getblockhash(previous_block_index + 1) - raw_current_block = backend.bitcoind.getblock(current_block_hash) - decoded_block = deserialize.deserialize_block( - raw_current_block, - parse_vouts=True, - block_index=previous_block_index + 1, - ) + continue + + if previous_block_hash != ledger.get_block_hash(db, previous_block_index): + # hashes don't match + previous_block_index -= 1 + continue + + break + logger.warning("Blockchain reorganization detected at block %s.", previous_block_index) + # rollback to the previous block - rollback(db, block_index=previous_block_index + 1) - previous_block = ledger.get_block(db, previous_block_index) - util.CURRENT_BLOCK_INDEX = previous_block_index + 1 - tx_index = get_next_tx_index(db) + current_block_index = previous_block_index + 1 + rollback(db, block_index=current_block_index) + util.CURRENT_BLOCK_INDEX = previous_block_index - return decoded_block, previous_block, tx_index + current_block = deserialize.deserialize_block( + backend.bitcoind.getblock(current_block_hash), + parse_vouts=True, + block_index=current_block_index, + ) + + return current_block def parse_new_block(db, decoded_block, tx_index=None): @@ -1324,12 +1333,8 @@ def parse_new_block(db, decoded_block, tx_index=None): previous_block = ledger.get_block(db, util.CURRENT_BLOCK_INDEX - 1) # check if reorg is needed if decoded_block["hash_prev"] != previous_block["block_hash"]: - decoded_block, previous_block, tx_index = handle_reorg(db) - - if "height" not in decoded_block: - decoded_block["block_index"] = util.CURRENT_BLOCK_INDEX - else: - decoded_block["block_index"] = decoded_block["height"] + new_current_block = handle_reorg(db) + return parse_new_block(db, new_current_block) # Sanity checks if decoded_block["block_index"] != config.BLOCK_FIRST: diff --git a/counterparty-core/counterpartycore/lib/deserialize.py b/counterparty-core/counterpartycore/lib/deserialize.py index d0a27bcac3..ffdd38326c 100644 --- a/counterparty-core/counterpartycore/lib/deserialize.py +++ b/counterparty-core/counterpartycore/lib/deserialize.py @@ -32,4 +32,6 @@ def deserialize_block(block_hex, parse_vouts=False, block_index=None): } ) current_block_index = block_index or util.CURRENT_BLOCK_INDEX - return deserializer.parse_block(block_hex, current_block_index, parse_vouts) + decoded_block = deserializer.parse_block(block_hex, current_block_index, parse_vouts) + decoded_block["block_index"] = decoded_block["height"] + return decoded_block From 2988ab4c215891e507843c4f42f5b2e43d2eae2f Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Fri, 10 Jan 2025 10:27:22 +0000 Subject: [PATCH 09/31] tweak --- counterparty-core/counterpartycore/lib/blocks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/counterparty-core/counterpartycore/lib/blocks.py b/counterparty-core/counterpartycore/lib/blocks.py index e5c2969b0a..5366e57271 100644 --- a/counterparty-core/counterpartycore/lib/blocks.py +++ b/counterparty-core/counterpartycore/lib/blocks.py @@ -1295,8 +1295,6 @@ def handle_reorg(db): break - logger.warning("Blockchain reorganization detected at block %s.", previous_block_index) - # rollback to the previous block current_block_index = previous_block_index + 1 rollback(db, block_index=current_block_index) @@ -1333,6 +1331,9 @@ def parse_new_block(db, decoded_block, tx_index=None): previous_block = ledger.get_block(db, util.CURRENT_BLOCK_INDEX - 1) # check if reorg is needed if decoded_block["hash_prev"] != previous_block["block_hash"]: + logger.warning( + "Blockchain reorganization detected at block %s.", util.CURRENT_BLOCK_INDEX + ) new_current_block = handle_reorg(db) return parse_new_block(db, new_current_block) From a179b6582a0362fe066ba9291e729c271421eaab Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Fri, 10 Jan 2025 12:06:31 +0000 Subject: [PATCH 10/31] inject block_index in rust --- counterparty-core/counterpartycore/lib/deserialize.py | 1 - counterparty-rs/src/indexer/block.rs | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/counterparty-core/counterpartycore/lib/deserialize.py b/counterparty-core/counterpartycore/lib/deserialize.py index ffdd38326c..d78e570204 100644 --- a/counterparty-core/counterpartycore/lib/deserialize.py +++ b/counterparty-core/counterpartycore/lib/deserialize.py @@ -33,5 +33,4 @@ def deserialize_block(block_hex, parse_vouts=False, block_index=None): ) current_block_index = block_index or util.CURRENT_BLOCK_INDEX decoded_block = deserializer.parse_block(block_hex, current_block_index, parse_vouts) - decoded_block["block_index"] = decoded_block["height"] return decoded_block diff --git a/counterparty-rs/src/indexer/block.rs b/counterparty-rs/src/indexer/block.rs index 7eeb8b970a..9a5c428281 100644 --- a/counterparty-rs/src/indexer/block.rs +++ b/counterparty-rs/src/indexer/block.rs @@ -156,6 +156,7 @@ impl IntoPy for Block { fn into_py(self, py: Python<'_>) -> PyObject { let dict = PyDict::new_bound(py); dict.set_item("height", self.height).unwrap(); + dict.set_item("block_index", self.height).unwrap(); dict.set_item("version", self.version).unwrap(); dict.set_item("hash_prev", self.hash_prev).unwrap(); dict.set_item("hash_merkle_root", self.hash_merkle_root) From 447f69c8224d816bdb39acfe008c55ad4df508f6 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Fri, 10 Jan 2025 13:58:05 +0000 Subject: [PATCH 11/31] more reorg tests --- .../counterpartycore/lib/blocks.py | 3 ++ .../test/regtest/regtestnode.py | 29 ++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/counterparty-core/counterpartycore/lib/blocks.py b/counterparty-core/counterpartycore/lib/blocks.py index 5366e57271..d71c2227f1 100644 --- a/counterparty-core/counterpartycore/lib/blocks.py +++ b/counterparty-core/counterpartycore/lib/blocks.py @@ -1278,6 +1278,7 @@ def handle_reorg(db): previous_block_hash = backend.bitcoind.getblockhash(previous_block_index) except exceptions.BlockOutOfRange: # previous block and current block are not in the blockchain + logger.debug(f"Previous block is not in the blockchain ({previous_block_index}).") previous_block_index -= 2 continue @@ -1285,11 +1286,13 @@ def handle_reorg(db): current_block_hash = backend.bitcoind.getblockhash(previous_block_index + 1) except exceptions.BlockOutOfRange: # current block is not in the blockchain + logger.debug(f"Current block is not in the blockchain ({previous_block_index + 1}).") previous_block_index -= 1 continue if previous_block_hash != ledger.get_block_hash(db, previous_block_index): # hashes don't match + logger.debug(f"Hashes don't match ({previous_block_index}).") previous_block_index -= 1 continue diff --git a/counterparty-core/counterpartycore/test/regtest/regtestnode.py b/counterparty-core/counterpartycore/test/regtest/regtestnode.py index 8086776f22..39b1b475f4 100644 --- a/counterparty-core/counterpartycore/test/regtest/regtestnode.py +++ b/counterparty-core/counterpartycore/test/regtest/regtestnode.py @@ -662,6 +662,7 @@ def test_reorg(self): assert intermediate_xcp_balance > initial_xcp_balance print("Mine a longest chain on the second node...") + before_reorg_block = int(self.bitcoin_cli_2("getblockcount").strip()) self.bitcoin_cli_2("generatetoaddress", 6, self.addresses[0]) print("Re-connect to the first node...") @@ -679,13 +680,39 @@ def test_reorg(self): self.wait_for_counterparty_server(block=last_block) print("Burn count after reorganization: ", self.get_burn_count(self.addresses[0])) - assert "Blockchain reorganization detected" in self.server_out.getvalue() + + assert ( + f"Blockchain reorganization detected at block {last_block - 1}" + in self.server_out.getvalue() + ) + assert f"Hashes don't match ({before_reorg_block + 1})" in self.server_out.getvalue() assert self.get_burn_count(self.addresses[0]) == 1 final_xcp_balance = self.get_xcp_balance(self.addresses[0]) print("Final XCP balance: ", final_xcp_balance) assert final_xcp_balance == initial_xcp_balance + # Test reorg with a shorter chain of one block + print("Disconnect from the first node...") + self.bitcoin_cli_2("disconnectnode", "localhost:18445") + + print("Invalidate the last block on the first node...") + best_block_hash = self.bitcoin_cli("getbestblockhash").strip() + self.bitcoin_cli("invalidateblock", best_block_hash) + + self.mine_blocks(1) + retry = 0 + while ( + f"Current block is not in the blockchain ({last_block + 1})." + not in self.server_out.getvalue() + ): + time.sleep(1) + retry += 1 + print("Waiting for reorg...") + assert retry < 100 + + print("Reorg test successful") + def test_electrs(self): self.start_and_wait_second_node() From e6bd18456561c4099c4997c9838fccedf0fa4f25 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Fri, 10 Jan 2025 14:43:54 +0000 Subject: [PATCH 12/31] more reorg tests --- .../counterpartycore/test/regtest/regtestnode.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/counterparty-core/counterpartycore/test/regtest/regtestnode.py b/counterparty-core/counterpartycore/test/regtest/regtestnode.py index 39b1b475f4..49339b458c 100644 --- a/counterparty-core/counterpartycore/test/regtest/regtestnode.py +++ b/counterparty-core/counterpartycore/test/regtest/regtestnode.py @@ -692,7 +692,7 @@ def test_reorg(self): print("Final XCP balance: ", final_xcp_balance) assert final_xcp_balance == initial_xcp_balance - # Test reorg with a shorter chain of one block + # Test reorg with same length chain but one different block print("Disconnect from the first node...") self.bitcoin_cli_2("disconnectnode", "localhost:18445") @@ -711,6 +711,20 @@ def test_reorg(self): print("Waiting for reorg...") assert retry < 100 + # Test reorg with a same length chain but three different blocks + print("Invalidate block ", last_block - 3) + previous_block_hash = self.bitcoin_cli("getblockhash", last_block - 3).strip() + self.bitcoin_cli("invalidateblock", previous_block_hash) + self.mine_blocks(3) + retry = 0 + while len(self.server_out.getvalue().split(f"Hashes don't match ({last_block - 3})")) < 3: + time.sleep(1) + retry += 1 + print("Waiting for reorg...") + assert retry < 100 + + self.wait_for_counterparty_server(last_block) + print("Reorg test successful") def test_electrs(self): From 2674038c24927b14b5a6f61cad5960a587e83710 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Fri, 10 Jan 2025 14:47:03 +0000 Subject: [PATCH 13/31] don't catch unexpected error --- counterparty-core/counterpartycore/lib/blocks.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/counterparty-core/counterpartycore/lib/blocks.py b/counterparty-core/counterpartycore/lib/blocks.py index d71c2227f1..ed750881b5 100644 --- a/counterparty-core/counterpartycore/lib/blocks.py +++ b/counterparty-core/counterpartycore/lib/blocks.py @@ -1274,13 +1274,7 @@ def handle_reorg(db): # search last block with the correct hash previous_block_index = util.CURRENT_BLOCK_INDEX - 1 while True: - try: - previous_block_hash = backend.bitcoind.getblockhash(previous_block_index) - except exceptions.BlockOutOfRange: - # previous block and current block are not in the blockchain - logger.debug(f"Previous block is not in the blockchain ({previous_block_index}).") - previous_block_index -= 2 - continue + previous_block_hash = backend.bitcoind.getblockhash(previous_block_index) try: current_block_hash = backend.bitcoind.getblockhash(previous_block_index + 1) @@ -1303,6 +1297,7 @@ def handle_reorg(db): rollback(db, block_index=current_block_index) util.CURRENT_BLOCK_INDEX = previous_block_index + # get the new deserialized current block current_block = deserialize.deserialize_block( backend.bitcoind.getblock(current_block_hash), parse_vouts=True, From 0670fe0db8ffc021e16370e437acf17c7ce04ec5 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Fri, 10 Jan 2025 15:00:19 +0000 Subject: [PATCH 14/31] fix bootstrap cleaning --- counterparty-core/counterpartycore/lib/bootstrap.py | 8 +++++--- release-notes/release-notes-v10.9.0.md | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/counterparty-core/counterpartycore/lib/bootstrap.py b/counterparty-core/counterpartycore/lib/bootstrap.py index be1cddc735..a0f6de28ca 100644 --- a/counterparty-core/counterpartycore/lib/bootstrap.py +++ b/counterparty-core/counterpartycore/lib/bootstrap.py @@ -114,9 +114,11 @@ def clean_data_dir(data_dir): if not os.path.exists(data_dir): os.makedirs(data_dir, mode=0o755) return - files_to_delete = glob.glob(os.path.join(data_dir, "*.db")) - files_to_delete += glob.glob(os.path.join(data_dir, "*.db-wal")) - files_to_delete += glob.glob(os.path.join(data_dir, "*.db-shm")) + network = "" if config.NETWORK_NAME == "mainnet" else f".{config.NETWORK_NAME}" + files_to_delete = [] + for db_name in ["counterparty", "state"]: + for ext in ["db", "db-wal", "db-shm"]: + files_to_delete += glob.glob(os.path.join(data_dir, f"{db_name}{network}.{ext}")) for file in files_to_delete: os.remove(file) diff --git a/release-notes/release-notes-v10.9.0.md b/release-notes/release-notes-v10.9.0.md index 0e2be950f6..03dffe81ea 100644 --- a/release-notes/release-notes-v10.9.0.md +++ b/release-notes/release-notes-v10.9.0.md @@ -37,7 +37,7 @@ The following transaction construction parameters have been deprecated (but rema - Fix the `dispensers` table in State DB: include dispensers with same the `source` and `asset` but a different `tx_hash` - Fix endpoint to get info from raw transaction when block index is not provided - Fix issue where composed transactions contained `script_pubkey` (lock script) where the `script_sig` (unlock script) should be -- Fix bootstrap when using `--bootstrap-url` flag +- Fix bootstrap when using `--bootstrap-url` flag and don't clean other networks files - Fix logic for blockchain reorgs of several blocks - Have the node terminate when the `follow` loop raises an error - Don't stop the server on "No such mempool or blockchain" error From 8c44857c9b68f03c42fbf15421e3c6b353a5dc7b Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Fri, 10 Jan 2025 15:58:13 +0000 Subject: [PATCH 15/31] fix regtest --- .../counterpartycore/test/regtest/regtestnode.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/counterparty-core/counterpartycore/test/regtest/regtestnode.py b/counterparty-core/counterpartycore/test/regtest/regtestnode.py index 49339b458c..2619a24e26 100644 --- a/counterparty-core/counterpartycore/test/regtest/regtestnode.py +++ b/counterparty-core/counterpartycore/test/regtest/regtestnode.py @@ -725,6 +725,21 @@ def test_reorg(self): self.wait_for_counterparty_server(last_block) + # other tests expect the second node to be connected if running + print("Re-connect to the first node...") + self.bitcoin_cli_2( + "addnode", + "localhost:18445", + "onetry", + _out=sys.stdout, + _err=sys.stdout, + ) + # fix block count for other tests + self.block_count -= 1 + + print("Wait for the two nodes to sync...") + last_block = self.wait_for_node_to_sync() + print("Reorg test successful") def test_electrs(self): From c60f3b7d978dfa83ed9f08bfa2e6ebca808632b3 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Fri, 10 Jan 2025 16:37:14 +0000 Subject: [PATCH 16/31] Add test for rpc_call() function --- .../counterpartycore/lib/backend/bitcoind.py | 21 ++-- .../counterpartycore/test/rpc_test.py | 97 +++++++++++++++++++ 2 files changed, 111 insertions(+), 7 deletions(-) create mode 100644 counterparty-core/counterpartycore/test/rpc_test.py diff --git a/counterparty-core/counterpartycore/lib/backend/bitcoind.py b/counterparty-core/counterpartycore/lib/backend/bitcoind.py index c558e8f3ed..7520d51fd6 100644 --- a/counterparty-core/counterpartycore/lib/backend/bitcoind.py +++ b/counterparty-core/counterpartycore/lib/backend/bitcoind.py @@ -21,6 +21,11 @@ TRANSACTIONS_CACHE_MAX_SIZE = 10000 +# for testing +def should_retry(): + return True + + def rpc_call(payload, retry=0): """Calls to bitcoin core and returns the response""" url = config.BACKEND_URL @@ -84,14 +89,16 @@ def rpc_call(payload, retry=0): raise exceptions.BlockOutOfRange(response_json["error"]["message"]) elif response_json["error"]["code"] in [-28, -8, -5, -2]: # "Verifying blocks..." or "Block height out of range" or "The network does not appear to fully agree!"" - logger.debug(f"Error calling {payload}: {response_json['error']}") - logger.debug("Sleeping for ten seconds and retrying...") + warning_message = f"Error calling {payload}: {response_json['error']}. Sleeping for ten seconds and retrying." if response_json["error"]["code"] == -5: # RPC_INVALID_ADDRESS_OR_KEY - logger.warning(f"Is `txindex` enabled in {config.BTC_NAME} Core?") - # If Bitcoin Core takes more than `sys.getrecursionlimit() * 10 = 9970` - # seconds to start, this'll hit the maximum recursion depth limit. - time.sleep(10) - return rpc_call(payload, retry=retry + 1) + warning_message += f" Is `txindex` enabled in {config.BTC_NAME} Core?" + logger.warning(warning_message) + if should_retry(): + # If Bitcoin Core takes more than `sys.getrecursionlimit() * 10 = 9970` + # seconds to start, this'll hit the maximum recursion depth limit. + time.sleep(10) + return rpc_call(payload, retry=retry + 1) + raise exceptions.BitcoindRPCError(warning_message) else: raise exceptions.BitcoindRPCError(response_json["error"]["message"]) diff --git a/counterparty-core/counterpartycore/test/rpc_test.py b/counterparty-core/counterpartycore/test/rpc_test.py new file mode 100644 index 0000000000..8b7f958c4c --- /dev/null +++ b/counterparty-core/counterpartycore/test/rpc_test.py @@ -0,0 +1,97 @@ +import json + +import pytest + +from counterpartycore.lib import config, exceptions +from counterpartycore.lib.backend import bitcoind + + +class MockResponse: + def __init__(self, status_code, json_data, reason=None): + self.status_code = status_code + self.json_data = json_data + self.reason = reason + + def json(self): + return self.json_data + + +def mock_requests_post(*args, **kwargs): + payload = json.loads(kwargs["data"]) + if payload["method"] == "getblockhash": + return MockResponse(200, {"error": {"message": "Block height out of range", "code": -8}}) + if payload["method"] == "return_none": + return None + if payload["method"] == "return_401": + return MockResponse(401, {}, "Unauthorized") + if payload["method"] == "return_800": + return MockResponse(800, {}, "because I want a 500") + if payload["method"] == "return_batch_list": + return MockResponse(200, ["ok", "ok"]) + if payload["method"] == "return_200": + return MockResponse(200, {"result": "ok"}) + if payload["method"] == "return_code_28": + return MockResponse(200, {"error": {"message": "Error 28", "code": -28}}) + if payload["method"] == "return_code_5": + return MockResponse(200, {"error": {"message": "Error 5", "code": -5}}) + if payload["method"] == "return_code_30": + return MockResponse(200, {"error": {"message": "Error 30", "code": -30}}) + + +@pytest.fixture(scope="function") +def init_mock(monkeypatch): + monkeypatch.setattr("requests.post", mock_requests_post) + monkeypatch.setattr("counterpartycore.lib.backend.bitcoind.should_retry", lambda: False) + config.BACKEND_URL = "http://localhost:14000" + config.BACKEND_SSL_NO_VERIFY = True + config.REQUESTS_TIMEOUT = 5 + + +def test_rpc_call(init_mock): + with pytest.raises(exceptions.BlockOutOfRange): + bitcoind.rpc("getblockhash", [1]) + + with pytest.raises(exceptions.BitcoindRPCError) as exc_info: + bitcoind.rpc("return_none", []) + assert ( + str(exc_info.value) + == "Cannot communicate with Bitcoin Core at `http://localhost:14000`. (server is set to run on testnet, is backend?)" + ) + + with pytest.raises(exceptions.BitcoindRPCError) as exc_info: + bitcoind.rpc("return_401", []) + assert ( + str(exc_info.value) + == "Authorization error connecting to http://localhost:14000: 401 Unauthorized" + ) + + with pytest.raises(exceptions.BitcoindRPCError) as exc_info: + bitcoind.rpc("return_500", []) + assert ( + str(exc_info.value) + == "Cannot communicate with Bitcoin Core at `http://localhost:14000`. (server is set to run on testnet, is backend?)" + ) + + result = bitcoind.rpc("return_batch_list", []) + assert result == ["ok", "ok"] + + result = bitcoind.rpc("return_200", []) + assert result == "ok" + + with pytest.raises(exceptions.BitcoindRPCError) as exc_info: + bitcoind.rpc("return_code_28", []) + assert ( + str(exc_info.value) + == "Error calling {'method': 'return_code_28', 'params': [], 'jsonrpc': '2.0', 'id': 0}: {'message': 'Error 28', 'code': -28}. Sleeping for ten seconds and retrying." + ) + + with pytest.raises(exceptions.BitcoindRPCError) as exc_info: + bitcoind.rpc("return_code_5", []) + assert ( + str(exc_info.value) + == "Error calling {'method': 'return_code_5', 'params': [], 'jsonrpc': '2.0', 'id': 0}: {'message': 'Error 5', 'code': -5}. Sleeping for ten seconds and retrying. Is `txindex` enabled in Bitcoin Core?" + ) + + with pytest.raises(exceptions.BitcoindRPCError) as exc_info: + bitcoind.rpc("return_code_30", []) + assert str(exc_info.value) == "Error 30" From bf80de87cb4cc7f393fdb1b145300542207b0d26 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Sat, 11 Jan 2025 19:03:31 +0000 Subject: [PATCH 17/31] Fix get_vin_info() function --- .../counterpartycore/lib/gettxinfo.py | 3 ++- .../counterpartycore/test/deserialize_test.py | 19 +++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/counterparty-core/counterpartycore/lib/gettxinfo.py b/counterparty-core/counterpartycore/lib/gettxinfo.py index 670af4fbb7..63cd78e421 100644 --- a/counterparty-core/counterpartycore/lib/gettxinfo.py +++ b/counterparty-core/counterpartycore/lib/gettxinfo.py @@ -127,7 +127,7 @@ def get_vin_info(vin): # have been from a while ago, so this call may not hit the cache. vin_ctx = backend.bitcoind.get_decoded_transaction(vin["hash"]) - is_segwit = len(vin_ctx["vtxinwit"]) > 0 + is_segwit = vin_ctx["segwit"] vout = vin_ctx["vout"][vin["n"]] return vout["value"], vout["script_pub_key"], is_segwit @@ -393,6 +393,7 @@ def get_tx_info_new(db, decoded_tx, block_index, p2sh_is_segwit=False, composing The destinations, if they exists, always comes before the data output; the change, if it exists, always comes after. """ + # Ignore coinbase transactions. if decoded_tx["coinbase"]: raise DecodeError("coinbase transaction") diff --git a/counterparty-core/counterpartycore/test/deserialize_test.py b/counterparty-core/counterpartycore/test/deserialize_test.py index 16ea0ab458..a7be3ce0f2 100644 --- a/counterparty-core/counterpartycore/test/deserialize_test.py +++ b/counterparty-core/counterpartycore/test/deserialize_test.py @@ -3,8 +3,9 @@ from io import BytesIO import bitcoin as bitcoinlib +import pytest -from counterpartycore.lib import deserialize, util +from counterpartycore.lib import config, deserialize, gettxinfo, util from counterpartycore.lib.util import inverse_hash @@ -13,7 +14,6 @@ def deserialize_bitcoinlib(tx_hex): def deserialize_rust(tx_hex): - # config.NETWORK_NAME = "mainnet" return deserialize.deserialize_tx(tx_hex, parse_vouts=True, block_index=900000) @@ -34,6 +34,21 @@ def create_block_hex(transactions_hex): return block_hex +@pytest.mark.skip +def test_deserialize_mpma(): + config.PREFIX = b"CNTRPRTY" + config.NETWORK_NAME = "mainnet" + config.ADDRESSVERSION = config.ADDRESSVERSION_MAINNET + + hex = "0100000001f9cf03a71930731618f2e0ff897db75d208a587129b96296f3958b0dc146420900000000e5483045022100a72e4be0a0f581e1c438c7048413c65c05793e8328a7acaa1ef081cc8c44909a0220718e772276aaa7adf8392a1d39ab44fc8778f622ee0dea9858cd5894290abb2b014c9a4c6f434e545250525459030003000fc815eeb3172efc23fbd39c41189e83e4e0c8150033dafc6a4dcd8bce30b038305e30e5defad4acd6009081f7ee77f0ef849a213670d4e785c26d71375d40467e543326526fa800000000000000060100000000000000018000000000000000006000752102e6dd23598e1d2428ecf7eb59c27fdfeeb7a27c26906e96dc1f3d5ebba6e54d08ad0075740087ffffffff0100000000000000000e6a0c2bb584c84ba87a60dcab46c100000000" + decoded_tx = deserialize_rust(hex) + assert not decoded_tx["segwit"] + p2sh_encoding_source, data, outputs_value = gettxinfo.get_transaction_source_from_p2sh( + decoded_tx, False + ) + assert p2sh_encoding_source == "18b7eyatTwZ8mvSCXRRxjNjvr3DPwhh6bU" + + def test_deserialize(): hex = "0100000001db3acf37743ac015808f7911a88761530c801819b3b907340aa65dfb6d98ce24030000006a473044022002961f4800cb157f8c0913084db0ee148fa3e1130e0b5e40c3a46a6d4f83ceaf02202c3dd8e631bf24f4c0c5341b3e1382a27f8436d75f3e0a095915995b0bf7dc8e01210395c223fbf96e49e5b9e06a236ca7ef95b10bf18c074bd91a5942fc40360d0b68fdffffff040000000000000000536a4c5058325bd61325dc633fadf05bec9157c23106759cee40954d39d9dbffc17ec5851a2d1feb5d271da422e0e24c7ae8ad29d2eeabf7f9ca3de306bd2bc98e2a39e47731aa000caf400053000c1283000149c8000000000000001976a91462bef4110f98fdcb4aac3c1869dbed9bce8702ed88acc80000000000000017a9144317f779c0a2ccf8f6bc3d440bd9e536a5bff75287fa3e5100000000001976a914bf2646b8ba8b4a143220528bde9c306dac44a01c88ac00000000" decoded_tx = deserialize_rust(hex) From 0b2efb1a3a48a928e8e4d19c6793f8f0939322c8 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Sat, 11 Jan 2025 21:11:33 +0000 Subject: [PATCH 18/31] some tests for get_vin_info() --- .../counterpartycore/test/deserialize_test.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/counterparty-core/counterpartycore/test/deserialize_test.py b/counterparty-core/counterpartycore/test/deserialize_test.py index a7be3ce0f2..630817adc2 100644 --- a/counterparty-core/counterpartycore/test/deserialize_test.py +++ b/counterparty-core/counterpartycore/test/deserialize_test.py @@ -150,3 +150,59 @@ def test_deserialize(): print( f"Time to deserialize {4 * iterations} transactions with bitcoinlib: {end_time - start_time} seconds" ) + + +def mock_get_decoded_transaction(tx_hash): + txs = { + "094246c10d8b95f39662b92971588a205db77d89ffe0f21816733019a703cff9": "0100000001c47705b604b5b375fb43b6a7a632e20a7c10eb11d3202c00bd659e673d4d9396010000006a47304402204bc0847f52965c645e164078cfb5d743eb918c4fddaf4f592056b3470445e2c602202986c27c2f0f3b858b8fee94bf712338bc0ab8ff462edcea285a835143e10532012102e6dd23598e1d2428ecf7eb59c27fdfeeb7a27c26906e96dc1f3d5ebba6e54d08ffffffff02893000000000000017a9148760df63af4701313b244bf5ccd7479914843da18778cb0000000000001976a914533c940b158eae03f5bf71f1195d757c819c2e0c88ac00000000", + "05e7e9f59f155b28311a5e2860388783b839027b6529889de791351fe172752d": "020000000001016d72a3d323f82e76dcbf5fe9448a91ea9e68649e313d9a43822c0b27308a7b080200000017160014f6a785077f78695c12d51078ea7d9c10641f24acffffffff0208420000000000002251202525a906d3d870c6c00a2bfd63824c6597a4eddd8d24392f42ffbb2e6991fc5dcb8d04000000000017a914f54105af74fb10e70e899901b6ac4593ac20eea1870247304402205bd9f7e2ebe915532309548aad4e36f4b4feb856dab74f1b0e4df5292c0dbb4102202ca4d61fca54d08e2fd077c7e11154d2f271cf7102bb355c16dcb480d48dd57001210395c693bfc3a4d00e4380bec0d85871a1d0083618f8f01663199261d011e2a2bb00000000", + "c93934dc5149f771c0a9100302006058c51a13af5146ded1053dae2a219f7852": "020000000001019963e21ab347fbd1527f138a6788ad9d63b589fbab5a15a63ec4dc6f8318ffa34000000000ffffffff02315f00000000000016001457b185fde87fefac8aa2c7c823d4aae4c25aa8539f680000000000001600140b8846404281da37f3c4daa8da3b85d21293b97a024730440220662b27c5aa429153ebbe2ff3844efa7c493226c645573c04d6a4ebf404dc738702200f1c36ba63debb4d38d285980c3ecc8d7b20a70f88605b3358f8d10363d741cf012103c9d887d18d3c3a2bbdaf00c98b50863aa4d1d844e448aa4defe8fc4bdf9036b100000000", + } + decoded_tx = deserialize_rust(txs[tx_hash]) + return decoded_tx + + +@pytest.fixture(scope="function") +def init_mock(monkeypatch): + monkeypatch.setattr( + "counterpartycore.lib.backend.bitcoind.get_decoded_transaction", + mock_get_decoded_transaction, + ) + + +def test_get_vin_info(init_mock): + vout_value, script_pubkey, is_segwit = gettxinfo.get_vin_info( + { + "hash": "094246c10d8b95f39662b92971588a205db77d89ffe0f21816733019a703cff9", + "n": 0, + } + ) + assert vout_value == 12425 + assert script_pubkey == b"\xa9\x14\x87`\xdfc\xafG\x011;$K\xf5\xcc\xd7G\x99\x14\x84=\xa1\x87" + assert not is_segwit + + vout_value, script_pubkey, is_segwit = gettxinfo.get_vin_info( + { + "hash": "05e7e9f59f155b28311a5e2860388783b839027b6529889de791351fe172752d", + "n": 0, + } + ) + assert vout_value == 16904 + assert ( + script_pubkey + == b"Q %%\xa9\x06\xd3\xd8p\xc6\xc0\n+\xfdc\x82Le\x97\xa4\xed\xdd\x8d$9/B\xff\xbb.i\x91\xfc]" + ) + assert is_segwit + + vout_value, script_pubkey, is_segwit = gettxinfo.get_vin_info( + { + "hash": "c93934dc5149f771c0a9100302006058c51a13af5146ded1053dae2a219f7852", + "n": 0, + } + ) + assert vout_value == 24369 + assert ( + script_pubkey + == b"\x00\x14W\xb1\x85\xfd\xe8\x7f\xef\xac\x8a\xa2\xc7\xc8#\xd4\xaa\xe4\xc2Z\xa8S" + ) + assert is_segwit From 73e20a7ad83e119ebc9e8a53dea2418c2378e63a Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Mon, 13 Jan 2025 07:52:31 +0000 Subject: [PATCH 19/31] Handle correctly RPC call errors from the API --- counterparty-core/counterpartycore/lib/backend/bitcoind.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/counterparty-core/counterpartycore/lib/backend/bitcoind.py b/counterparty-core/counterpartycore/lib/backend/bitcoind.py index 7520d51fd6..4b68c15859 100644 --- a/counterparty-core/counterpartycore/lib/backend/bitcoind.py +++ b/counterparty-core/counterpartycore/lib/backend/bitcoind.py @@ -156,6 +156,8 @@ def safe_rpc(method, params): verify=(not config.BACKEND_SSL_NO_VERIFY), timeout=config.REQUESTS_TIMEOUT, ).json() + if "error" in response: + raise exceptions.BitcoindRPCError(response["error"]["message"]) return response["result"] except (requests.exceptions.RequestException, json.decoder.JSONDecodeError, KeyError) as e: raise exceptions.BitcoindRPCError(f"Error calling {method}: {str(e)}") from e @@ -376,7 +378,10 @@ def sendrawtransaction(signedhex: str): Proxy to `sendrawtransaction` RPC call. :param signedhex: The signed transaction hex. """ - return rpc("sendrawtransaction", [signedhex]) + try: + return rpc("sendrawtransaction", [signedhex]) + except Exception as e: + raise exceptions.BitcoindRPCError(f"Error broadcasting transaction: {str(e)}") from e def decoderawtransaction(rawtx: str): From 562a8f088bdd0386adc37b744013416ab309013c Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Mon, 13 Jan 2025 07:53:34 +0000 Subject: [PATCH 20/31] Handle correctly RPC call errors from the API --- release-notes/release-notes-v10.9.0.md | 1 + 1 file changed, 1 insertion(+) diff --git a/release-notes/release-notes-v10.9.0.md b/release-notes/release-notes-v10.9.0.md index 03dffe81ea..d994036250 100644 --- a/release-notes/release-notes-v10.9.0.md +++ b/release-notes/release-notes-v10.9.0.md @@ -41,6 +41,7 @@ The following transaction construction parameters have been deprecated (but rema - Fix logic for blockchain reorgs of several blocks - Have the node terminate when the `follow` loop raises an error - Don't stop the server on "No such mempool or blockchain" error +- Handle correctly RPC call errors from the API ## Codebase From cc12a62534b1213046c04308c5154da55a055950 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Mon, 13 Jan 2025 09:30:22 +0000 Subject: [PATCH 21/31] Don't clean mempool on catch-up --- counterparty-core/counterpartycore/lib/blocks.py | 2 -- counterparty-core/counterpartycore/lib/follow.py | 3 ++- release-notes/release-notes-v10.9.0.md | 1 + 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/counterparty-core/counterpartycore/lib/blocks.py b/counterparty-core/counterpartycore/lib/blocks.py index ed750881b5..17896d77d9 100644 --- a/counterparty-core/counterpartycore/lib/blocks.py +++ b/counterparty-core/counterpartycore/lib/blocks.py @@ -24,7 +24,6 @@ gas, ledger, log, - mempool, message_type, util, ) @@ -1503,7 +1502,6 @@ def catch_up(db, check_asset_conservation=True): fetcher = start_rsfetcher() else: assert parsed_block_index == block_height - mempool.clean_mempool(db) parsed_blocks += 1 formatted_duration = util.format_duration(time.time() - start_time) diff --git a/counterparty-core/counterpartycore/lib/follow.py b/counterparty-core/counterpartycore/lib/follow.py index ba9646e0c6..af287d7f74 100644 --- a/counterparty-core/counterpartycore/lib/follow.py +++ b/counterparty-core/counterpartycore/lib/follow.py @@ -138,7 +138,8 @@ def receive_rawblock(self, body): blocks.catch_up(self.db, check_asset_conservation=False) else: blocks.parse_new_block(self.db, decoded_block) - mempool.clean_mempool(self.db) + if not config.NO_MEMPOOL: + mempool.clean_mempool(self.db) if not config.NO_TELEMETRY: TelemetryOneShot().submit() diff --git a/release-notes/release-notes-v10.9.0.md b/release-notes/release-notes-v10.9.0.md index d994036250..ee00c1173b 100644 --- a/release-notes/release-notes-v10.9.0.md +++ b/release-notes/release-notes-v10.9.0.md @@ -42,6 +42,7 @@ The following transaction construction parameters have been deprecated (but rema - Have the node terminate when the `follow` loop raises an error - Don't stop the server on "No such mempool or blockchain" error - Handle correctly RPC call errors from the API +- Don't clean mempool on catchup ## Codebase From 00edf5fba45a6e7e74b19624711876da5accdabf Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Mon, 13 Jan 2025 09:47:10 +0000 Subject: [PATCH 22/31] Retry 5 times when getting invalid Json with status 200 from Bitcoin Core --- .../counterpartycore/lib/backend/bitcoind.py | 23 ++++++++++++++----- release-notes/release-notes-v10.9.0.md | 1 + 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/counterparty-core/counterpartycore/lib/backend/bitcoind.py b/counterparty-core/counterpartycore/lib/backend/bitcoind.py index 4b68c15859..bdb40653f1 100644 --- a/counterparty-core/counterpartycore/lib/backend/bitcoind.py +++ b/counterparty-core/counterpartycore/lib/backend/bitcoind.py @@ -26,6 +26,22 @@ def should_retry(): return True +def get_json_response(response, retry=0): + try: + return response.json() + except json.decoder.JSONDecodeError as e: # noqa: F841 + if response.status_code == 200: + logger.warning( + f"Received invalid JSON with status 200 from Bitcoin Core: {response.text}. Retrying in 5 seconds..." + ) + time.sleep(5) + if retry < 5: + return get_json_response(response, retry=retry + 1) + raise exceptions.BitcoindRPCError( # noqa: B904 + f"Received invalid JSON from backend with a response of {str(response.status_code)}: {response.text}" + ) from e + + def rpc_call(payload, retry=0): """Calls to bitcoin core and returns the response""" url = config.BACKEND_URL @@ -72,12 +88,7 @@ def rpc_call(payload, retry=0): raise broken_error # Handle json decode errors - try: - response_json = response.json() - except json.decoder.JSONDecodeError as e: # noqa: F841 - raise exceptions.BitcoindRPCError( # noqa: B904 - f"Received invalid JSON from backend with a response of {str(response.status_code) + ' ' + response.reason}" - ) from e + response_json = get_json_response(response) # Batch query returns a list if isinstance(response_json, list): diff --git a/release-notes/release-notes-v10.9.0.md b/release-notes/release-notes-v10.9.0.md index ee00c1173b..46d24d250b 100644 --- a/release-notes/release-notes-v10.9.0.md +++ b/release-notes/release-notes-v10.9.0.md @@ -43,6 +43,7 @@ The following transaction construction parameters have been deprecated (but rema - Don't stop the server on "No such mempool or blockchain" error - Handle correctly RPC call errors from the API - Don't clean mempool on catchup +- Retry 5 times when getting invalid Json with status 200 from Bitcoin Core ## Codebase From de93828f9ce7579141eea3a434fae442a465ea66 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Mon, 13 Jan 2025 10:03:17 +0000 Subject: [PATCH 23/31] Add mainnet checkoint for block 879058 and testnet4 checkpoint for block 64493 --- counterparty-core/counterpartycore/lib/check.py | 8 ++++++++ release-notes/release-notes-v10.9.0.md | 1 + 2 files changed, 9 insertions(+) diff --git a/counterparty-core/counterpartycore/lib/check.py b/counterparty-core/counterpartycore/lib/check.py index 3b5d5c064b..f475f44116 100644 --- a/counterparty-core/counterpartycore/lib/check.py +++ b/counterparty-core/counterpartycore/lib/check.py @@ -695,6 +695,10 @@ "ledger_hash": "4c4d6b660af23bb03a04bbf93ddd0a4b8e615dd7b883ecf827274cabe658bfc2", "txlist_hash": "f6a99d60337c33c1822c048f56e241455cd7e45bb5a9515096f1ac609d50f669", }, + 879058: { + "ledger_hash": "e6bf730a18c148adbd2cce9bd0f361e595e44d53fa98a9d9bdbf4c944f6c233b", + "txlist_hash": "e9946ac128405885f251fbb98a952ed554ea6cc25973261da9f40eabf4f3b429", + }, } CONSENSUS_HASH_VERSION_TESTNET = 7 @@ -875,6 +879,10 @@ "ledger_hash": "33cf0669a0d309d7e6b1bf79494613b69262b58c0ea03c9c221d955eb4c84fe5", "txlist_hash": "33cf0669a0d309d7e6b1bf79494613b69262b58c0ea03c9c221d955eb4c84fe5", }, + 64493: { + "ledger_hash": "af481088fb9303d0f61543fb3646110fcd27d5ebd3ac40e04397000320216699", + "txlist_hash": "4b610518a76f59e8f98922f37434be9bb5567040afebabda6ee69b4f92b80434", + }, } diff --git a/release-notes/release-notes-v10.9.0.md b/release-notes/release-notes-v10.9.0.md index 46d24d250b..bdb6a7ef71 100644 --- a/release-notes/release-notes-v10.9.0.md +++ b/release-notes/release-notes-v10.9.0.md @@ -60,6 +60,7 @@ The following transaction construction parameters have been deprecated (but rema - Add `testnet4` support - Repeat the RPC call to Bitcoin Core indefinitely until it succeeds - Raise a specific `BlockOutOfRange` error when querying an unknown block +- Add mainnet checkoint for block 879058 and testnet4 checkpoint for block 64493 ## API From 6e2b6c75ab316e3d079ff5f156d8a4d9b9c6e35d Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Mon, 13 Jan 2025 10:04:16 +0000 Subject: [PATCH 24/31] fix typo --- release-notes/release-notes-v10.9.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-notes/release-notes-v10.9.0.md b/release-notes/release-notes-v10.9.0.md index bdb6a7ef71..6c1669d9b2 100644 --- a/release-notes/release-notes-v10.9.0.md +++ b/release-notes/release-notes-v10.9.0.md @@ -60,7 +60,7 @@ The following transaction construction parameters have been deprecated (but rema - Add `testnet4` support - Repeat the RPC call to Bitcoin Core indefinitely until it succeeds - Raise a specific `BlockOutOfRange` error when querying an unknown block -- Add mainnet checkoint for block 879058 and testnet4 checkpoint for block 64493 +- Add mainnet checkpoint for block 879058 and testnet4 checkpoint for block 64493 ## API From 5290fb51299ab241a23b00c16396ab9b35a1095d Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Mon, 13 Jan 2025 21:14:28 +0000 Subject: [PATCH 25/31] Add test for utxo move in mempool --- .../regtest/scenarios/scenario_23_detach.py | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/counterparty-core/counterpartycore/test/regtest/scenarios/scenario_23_detach.py b/counterparty-core/counterpartycore/test/regtest/scenarios/scenario_23_detach.py index aa253b1713..f44d1f1ea2 100644 --- a/counterparty-core/counterpartycore/test/regtest/scenarios/scenario_23_detach.py +++ b/counterparty-core/counterpartycore/test/regtest/scenarios/scenario_23_detach.py @@ -173,4 +173,52 @@ } ], }, + { + "title": "Attach DETACHA asset to UTXO", + "transaction": "attach", + "source": "$ADDRESS_10", + "params": { + "asset": "DETACHA", + "quantity": 1 * 10**8, + }, + "set_variables": { + "ATTACH2_DETACHA_TX_HASH": "$TX_HASH", + }, + "controls": [], + }, + { + "title": "Move no confirmation", + "transaction": "movetoutxo", + "no_confirmation": True, + "source": "$ATTACH2_DETACHA_TX_HASH:0", + "params": { + "destination": "$ADDRESS_9", + "quantity": 1 * 10**8, + }, + "controls": [ + { + "url": "mempool/events?event_name=UTXO_MOVE", + "result": [ + { + "event": "UTXO_MOVE", + "params": { + "asset": "DETACHA", + "block_index": 9999999, + "destination": "$TX_HASH:0", + "destination_address": "$ADDRESS_9", + "msg_index": 0, + "quantity": 100000000, + "send_type": "move", + "source": "$ATTACH2_DETACHA_TX_HASH:0", + "source_address": "$ADDRESS_10", + "status": "valid", + "tx_hash": "$TX_HASH", + "tx_index": "$TX_INDEX", + }, + "tx_hash": "$TX_HASH", + } + ], + } + ], + }, ] From de28243f8b70cb8fb3ce3952dd3cd9da3ace8d19 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Tue, 14 Jan 2025 17:52:50 +0000 Subject: [PATCH 26/31] Bump version --- apiary.apib | 2 +- counterparty-core/counterpartycore/lib/config.py | 6 +++--- .../test/regtest/apidoc/blueprint-template.md | 2 +- counterparty-core/requirements.txt | 2 +- counterparty-rs/Cargo.lock | 2 +- counterparty-rs/Cargo.toml | 2 +- counterparty-wallet/requirements.txt | 2 +- docker-compose.yml | 2 +- release-notes/release-notes-v10.9.0.md | 2 +- 9 files changed, 11 insertions(+), 11 deletions(-) diff --git a/apiary.apib b/apiary.apib index 9c5e7a59ba..3a021154f6 100644 --- a/apiary.apib +++ b/apiary.apib @@ -1466,7 +1466,7 @@ Returns server information and the list of documented routes in JSON format. "result": { "server_ready": true, "network": "mainnet", - "version": "10.9.0-rc.1", + "version": "10.9.0", "backend_height": 850214, "counterparty_height": 850214, "documentation": "https://counterpartycore.docs.apiary.io/", diff --git a/counterparty-core/counterpartycore/lib/config.py b/counterparty-core/counterpartycore/lib/config.py index 18d9684f1c..b0e2415bc4 100644 --- a/counterparty-core/counterpartycore/lib/config.py +++ b/counterparty-core/counterpartycore/lib/config.py @@ -5,7 +5,7 @@ # Semantic Version -__version__ = "10.9.0-rc.1" # for hatch +__version__ = "10.9.0" # for hatch VERSION_STRING = __version__ version = VERSION_STRING.split("-")[0].split(".") VERSION_MAJOR = int(version[0]) @@ -30,8 +30,8 @@ ] NEED_REPARSE_IF_MINOR_IS_LESS_THAN_TESTNET4 = None -NEED_ROLLBACK_IF_MINOR_IS_LESS_THAN = [(8, 871780)] -NEED_ROLLBACK_IF_MINOR_IS_LESS_THAN_TESTNET = [(8, 3522632)] +NEED_ROLLBACK_IF_MINOR_IS_LESS_THAN = [(8, 871780), (9, 871780)] +NEED_ROLLBACK_IF_MINOR_IS_LESS_THAN_TESTNET = [(8, 3522632), (9, 3522632)] NEED_ROLLBACK_IF_MINOR_IS_LESS_THAN_TESTNET4 = None STATE_DB_NEED_REFRESH_ON_VERSION_UPDATE = ["10.9.0-rc.1", "10.9.0"] diff --git a/counterparty-core/counterpartycore/test/regtest/apidoc/blueprint-template.md b/counterparty-core/counterpartycore/test/regtest/apidoc/blueprint-template.md index 3777c0b0a5..88fb9ded85 100644 --- a/counterparty-core/counterpartycore/test/regtest/apidoc/blueprint-template.md +++ b/counterparty-core/counterpartycore/test/regtest/apidoc/blueprint-template.md @@ -165,7 +165,7 @@ Returns server information and the list of documented routes in JSON format. "result": { "server_ready": true, "network": "mainnet", - "version": "10.9.0-rc.1", + "version": "10.9.0", "backend_height": 850214, "counterparty_height": 850214, "documentation": "https://counterpartycore.docs.apiary.io/", diff --git a/counterparty-core/requirements.txt b/counterparty-core/requirements.txt index 477670203d..3554ef0dae 100644 --- a/counterparty-core/requirements.txt +++ b/counterparty-core/requirements.txt @@ -38,4 +38,4 @@ hypothesis==6.116.0 bitcoin-utils==0.7.1 pyzstd==0.16.2 dredd_hooks==0.2.0 -counterparty-rs==10.9.0-rc.1 +counterparty-rs==10.9.0 diff --git a/counterparty-rs/Cargo.lock b/counterparty-rs/Cargo.lock index 267a4066a6..e83c558ae8 100644 --- a/counterparty-rs/Cargo.lock +++ b/counterparty-rs/Cargo.lock @@ -394,7 +394,7 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "counterparty-rs" -version = "10.9.0-rc.1" +version = "10.9.0" dependencies = [ "bip32", "bitcoin", diff --git a/counterparty-rs/Cargo.toml b/counterparty-rs/Cargo.toml index 67695d2a8c..32faad9764 100644 --- a/counterparty-rs/Cargo.toml +++ b/counterparty-rs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "counterparty-rs" -version = "10.9.0-rc.1" +version = "10.9.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/counterparty-wallet/requirements.txt b/counterparty-wallet/requirements.txt index 0bf1e683ba..96cc1999f1 100644 --- a/counterparty-wallet/requirements.txt +++ b/counterparty-wallet/requirements.txt @@ -5,4 +5,4 @@ colorlog==6.8.0 python-dateutil==2.8.2 requests==2.32.0 termcolor==2.4.0 -counterparty-core==10.9.0-rc.1 +counterparty-core==10.9.0 diff --git a/docker-compose.yml b/docker-compose.yml index 33c42103d3..3062f27330 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -6,7 +6,7 @@ x-bitcoind-common: &bitcoind-common restart: unless-stopped x-counterparty-common: &counterparty-common - image: counterparty/counterparty:v10.9.0-rc.1 + image: counterparty/counterparty:v10.9.0 stop_grace_period: 1m volumes: - data:/root/.bitcoin diff --git a/release-notes/release-notes-v10.9.0.md b/release-notes/release-notes-v10.9.0.md index 6c1669d9b2..8dd2c63e76 100644 --- a/release-notes/release-notes-v10.9.0.md +++ b/release-notes/release-notes-v10.9.0.md @@ -1,4 +1,4 @@ -# Release Notes - Counterparty Core v10.9.0 (2025-01-??) +# Release Notes - Counterparty Core v10.9.0 (2025-01-14) This release represents a major technical milestone in the development of Counterparty Core: Counterparty no longer has AddrIndexRs as an external dependency. Originally, AddrIndexRs was used for transaction construction, and at the end of 2023 it was accidentally turned into a consensus-critical dependency (causing a number of subsequent consensus breaks and reliability issues). As of today, the only external dependency for a Counterparty node is Bitcoin Core itself. From 70d20e0efad9c0590cfadf3590dcb85b0e7735da Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Tue, 14 Jan 2025 20:56:44 +0000 Subject: [PATCH 27/31] No RPC retry from mempool follower --- .../counterpartycore/lib/backend/bitcoind.py | 8 ++++---- counterparty-core/counterpartycore/lib/follow.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/counterparty-core/counterpartycore/lib/backend/bitcoind.py b/counterparty-core/counterpartycore/lib/backend/bitcoind.py index bdb40653f1..39d3fe9fba 100644 --- a/counterparty-core/counterpartycore/lib/backend/bitcoind.py +++ b/counterparty-core/counterpartycore/lib/backend/bitcoind.py @@ -137,8 +137,8 @@ def is_api_request(): return False -def rpc(method, params): - if is_api_request(): +def rpc(method, params, no_retry=False): + if is_api_request() or no_retry: return safe_rpc(method, params) payload = { @@ -199,8 +199,8 @@ def convert_to_psbt(rawtx): @functools.lru_cache(maxsize=10000) -def getrawtransaction(tx_hash, verbose=False): - return rpc("getrawtransaction", [tx_hash, 1 if verbose else 0]) +def getrawtransaction(tx_hash, verbose=False, no_retry=False): + return rpc("getrawtransaction", [tx_hash, 1 if verbose else 0], no_retry=no_retry) def getrawtransaction_batch(tx_hashes, verbose=False, return_dict=False): diff --git a/counterparty-core/counterpartycore/lib/follow.py b/counterparty-core/counterpartycore/lib/follow.py index af287d7f74..0dc17a3233 100644 --- a/counterparty-core/counterpartycore/lib/follow.py +++ b/counterparty-core/counterpartycore/lib/follow.py @@ -185,7 +185,7 @@ def receive_sequence(self, body): raw_tx = self.raw_tx_cache.get(item_hash) if raw_tx is None: try: - raw_tx = backend.bitcoind.getrawtransaction(item_hash) + raw_tx = backend.bitcoind.getrawtransaction(item_hash, no_retry=True) except exceptions.BitcoindRPCError: logger.trace("Transaction not found in bitcoind: %s", item_hash) return From 69bf7451231a287758cb08c57cecfa3cf8b93155 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Tue, 14 Jan 2025 21:00:02 +0000 Subject: [PATCH 28/31] update release notes --- release-notes/release-notes-v10.9.0.md | 1 + 1 file changed, 1 insertion(+) diff --git a/release-notes/release-notes-v10.9.0.md b/release-notes/release-notes-v10.9.0.md index 6c1669d9b2..2322746c71 100644 --- a/release-notes/release-notes-v10.9.0.md +++ b/release-notes/release-notes-v10.9.0.md @@ -44,6 +44,7 @@ The following transaction construction parameters have been deprecated (but rema - Handle correctly RPC call errors from the API - Don't clean mempool on catchup - Retry 5 times when getting invalid Json with status 200 from Bitcoin Core +- Don't retry RPC call when parsing Mempool transactions ## Codebase From d64757c57f295875071dab17f3f00683e60f739e Mon Sep 17 00:00:00 2001 From: Adam Krellenstein Date: Tue, 14 Jan 2025 16:10:48 -0500 Subject: [PATCH 29/31] typo --- release-notes/release-notes-v10.9.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-notes/release-notes-v10.9.0.md b/release-notes/release-notes-v10.9.0.md index 2322746c71..55eedd810d 100644 --- a/release-notes/release-notes-v10.9.0.md +++ b/release-notes/release-notes-v10.9.0.md @@ -44,7 +44,7 @@ The following transaction construction parameters have been deprecated (but rema - Handle correctly RPC call errors from the API - Don't clean mempool on catchup - Retry 5 times when getting invalid Json with status 200 from Bitcoin Core -- Don't retry RPC call when parsing Mempool transactions +- Don't retry RPC call when parsing mempool transactions ## Codebase From 0bed8366c08df6b3ee3c32fc4bf7ca5f5844f79a Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Wed, 15 Jan 2025 09:43:21 +0000 Subject: [PATCH 30/31] Add regtest tests for RBF --- .../counterpartycore/lib/follow.py | 3 +- .../test/regtest/regtestnode.py | 65 +++++++++++++++++++ .../test/regtest/testscenarios.py | 2 + 3 files changed, 69 insertions(+), 1 deletion(-) diff --git a/counterparty-core/counterpartycore/lib/follow.py b/counterparty-core/counterpartycore/lib/follow.py index 0dc17a3233..e050b433ea 100644 --- a/counterparty-core/counterpartycore/lib/follow.py +++ b/counterparty-core/counterpartycore/lib/follow.py @@ -187,7 +187,7 @@ def receive_sequence(self, body): try: raw_tx = backend.bitcoind.getrawtransaction(item_hash, no_retry=True) except exceptions.BitcoindRPCError: - logger.trace("Transaction not found in bitcoind: %s", item_hash) + logger.warning("Transaction not found in bitcoind: %s", item_hash) return # add transaction to mempool block # logger.trace("Adding transaction to mempool block: %s", item_hash) @@ -205,6 +205,7 @@ def receive_sequence(self, body): logger.trace("Waiting for new transactions in the mempool or a new block...") # transaction removed from mempool for non-block inclusion reasons elif label == "R": + logger.debug("Removing transaction from mempool: %s", item_hash) mempool.clean_transaction_events(self.db, item_hash) def receive_message(self, topic, body, seq): diff --git a/counterparty-core/counterpartycore/test/regtest/regtestnode.py b/counterparty-core/counterpartycore/test/regtest/regtestnode.py index 2619a24e26..99d6e7f2a2 100644 --- a/counterparty-core/counterpartycore/test/regtest/regtestnode.py +++ b/counterparty-core/counterpartycore/test/regtest/regtestnode.py @@ -361,6 +361,7 @@ def start_bitcoin_node(self): "-acceptnonstdtxn", "-minrelaytxfee=0", "-blockmintxfee=0", + "-mempoolfullrbf", f"-datadir={self.datadir}", _bg=True, _out=sys.stdout, @@ -384,6 +385,7 @@ def start_bitcoin_node_2(self): "-minrelaytxfee=0", "-blockmintxfee=0", "-bind=127.0.0.1:2223=onion", + "-mempoolfullrbf", _bg=True, _out=sys.stdout, ) @@ -1134,6 +1136,69 @@ def test_fee_calculation(self): ) assert size * 3 - 3 <= unsigned_tx["btc_fee"] <= size * 3 + 3 + def test_rbf(self): + self.start_and_wait_second_node() + + unsigned_tx = self.compose( + self.addresses[0], + "send", + { + "destination": self.addresses[1], + "quantity": 1, + "asset": "XCP", + "exact_fee": 1, + "verbose": True, + "validate": False, + }, + )["result"] + transaction = Transaction.from_raw(unsigned_tx["rawtransaction"]) + raw_hexs = [] + # create 10 transactions with increasing fees + for _i in range(10): + transaction.outputs[1].amount -= 170 + new_raw_transaction = transaction.to_hex() + signed_tx = json.loads( + self.bitcoin_wallet("signrawtransactionwithwallet", new_raw_transaction).strip() + )["hex"] + raw_hexs.append(signed_tx) + + # check that no transaction is in the mempool + mempool_event_count_before = self.api_call("mempool/events?event_name=TRANSACTION_PARSED")[ + "result_count" + ] + assert mempool_event_count_before == 0 + + # broadcast the transactions to the two nodes + tx_hahses = [] + for i, raw_hex in enumerate(raw_hexs): + tx_hash = self.bitcoin_wallet("sendrawtransaction", raw_hex, 0).strip() + tx_hahses.append(tx_hash) + print(f"Transaction {i} sent: {tx_hash}") + + # check that all transactions are in the mempool + mempool_event_count_after = self.api_call("mempool/events?event_name=TRANSACTION_PARSED")[ + "result_count" + ] + while mempool_event_count_after == 0: + time.sleep(1) + mempool_event_count_after = self.api_call( + "mempool/events?event_name=TRANSACTION_PARSED" + )["result_count"] + time.sleep(10) + + print("Mempool event count: ", mempool_event_count_after) + + # only one event should be in the mempool + assert mempool_event_count_after == 1 + # check that RBFed transactions are removed from the mempool + for tx_hash in tx_hahses[:-1]: + assert f"Removing transaction from mempool: {tx_hash}" in self.server_out.getvalue() + event = self.api_call("mempool/events?event_name=TRANSACTION_PARSED")["result"][0] + # check that the last transaction is the one in the mempool + assert event["tx_hash"] == tx_hahses[-1] + + print("RBF test successful") + class RegtestNodeThread(threading.Thread): def __init__(self, wsgi_server="waitress", burn_in_one_block=True): diff --git a/counterparty-core/counterpartycore/test/regtest/testscenarios.py b/counterparty-core/counterpartycore/test/regtest/testscenarios.py index b53a19bb68..dce67245b0 100644 --- a/counterparty-core/counterpartycore/test/regtest/testscenarios.py +++ b/counterparty-core/counterpartycore/test/regtest/testscenarios.py @@ -445,6 +445,8 @@ def run_scenarios(serve=False, wsgi_server="gunicorn"): regtest_node_thread.node.test_electrs() print("Testing fee calculation...") regtest_node_thread.node.test_fee_calculation() + print("Testing RBF...") + regtest_node_thread.node.test_rbf() except KeyboardInterrupt: print(regtest_node_thread.node.server_out.getvalue()) pass From 62ce9338c4aa2d22b3e47a41c06d146d10ee50a2 Mon Sep 17 00:00:00 2001 From: Adam Krellenstein Date: Wed, 15 Jan 2025 09:34:28 -0500 Subject: [PATCH 31/31] Bump Date for Release --- release-notes/release-notes-v10.9.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-notes/release-notes-v10.9.0.md b/release-notes/release-notes-v10.9.0.md index 8dd2c63e76..4f2ec4df2c 100644 --- a/release-notes/release-notes-v10.9.0.md +++ b/release-notes/release-notes-v10.9.0.md @@ -1,4 +1,4 @@ -# Release Notes - Counterparty Core v10.9.0 (2025-01-14) +# Release Notes - Counterparty Core v10.9.0 (2025-01-15) This release represents a major technical milestone in the development of Counterparty Core: Counterparty no longer has AddrIndexRs as an external dependency. Originally, AddrIndexRs was used for transaction construction, and at the end of 2023 it was accidentally turned into a consensus-critical dependency (causing a number of subsequent consensus breaks and reliability issues). As of today, the only external dependency for a Counterparty node is Bitcoin Core itself.