diff --git a/.github/workflows/test_compose.sh b/.github/workflows/test_compose.sh index 5ab69742db..decb3d9111 100644 --- a/.github/workflows/test_compose.sh +++ b/.github/workflows/test_compose.sh @@ -3,7 +3,7 @@ set -e set -x -#exit 0 +exit 0 export PATH="/snap/bin:$PATH" diff --git a/.github/workflows/test_compose.yml b/.github/workflows/test_compose.yml deleted file mode 100644 index af016b5c87..0000000000 --- a/.github/workflows/test_compose.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Docker Compose - -on: - push: - branches: ['develop', 'master', 'preprelease'] - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Get branch names. - id: branch-names - uses: tj-actions/branch-names@v8 - - uses: alinz/ssh-scp-action@master - with: - key: ${{ secrets.TEST_SERVER_KEY }} - host: ${{ secrets.TEST_SERVER_IP }} - user: ${{ secrets.TEST_SERVER_USER }} - ssh_before: | - rm -f test_compose.sh - scp: | - .github/workflows/test_compose.sh ${{ secrets.TEST_SERVER_USER }}@${{ secrets.TEST_SERVER_IP }}:~/test_compose.sh - ssh_after: | - sh test_compose.sh ${{ steps.branch-names.outputs.current_branch }} diff --git a/apiary.apib b/apiary.apib index da8ebaf086..da13d18863 100644 --- a/apiary.apib +++ b/apiary.apib @@ -1456,7 +1456,7 @@ Returns server information and the list of documented routes in JSON format. "result": { "server_ready": true, "network": "mainnet", - "version": "10.8.0-rc.2", + "version": "10.8.0", "backend_height": 850214, "counterparty_height": 850214, "documentation": "https://counterpartycore.docs.apiary.io/", diff --git a/counterparty-core/counterpartycore/lib/api/api_server.py b/counterparty-core/counterpartycore/lib/api/api_server.py index 1c0b851116..9ec3d6854f 100644 --- a/counterparty-core/counterpartycore/lib/api/api_server.py +++ b/counterparty-core/counterpartycore/lib/api/api_server.py @@ -363,6 +363,8 @@ def handle_route(**kwargs): except Exception as e: capture_exception(e) logger.error("Error in API: %s", e) + # import traceback + # print(traceback.format_exc()) return return_result( 503, error="Unknown error", start_time=start_time, query_args=query_args ) diff --git a/counterparty-core/counterpartycore/lib/backend/addrindexrs.py b/counterparty-core/counterpartycore/lib/backend/addrindexrs.py index 08e84e679c..fa2d6031ba 100644 --- a/counterparty-core/counterpartycore/lib/backend/addrindexrs.py +++ b/counterparty-core/counterpartycore/lib/backend/addrindexrs.py @@ -692,6 +692,10 @@ def get_oldest_tx(self, address, block_index, timeout=ADDRINDEXRS_CLIENT_TIMEOUT "block_index": 820321, "tx_hash": "b61ac3ab1ba9d63d484e8f83e8b9607bd932c8f4b742095445c3527ab575d972", }, # {} + "842375-bc1q2rn0c89mylzj6c26exda5y2nkezmzh4lh5rkkl": { + "block_index": 842374, + "tx_hash": "3043db292b7f64cbe1cafa32bc733316b162d8a1f41f31a1e6cd224fd5b72415", + }, } ADDRINDEXRS_CLIENT = None diff --git a/counterparty-core/counterpartycore/lib/backend/bitcoind.py b/counterparty-core/counterpartycore/lib/backend/bitcoind.py index 376306a772..2d3551fb3e 100644 --- a/counterparty-core/counterpartycore/lib/backend/bitcoind.py +++ b/counterparty-core/counterpartycore/lib/backend/bitcoind.py @@ -87,7 +87,7 @@ def rpc_call(payload, retry=0): f"{response_json['error']} Is `txindex` enabled in {config.BTC_NAME} Core?" ) elif response_json["error"]["code"] in [-28, -8, -2]: - # "Verifying blocks..." or "Block height out of range" or "The network does not appear to fully agree!" + # "Verifying blocks..." or "Block height out of range" or "The network does not appear to fully agree!"" logger.debug(f"Backend not ready. Sleeping for ten seconds. ({response_json['error']})") logger.debug(f"Payload: {payload}") if retry >= 10: @@ -101,8 +101,15 @@ def rpc_call(payload, retry=0): else: raise exceptions.BitcoindRPCError(response_json["error"]["message"]) + if isinstance(payload, dict): + method = payload["method"] + elif isinstance(payload, list): + method = payload[0]["method"] + else: + method = "unknown" elapsed = time.time() - start_time - logger.trace(f"Bitcoin Core RPC call {payload['method']} took {elapsed:.3f}s") + logger.trace(f"Bitcoin Core RPC call {method} took {elapsed:.3f}s") + return result diff --git a/counterparty-core/counterpartycore/lib/check.py b/counterparty-core/counterpartycore/lib/check.py index 412fc0f89f..28410f317f 100644 --- a/counterparty-core/counterpartycore/lib/check.py +++ b/counterparty-core/counterpartycore/lib/check.py @@ -691,6 +691,10 @@ "ledger_hash": "8e55b64d0dfd85a58e3a9dd27ce49efd98559d96f16f53beb66a10b7671ea857", "txlist_hash": "b3f549168f56702287c7b06c0348c4ac0adffcd219bab386d2f19326c0cd491c", }, + 874883: { + "ledger_hash": "4c4d6b660af23bb03a04bbf93ddd0a4b8e615dd7b883ecf827274cabe658bfc2", + "txlist_hash": "f6a99d60337c33c1822c048f56e241455cd7e45bb5a9515096f1ac609d50f669", + }, } CONSENSUS_HASH_VERSION_TESTNET = 7 @@ -1114,5 +1118,6 @@ def database_version(db): message = "`VERSION_STRING` not found in dataase. " else: message = f"Client pre-release version number mismatch: {version_pre_release} ≠ {config.VERSION_PRE_RELEASE}. " - message += "Checking if a reparse is needed..." + message += "Checking if a rollback or a reparse is needed..." + check_need_rollback(version_minor, message) check_need_reparse(version_minor, message) diff --git a/counterparty-core/counterpartycore/lib/config.py b/counterparty-core/counterpartycore/lib/config.py index ff0f656ddd..300a8bf12e 100644 --- a/counterparty-core/counterpartycore/lib/config.py +++ b/counterparty-core/counterpartycore/lib/config.py @@ -5,7 +5,7 @@ # Semantic Version -__version__ = "10.8.0-rc.2" # for hatch +__version__ = "10.8.0" # for hatch VERSION_STRING = __version__ version = VERSION_STRING.split("-")[0].split(".") VERSION_MAJOR = int(version[0]) @@ -155,7 +155,7 @@ # Custom exit codes EXITCODE_UPDATE_REQUIRED = 5 -BACKEND_RAW_TRANSACTIONS_CACHE_SIZE = 20000 +BACKEND_RAW_TRANSACTIONS_CACHE_SIZE = 1000 BACKEND_RPC_BATCH_NUM_WORKERS = 6 DEFAULT_UTXO_LOCKS_MAX_ADDRESSES = 1000 diff --git a/counterparty-core/counterpartycore/lib/gettxinfo.py b/counterparty-core/counterpartycore/lib/gettxinfo.py index 2eae2f109d..9c7b18a218 100644 --- a/counterparty-core/counterpartycore/lib/gettxinfo.py +++ b/counterparty-core/counterpartycore/lib/gettxinfo.py @@ -741,7 +741,8 @@ def get_op_return_vout(decoded_tx): KNOWN_SOURCES = { - "92ad58f5aa35c503489efbdd2a466e942baa9ac5cd67cb7544adf03e47a457d0": "a71da7169db3672408c7b25f84be425839548e63fa480c0478f91e3c2aa3ec67:0" + "92ad58f5aa35c503489efbdd2a466e942baa9ac5cd67cb7544adf03e47a457d0": "a71da7169db3672408c7b25f84be425839548e63fa480c0478f91e3c2aa3ec67:0", + "c80143886181ebbc782d23a50acca0f5ea7ac005d3164d7c76fc5e14f72d47c8": "", } diff --git a/counterparty-core/counterpartycore/lib/mempool.py b/counterparty-core/counterpartycore/lib/mempool.py index 65f4692711..909fdfafcc 100644 --- a/counterparty-core/counterpartycore/lib/mempool.py +++ b/counterparty-core/counterpartycore/lib/mempool.py @@ -131,22 +131,22 @@ def parse_raw_mempool(db): timestamps = {} cursor = db.cursor() logger.debug(f"Found {len(raw_mempool)} transaction(s) in the mempool...") + txhash_list = [] for txid, tx_info in raw_mempool.items(): existing_tx_in_mempool = cursor.execute( "SELECT * FROM mempool WHERE tx_hash = ? LIMIT 1", (txid,) ).fetchone() if existing_tx_in_mempool: continue - try: - logger.trace(f"Getting raw transaction `{txid}` from the mempool...") - raw_tx = backend.bitcoind.getrawtransaction(txid) - raw_tx_list.append(raw_tx) - timestamps[txid] = tx_info["time"] - except exceptions.BitcoindRPCError as e: - if "No such mempool or blockchain transaction" in str(e): - pass - else: - raise e + txhash_list.append(txid) + timestamps[txid] = tx_info["time"] + + logger.debug(f"Getting {len(txhash_list)} raw transactions by batch from the mempool...") + raw_transactions_by_hash = backend.addrindexrs.getrawtransaction_batch( + txhash_list, skip_missing=True + ) + raw_tx_list = [raw_hex for raw_hex in raw_transactions_by_hash.values() if raw_hex is not None] + logger.debug(f"Parsing {len(raw_tx_list)} transaction(s) from the mempool...") parse_mempool_transactions(db, raw_tx_list, timestamps) logger.debug("Raw mempool parsed successfully.") diff --git a/counterparty-core/counterpartycore/server.py b/counterparty-core/counterpartycore/server.py index 7b0c51d8a7..087c3358f3 100755 --- a/counterparty-core/counterpartycore/server.py +++ b/counterparty-core/counterpartycore/server.py @@ -988,13 +988,18 @@ def bootstrap(no_confirm=False, snapshot_url=None): if config.TESTNET: ledger_database_path += ".testnet" ledger_database_path += ".db" - api_database_path = ledger_database_path.replace(".db", ".api.db") + + old_api_database_path = ledger_database_path.replace(".db", ".api.db") + if config.TESTNET: + api_database_path = os.path.join(config.DATA_DIR, "state.testnet.db") + else: + api_database_path = os.path.join(config.DATA_DIR, "state.db") # Prepare Directory. if not os.path.exists(config.DATA_DIR): os.makedirs(config.DATA_DIR, mode=0o755) - for database_path in [ledger_database_path, api_database_path]: + for database_path in [ledger_database_path, api_database_path, old_api_database_path]: if os.path.exists(database_path): os.remove(database_path) # Delete SQLite Write-Ahead-Log @@ -1037,10 +1042,13 @@ def bootstrap_progress(blocknum, blocksize, totalsize): tar_file.extractall(path=config.DATA_DIR) # nosec B202 # noqa: S202 assert os.path.exists(ledger_database_path) - assert os.path.exists(api_database_path) + assert os.path.exists(api_database_path) or os.path.exists(old_api_database_path) # user and group have "rw" access os.chmod(ledger_database_path, 0o660) # nosec B103 - os.chmod(api_database_path, 0o660) # nosec B103 + if os.path.exists(api_database_path): + os.chmod(api_database_path, 0o660) # nosec B103 + if os.path.exists(old_api_database_path): + os.chmod(old_api_database_path, 0o660) # nosec B103 with log.Spinner("Cleaning up..."): os.remove(tarball_path) diff --git a/counterparty-core/counterpartycore/test/regtest/apidoc/blueprint-template.md b/counterparty-core/counterpartycore/test/regtest/apidoc/blueprint-template.md index 5ae6214d8c..f0531fe2de 100644 --- a/counterparty-core/counterpartycore/test/regtest/apidoc/blueprint-template.md +++ b/counterparty-core/counterpartycore/test/regtest/apidoc/blueprint-template.md @@ -165,7 +165,7 @@ Returns server information and the list of documented routes in JSON format. "result": { "server_ready": true, "network": "mainnet", - "version": "10.8.0-rc.2", + "version": "10.8.0", "backend_height": 850214, "counterparty_height": 850214, "documentation": "https://counterpartycore.docs.apiary.io/", diff --git a/counterparty-core/requirements.txt b/counterparty-core/requirements.txt index d85df46ed7..a7f36ff46a 100644 --- a/counterparty-core/requirements.txt +++ b/counterparty-core/requirements.txt @@ -36,4 +36,4 @@ gunicorn==23.0.0 waitress==3.0.1 hypothesis==6.116.0 bitcoin-utils==0.7.1 -counterparty-rs==10.8.0-rc.2 +counterparty-rs==10.8.0 diff --git a/counterparty-rs/Cargo.lock b/counterparty-rs/Cargo.lock index 3d224c55aa..1ebfe31532 100644 --- a/counterparty-rs/Cargo.lock +++ b/counterparty-rs/Cargo.lock @@ -394,7 +394,7 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "counterparty-rs" -version = "10.8.0-rc.2" +version = "10.8.0" dependencies = [ "bip32", "bitcoin", diff --git a/counterparty-rs/Cargo.toml b/counterparty-rs/Cargo.toml index 3154735637..4a8753548c 100644 --- a/counterparty-rs/Cargo.toml +++ b/counterparty-rs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "counterparty-rs" -version = "10.8.0-rc.2" +version = "10.8.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/counterparty-wallet/requirements.txt b/counterparty-wallet/requirements.txt index 27c6f0a661..6c51606c7d 100644 --- a/counterparty-wallet/requirements.txt +++ b/counterparty-wallet/requirements.txt @@ -5,4 +5,4 @@ colorlog==6.8.0 python-dateutil==2.8.2 requests==2.32.0 termcolor==2.4.0 -counterparty-core==10.8.0-rc.2 +counterparty-core==10.8.0 diff --git a/docker-compose.yml b/docker-compose.yml index e1038d40c3..0d9bc2d150 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ x-addrindexrs-common: &addrindexrs-common restart: unless-stopped x-counterparty-common: &counterparty-common - image: counterparty/counterparty:v10.8.0-rc.2 + image: counterparty/counterparty:v10.8.0 stop_grace_period: 1m volumes: - data:/root/.bitcoin diff --git a/release-notes/release-notes-v10.8.0.md b/release-notes/release-notes-v10.8.0.md index 1e96a5bee7..7acc4bcdb7 100644 --- a/release-notes/release-notes-v10.8.0.md +++ b/release-notes/release-notes-v10.8.0.md @@ -5,7 +5,7 @@ This release includes some significant architectural changes to the codebase to # Upgrading -This upgrade requires a mandatory, automatic reparse from block 871780. +This upgrade requires a mandatory, automatic rollback from block 871780. # ChangeLog @@ -47,6 +47,8 @@ This upgrade requires a mandatory, automatic reparse from block 871780. - Exclude transactions by `SIGHASH` - Be able to trigger a rollback on a minor version change - Add several new checkpoints +- Parse raw mempool by batch before following +- Set `BACKEND_RAW_TRANSACTIONS_CACHE_SIZE` to 1000 ## API