From e6e26fd28ff541697e5a04265891271cc847a62b Mon Sep 17 00:00:00 2001 From: Adam Krellenstein Date: Fri, 15 Mar 2024 17:24:06 -0400 Subject: [PATCH 01/37] Tweak Link in v10.0.0 Release Notes --- release-notes/release-notes-v10.0.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-notes/release-notes-v10.0.0.md b/release-notes/release-notes-v10.0.0.md index 45979cbb04..b4cc8bd3e3 100644 --- a/release-notes/release-notes-v10.0.0.md +++ b/release-notes/release-notes-v10.0.0.md @@ -5,7 +5,7 @@ Counterparty Core v10.0.0 is a very large release comprising many improvements a # Upgrade Procedure This release does not include any protocol changes, so there is no deadline for upgrading. However it is **strongly recommended** that all users upgrade as soon as possible, in particular to avoid consensus problems due to non-determinism in previous versions. The Counterparty Core API is also unchanged for this release. -Because this release includes numerous changes to the database schema, a full database rebuild is required and the major version number has been bumped from 9 to 10. Follow the updated installation instructions in the [README](../README.md) to download and install the latest version of Counterparty Core, run `counterparty-server kickstart` (while `bitcoind` is not running), then start the server with `counterparty-server start`. The rebuild should happen automatically, and it should take between 8 and 24 hours hours to complete. +Because this release includes numerous changes to the database schema, a full database rebuild is required and the major version number has been bumped from 9 to 10. Follow the updated installation instructions in the [README](/README.md) to download and install the latest version of Counterparty Core, run `counterparty-server kickstart` (while `bitcoind` is not running), then start the server with `counterparty-server start`. The rebuild should happen automatically, and it should take between 8 and 24 hours hours to complete. **IMPORTANT** Be certain that your are running the latest version of AddrIndexRs (v0.4.3). From 56e93a82274d96014c6b2aa7b4db056f05f57836 Mon Sep 17 00:00:00 2001 From: Adam Krellenstein Date: Fri, 15 Mar 2024 17:29:38 -0400 Subject: [PATCH 02/37] Typo in Release Procedure --- RELEASE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index e1e00ff590..66a62180ea 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,7 +1,7 @@ # Codebase - [ ] Update `VERSION_` variables in `lib/config.py` - [ ] Update `protocol_changes.json` (as necessary) -- [ ] Update Counterparty packages version in `requirement.txt` files +- [ ] Update Counterparty package versions in the `requirements.txt` files - [ ] Review all open pull requests - [ ] Write release notes - [ ] Create pull request against `master` From 4f1353d5eba3bb07f6fc9de25aa5a90c85543586 Mon Sep 17 00:00:00 2001 From: Adam Krellenstein Date: Fri, 15 Mar 2024 17:31:13 -0400 Subject: [PATCH 03/37] Tweak Release Procedure --- RELEASE.md | 1 + 1 file changed, 1 insertion(+) diff --git a/RELEASE.md b/RELEASE.md index 66a62180ea..90f7850d92 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -12,6 +12,7 @@ - [ ] Upload (signed) package to PyPi * `sudo python3 setup.py sdist build` * `sudo python3 setup.py sdist upload -r pypi` +- [ ] Publish bootstrap files - [ ] Publish Docker images - [ ] Update documentation From 01be5945c8c6144e38070fb2494d9f988a2f8808 Mon Sep 17 00:00:00 2001 From: Adam Krellenstein Date: Sat, 16 Mar 2024 07:18:09 -0400 Subject: [PATCH 04/37] Typo in v10.0.0 Release Notes --- release-notes/release-notes-v10.0.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-notes/release-notes-v10.0.0.md b/release-notes/release-notes-v10.0.0.md index b4cc8bd3e3..2987ab3036 100644 --- a/release-notes/release-notes-v10.0.0.md +++ b/release-notes/release-notes-v10.0.0.md @@ -93,4 +93,4 @@ Because this release includes numerous changes to the database schema, a full da # Credits * Ouziel Slama * Adam Krellenstein -* Warren Puffet +* Warren Puffett From 126e9586a6d742c445f1d88e28a2d6918b1d0b0a Mon Sep 17 00:00:00 2001 From: Adam Krellenstein Date: Sat, 16 Mar 2024 07:19:05 -0400 Subject: [PATCH 05/37] Typo in v10.0.0 Release Notes --- release-notes/release-notes-v10.0.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release-notes/release-notes-v10.0.0.md b/release-notes/release-notes-v10.0.0.md index 2987ab3036..1bddb6c398 100644 --- a/release-notes/release-notes-v10.0.0.md +++ b/release-notes/release-notes-v10.0.0.md @@ -7,7 +7,7 @@ This release does not include any protocol changes, so there is no deadline for Because this release includes numerous changes to the database schema, a full database rebuild is required and the major version number has been bumped from 9 to 10. Follow the updated installation instructions in the [README](/README.md) to download and install the latest version of Counterparty Core, run `counterparty-server kickstart` (while `bitcoind` is not running), then start the server with `counterparty-server start`. The rebuild should happen automatically, and it should take between 8 and 24 hours hours to complete. -**IMPORTANT** Be certain that your are running the latest version of AddrIndexRs (v0.4.3). +**IMPORTANT** Be certain that you are running the latest version of AddrIndexRs (v0.4.3). # ChangeLog From 607f6fd5ed4673c797b0d3d31aa28d46ff5e5adf Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Sat, 16 Mar 2024 12:54:33 +0100 Subject: [PATCH 06/37] Update Rust dependencies version --- counterparty-rs/Cargo.lock | 264 ++++++++++++++++--------------------- counterparty-rs/Cargo.toml | 10 +- 2 files changed, 115 insertions(+), 159 deletions(-) diff --git a/counterparty-rs/Cargo.lock b/counterparty-rs/Cargo.lock index 9ee2dd42c1..39f2bac181 100644 --- a/counterparty-rs/Cargo.lock +++ b/counterparty-rs/Cargo.lock @@ -40,7 +40,7 @@ checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn", ] [[package]] @@ -66,9 +66,9 @@ dependencies = [ [[package]] name = "base16ct" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" [[package]] name = "bech32" @@ -78,9 +78,9 @@ checksum = "d86b93f97252c47b41663388e6d155714a9d0c398b99f1005cbc5f978b29f445" [[package]] name = "bip32" -version = "0.4.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b30ed1d6f8437a487a266c8293aeb95b61a23261273e3e02912cdb8b68bf798b" +checksum = "7e141fb0f8be1c7b45887af94c88b182472b57c96b56773250ae00cd6a14a164" dependencies = [ "bs58", "hmac", @@ -89,8 +89,8 @@ dependencies = [ "pbkdf2", "rand_core", "ripemd", - "secp256k1 0.21.3", - "sha2 0.10.8", + "secp256k1 0.27.0", + "sha2", "subtle", "zeroize", ] @@ -118,15 +118,6 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" -[[package]] -name = "block-buffer" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" -dependencies = [ - "generic-array", -] - [[package]] name = "block-buffer" version = "0.10.4" @@ -138,11 +129,12 @@ dependencies = [ [[package]] name = "bs58" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" +checksum = "f5353f36341f7451062466f0b755b96ac3a9547e4d7f6b70d603fc721a7d7896" dependencies = [ - "sha2 0.9.9", + "sha2", + "tinyvec", ] [[package]] @@ -213,7 +205,7 @@ dependencies = [ "bs58", "par-map", "pyo3", - "pyo3-build-config 0.20.3", + "pyo3-build-config", "ripemd", "sha256", "vergen", @@ -230,9 +222,9 @@ dependencies = [ [[package]] name = "crypto-bigint" -version = "0.4.9" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef2b4b23cddf68b89b8f8069890e8c270d54e2d5fe1b143820234805e4cb17ef" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ "generic-array", "rand_core", @@ -252,11 +244,12 @@ dependencies = [ [[package]] name = "der" -version = "0.6.1" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1a467a65c5e759bce6e65eaf91cc29f466cdc57cb65777bd646872a8a1fd4de" +checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c" dependencies = [ "const-oid", + "zeroize", ] [[package]] @@ -268,33 +261,26 @@ dependencies = [ "powerfmt", ] -[[package]] -name = "digest" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" -dependencies = [ - "generic-array", -] - [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer 0.10.4", + "block-buffer", + "const-oid", "crypto-common", "subtle", ] [[package]] name = "ecdsa" -version = "0.14.8" +version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "413301934810f597c1d19ca71c8710e99a3f1ba28a0d2ebc01551a2daeea3c5c" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ "der", + "digest", "elliptic-curve", "rfc6979", "signature", @@ -302,14 +288,13 @@ dependencies = [ [[package]] name = "elliptic-curve" -version = "0.12.3" +version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7bb888ab5300a19b8e5bceef25ac745ad065f3c9f7efc6de1b91958110891d3" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ "base16ct", "crypto-bigint", - "der", - "digest 0.10.7", + "digest", "ff", "generic-array", "group", @@ -321,9 +306,9 @@ dependencies = [ [[package]] name = "ff" -version = "0.12.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d013fc25338cc558c5c2cfbad646908fb23591e2404481826742b651c9af7160" +checksum = "ded41244b729663b1e574f1b4fb731469f69f79c17667b5d776b16cda0479449" dependencies = [ "rand_core", "subtle", @@ -353,6 +338,7 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", + "zeroize", ] [[package]] @@ -363,15 +349,21 @@ checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" [[package]] name = "group" -version = "0.12.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dfbfb3a6cfbd390d5c9564ab283a0349b9b9fcd46a706c1eb10e0db70bfbac7" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ "ff", "rand_core", "subtle", ] +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "hermit-abi" version = "0.3.6" @@ -390,14 +382,14 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest 0.10.7", + "digest", ] [[package]] name = "indoc" -version = "1.0.9" +version = "2.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" +checksum = "1e186cfbae8084e513daff4240b4797e342f988cecda4fb6c939150f96315fd8" [[package]] name = "itoa" @@ -407,24 +399,14 @@ checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" [[package]] name = "k256" -version = "0.11.6" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72c1e0b51e7ec0a97369623508396067a486bd0cbed95a2659a4b863d28cfc8b" +checksum = "956ff9b67e26e1a6a866cb758f12c6f8746208489e3e4a4b5580802f2f0a587b" dependencies = [ "cfg-if", "ecdsa", "elliptic-curve", - "sha2 0.10.8", - "sha3", -] - -[[package]] -name = "keccak" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ecc2af9a1119c51f12a14607e783cb977bde58bc069ff0c3da1095e635d70654" -dependencies = [ - "cpufeatures", + "sha2", ] [[package]] @@ -451,9 +433,9 @@ checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "memoffset" -version = "0.6.5" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" dependencies = [ "autocfg", ] @@ -507,12 +489,6 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" -[[package]] -name = "opaque-debug" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" - [[package]] name = "par-map" version = "0.1.4" @@ -550,11 +526,12 @@ dependencies = [ [[package]] name = "pbkdf2" -version = "0.11.0" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" dependencies = [ - "digest 0.10.7", + "digest", + "hmac", ] [[package]] @@ -563,6 +540,12 @@ version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" +[[package]] +name = "portable-atomic" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" + [[package]] name = "powerfmt" version = "0.2.0" @@ -586,31 +569,22 @@ checksum = "858afdbecdce657c6e32031348cf7326da7700c869c368a136d31565972f7018" [[package]] name = "pyo3" -version = "0.17.3" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "268be0c73583c183f2b14052337465768c07726936a260f480f0857cb95ba543" +checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233" dependencies = [ "cfg-if", "indoc", "libc", "memoffset", "parking_lot", - "pyo3-build-config 0.17.3", + "portable-atomic", + "pyo3-build-config", "pyo3-ffi", "pyo3-macros", "unindent", ] -[[package]] -name = "pyo3-build-config" -version = "0.17.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28fcd1e73f06ec85bf3280c48c67e731d8290ad3d730f8be9dc07946923005c8" -dependencies = [ - "once_cell", - "target-lexicon", -] - [[package]] name = "pyo3-build-config" version = "0.20.3" @@ -623,35 +597,37 @@ dependencies = [ [[package]] name = "pyo3-ffi" -version = "0.17.3" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f6cb136e222e49115b3c51c32792886defbfb0adead26a688142b346a0b9ffc" +checksum = "62b42531d03e08d4ef1f6e85a2ed422eb678b8cd62b762e53891c05faf0d4afa" dependencies = [ "libc", - "pyo3-build-config 0.17.3", + "pyo3-build-config", ] [[package]] name = "pyo3-macros" -version = "0.17.3" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94144a1266e236b1c932682136dc35a9dee8d3589728f68130c7c3861ef96b28" +checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 1.0.109", + "syn", ] [[package]] name = "pyo3-macros-backend" -version = "0.17.3" +version = "0.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8df9be978a2d2f0cdebabb03206ed73b11314701a5bfe71b0d753b81997777f" +checksum = "7c7e9b68bb9c3149c5b0cade5d07f953d6d125eb4337723c4ccdb665f1f96185" dependencies = [ + "heck", "proc-macro2", + "pyo3-build-config", "quote", - "syn 1.0.109", + "syn", ] [[package]] @@ -709,13 +685,12 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "rfc6979" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" dependencies = [ - "crypto-bigint", "hmac", - "zeroize", + "subtle", ] [[package]] @@ -724,7 +699,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd124222d17ad93a644ed9d011a40f4fb64aa54275c08cc216524a9ea82fb09f" dependencies = [ - "digest 0.10.7", + "digest", ] [[package]] @@ -762,9 +737,9 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "sec1" -version = "0.3.0" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be24c1842290c45df0a7bf069e0c268a747ad05a192f2fd7dcfdbc1cba40928" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct", "der", @@ -775,37 +750,37 @@ dependencies = [ [[package]] name = "secp256k1" -version = "0.21.3" +version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c42e6f1735c5f00f51e43e28d6634141f2bcad10931b2609ddd74a86d751260" +checksum = "6b1629c9c557ef9b293568b338dddfc8208c98a18c59d722a9d53f859d9c9b62" dependencies = [ - "secp256k1-sys 0.4.2", + "bitcoin_hashes", + "secp256k1-sys 0.6.1", ] [[package]] name = "secp256k1" -version = "0.24.3" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b1629c9c557ef9b293568b338dddfc8208c98a18c59d722a9d53f859d9c9b62" +checksum = "25996b82292a7a57ed3508f052cfff8640d38d32018784acd714758b43da9c8f" dependencies = [ - "bitcoin_hashes", - "secp256k1-sys 0.6.1", + "secp256k1-sys 0.8.1", ] [[package]] name = "secp256k1-sys" -version = "0.4.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "957da2573cde917463ece3570eab4a0b3f19de6f1646cde62e6fd3868f566036" +checksum = "83080e2c2fc1006e625be82e5d1eb6a43b7fd9578b617fcc55814daf286bba4b" dependencies = [ "cc", ] [[package]] name = "secp256k1-sys" -version = "0.6.1" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83080e2c2fc1006e625be82e5d1eb6a43b7fd9578b617fcc55814daf286bba4b" +checksum = "70a129b9e9efbfb223753b9163c4ab3b13cff7fd9c7f010fbac25ab4099fa07e" dependencies = [ "cc", ] @@ -836,7 +811,7 @@ checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn", ] [[package]] @@ -850,19 +825,6 @@ dependencies = [ "serde", ] -[[package]] -name = "sha2" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" -dependencies = [ - "block-buffer 0.9.0", - "cfg-if", - "cpufeatures", - "digest 0.9.0", - "opaque-debug", -] - [[package]] name = "sha2" version = "0.10.8" @@ -871,7 +833,7 @@ checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.7", + "digest", ] [[package]] @@ -883,27 +845,17 @@ dependencies = [ "async-trait", "bytes", "hex", - "sha2 0.10.8", + "sha2", "tokio", ] -[[package]] -name = "sha3" -version = "0.10.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75872d278a8f37ef87fa0ddbda7802605cb18344497949862c0d4dcb291eba60" -dependencies = [ - "digest 0.10.7", - "keccak", -] - [[package]] name = "signature" -version = "1.6.4" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "digest 0.10.7", + "digest", "rand_core", ] @@ -919,17 +871,6 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "syn" version = "2.0.48" @@ -964,7 +905,7 @@ checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn", ] [[package]] @@ -1000,6 +941,21 @@ dependencies = [ "time-core", ] +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.36.0" @@ -1025,9 +981,9 @@ checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unindent" -version = "0.1.11" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1766d682d402817b5ac4490b3c3002d91dfa0d22812f341609f97b08757359c" +checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" [[package]] name = "vergen" diff --git a/counterparty-rs/Cargo.toml b/counterparty-rs/Cargo.toml index 33de019577..1903b6db44 100644 --- a/counterparty-rs/Cargo.toml +++ b/counterparty-rs/Cargo.toml @@ -9,15 +9,15 @@ name = "counterparty_rs" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.17.3", features = ["extension-module", "abi3-py37"] } -bs58 = { version = "0.4.0", features = ["check"] } +pyo3 = { version = "0.20.3", features = ["extension-module", "abi3-py37"] } +bs58 = { version = "0.5.0", features = ["check"] } # secp256k1 = "0.25.0" -bip32 = { version = "0.4.0", features = ["secp256k1-ffi"] } +bip32 = { version = "0.5.1", features = ["secp256k1-ffi"] } ripemd = "0.1.3" -sha256 = "1.1.1" +sha256 = "1.5.0" bitcoin = "0.29.2" par-map = "0.1.4" [build-dependencies] -vergen = { version = "8.2.6", features = ["build", "cargo", "git", "gitcl", "rustc"] } +vergen = { version = "8.3.1", features = ["build", "cargo", "git", "gitcl", "rustc"] } pyo3-build-config = "0.20.3" From c3e10a94556f7a6e6365a1608d96280033755a04 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Sat, 16 Mar 2024 16:10:26 +0100 Subject: [PATCH 07/37] Add testnet checkpoint at 2580000 --- counterparty-lib/counterpartylib/lib/check.py | 1 + 1 file changed, 1 insertion(+) diff --git a/counterparty-lib/counterpartylib/lib/check.py b/counterparty-lib/counterpartylib/lib/check.py index 84eaa89dc8..af4a39c5e6 100644 --- a/counterparty-lib/counterpartylib/lib/check.py +++ b/counterparty-lib/counterpartylib/lib/check.py @@ -209,6 +209,7 @@ 2400000: {'ledger_hash': '5c0606e2729d9b2a2181388231fd816ce3279c4183137bf62e9c699dbdc2f140', 'txlist_hash': '36bbd14c69e5fc17cb1e69303affcb808909757395d28e3e3da83394260bf0dd'}, 2500000: {'ledger_hash': '76262b272c47b5a17f19ffa0ba72256617bd18e51fad4c3c5b3c776cb3a1037b', 'txlist_hash': '26567e5c45a59426b2bcdeb177167a92c5fc21e8fd2000ae9a24eb09e3945f70'}, 2540000: {'ledger_hash': '6d3e77f1c059b062f4eb131cc8eb1d8355598de756905d83803df0009a514f48', 'txlist_hash': '45e134cb4196bc5cfb4ef9b356e02025f752a9bc0ae635bc9ced2c471ecfcb6c'}, + 2580000: {'ledger_hash': 'f84878f8e1293cd5365fc66ec7acbc24b4201b79ab3125d7e60e26624f49037a', 'txlist_hash': 'dc1f037beb8a632c5277fa0e6fd52866f0370d1f93de024420a7c31ce956c812'}, } CONSENSUS_HASH_VERSION_REGTEST = 1 From 340844075ef6d35b563e4987ea6b1140c04eed36 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Sat, 16 Mar 2024 16:13:31 +0100 Subject: [PATCH 08/37] Mock addrindexrs for test book --- .../counterpartylib/lib/backend/addrindexrs.py | 12 +++++++----- .../counterpartylib/test/book_test.py | 11 +++++------ .../counterpartylib/test/conftest.py | 4 ++-- .../counterpartylib/test/util_test.py | 17 ++++++++++++++++- 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/counterparty-lib/counterpartylib/lib/backend/addrindexrs.py b/counterparty-lib/counterpartylib/lib/backend/addrindexrs.py index 6641c2f2c3..31e9615231 100644 --- a/counterparty-lib/counterpartylib/lib/backend/addrindexrs.py +++ b/counterparty-lib/counterpartylib/lib/backend/addrindexrs.py @@ -639,9 +639,11 @@ def get_oldest_tx(address, block_index=None): current_block_index = block_index or ledger.CURRENT_BLOCK_INDEX hardcoded_key = f"{current_block_index}-{address}" if hardcoded_key in GET_OLDEST_TX_HARDCODED: - return GET_OLDEST_TX_HARDCODED[hardcoded_key] + result = GET_OLDEST_TX_HARDCODED[hardcoded_key] + else: + global ADDRINDEXRS_CLIENT + if ADDRINDEXRS_CLIENT is None: + ADDRINDEXRS_CLIENT = AddrindexrsSocket() + result = ADDRINDEXRS_CLIENT.get_oldest_tx(address, block_index=current_block_index) - global ADDRINDEXRS_CLIENT - if ADDRINDEXRS_CLIENT is None: - ADDRINDEXRS_CLIENT = AddrindexrsSocket() - return ADDRINDEXRS_CLIENT.get_oldest_tx(address, block_index=current_block_index) + return result diff --git a/counterparty-lib/counterpartylib/test/book_test.py b/counterparty-lib/counterpartylib/test/book_test.py index 9f82c1e6d9..db0aa0cb25 100644 --- a/counterparty-lib/counterpartylib/test/book_test.py +++ b/counterparty-lib/counterpartylib/test/book_test.py @@ -6,10 +6,9 @@ from counterpartylib.test import util_test -def test_book(book): - if book == 'testnet': - util_test.reparse(testnet=True) - elif book == 'mainnet': - util_test.reparse(testnet=False) - else: +def test_book(skip): + print("Skip: ", skip) + if skip: pytest.skip("Skipping test book") + else: + util_test.reparse(testnet=True) diff --git a/counterparty-lib/counterpartylib/test/conftest.py b/counterparty-lib/counterpartylib/test/conftest.py index ee9b941b3a..e5484a6978 100644 --- a/counterparty-lib/counterpartylib/test/conftest.py +++ b/counterparty-lib/counterpartylib/test/conftest.py @@ -116,7 +116,7 @@ def pytest_generate_tests(metafunc): args.append((scenario_name, INTEGRATION_SCENARIOS[scenario_name][1], INTEGRATION_SCENARIOS[scenario_name][0], metafunc.config)) metafunc.parametrize('scenario_name, base_scenario_name, transactions, pytest_config', args) elif metafunc.function.__name__ == 'test_book': - metafunc.parametrize('book', [(metafunc.config.getoption("testbook"),)]) + metafunc.parametrize('skip', [not metafunc.config.getoption("testbook")]) def pytest_addoption(parser): @@ -126,7 +126,7 @@ def pytest_addoption(parser): parser.addoption("--gentxhex", action='store_true', default=False, help="Generate and print unsigned hex for *.compose() tests") parser.addoption("--savescenarios", action='store_true', default=False, help="Generate sql dump and log in .new files") parser.addoption("--alternative", action='store_true', default=False) - parser.addoption("--testbook", default='no', help="Include test book (use with one of the following values: `testnet` or `mainnet`)") + parser.addoption("--testbook", action='store_true', default=False, help="Include testnet test book") @pytest.fixture(scope="function") diff --git a/counterparty-lib/counterpartylib/test/util_test.py b/counterparty-lib/counterpartylib/test/util_test.py index 6257fc0bf8..082b138dc0 100644 --- a/counterparty-lib/counterpartylib/test/util_test.py +++ b/counterparty-lib/counterpartylib/test/util_test.py @@ -742,12 +742,26 @@ def __exit__(self, exc_type, exc_val, exc_tb): for k in self._before_empty: del self.mock_protocol_changes[k] -def reparse(testnet=True, checkpoint_count=10): + +def connect_to_addrindexrs_mock(): + return True + +def get_oldest_tx_mock(address, block_index=None): + if address == "mrHFGUKSiNMeErqByjX97qPKfumdZxe6mC" and block_index==99999999999: + return {"block_index": 2576100, "tx_hash": "d6751cc77da77d9270e16046a19954bfaffc005269f0c6a9248e680be6d1f468"} + return {} + +def reparse(testnet=True, checkpoint_count=5): """ Reparse all transaction from the database. - Create a new in-memory DB, copy the DB that is on-disk - Reparse DB, automatically compares consensus hashes to the original ones from the on-disk DB """ + + # mock the backend + server.connect_to_addrindexrs = connect_to_addrindexrs_mock + backend.get_oldest_tx = get_oldest_tx_mock + # create a new in-memory DB options = dict(COUNTERPARTYD_OPTIONS) server.initialise(database_file=':memory:', testnet=testnet, **options) @@ -775,6 +789,7 @@ def reparse(testnet=True, checkpoint_count=10): # we start one block after the checkpoint before the first one we want to check block_index = sorted(list(CHECKPOINTS.keys()))[-checkpoint_count - 1] + print(f"Checking from block {block_index}") # Initialise missing tables blocks.initialise(memory_db) From c8617a04cabb9a06660b431716ed8bcaad7f99df Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Sat, 16 Mar 2024 16:48:12 +0100 Subject: [PATCH 09/37] Test book workflow --- .github/workflows/test_book.yml | 40 +++++++++++++++++++++++++ counterparty-lib/tools/compareledger.py | 6 ++-- 2 files changed, 43 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/test_book.yml diff --git a/.github/workflows/test_book.yml b/.github/workflows/test_book.yml new file mode 100644 index 0000000000..e57eb6b3df --- /dev/null +++ b/.github/workflows/test_book.yml @@ -0,0 +1,40 @@ +name: Testnet Test Book + +on: + push: + branches: ["testbook"] + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11"] + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + - name: Install Rust toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + profile: minimal + default: true + - name: Install dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y libgirepository1.0-dev libleveldb-dev + python -m pip install --upgrade pip + pip install pylint maturin pytest + cd counterparty-rs && pip install -e . && cd .. + cd counterparty-lib && pip install -e . && cd .. + cd counterparty-cli && pip install -e . && cd .. + - name: Bootstrap testnet database + run: | + counterparty-server --backend-password=rpc --testnet bootstrap + - name: Run tests + run: | + cd counterparty-lib + pytest counterpartylib/test/book_test.py --testbook -s diff --git a/counterparty-lib/tools/compareledger.py b/counterparty-lib/tools/compareledger.py index 6dd2aa75e1..b6d8b2f296 100644 --- a/counterparty-lib/tools/compareledger.py +++ b/counterparty-lib/tools/compareledger.py @@ -109,7 +109,7 @@ def check_hashes(database_file_1, database_file_2, hash_name="ledger_hash"): def get_checkpoints(database_file): - checkpoints = [1600000, 1700000, 1800000, 1900000, 2000000, 2200000, 2400000, 2500000, 2540000] + checkpoints = [2580000] db = apsw.Connection(database_file_1, flags=apsw.SQLITE_OPEN_READONLY) cursor = db.cursor() for checkpoint in checkpoints: @@ -138,6 +138,6 @@ def get_last_block(database_file_1, database_file_2): LAST_BLOCK = 290000 #compare_ledger(database_file_1, database_file_2) -check_hashes(database_file_1, database_file_2, "txlist_hash") -#get_checkpoints(database_file_1) +#check_hashes(database_file_1, database_file_2, "txlist_hash") +get_checkpoints(database_file_1) #get_last_block(database_file_1, database_file_2) From 40ead2d55a6c927c2ccd946b7c94777c3b4f35ac Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Sat, 16 Mar 2024 17:01:58 +0100 Subject: [PATCH 10/37] clean debug --- counterparty-lib/counterpartylib/test/book_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/counterparty-lib/counterpartylib/test/book_test.py b/counterparty-lib/counterpartylib/test/book_test.py index db0aa0cb25..b77c5ca71f 100644 --- a/counterparty-lib/counterpartylib/test/book_test.py +++ b/counterparty-lib/counterpartylib/test/book_test.py @@ -7,7 +7,6 @@ def test_book(skip): - print("Skip: ", skip) if skip: pytest.skip("Skipping test book") else: From 3b38cff162108cbaff7489b1215545111530f45e Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Sat, 16 Mar 2024 19:31:33 +0100 Subject: [PATCH 11/37] restore default password --- .github/workflows/test_book.yml | 2 +- counterparty-cli/counterpartycli/server.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_book.yml b/.github/workflows/test_book.yml index e57eb6b3df..9e18c5b9e3 100644 --- a/.github/workflows/test_book.yml +++ b/.github/workflows/test_book.yml @@ -33,7 +33,7 @@ jobs: cd counterparty-cli && pip install -e . && cd .. - name: Bootstrap testnet database run: | - counterparty-server --backend-password=rpc --testnet bootstrap + counterparty-server --testnet bootstrap - name: Run tests run: | cd counterparty-lib diff --git a/counterparty-cli/counterpartycli/server.py b/counterparty-cli/counterpartycli/server.py index b0a9b6bcc3..70a962addb 100755 --- a/counterparty-cli/counterpartycli/server.py +++ b/counterparty-cli/counterpartycli/server.py @@ -32,7 +32,7 @@ [('--backend-connect',), {'default': 'localhost', 'help': 'the hostname or IP of the backend server'}], [('--backend-port',), {'type': int, 'help': 'the backend port to connect to'}], [('--backend-user',), {'default': 'bitcoinrpc', 'help': 'the username used to communicate with backend'}], - [('--backend-password',), {'help': 'the password used to communicate with backend'}], + [('--backend-password',), {'default': 'rpc', 'help': 'the password used to communicate with backend'}], [('--backend-ssl',), {'action': 'store_true', 'default': False, 'help': 'use SSL to connect to backend (default: false)'}], [('--backend-ssl-no-verify',), {'action': 'store_true', 'default': False, 'help': 'verify SSL certificate of backend; disallow use of self‐signed certificates (default: true)'}], [('--backend-poll-interval',), {'type': float, 'default': 0.5, 'help': 'poll interval, in seconds (default: 0.5)'}], From 074c172b26b68f07cf53a65cb1da1ef219bda28a Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Sat, 16 Mar 2024 19:47:43 +0100 Subject: [PATCH 12/37] Restore 'check.software version()' but only every 24 hours --- counterparty-lib/counterpartylib/lib/blocks.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/counterparty-lib/counterpartylib/lib/blocks.py b/counterparty-lib/counterpartylib/lib/blocks.py index 700816fc55..38613cccf8 100644 --- a/counterparty-lib/counterpartylib/lib/blocks.py +++ b/counterparty-lib/counterpartylib/lib/blocks.py @@ -738,9 +738,6 @@ def follow(db): # Get new blocks. if block_index <= block_count: - #mempool_spinner.stop() - #block_spinner = Halo(text=f"Parsing block {block_index}...", spinner=SPINNER_STYLE) - #block_spinner.start() # Backwards check for incorrect blocks due to chain reorganisation, and stop when a common parent is found. current_index = block_index @@ -783,9 +780,11 @@ def follow(db): tx_index = get_next_tx_index(db) continue - # Check version. (Don’t add any blocks to the database while + # Check version every 144 blocks (around 24H). + # (Don’t add more blocks to the database while # running an out‐of‐date client!) - # TODO: check.software_version() # This is too much! + if block_index % 144 == 0: + check.software_version() # Get and parse transactions in this block (atomically). # logger.debug(f'Blockchain cache size: {len(prefetcher.BLOCKCHAIN_CACHE)}') From 5157bb36e59e547453d46ea4ec8ee7cc86dfd560 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Mon, 18 Mar 2024 09:37:55 +0100 Subject: [PATCH 13/37] use time instead blocks --- counterparty-lib/counterpartylib/lib/blocks.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/counterparty-lib/counterpartylib/lib/blocks.py b/counterparty-lib/counterpartylib/lib/blocks.py index 38613cccf8..773fe8e5ae 100644 --- a/counterparty-lib/counterpartylib/lib/blocks.py +++ b/counterparty-lib/counterpartylib/lib/blocks.py @@ -673,6 +673,7 @@ class MempoolError(Exception): def follow(db): # Check software version. check.software_version() + last_software_check = time.time() # Initialise. initialise(db) @@ -780,10 +781,11 @@ def follow(db): tx_index = get_next_tx_index(db) continue - # Check version every 144 blocks (around 24H). + # Check version every 24H. # (Don’t add more blocks to the database while # running an out‐of‐date client!) - if block_index % 144 == 0: + if time.time() - last_software_check > 86400: + last_software_check = time.time() check.software_version() # Get and parse transactions in this block (atomically). From a7184f033b063c211f6b868467a9ef1988246f9f Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Mon, 18 Mar 2024 12:03:00 +0100 Subject: [PATCH 14/37] Move bootstrap in -lib; Add 'catch-up' flag --- counterparty-cli/counterpartycli/server.py | 22 +++--- counterparty-cli/counterpartycli/util.py | 70 ----------------- counterparty-lib/counterpartylib/server.py | 89 +++++++++++++++++++--- 3 files changed, 89 insertions(+), 92 deletions(-) diff --git a/counterparty-cli/counterpartycli/server.py b/counterparty-cli/counterpartycli/server.py index 70a962addb..a31714ce9a 100755 --- a/counterparty-cli/counterpartycli/server.py +++ b/counterparty-cli/counterpartycli/server.py @@ -11,7 +11,7 @@ from counterpartylib.lib import log from counterpartylib import server from counterpartylib.lib import config -from counterpartycli.util import add_config_arguments, bootstrap +from counterpartycli.util import add_config_arguments from counterpartycli.setup import generate_config_files from counterpartycli import APP_VERSION @@ -60,7 +60,6 @@ [('--utxo-locks-max-age',), {'type': int, 'default': config.DEFAULT_UTXO_LOCKS_MAX_AGE, 'help': 'how long to keep a lock on a UTXO being tracked'}], ] -COMMANDS_WITH_DB = ['reparse', 'rollback', 'start', 'vacuum', 'check-db'] class VersionError(Exception): pass @@ -86,6 +85,7 @@ def main(): subparsers = parser.add_subparsers(dest='action', help='the action to be taken') parser_server = subparsers.add_parser('start', help='run the server') + parser_server.add_argument('--catch-up', choices=['normal', 'bootstrap'], default='normal', help='Catch up mode (default: normal)') parser_reparse = subparsers.add_parser('reparse', help='reparse all transactions in the database') parser_reparse.add_argument('block_index', type=int, help='the index of the last known good block') @@ -138,11 +138,7 @@ def main(): utxo_locks_max_addresses=args.utxo_locks_max_addresses, utxo_locks_max_age=args.utxo_locks_max_age) - if args.action in COMMANDS_WITH_DB: - # server.initialise_config() is called in server.initialise() - db = server.initialise(**init_args) - else: - server.initialise_config(**init_args) + server.initialise_config(**init_args) # set up logging log.set_up( @@ -162,14 +158,14 @@ def main(): # Bootstrapping if args.action == 'bootstrap': - bootstrap(testnet=args.testnet) + server.bootstrap() # PARSING elif args.action == 'reparse': - server.reparse(db, block_index=args.block_index) + server.reparse(block_index=args.block_index) elif args.action == 'rollback': - server.rollback(db, block_index=args.block_index) + server.rollback(block_index=args.block_index) elif args.action == 'kickstart': server.kickstart( @@ -179,16 +175,16 @@ def main(): debug_block=args.debug_block) elif args.action == 'start': - server.start_all(db) + server.start_all(catch_up=args.catch_up) elif args.action == 'show-config': server.show_config() elif args.action == 'vacuum': - server.vacuum(db) + server.vacuum() elif args.action == 'check-db': - server.check_database(db) + server.check_database() else: parser.print_help() diff --git a/counterparty-cli/counterpartycli/util.py b/counterparty-cli/counterpartycli/util.py index 0d78706a2b..7c2fb5534b 100644 --- a/counterparty-cli/counterpartycli/util.py +++ b/counterparty-cli/counterpartycli/util.py @@ -122,76 +122,6 @@ def value_out(quantity, asset, divisible=None): return value_output(quantity, asset, divisible) -def bootstrap(testnet=False, overwrite=True): - data_dir = appdirs.user_data_dir(appauthor=config.XCP_NAME, appname=config.APP_NAME, roaming=True) - - # Set Constants. - bootstrap_url = config.BOOTSTRAP_URL_TESTNET if testnet else config.BOOTSTRAP_URL_MAINNET - tar_filename = os.path.basename(bootstrap_url) - tarball_path = os.path.join(tempfile.gettempdir(), tar_filename) - database_path = os.path.join(data_dir, config.APP_NAME) - if testnet: - database_path += '.testnet' - database_path += '.db' - - # Delete SQLite Write-Ahead-Log - wal_path = database_path + "-wal" - shm_path = database_path + "-shm" - try: - os.remove(wal_path) - except OSError: - pass - try: - os.remove(shm_path) - except OSError: - pass - - # Prepare Directory. - if not os.path.exists(data_dir): - os.makedirs(data_dir, mode=0o755) - if not overwrite and os.path.exists(database_path): - return - - # Define Progress Bar. - step = f'Downloading database from {bootstrap_url}...' - spinner = Halo(text=step, spinner=SPINNER_STYLE) - - def bootstrap_progress(blocknum, blocksize, totalsize): - readsofar = blocknum * blocksize - if totalsize > 0: - percent = readsofar * 1e2 / totalsize - message = f"Downloading database: {percent:5.1f}% {readsofar} / {totalsize}" - spinner.text = message - - # Downloading - spinner.start() - urllib.request.urlretrieve( - bootstrap_url, - tarball_path, - bootstrap_progress - ) # nosec B310 - spinner.stop() - print(f"{OK_GREEN} {step}") - - # TODO: check checksum, filenames, etc. - step = f'Extracting database to {data_dir}...' - with Halo(text=step, spinner=SPINNER_STYLE): - with tarfile.open(tarball_path, 'r:gz') as tar_file: - tar_file.extractall(path=data_dir) # nosec B202 - print(f"{OK_GREEN} {step}") - - assert os.path.exists(database_path) - # user and group have "rw" access - os.chmod(database_path, 0o660) # nosec B103 - - step = 'Cleaning up...' - with Halo(text=step, spinner=SPINNER_STYLE): - os.remove(tarball_path) - print(f"{OK_GREEN} {step}") - - cprint(f"Database has been successfully bootstrapped to {database_path}.", "green") - - # Set default values of command line arguments with config file def add_config_arguments(arg_parser, config_args, default_config_file, config_file_arg_name='config_file'): cmd_args = arg_parser.parse_known_args()[0] diff --git a/counterparty-lib/counterpartylib/server.py b/counterparty-lib/counterpartylib/server.py index bdd9f39d12..5a04f736cc 100755 --- a/counterparty-lib/counterpartylib/server.py +++ b/counterparty-lib/counterpartylib/server.py @@ -2,7 +2,7 @@ import os import decimal -import pprint +import tempfile import sys import apsw import time @@ -13,7 +13,8 @@ import platform import bitcoin as bitcoinlib import logging -import traceback +import urllib +import tarfile from urllib.parse import quote_plus as urlencode from halo import Halo @@ -478,7 +479,7 @@ def connect_to_addrindexrs(): print(f'{OK_GREEN} {step}') -def start_all(db): +def start_all(catch_up='normal'): # Backend. connect_to_backend() @@ -492,15 +493,22 @@ def start_all(db): api_server.daemon = True api_server.start() + if not os.path.exists(config.DATABASE) and catch_up == 'bootstrap': + bootstrap() + + db = initialise_db() + # Server blocks.follow(db) -def reparse(db, block_index): +def reparse(block_index): + db = initialise_db() blocks.reparse(db, block_index=block_index) -def rollback(db, block_index=None): +def rollback(block_index=None): + db = initialise_db() blocks.rollback(db, block_index=block_index) @@ -513,15 +521,16 @@ def kickstart(bitcoind_dir, force=False, max_queue_size=None, debug_block=None): ) -def vacuum(db): +def vacuum(): + db = initialise_db() step = 'Vacuuming database...' with Halo(text=step, spinner=SPINNER_STYLE): database.vacuum(db) print(f'{OK_GREEN} {step}') -def check_database(db): - ledger.CURRENT_BLOCK_INDEX = blocks.last_db_index(db) +def check_database(): + db = initialise_db() step = 'Checking asset conservation...' with Halo(text=step, spinner=SPINNER_STYLE): @@ -564,4 +573,66 @@ def configure_rpc(rpc_password=None): config.RPC = 'http://' + config.RPC_HOST + ':' + str(config.RPC_PORT) + config.RPC_WEBROOT -# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 +def bootstrap(): + data_dir = appdirs.user_data_dir(appauthor=config.XCP_NAME, appname=config.APP_NAME, roaming=True) + + # Set Constants. + bootstrap_url = config.BOOTSTRAP_URL_TESTNET if config.TESTNET else config.BOOTSTRAP_URL_MAINNET + tar_filename = os.path.basename(bootstrap_url) + tarball_path = os.path.join(tempfile.gettempdir(), tar_filename) + database_path = os.path.join(data_dir, config.APP_NAME) + if config.TESTNET: + database_path += '.testnet' + database_path += '.db' + + # Prepare Directory. + if not os.path.exists(data_dir): + os.makedirs(data_dir, mode=0o755) + if os.path.exists(database_path): + os.remove(database_path) + # Delete SQLite Write-Ahead-Log + wal_path = database_path + "-wal" + shm_path = database_path + "-shm" + if os.path.exists(wal_path): + os.remove(wal_path) + if os.path.exists(shm_path): + os.remove(shm_path) + + # Define Progress Bar. + step = f'Downloading database from {bootstrap_url}...' + spinner = Halo(text=step, spinner=SPINNER_STYLE) + + def bootstrap_progress(blocknum, blocksize, totalsize): + readsofar = blocknum * blocksize + if totalsize > 0: + percent = readsofar * 1e2 / totalsize + message = f"Downloading database: {percent:5.1f}% {readsofar} / {totalsize}" + spinner.text = message + + # Downloading + spinner.start() + urllib.request.urlretrieve( + bootstrap_url, + tarball_path, + bootstrap_progress + ) # nosec B310 + spinner.stop() + print(f"{OK_GREEN} {step}") + + # TODO: check checksum, filenames, etc. + step = f'Extracting database to {data_dir}...' + with Halo(text=step, spinner=SPINNER_STYLE): + with tarfile.open(tarball_path, 'r:gz') as tar_file: + tar_file.extractall(path=data_dir) # nosec B202 + print(f"{OK_GREEN} {step}") + + assert os.path.exists(database_path) + # user and group have "rw" access + os.chmod(database_path, 0o660) # nosec B103 + + step = 'Cleaning up...' + with Halo(text=step, spinner=SPINNER_STYLE): + os.remove(tarball_path) + print(f"{OK_GREEN} {step}") + + cprint(f"Database has been successfully bootstrapped to {database_path}.", "green") From 635268ff064a207cb7911bfa3779e4c3035046d5 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Mon, 18 Mar 2024 12:09:50 +0100 Subject: [PATCH 15/37] Bootstrap before starting API --- counterparty-lib/counterpartylib/server.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/counterparty-lib/counterpartylib/server.py b/counterparty-lib/counterpartylib/server.py index 5a04f736cc..87d5b16403 100755 --- a/counterparty-lib/counterpartylib/server.py +++ b/counterparty-lib/counterpartylib/server.py @@ -483,6 +483,11 @@ def start_all(catch_up='normal'): # Backend. connect_to_backend() + if not os.path.exists(config.DATABASE) and catch_up == 'bootstrap': + bootstrap() + + db = initialise_db() + # API Status Poller. api_status_poller = api.APIStatusPoller() api_status_poller.daemon = True @@ -493,11 +498,6 @@ def start_all(catch_up='normal'): api_server.daemon = True api_server.start() - if not os.path.exists(config.DATABASE) and catch_up == 'bootstrap': - bootstrap() - - db = initialise_db() - # Server blocks.follow(db) From 278a3148812b750e09d05b1e2abfd923a4bd7b2d Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Mon, 18 Mar 2024 12:18:03 +0100 Subject: [PATCH 16/37] Add '--catch-up=bootstrap' in simplenode --- simplenode/compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/simplenode/compose.yml b/simplenode/compose.yml index 61e923e666..05eb73c5de 100644 --- a/simplenode/compose.yml +++ b/simplenode/compose.yml @@ -68,6 +68,7 @@ services: - "--log-file=1" - "--api-log-file=1" - "start" + - "--catch-up=bootstrap" environment: - "XDG_DATA_HOME=/data/" - "XDG_LOG_HOME=/data/" From 20f0f635f80d6f8fa459c31e0f85f26e2abd964b Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Mon, 18 Mar 2024 12:39:33 +0100 Subject: [PATCH 17/37] On minor version change, reparse from a given block if required --- counterparty-lib/counterpartylib/lib/check.py | 15 ++++++++++++--- counterparty-lib/counterpartylib/lib/config.py | 6 ++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/counterparty-lib/counterpartylib/lib/check.py b/counterparty-lib/counterpartylib/lib/check.py index af4a39c5e6..6880c9fdd2 100644 --- a/counterparty-lib/counterpartylib/lib/check.py +++ b/counterparty-lib/counterpartylib/lib/check.py @@ -364,7 +364,16 @@ def database_version(db): ) elif version_minor != config.VERSION_MINOR: # Reparse transactions from the vesion block if minor version has changed. + message = f'Client minor version number mismatch ({version_minor} ≠ {config.VERSION_MINOR}).' + if config.NEED_REPARSE_IF_MINOR_IS_LESS_THAN is not None: + min_version_minor, min_version_block_index = config.NEED_REPARSE_IF_MINOR_IS_LESS_THAN + if version_minor < min_version_minor: + raise DatabaseVersionError( + message=message, + required_action='reparse', + from_block_index=min_version_block_index + ) raise DatabaseVersionError( - message=f'Client minor version number mismatch ({version_minor} ≠ {config.VERSION_MINOR}).', - required_action='reparse', - from_block_index=config.BLOCK_FIRST) + message=message, + required_action=None + ) diff --git a/counterparty-lib/counterpartylib/lib/config.py b/counterparty-lib/counterpartylib/lib/config.py index 6201f29ef1..3285d89e9c 100644 --- a/counterparty-lib/counterpartylib/lib/config.py +++ b/counterparty-lib/counterpartylib/lib/config.py @@ -17,6 +17,12 @@ ADDRINDEXRS_VERSION = "0.4.3" +# When updating to a new verion, we are making a rollback if major version changes. +# If minor version changes and if needed, we are making a reparse from a given block. +# Fo example: +# NEED_REPARSE_IF_MINOR_IS_LESS_THAN = (1, 800000) +# means that we need to reparse from block 800000 if database minor version is less than 1 +NEED_REPARSE_IF_MINOR_IS_LESS_THAN = None # Counterparty protocol TXTYPE_FORMAT = '>I' From 4d3a1418c0f5e6e4fbe1a74c738135412c0ba619 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Mon, 18 Mar 2024 13:26:56 +0100 Subject: [PATCH 18/37] fix typo --- .github/workflows/build_docker_image.yml | 0 .github/workflows/test_book.yml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 .github/workflows/build_docker_image.yml diff --git a/.github/workflows/build_docker_image.yml b/.github/workflows/build_docker_image.yml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.github/workflows/test_book.yml b/.github/workflows/test_book.yml index 9e18c5b9e3..9bcd60d169 100644 --- a/.github/workflows/test_book.yml +++ b/.github/workflows/test_book.yml @@ -2,7 +2,7 @@ name: Testnet Test Book on: push: - branches: ["testbook"] + branches: "**" jobs: build: From 1173ceffb84ac79d75b86747f41ea37a433dde93 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Mon, 18 Mar 2024 13:31:41 +0100 Subject: [PATCH 19/37] Add docker github workflow --- .github/workflows/build_docker_image.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/workflows/build_docker_image.yml b/.github/workflows/build_docker_image.yml index e69de29bb2..46b166500e 100644 --- a/.github/workflows/build_docker_image.yml +++ b/.github/workflows/build_docker_image.yml @@ -0,0 +1,21 @@ +name: Build Docker Image + +on: + push: + branches: "**" + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install Docker + run: | + curl -fsSL https://get.docker.com -o get-docker.sh + sudo sh get-docker.sh + - name: Build image + run: | + docker build -t counterparty . + - name: Test image + run: | + docker run --rm counterparty counterparty-server -h From 0fb4ea7289d0b169513bcae84434c6a87f71d722 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Mon, 18 Mar 2024 14:57:31 +0100 Subject: [PATCH 20/37] publish Docker image on release --- .github/workflows/publish_docker_image.yml | 28 ++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/publish_docker_image.yml diff --git a/.github/workflows/publish_docker_image.yml b/.github/workflows/publish_docker_image.yml new file mode 100644 index 0000000000..f884f45675 --- /dev/null +++ b/.github/workflows/publish_docker_image.yml @@ -0,0 +1,28 @@ +name: Publish Docker Image + +on: + release: + types: [published] + +env: + DOCKER_REPO: counterparty/counterparty + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install Docker + run: | + curl -fsSL https://get.docker.com -o get-docker.sh + sudo sh get-docker.sh + - name: Build, tag, login and push image + run: | + export VERSION=v$(cat counterparty-lib/counterpartylib/lib/config.py | grep '__version__ =' | awk -F '"' '{print $2}') + docker build -t $DOCKER_REPO:$VERSION . + docker tag $DOCKER_REPO:$VERSION $DOCKER_REPO:latest + docker login -u "$DOCKER_USERNAME" -p "$DOCKER_PASSWORD" + docker push $DOCKER_REPO:$VERSION + docker push $DOCKER_REPO:latest From ec537fdeaf867eff8d78dc8eef2c9f6e93175e1d Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Mon, 18 Mar 2024 20:35:49 +0100 Subject: [PATCH 21/37] Accept config args before and after the command --- counterparty-cli/counterpartycli/server.py | 31 +++++++++++++++++----- counterparty-cli/counterpartycli/util.py | 27 +++++++++---------- 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/counterparty-cli/counterpartycli/server.py b/counterparty-cli/counterpartycli/server.py index a31714ce9a..9c0726264e 100755 --- a/counterparty-cli/counterpartycli/server.py +++ b/counterparty-cli/counterpartycli/server.py @@ -3,6 +3,7 @@ import os import sys import argparse +import optparse import logging @@ -11,7 +12,7 @@ from counterpartylib.lib import log from counterpartylib import server from counterpartylib.lib import config -from counterpartycli.util import add_config_arguments +from counterpartycli.util import add_config_arguments, read_config_file from counterpartycli.setup import generate_config_files from counterpartycli import APP_VERSION @@ -75,36 +76,54 @@ def main(): generate_config_files() # Parse command-line arguments. - parser = argparse.ArgumentParser(prog=APP_NAME, description=f'Server for the {config.XCP_NAME} protocol', add_help=False) + parser = argparse.ArgumentParser( + prog=APP_NAME, + description=f'Server for the {config.XCP_NAME} protocol', + add_help=False, + exit_on_error=False + ) parser.add_argument('-h', '--help', dest='help', action='store_true', help='show this help message and exit') parser.add_argument('-V', '--version', action='version', version=f"{APP_NAME} v{APP_VERSION}; counterparty-lib v{config.VERSION_STRING}") parser.add_argument('--config-file', help='the path to the configuration file') - add_config_arguments(parser, CONFIG_ARGS, 'server.conf') + cmd_args = parser.parse_known_args()[0] + config_file_path = getattr(cmd_args, 'config_file', None) + configfile = read_config_file('server.conf', config_file_path) + + add_config_arguments(parser, CONFIG_ARGS, configfile) subparsers = parser.add_subparsers(dest='action', help='the action to be taken') parser_server = subparsers.add_parser('start', help='run the server') + parser_server.add_argument('--config-file', help='the path to the configuration file') parser_server.add_argument('--catch-up', choices=['normal', 'bootstrap'], default='normal', help='Catch up mode (default: normal)') + add_config_arguments(parser_server, CONFIG_ARGS, configfile) parser_reparse = subparsers.add_parser('reparse', help='reparse all transactions in the database') parser_reparse.add_argument('block_index', type=int, help='the index of the last known good block') + add_config_arguments(parser_reparse, CONFIG_ARGS, configfile) parser_vacuum = subparsers.add_parser('vacuum', help='VACUUM the database (to improve performance)') + add_config_arguments(parser_vacuum, CONFIG_ARGS, configfile) parser_rollback = subparsers.add_parser('rollback', help='rollback database') parser_rollback.add_argument('block_index', type=int, help='the index of the last known good block') + add_config_arguments(parser_rollback, CONFIG_ARGS, configfile) parser_kickstart = subparsers.add_parser('kickstart', help='rapidly build database by reading from Bitcoin Core blockchain') parser_kickstart.add_argument('--bitcoind-dir', help='Bitcoin Core data directory') parser_kickstart.add_argument('--max-queue-size', type=int, help='Size of the multiprocessing.Queue for parsing blocks') parser_kickstart.add_argument('--debug-block', type=int, help='Rollback and run kickstart for a single block;') + add_config_arguments(parser_kickstart, CONFIG_ARGS, configfile) - subparsers.add_parser('bootstrap', help='bootstrap database with hosted snapshot') + parser_bootstrap = subparsers.add_parser('bootstrap', help='bootstrap database with hosted snapshot') + add_config_arguments(parser_bootstrap, CONFIG_ARGS, configfile) - subparsers.add_parser('check-db', help='do an integrity check on the database') + parser_checkdb = subparsers.add_parser('check-db', help='do an integrity check on the database') + add_config_arguments(parser_checkdb, CONFIG_ARGS, configfile) - subparsers.add_parser('show-config', help='Show counterparty-server configuration') + parser_show_config = subparsers.add_parser('show-config', help='Show counterparty-server configuration') + add_config_arguments(parser_show_config, CONFIG_ARGS, configfile) args = parser.parse_args() diff --git a/counterparty-cli/counterpartycli/util.py b/counterparty-cli/counterpartycli/util.py index 7c2fb5534b..15fd1f6f01 100644 --- a/counterparty-cli/counterpartycli/util.py +++ b/counterparty-cli/counterpartycli/util.py @@ -122,21 +122,17 @@ def value_out(quantity, asset, divisible=None): return value_output(quantity, asset, divisible) -# Set default values of command line arguments with config file -def add_config_arguments(arg_parser, config_args, default_config_file, config_file_arg_name='config_file'): - cmd_args = arg_parser.parse_known_args()[0] - - config_file = getattr(cmd_args, config_file_arg_name, None) - if not config_file: +def read_config_file(default_config_file, config_file_path=None): + if not config_file_path: config_dir = appdirs.user_config_dir(appauthor=config.XCP_NAME, appname=config.APP_NAME, roaming=True) if not os.path.isdir(config_dir): os.makedirs(config_dir, mode=0o755) - config_file = os.path.join(config_dir, default_config_file) + config_file_path = os.path.join(config_dir, default_config_file) # clean BOM bufsize = 4096 bomlen = len(codecs.BOM_UTF8) - with codecs.open(config_file, 'r+b') as fp: + with codecs.open(config_file_path, 'r+b') as fp: chunk = fp.read(bufsize) if chunk.startswith(codecs.BOM_UTF8): i = 0 @@ -150,16 +146,19 @@ def add_config_arguments(arg_parser, config_args, default_config_file, config_fi fp.seek(-bomlen, os.SEEK_CUR) fp.truncate() - logger.debug(f'Loading configuration file: `{config_file}`') + logger.debug(f'Loading configuration file: `{config_file_path}`') configfile = configparser.SafeConfigParser(allow_no_value=True, inline_comment_prefixes=('#', ';')) - with codecs.open(config_file, 'r', encoding='utf8') as fp: + with codecs.open(config_file_path, 'r', encoding='utf8') as fp: configfile.readfp(fp) if not 'Default' in configfile: configfile['Default'] = {} + + return configfile + - # Initialize default values with the config file. - for arg in config_args: +def add_config_arguments(parser, args, configfile): + for arg in args: key = arg[0][-1].replace('--', '') if 'action' in arg[1] and arg[1]['action'] == 'store_true' and key in configfile['Default']: arg[1]['default'] = configfile['Default'].getboolean(key) @@ -167,6 +166,4 @@ def add_config_arguments(arg_parser, config_args, default_config_file, config_fi arg[1]['default'] = configfile['Default'][key] elif key in configfile['Default'] and arg[1].get('nargs', '') == '?' and 'const' in arg[1]: arg[1]['default'] = arg[1]['const'] # bit of a hack - arg_parser.add_argument(*arg[0], **arg[1]) - -# vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4 + parser.add_argument(*arg[0], **arg[1]) From be3b10f0408bd69fc944a4810d81f6c23e1f28e5 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Mon, 18 Mar 2024 21:22:16 +0100 Subject: [PATCH 22/37] add default only the first time --- counterparty-cli/counterpartycli/server.py | 2 +- counterparty-cli/counterpartycli/util.py | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/counterparty-cli/counterpartycli/server.py b/counterparty-cli/counterpartycli/server.py index 9c0726264e..00dc6cec7c 100755 --- a/counterparty-cli/counterpartycli/server.py +++ b/counterparty-cli/counterpartycli/server.py @@ -90,7 +90,7 @@ def main(): config_file_path = getattr(cmd_args, 'config_file', None) configfile = read_config_file('server.conf', config_file_path) - add_config_arguments(parser, CONFIG_ARGS, configfile) + add_config_arguments(parser, CONFIG_ARGS, configfile, add_default=True) subparsers = parser.add_subparsers(dest='action', help='the action to be taken') diff --git a/counterparty-cli/counterpartycli/util.py b/counterparty-cli/counterpartycli/util.py index 15fd1f6f01..15ed11083b 100644 --- a/counterparty-cli/counterpartycli/util.py +++ b/counterparty-cli/counterpartycli/util.py @@ -157,13 +157,16 @@ def read_config_file(default_config_file, config_file_path=None): return configfile -def add_config_arguments(parser, args, configfile): +def add_config_arguments(parser, args, configfile, add_default=False): for arg in args: - key = arg[0][-1].replace('--', '') - if 'action' in arg[1] and arg[1]['action'] == 'store_true' and key in configfile['Default']: - arg[1]['default'] = configfile['Default'].getboolean(key) - elif key in configfile['Default'] and configfile['Default'][key]: - arg[1]['default'] = configfile['Default'][key] - elif key in configfile['Default'] and arg[1].get('nargs', '') == '?' and 'const' in arg[1]: - arg[1]['default'] = arg[1]['const'] # bit of a hack + if add_default: + key = arg[0][-1].replace('--', '') + if 'action' in arg[1] and arg[1]['action'] == 'store_true' and key in configfile['Default']: + arg[1]['default'] = configfile['Default'].getboolean(key) + elif key in configfile['Default'] and configfile['Default'][key]: + arg[1]['default'] = configfile['Default'][key] + elif key in configfile['Default'] and arg[1].get('nargs', '') == '?' and 'const' in arg[1]: + arg[1]['default'] = arg[1]['const'] # bit of a hack + else: + arg[1]['default'] = argparse.SUPPRESS parser.add_argument(*arg[0], **arg[1]) From 6a2b2f64e691563c1aebcf96ff130e8cc6160103 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Mon, 18 Mar 2024 21:30:01 +0100 Subject: [PATCH 23/37] clean imports --- counterparty-cli/counterpartycli/server.py | 3 --- counterparty-cli/counterpartycli/util.py | 13 ------------- 2 files changed, 16 deletions(-) diff --git a/counterparty-cli/counterpartycli/server.py b/counterparty-cli/counterpartycli/server.py index 00dc6cec7c..6f0949e555 100755 --- a/counterparty-cli/counterpartycli/server.py +++ b/counterparty-cli/counterpartycli/server.py @@ -1,12 +1,9 @@ #! /usr/bin/env python3 import os -import sys import argparse -import optparse import logging - from termcolor import cprint from counterpartylib.lib import log diff --git a/counterparty-cli/counterpartycli/util.py b/counterparty-cli/counterpartycli/util.py index 15ed11083b..74e545b3ff 100644 --- a/counterparty-cli/counterpartycli/util.py +++ b/counterparty-cli/counterpartycli/util.py @@ -1,26 +1,13 @@ -#! /usr/bin/python3 - -import sys import os -import threading import decimal import time import json -import re import requests -import collections import logging -import binascii -from datetime import datetime -from dateutil.tz import tzlocal import argparse import configparser import appdirs -import tarfile -import urllib.request -import shutil import codecs -import tempfile from halo import Halo from termcolor import colored, cprint From c284bcd018c2feeb5487f481355cdb6a5938f53f Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Tue, 19 Mar 2024 15:53:30 +0100 Subject: [PATCH 24/37] update release notes --- release-notes/release-notes-v10.0.0.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/release-notes/release-notes-v10.0.0.md b/release-notes/release-notes-v10.0.0.md index 1bddb6c398..c484987a9a 100644 --- a/release-notes/release-notes-v10.0.0.md +++ b/release-notes/release-notes-v10.0.0.md @@ -32,7 +32,9 @@ Because this release includes numerous changes to the database schema, a full da * Bandit * CodeQL * License Scanner -* Add checkpoints for `mainnet` up to block 825,000 and for `testnet` up to block 2,540,000 + * Build and publish Docker image + * Enable `testnet` test book +* Add checkpoints for `mainnet` up to block 834,500 and for `testnet` up to block 2,580,000 * Rewrite README @@ -60,6 +62,7 @@ Because this release includes numerous changes to the database schema, a full da * Always log to a file, unless `--no-log-files` is set * Fix and refactor `log.set_up()` * Improve thread shutdown logic +* Accept config args before and after the command ## Refactoring and Performance Optimizations @@ -88,7 +91,8 @@ Because this release includes numerous changes to the database schema, a full da * Isolate transaction parsing inside `gettxinfo.py` module * Heavily refactor code; eliminate unused code blocks * Isolate dispenser logic in `get_dispensers_outputs()` and `get_dispensers_tx_info()` - +* Activate check software version every 24H +* Add the possibility to reparse from a given block on minor version change # Credits * Ouziel Slama From c2660aeb2820b3c50b753b6e3a9296356f2f95f8 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Tue, 19 Mar 2024 19:52:33 +0100 Subject: [PATCH 25/37] Update Dockerfile and compose.yml --- Dockerfile | 3 +++ simplenode/compose.yml | 13 +++---------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 506acd4917..f496eb1258 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,3 +31,6 @@ RUN pip3 install . # install counterparty-cli WORKDIR /counterparty-cli RUN pip3 install . + +ENTRYPOINT [ "counterparty-server", "start" ] +CMD [ "-h" ] diff --git a/simplenode/compose.yml b/simplenode/compose.yml index 05eb73c5de..958440b339 100644 --- a/simplenode/compose.yml +++ b/simplenode/compose.yml @@ -11,7 +11,7 @@ services: # - "-chain=test" # testnet - "-rpcallowip=0.0.0.0/0" - "-rpcbind=0.0.0.0" - - "-rpcuser=rpc" + - "-rpcuser=bitcoinrpc" - "-rpcpassword=rpc" - "-listen=1" - "-server=1" @@ -38,7 +38,7 @@ services: # - "--network=testnet" # testnet - "--indexer-rpc-host=0.0.0.0" - "--daemon-rpc-host=0.0.0.0" - - "--cookie=rpc:rpc" + - "--cookie=bitcoinrpc:rpc" - "--daemon-dir=/bitcoin/.bitcoin" - "-vvv" - "--db-dir=/data/addrindexrs/db" @@ -55,19 +55,12 @@ services: ports: - "4000:4000" # mainnet # - "14000:14000" # testnet - entrypoint: "counterparty-server" # TODO: `start` has to come after the flags. command: # "--testnet" # testnet - "--backend-connect=bitcoind" - - "--backend-user=rpc" - - "--backend-password=rpc" - "--indexd-connect=addrindexrs" - - "--rpc-user=rpc" - - "--rpc-password=rpc" - "--rpc-host=0.0.0.0" - - "--log-file=1" - - "--api-log-file=1" - - "start" + - "--rpc-password=rpc" # optional - "--catch-up=bootstrap" environment: - "XDG_DATA_HOME=/data/" From ee3ec3054a7981fb13db7127c391ddb665e26278 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Wed, 20 Mar 2024 09:23:00 +0100 Subject: [PATCH 26/37] always 'rpc' for default user --- counterparty-cli/counterpartycli/client.py | 2 +- counterparty-cli/counterpartycli/clientapi.py | 2 +- counterparty-cli/counterpartycli/server.py | 4 ++-- simplenode/compose.yml | 5 ++--- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/counterparty-cli/counterpartycli/client.py b/counterparty-cli/counterpartycli/client.py index b287eddd8f..596e9dff2f 100755 --- a/counterparty-cli/counterpartycli/client.py +++ b/counterparty-cli/counterpartycli/client.py @@ -37,7 +37,7 @@ [('--wallet-name',), {'default': 'bitcoincore', 'help': 'the wallet name to connect to'}], [('--wallet-connect',), {'default': 'localhost', 'help': 'the hostname or IP of the wallet server'}], [('--wallet-port',), {'type': int, 'help': 'the wallet port to connect to'}], - [('--wallet-user',), {'default': 'bitcoinrpc', 'help': 'the username used to communicate with wallet'}], + [('--wallet-user',), {'default': 'rpc', 'help': 'the username used to communicate with wallet'}], [('--wallet-password',), {'help': 'the password used to communicate with wallet'}], [('--wallet-ssl',), {'action': 'store_true', 'default': False, 'help': 'use SSL to connect to wallet (default: false)'}], [('--wallet-ssl-verify',), {'action': 'store_true', 'default': False, 'help': 'verify SSL certificate of wallet; disallow use of self‐signed certificates (default: false)'}], diff --git a/counterparty-cli/counterpartycli/clientapi.py b/counterparty-cli/counterpartycli/clientapi.py index bf3014b7fe..cf1a85fa60 100755 --- a/counterparty-cli/counterpartycli/clientapi.py +++ b/counterparty-cli/counterpartycli/clientapi.py @@ -119,7 +119,7 @@ def handle_exception(exc_type, exc_value, exc_traceback): raise ConfigurationError("Please specific a valid port number wallet-port configuration parameter") # BTC Wallet user - config.WALLET_USER = wallet_user or 'bitcoinrpc' + config.WALLET_USER = wallet_user or 'rpc' # BTC Wallet password if wallet_password: diff --git a/counterparty-cli/counterpartycli/server.py b/counterparty-cli/counterpartycli/server.py index 6f0949e555..f49a0d1811 100755 --- a/counterparty-cli/counterpartycli/server.py +++ b/counterparty-cli/counterpartycli/server.py @@ -29,7 +29,7 @@ [('--backend-name',), {'default': 'addrindex', 'help': 'the backend name to connect to'}], [('--backend-connect',), {'default': 'localhost', 'help': 'the hostname or IP of the backend server'}], [('--backend-port',), {'type': int, 'help': 'the backend port to connect to'}], - [('--backend-user',), {'default': 'bitcoinrpc', 'help': 'the username used to communicate with backend'}], + [('--backend-user',), {'default': 'rpc', 'help': 'the username used to communicate with backend'}], [('--backend-password',), {'default': 'rpc', 'help': 'the password used to communicate with backend'}], [('--backend-ssl',), {'action': 'store_true', 'default': False, 'help': 'use SSL to connect to backend (default: false)'}], [('--backend-ssl-no-verify',), {'action': 'store_true', 'default': False, 'help': 'verify SSL certificate of backend; disallow use of self‐signed certificates (default: true)'}], @@ -43,7 +43,7 @@ [('--rpc-host',), {'default': 'localhost', 'help': 'the IP of the interface to bind to for providing JSON-RPC API access (0.0.0.0 for all interfaces)'}], [('--rpc-port',), {'type': int, 'help': f'port on which to provide the {config.APP_NAME} JSON-RPC API'}], [('--rpc-user',), {'default': 'rpc', 'help': f'required username to use the {config.APP_NAME} JSON-RPC API (via HTTP basic auth)'}], - [('--rpc-password',), {'help':f'required password (for rpc-user) to use the {config.APP_NAME} JSON-RPC API (via HTTP basic auth)'}], + [('--rpc-password',), {'default': 'rpc', 'help':f'required password (for rpc-user) to use the {config.APP_NAME} JSON-RPC API (via HTTP basic auth)'}], [('--rpc-no-allow-cors',), {'action': 'store_true', 'default': False, 'help': 'allow ajax cross domain request'}], [('--rpc-batch-size',), {'type': int, 'default': config.DEFAULT_RPC_BATCH_SIZE, 'help': f'number of RPC queries by batch (default: {config.DEFAULT_RPC_BATCH_SIZE})'}], [('--requests-timeout',), {'type': int, 'default': config.DEFAULT_REQUESTS_TIMEOUT, 'help': 'timeout value (in seconds) used for all HTTP requests (default: 5)'}], diff --git a/simplenode/compose.yml b/simplenode/compose.yml index 958440b339..0a331be5b1 100644 --- a/simplenode/compose.yml +++ b/simplenode/compose.yml @@ -11,7 +11,7 @@ services: # - "-chain=test" # testnet - "-rpcallowip=0.0.0.0/0" - "-rpcbind=0.0.0.0" - - "-rpcuser=bitcoinrpc" + - "-rpcuser=rpc" - "-rpcpassword=rpc" - "-listen=1" - "-server=1" @@ -38,7 +38,7 @@ services: # - "--network=testnet" # testnet - "--indexer-rpc-host=0.0.0.0" - "--daemon-rpc-host=0.0.0.0" - - "--cookie=bitcoinrpc:rpc" + - "--cookie=rpc:rpc" - "--daemon-dir=/bitcoin/.bitcoin" - "-vvv" - "--db-dir=/data/addrindexrs/db" @@ -60,7 +60,6 @@ services: - "--backend-connect=bitcoind" - "--indexd-connect=addrindexrs" - "--rpc-host=0.0.0.0" - - "--rpc-password=rpc" # optional - "--catch-up=bootstrap" environment: - "XDG_DATA_HOME=/data/" From 4c4dc79bce8982fb8a103d182530fad5587ba9c1 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Wed, 20 Mar 2024 10:45:48 +0100 Subject: [PATCH 27/37] Welcome message with more info --- counterparty-cli/counterpartycli/server.py | 44 +++++++++++++++++----- counterparty-cli/counterpartycli/setup.py | 1 + counterparty-lib/counterpartylib/server.py | 26 +++++++++++-- 3 files changed, 58 insertions(+), 13 deletions(-) diff --git a/counterparty-cli/counterpartycli/server.py b/counterparty-cli/counterpartycli/server.py index f49a0d1811..f160a6c1ee 100755 --- a/counterparty-cli/counterpartycli/server.py +++ b/counterparty-cli/counterpartycli/server.py @@ -3,6 +3,7 @@ import os import argparse import logging +from urllib.parse import quote_plus as urlencode from termcolor import cprint @@ -58,19 +59,48 @@ [('--utxo-locks-max-age',), {'type': int, 'default': config.DEFAULT_UTXO_LOCKS_MAX_AGE, 'help': 'how long to keep a lock on a UTXO being tracked'}], ] +def welcome_message(action, server_configfile): + cprint(f'Running v{config.__version__} of {config.FULL_APP_NAME}.', 'magenta') + + # print some info + cprint(f"Configuration file: {server_configfile}", 'light_grey') + cprint(f"Counterparty database: {config.DATABASE}", 'light_grey') + if config.LOG: + cprint(f'Writing log to file: `{config.LOG}`', 'light_grey') + else: + cprint('Warning: log disabled', 'yellow') + if config.API_LOG: + cprint(f'Writing API accesses log to file: `{config.API_LOG}`', 'light_grey') + else: + cprint('Warning: API log disabled', 'yellow') + + if config.VERBOSE: + if config.TESTNET: + cprint('NETWORK: Testnet', 'light_grey') + elif config.REGTEST: + cprint('NETWORK: Regtest', 'light_grey') + else: + cprint('NETWORK: Mainnet', 'light_grey') + + pass_str = f":{urlencode(config.BACKEND_PASSWORD)}@" + cprint(f'BACKEND_URL: {config.BACKEND_URL.replace(pass_str, ":*****@")}', 'light_grey') + cprint(f'INDEXD_URL: {config.INDEXD_URL}', 'light_grey') + pass_str = f":{urlencode(config.RPC_PASSWORD)}@" + cprint(f'RPC: {config.RPC.replace(pass_str, ":*****@")}', 'light_grey') + + cprint(f"{'-' * 30} {action} {'-' * 30}\n", 'green') + class VersionError(Exception): pass def main(): - cprint(f'Running v{config.__version__} of {config.FULL_APP_NAME}.', 'magenta') - if os.name == 'nt': from counterpartylib.lib import util_windows #patch up cmd.exe's "challenged" (i.e. broken/non-existent) UTF-8 logging util_windows.fix_win32_unicode() # Post installation tasks - generate_config_files() + server_configfile = generate_config_files() # Parse command-line arguments. parser = argparse.ArgumentParser( @@ -163,14 +193,10 @@ def main(): log_file=config.LOG, log_in_console=args.action == 'start' ) + logger.info(f'Running v{APP_VERSION} of {APP_NAME}.') - # print some info - if config.LOG: - cprint(f'Writing log to file: `{config.LOG}`', 'light_grey') - if args.action == 'start' and config.API_LOG: - cprint(f'Writing API accesses log to file: `{config.API_LOG}`', 'light_grey') - cprint(f"{'-' * 30} {args.action} {'-' * 30}\n", 'green') + welcome_message(args.action, server_configfile) # Bootstrapping if args.action == 'bootstrap': diff --git a/counterparty-cli/counterpartycli/setup.py b/counterparty-cli/counterpartycli/setup.py index 00b5a4e371..1ddcdf02ef 100644 --- a/counterparty-cli/counterpartycli/setup.py +++ b/counterparty-cli/counterpartycli/setup.py @@ -156,6 +156,7 @@ def generate_config_files(): if not os.path.exists(client_configfile): client_known_config = server_to_client_config(server_known_config) generate_config_file(client_configfile, CLIENT_CONFIG_ARGS, client_known_config) + return server_configfile def zip_folder(folder_path, zip_path): zip_file = zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) diff --git a/counterparty-lib/counterpartylib/server.py b/counterparty-lib/counterpartylib/server.py index 87d5b16403..53a1ad37fc 100755 --- a/counterparty-lib/counterpartylib/server.py +++ b/counterparty-lib/counterpartylib/server.py @@ -140,10 +140,13 @@ def initialise_config(database_file=None, log_file=None, api_log_file=None, no_l if config.TESTNET: bitcoinlib.SelectParams('testnet') + logger.debug('NETWORK: testnet') elif config.REGTEST: bitcoinlib.SelectParams('regtest') + logger.debug('NETWORK: regtest') else: bitcoinlib.SelectParams('mainnet') + logger.debug('NETWORK: mainnet') network = '' if config.TESTNET: @@ -160,11 +163,20 @@ def initialise_config(database_file=None, log_file=None, api_log_file=None, no_l filename = f'{config.APP_NAME}{network}.db' config.DATABASE = os.path.join(data_dir, filename) + logger.debug('DATABASE: %s', config.DATABASE) + # Log directory log_dir = appdirs.user_log_dir(appauthor=config.XCP_NAME, appname=config.APP_NAME) if not os.path.isdir(log_dir): os.makedirs(log_dir, mode=0o755) + # Set up logging. + config.VERBOSE = verbose + config.QUIET = quiet + + logger.debug('VERBOSE: %s', config.VERBOSE) + logger.debug('QUIET: %s', config.QUIET) + # Log if no_log_files: config.LOG = None @@ -173,10 +185,8 @@ def initialise_config(database_file=None, log_file=None, api_log_file=None, no_l config.LOG = os.path.join(log_dir, filename) else: # user-specified location config.LOG = log_file - - # Set up logging. - config.VERBOSE = verbose - config.QUIET = quiet + + logger.debug('LOG: %s', config.LOG) if no_log_files: # no file logging config.API_LOG = None @@ -186,6 +196,8 @@ def initialise_config(database_file=None, log_file=None, api_log_file=None, no_l else: # user-specified location config.API_LOG = api_log_file + logger.debug('API_LOG: %s', config.API_LOG) + config.API_LIMIT_ROWS = api_limit_rows ############## @@ -258,6 +270,8 @@ def initialise_config(database_file=None, log_file=None, api_log_file=None, no_l config.BACKEND_URL = 'https://' + config.BACKEND_URL else: config.BACKEND_URL = 'http://' + config.BACKEND_URL + + logger.debug('BACKEND_URL: %s', config.BACKEND_URL.replace(f':{config.BACKEND_PASSWORD}@', ':*****@')) # Indexd RPC host if indexd_connect: @@ -286,6 +300,8 @@ def initialise_config(database_file=None, log_file=None, api_log_file=None, no_l # Construct Indexd URL. config.INDEXD_URL = 'http://' + config.INDEXD_CONNECT + ':' + str(config.INDEXD_PORT) + logger.debug('INDEXD_URL: %s', config.INDEXD_URL) + ############## # THINGS WE SERVE @@ -572,6 +588,8 @@ def configure_rpc(rpc_password=None): else: config.RPC = 'http://' + config.RPC_HOST + ':' + str(config.RPC_PORT) + config.RPC_WEBROOT + logger.debug('RPC: %s', config.RPC.replace(f':{urlencode(config.RPC_PASSWORD)}@', ':*****@')) + def bootstrap(): data_dir = appdirs.user_data_dir(appauthor=config.XCP_NAME, appname=config.APP_NAME, roaming=True) From e68339d191d27b34fa5a6d6aca72d45f5f1d91f2 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Wed, 20 Mar 2024 11:08:42 +0100 Subject: [PATCH 28/37] Set up log as soon as possible --- counterparty-cli/counterpartycli/server.py | 12 ++-- counterparty-lib/counterpartylib/server.py | 84 ++++++++++++---------- 2 files changed, 56 insertions(+), 40 deletions(-) diff --git a/counterparty-cli/counterpartycli/server.py b/counterparty-cli/counterpartycli/server.py index f160a6c1ee..3b2de529a7 100755 --- a/counterparty-cli/counterpartycli/server.py +++ b/counterparty-cli/counterpartycli/server.py @@ -161,11 +161,9 @@ def main(): # Configuration init_args = dict(database_file=args.database_file, - log_file=args.log_file, api_log_file=args.api_log_file, no_log_files=args.no_log_files, testnet=args.testnet, testcoin=args.testcoin, regtest=args.regtest, customnet=args.customnet, api_limit_rows=args.api_limit_rows, - backend_name=args.backend_name, backend_connect=args.backend_connect, backend_port=args.backend_port, backend_user=args.backend_user, @@ -179,12 +177,16 @@ def main(): requests_timeout=args.requests_timeout, rpc_batch_size=args.rpc_batch_size, check_asset_conservation=not args.no_check_asset_conservation, - force=args.force, verbose=args.verbose, quiet=args.quiet, + force=args.force, p2sh_dust_return_pubkey=args.p2sh_dust_return_pubkey, utxo_locks_max_addresses=args.utxo_locks_max_addresses, utxo_locks_max_age=args.utxo_locks_max_age) - server.initialise_config(**init_args) + server.initialise_log_config( + verbose=args.verbose, quiet=args.quiet, + log_file=args.log_file, api_log_file=args.api_log_file, no_log_files=args.no_log_files, + testnet=args.testnet, testcoin=args.testcoin, regtest=args.regtest, + ) # set up logging log.set_up( @@ -194,6 +196,8 @@ def main(): log_in_console=args.action == 'start' ) + server.initialise_config(**init_args) + logger.info(f'Running v{APP_VERSION} of {APP_NAME}.') welcome_message(args.action, server_configfile) diff --git a/counterparty-lib/counterpartylib/server.py b/counterparty-lib/counterpartylib/server.py index 53a1ad37fc..229ce9ec3a 100755 --- a/counterparty-lib/counterpartylib/server.py +++ b/counterparty-lib/counterpartylib/server.py @@ -88,10 +88,49 @@ def initialise(*args, **kwargs): return initialise_db() -def initialise_config(database_file=None, log_file=None, api_log_file=None, no_log_files=False, +def initialise_log_config( + verbose=False, quiet=False, log_file=None, api_log_file=None, no_log_files=False, + testnet=False, testcoin=False, regtest=False + ): + # Log directory + log_dir = appdirs.user_log_dir(appauthor=config.XCP_NAME, appname=config.APP_NAME) + if not os.path.isdir(log_dir): + os.makedirs(log_dir, mode=0o755) + + # Set up logging. + config.VERBOSE = verbose + config.QUIET = quiet + + network = '' + if testnet: + network += '.testnet' + if regtest: + network += '.regtest' + if testcoin: + network += '.testcoin' + + # Log + if no_log_files: + config.LOG = None + elif not log_file: # default location + filename = f'server{network}.log' + config.LOG = os.path.join(log_dir, filename) + else: # user-specified location + config.LOG = log_file + + if no_log_files: # no file logging + config.API_LOG = None + elif not api_log_file: # default location + filename = f'server{network}.access.log' + config.API_LOG = os.path.join(log_dir, filename) + else: # user-specified location + config.API_LOG = api_log_file + + +def initialise_config(database_file=None, testnet=False, testcoin=False, regtest=False, api_limit_rows=1000, - backend_name=None, backend_connect=None, backend_port=None, + backend_connect=None, backend_port=None, backend_user=None, backend_password=None, indexd_connect=None, indexd_port=None, backend_ssl=False, backend_ssl_no_verify=False, @@ -99,7 +138,7 @@ def initialise_config(database_file=None, log_file=None, api_log_file=None, no_l rpc_host=None, rpc_port=None, rpc_user=None, rpc_password=None, rpc_no_allow_cors=False, - force=False, verbose=False, quiet=False, + force=False, requests_timeout=config.DEFAULT_REQUESTS_TIMEOUT, rpc_batch_size=config.DEFAULT_RPC_BATCH_SIZE, check_asset_conservation=config.DEFAULT_CHECK_ASSET_CONSERVATION, @@ -109,6 +148,12 @@ def initialise_config(database_file=None, log_file=None, api_log_file=None, no_l estimate_fee_per_kb=None, customnet=None): + # log config alreasdy initialized + logger.debug('VERBOSE: %s', config.VERBOSE) + logger.debug('QUIET: %s', config.QUIET) + logger.debug('LOG: %s', config.LOG) + logger.debug('API_LOG: %s', config.API_LOG) + # Data directory data_dir = appdirs.user_data_dir(appauthor=config.XCP_NAME, appname=config.APP_NAME, roaming=True) if not os.path.isdir(data_dir): @@ -165,39 +210,6 @@ def initialise_config(database_file=None, log_file=None, api_log_file=None, no_l logger.debug('DATABASE: %s', config.DATABASE) - # Log directory - log_dir = appdirs.user_log_dir(appauthor=config.XCP_NAME, appname=config.APP_NAME) - if not os.path.isdir(log_dir): - os.makedirs(log_dir, mode=0o755) - - # Set up logging. - config.VERBOSE = verbose - config.QUIET = quiet - - logger.debug('VERBOSE: %s', config.VERBOSE) - logger.debug('QUIET: %s', config.QUIET) - - # Log - if no_log_files: - config.LOG = None - elif not log_file: # default location - filename = f'server{network}.log' - config.LOG = os.path.join(log_dir, filename) - else: # user-specified location - config.LOG = log_file - - logger.debug('LOG: %s', config.LOG) - - if no_log_files: # no file logging - config.API_LOG = None - elif not api_log_file: # default location - filename = f'server{network}.access.log' - config.API_LOG = os.path.join(log_dir, filename) - else: # user-specified location - config.API_LOG = api_log_file - - logger.debug('API_LOG: %s', config.API_LOG) - config.API_LIMIT_ROWS = api_limit_rows ############## From 568df44c47926f2575a1e64559c77a08c4875f73 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Wed, 20 Mar 2024 11:10:36 +0100 Subject: [PATCH 29/37] try to explain to CodeQL --- counterparty-cli/counterpartycli/server.py | 6 ++++-- counterparty-lib/counterpartylib/server.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/counterparty-cli/counterpartycli/server.py b/counterparty-cli/counterpartycli/server.py index 3b2de529a7..dfd3475449 100755 --- a/counterparty-cli/counterpartycli/server.py +++ b/counterparty-cli/counterpartycli/server.py @@ -83,10 +83,12 @@ def welcome_message(action, server_configfile): cprint('NETWORK: Mainnet', 'light_grey') pass_str = f":{urlencode(config.BACKEND_PASSWORD)}@" - cprint(f'BACKEND_URL: {config.BACKEND_URL.replace(pass_str, ":*****@")}', 'light_grey') + cleaned_backend_url = config.BACKEND_URL.replace(pass_str, ":*****@") + cprint(f'BACKEND_URL: {cleaned_backend_url}', 'light_grey') cprint(f'INDEXD_URL: {config.INDEXD_URL}', 'light_grey') pass_str = f":{urlencode(config.RPC_PASSWORD)}@" - cprint(f'RPC: {config.RPC.replace(pass_str, ":*****@")}', 'light_grey') + cleaned_rpc_url = config.RPC.replace(pass_str, ":*****@") + cprint(f'RPC: {cleaned_rpc_url}', 'light_grey') cprint(f"{'-' * 30} {action} {'-' * 30}\n", 'green') diff --git a/counterparty-lib/counterpartylib/server.py b/counterparty-lib/counterpartylib/server.py index 229ce9ec3a..8ab6c5a6cc 100755 --- a/counterparty-lib/counterpartylib/server.py +++ b/counterparty-lib/counterpartylib/server.py @@ -283,7 +283,8 @@ def initialise_config(database_file=None, else: config.BACKEND_URL = 'http://' + config.BACKEND_URL - logger.debug('BACKEND_URL: %s', config.BACKEND_URL.replace(f':{config.BACKEND_PASSWORD}@', ':*****@')) + cleaned_backend_url = config.BACKEND_URL.replace(f':{config.BACKEND_PASSWORD}@', ':*****@') + logger.debug('BACKEND_URL: %s', cleaned_backend_url) # Indexd RPC host if indexd_connect: @@ -600,7 +601,8 @@ def configure_rpc(rpc_password=None): else: config.RPC = 'http://' + config.RPC_HOST + ':' + str(config.RPC_PORT) + config.RPC_WEBROOT - logger.debug('RPC: %s', config.RPC.replace(f':{urlencode(config.RPC_PASSWORD)}@', ':*****@')) + cleaned_rpc_url = config.RPC.replace(f':{urlencode(config.RPC_PASSWORD)}@', ':*****@') + logger.debug('RPC: %s', cleaned_rpc_url) def bootstrap(): From 6fae3d9508c32d4c6fa3a2687d770ed0db8ab497 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Wed, 20 Mar 2024 11:29:12 +0100 Subject: [PATCH 30/37] fix server.initialise() --- counterparty-lib/counterpartylib/server.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/counterparty-lib/counterpartylib/server.py b/counterparty-lib/counterpartylib/server.py index 8ab6c5a6cc..18646124dc 100755 --- a/counterparty-lib/counterpartylib/server.py +++ b/counterparty-lib/counterpartylib/server.py @@ -84,6 +84,16 @@ def get_lock(): def initialise(*args, **kwargs): + initialise_log_config( + verbose=kwargs.pop('verbose', False), + quiet=kwargs.pop('quiet', False), + log_file=kwargs.pop('log_file', None), + api_log_file=kwargs.pop('api_log_file', None), + no_log_files=kwargs.pop('no_log_files', False), + testnet=kwargs.get('testnet', False), + testcoin=kwargs.get('testcoin', False), + regtest=kwargs.get('regtest', False), + ) initialise_config(*args, **kwargs) return initialise_db() From 2497b83f39951764c315f6807128d5c651015af6 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Wed, 20 Mar 2024 11:36:19 +0100 Subject: [PATCH 31/37] rename 'show-config' to 'show-params' --- counterparty-cli/counterpartycli/server.py | 6 +++--- counterparty-lib/counterpartylib/server.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/counterparty-cli/counterpartycli/server.py b/counterparty-cli/counterpartycli/server.py index dfd3475449..4fccbc3117 100755 --- a/counterparty-cli/counterpartycli/server.py +++ b/counterparty-cli/counterpartycli/server.py @@ -151,7 +151,7 @@ def main(): parser_checkdb = subparsers.add_parser('check-db', help='do an integrity check on the database') add_config_arguments(parser_checkdb, CONFIG_ARGS, configfile) - parser_show_config = subparsers.add_parser('show-config', help='Show counterparty-server configuration') + parser_show_config = subparsers.add_parser('show-params', help='Show counterparty-server configuration') add_config_arguments(parser_show_config, CONFIG_ARGS, configfile) args = parser.parse_args() @@ -225,8 +225,8 @@ def main(): elif args.action == 'start': server.start_all(catch_up=args.catch_up) - elif args.action == 'show-config': - server.show_config() + elif args.action == 'show-params': + server.show_params() elif args.action == 'vacuum': server.vacuum() diff --git a/counterparty-lib/counterpartylib/server.py b/counterparty-lib/counterpartylib/server.py index 18646124dc..06e4c3c9c4 100755 --- a/counterparty-lib/counterpartylib/server.py +++ b/counterparty-lib/counterpartylib/server.py @@ -589,7 +589,7 @@ def check_database(): cprint('Database checks complete.', 'green') -def show_config(): +def show_params(): output = vars(config) for k in list(output.keys()): if k.isupper(): From 588843c70c0e91b24d7325bd8bdf5063ba52d0b8 Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Wed, 20 Mar 2024 13:12:08 +0100 Subject: [PATCH 32/37] Add Warning with Confirmation Dialogue to bootstrap Command --- counterparty-lib/counterpartylib/server.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/counterparty-lib/counterpartylib/server.py b/counterparty-lib/counterpartylib/server.py index 06e4c3c9c4..9410e66352 100755 --- a/counterparty-lib/counterpartylib/server.py +++ b/counterparty-lib/counterpartylib/server.py @@ -523,7 +523,7 @@ def start_all(catch_up='normal'): connect_to_backend() if not os.path.exists(config.DATABASE) and catch_up == 'bootstrap': - bootstrap() + bootstrap(no_confirm=True) db = initialise_db() @@ -615,7 +615,19 @@ def configure_rpc(rpc_password=None): logger.debug('RPC: %s', cleaned_rpc_url) -def bootstrap(): +def bootstrap(no_confirm=False): + warning_message = '''WARNING: `counterparty-server bootstrap` downloads a recent snapshot of a Counterparty database +from a centralized server maintained by the Counterparty Core development team. +Because this method does not involve verifying the history of Counterparty transactions yourself, +the `bootstrap` command should not be used for mission-critical, commercial or public-facing nodes. + ''' + cprint(warning_message, 'yellow') + + if not no_confirm: + confirmation_message = colored('Continue? (y/N): ', "magenta") + if input(confirmation_message).lower() != 'y': + exit() + data_dir = appdirs.user_data_dir(appauthor=config.XCP_NAME, appname=config.APP_NAME, roaming=True) # Set Constants. From f3f3ead567bdb25047b5b58d86827f25508b515f Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Wed, 20 Mar 2024 13:21:05 +0100 Subject: [PATCH 33/37] Update README with supported OS --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 19c932b41e..c5ceae67a1 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,8 @@ Then wait for your node to catch up with the network. Note: this process current # Manual Installation +Counterparty Core can be installed on most platforms but, for now, manual installation is being tested and is only officially supported on Ubuntu 22.04 and MacOS. + Dependencies: - Bitcoin Core From ba36aba04e7565bc5c9cc3790dacfa83a5d7361d Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Wed, 20 Mar 2024 13:26:49 +0100 Subject: [PATCH 34/37] Add --no-confirm flag --- .github/workflows/test_book.yml | 2 +- counterparty-cli/counterpartycli/server.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_book.yml b/.github/workflows/test_book.yml index 9bcd60d169..1b790f5fae 100644 --- a/.github/workflows/test_book.yml +++ b/.github/workflows/test_book.yml @@ -33,7 +33,7 @@ jobs: cd counterparty-cli && pip install -e . && cd .. - name: Bootstrap testnet database run: | - counterparty-server --testnet bootstrap + counterparty-server --testnet bootstrap --no-confirm - name: Run tests run: | cd counterparty-lib diff --git a/counterparty-cli/counterpartycli/server.py b/counterparty-cli/counterpartycli/server.py index 4fccbc3117..719dddf42f 100755 --- a/counterparty-cli/counterpartycli/server.py +++ b/counterparty-cli/counterpartycli/server.py @@ -50,6 +50,7 @@ [('--requests-timeout',), {'type': int, 'default': config.DEFAULT_REQUESTS_TIMEOUT, 'help': 'timeout value (in seconds) used for all HTTP requests (default: 5)'}], [('--force',), {'action': 'store_true', 'default': False, 'help': 'skip backend check, version check, process lock (NOT FOR USE ON PRODUCTION SYSTEMS)'}], + [('--no-confirm',), {'action': 'store_true', 'default': False, 'help': 'don\'t ask for confirmation'}], [('--database-file',), {'default': None, 'help': 'the path to the SQLite3 database file'}], [('--log-file',), {'nargs': '?', 'const': None, 'default': False, 'help': 'log to the specified file'}], [('--api-log-file',), {'nargs': '?', 'const': None, 'default': False, 'help': 'log API requests to the specified file'}], @@ -206,7 +207,7 @@ def main(): # Bootstrapping if args.action == 'bootstrap': - server.bootstrap() + server.bootstrap(no_confirm=args.no_confirm) # PARSING elif args.action == 'reparse': From 0ea70bbbf38d60291b451c46e978e282e2de5a1e Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Wed, 20 Mar 2024 13:29:55 +0100 Subject: [PATCH 35/37] update release notes --- release-notes/release-notes-v10.0.0.md | 1 + 1 file changed, 1 insertion(+) diff --git a/release-notes/release-notes-v10.0.0.md b/release-notes/release-notes-v10.0.0.md index c484987a9a..40c813925c 100644 --- a/release-notes/release-notes-v10.0.0.md +++ b/release-notes/release-notes-v10.0.0.md @@ -93,6 +93,7 @@ Because this release includes numerous changes to the database schema, a full da * Isolate dispenser logic in `get_dispensers_outputs()` and `get_dispensers_tx_info()` * Activate check software version every 24H * Add the possibility to reparse from a given block on minor version change +* Add Warning with Confirmation Dialogue to bootstrap Command and `--no-confirm` flag # Credits * Ouziel Slama From 23a99bcee9372365c1a31d672b4a0b3d6b993a6d Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Wed, 20 Mar 2024 15:28:15 +0100 Subject: [PATCH 36/37] Fix compose.yml --- simplenode/compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simplenode/compose.yml b/simplenode/compose.yml index 0a331be5b1..5c10e30203 100644 --- a/simplenode/compose.yml +++ b/simplenode/compose.yml @@ -37,7 +37,7 @@ services: - "--network=bitcoin" # mainnet # - "--network=testnet" # testnet - "--indexer-rpc-host=0.0.0.0" - - "--daemon-rpc-host=0.0.0.0" + - "--daemon-rpc-host=bitcoind" - "--cookie=rpc:rpc" - "--daemon-dir=/bitcoin/.bitcoin" - "-vvv" From 056e57859469f48535f735546378f0de9f3a46ba Mon Sep 17 00:00:00 2001 From: Ouziel Slama Date: Wed, 20 Mar 2024 20:08:23 +0100 Subject: [PATCH 37/37] v10.0.0-beta.1 --- counterparty-cli/requirements.txt | 2 +- counterparty-lib/counterpartylib/lib/config.py | 4 ++-- counterparty-lib/requirements.txt | 2 +- counterparty-rs/Cargo.lock | 2 +- counterparty-rs/Cargo.toml | 2 +- release-notes/release-notes-v10.0.0.md | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/counterparty-cli/requirements.txt b/counterparty-cli/requirements.txt index 990bf8f87d..1a8a35db41 100644 --- a/counterparty-cli/requirements.txt +++ b/counterparty-cli/requirements.txt @@ -5,4 +5,4 @@ colorlog==6.8.0 python-dateutil==2.8.2 requests==2.31.0 termcolor==2.4.0 -counterparty-lib==10.0.0-alpha +counterparty-lib==10.0.0-beta.1 diff --git a/counterparty-lib/counterpartylib/lib/config.py b/counterparty-lib/counterpartylib/lib/config.py index 3285d89e9c..5a0f174509 100644 --- a/counterparty-lib/counterpartylib/lib/config.py +++ b/counterparty-lib/counterpartylib/lib/config.py @@ -7,7 +7,7 @@ # Semantic Version -__version__ = "10.0.0-alpha" # for hatch +__version__ = "10.0.0-beta.1" # for hatch VERSION_STRING = __version__ version = VERSION_STRING.split('-')[0].split('.') VERSION_MAJOR = int(version[0]) @@ -15,7 +15,7 @@ VERSION_REVISION = int(version[2]) VERSION_PRE_RELEASE = '-'.join(VERSION_STRING.split('-')[1:]) -ADDRINDEXRS_VERSION = "0.4.3" +ADDRINDEXRS_VERSION = "0.4.4" # When updating to a new verion, we are making a rollback if major version changes. # If minor version changes and if needed, we are making a reparse from a given block. diff --git a/counterparty-lib/requirements.txt b/counterparty-lib/requirements.txt index c827b94294..5923604bad 100644 --- a/counterparty-lib/requirements.txt +++ b/counterparty-lib/requirements.txt @@ -24,4 +24,4 @@ plyvel==1.5.1 arc4==0.4.0 halo==0.0.31 termcolor==2.4.0 -counterparty-rs==10.0.0-alpha +counterparty-rs==10.0.0-beta.1 diff --git a/counterparty-rs/Cargo.lock b/counterparty-rs/Cargo.lock index 39f2bac181..f2695080db 100644 --- a/counterparty-rs/Cargo.lock +++ b/counterparty-rs/Cargo.lock @@ -198,7 +198,7 @@ checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" [[package]] name = "counterparty-rs" -version = "10.0.0-alpha" +version = "10.0.0-beta.1" dependencies = [ "bip32", "bitcoin", diff --git a/counterparty-rs/Cargo.toml b/counterparty-rs/Cargo.toml index 1903b6db44..e8ae5c353e 100644 --- a/counterparty-rs/Cargo.toml +++ b/counterparty-rs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "counterparty-rs" -version = "10.0.0-alpha" +version = "10.0.0-beta.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/release-notes/release-notes-v10.0.0.md b/release-notes/release-notes-v10.0.0.md index 40c813925c..108a9dc308 100644 --- a/release-notes/release-notes-v10.0.0.md +++ b/release-notes/release-notes-v10.0.0.md @@ -7,7 +7,7 @@ This release does not include any protocol changes, so there is no deadline for Because this release includes numerous changes to the database schema, a full database rebuild is required and the major version number has been bumped from 9 to 10. Follow the updated installation instructions in the [README](/README.md) to download and install the latest version of Counterparty Core, run `counterparty-server kickstart` (while `bitcoind` is not running), then start the server with `counterparty-server start`. The rebuild should happen automatically, and it should take between 8 and 24 hours hours to complete. -**IMPORTANT** Be certain that you are running the latest version of AddrIndexRs (v0.4.3). +**IMPORTANT** Be certain that you are running the latest version of AddrIndexRs (v0.4.4). # ChangeLog