Merge pull request #1567 from CounterpartyXCP/develop

v10.0.0-rc.1

.github/workflows/build_docker_image.yml

@@ -18,4 +18,4 @@ jobs:
         docker build -t counterparty .
     - name: Test image
       run: |
-        docker run --rm counterparty counterparty-server -h
+        docker run --rm counterparty -h

.github/workflows/test_compose.sh

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+set -e
+set -x
+
+if [ -f "./DOCKER_COMPOSE_TEST_LOCK" ]; then
+    echo "A test is already running. Exiting."
+    exit 1
+fi
+touch "./DOCKER_COMPOSE_TEST_LOCK"
+
+GIT_BRANCH="$1"
+
+# pull the latest code
+rm -rf counterparty-core
+git clone --branch "$GIT_BRANCH" https://github.com/CounterpartyXCP/counterparty-core.git
+cd counterparty-core
+
+VERSION=$(cat docker-compose.yml | grep 'image: counterparty/counterparty:' | awk -F ":" '{print $3}')
+
+# stop the running containers
+docker compose stop
+
+# remove counterparty-core container
+docker rm counterparty-core-counterparty-core-1
+
+# remove counterparty-core image
+docker rmi counterparty/counterparty:$VERSION || true
+
+# build the counterparty-core new image
+docker build -t counterparty/counterparty:$VERSION .
+
+# remove the counterparty-core data
+sudo rm -rf ~/counterparty-docker-data/counterparty/*
+
+# re-start containers
+COUNTERPARTY_NETWORK=test docker compose up -d
+
+while [ "$(docker compose logs counterparty-core 2>&1 | grep 'Ready for queries')" = "" ]; do
+    echo "Waiting for counterparty-core to be ready"
+    sleep 1
+done
+
+rm -f ../DOCKER_COMPOSE_TEST_LOCK
+
+server_response=$(curl -X POST http://127.0.0.1:14000/api/ \
+                        --user rpc:rpc \
+                        -H 'Content-Type: application/json; charset=UTF-8'\
+                        -H 'Accept: application/json, text/javascript' \
+                        --data-binary '{ "jsonrpc": "2.0", "id": 0, "method": "get_running_info" }' \
+                        --write-out '%{http_code}' --silent --output /dev/null)
+
+if [ "$server_response" -ne 200 ]; then
+    echo "Failed to get_running_info"
+    exit 1
+fi

.github/workflows/test_compose.yml

@@ -0,0 +1,25 @@
+name: Test Docker Compose
+
+on:
+  push:
+    branches: ['develop']
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Get branch names.
+      id: branch-names
+      uses: tj-actions/branch-names@v8
+    - uses: alinz/ssh-scp-action@master
+      with:
+        key: ${{ secrets.TEST_SERVER_KEY }}
+        host: ${{ secrets.TEST_SERVER_IP }}
+        user: ${{ secrets.TEST_SERVER_USER }}
+        ssh_before: |
+          rm -f test_compose.sh
+        scp: |
+          .github/workflows/test_compose.sh ${{ secrets.TEST_SERVER_USER }}@${{ secrets.TEST_SERVER_IP }}:~/test_compose.sh
+        ssh_after: |
+          sh test_compose.sh ${{ steps.branch-names.outputs.current_branch }}

Dockerfile

@@ -32,5 +32,5 @@ RUN pip3 install .
 WORKDIR /counterparty-cli
 RUN pip3 install .
 
-ENTRYPOINT [ "counterparty-server", "start" ]
-CMD [ "-h" ]
+ENTRYPOINT [ "counterparty-server"]
+CMD [ "start" ]

README.md

@@ -13,18 +13,38 @@ The simplest way to get your Counterparty node up and running is to use Docker C
 sudo apt install docker-compose
 ```
 
-Then, for `mainnet`, run:
+Then run node services in background with:
 
 ```bash
-docker-compose -f simplenode/compose.yml up
+git clone [email protected]:CounterpartyXCP/counterparty-core.git
+cd counterparty-core
+mkdir ~/counterparty-docker-data
+docker-compose up -d
 ```
 
-For `testnet`, modify the Docker Compose file in `simplenode/` and then run:
+**To run a node you must have at least 1.5TB free.** By default all data is stored in the `~/counterparty-docker-data` folder. You can modify this folder with the environment variable `$COUNTERPARTY_DOCKER_DATA`. For example:
+
+```bash
+COUNTERPARTY_DOCKER_DATA=/var/data docker-compose up -d
+```
+
+Use `docker-compose logs` to view output from services. For example:
+
 ```bash
-docker-compose -f simplenode/compose.yml -p simplenode-testnet up
+docker-compose logs --tail=10 -f bitcoind
+docker-compose logs --tail=10 -f addrindexrs
+docker-compose logs --tail=10 -f counterparty-core
+```
+
+You can use the environment variable `COUNTERPARTY_NETWORK` to run a `testnet` node:
+
+```
+COUNTERPARTY_NETWORK=test docker-compose up -d
 ```
 
-Then wait for your node to catch up with the network. Note: this process currently takes a long time, beause it does not make use of the `bootstrap` or `kickstart` functionality. (See below.)
+NOTES:
+- By default, this Docker Compose script makes use of the `bootstrap` functionality, because Docker makes it hard to use `kickstart`. (See below.)
+- When working with a low-memory system, you can tell `addrindexrs` to use JSON-RPC to communicate with `bitcoind` using the environment variable `ADDRINDEXRS_JSONRPC_IMPORT`: `ADDRINDEXRS_JSONRPC_IMPORT=true docker-compose up -d`
 
 
 # Manual Installation

RELEASE.md

@@ -2,6 +2,7 @@
 - [ ] Update `VERSION_` variables in `lib/config.py`
 - [ ] Update `protocol_changes.json` (as necessary)
 - [ ] Update Counterparty package versions in the `requirements.txt` files
+- [ ] Update Counterparty Docker images versions in the `docker-compose.yml` files
 - [ ] Review all open pull requests
 - [ ] Write release notes
 - [ ] Create pull request against `master`

counterparty-cli/counterpartycli/client.py

@@ -14,7 +14,7 @@
 from counterpartylib.lib.util import make_id, BET_TYPE_NAME
 from counterpartylib.lib.log import isodt
 from counterpartylib.lib.exceptions import TransactionError
-from counterpartycli.util import add_config_arguments
+from counterpartycli.util import add_config_arguments, read_config_file
 from counterpartycli.setup import generate_config_files
 from counterpartycli import APP_VERSION, util, messages, wallet, console, clientapi
 
@@ -70,7 +70,11 @@ def main():
     parser.add_argument('-V', '--version', action='version', version=f"{APP_NAME} v{APP_VERSION}; counterparty-lib v{config.VERSION_STRING}")
     parser.add_argument('--config-file', help='the location of the configuration file')
 
-    add_config_arguments(parser, CONFIG_ARGS, 'client.conf')
+    cmd_args = parser.parse_known_args()[0]
+    config_file_path = getattr(cmd_args, 'config_file', None)
+    configfile = read_config_file('client.conf', config_file_path)
+
+    add_config_arguments(parser, CONFIG_ARGS, configfile, add_default=True)
 
     subparsers = parser.add_subparsers(dest='action', help='the action to be taken')
 

counterparty-cli/counterpartycli/server.py

@@ -22,6 +22,7 @@
 CONFIG_ARGS = [
     [('-v', '--verbose'), {'dest': 'verbose', 'action': 'store_true', 'default': False, 'help': 'sets log level to DEBUG'}],
     [('--quiet',), {'dest': 'quiet', 'action': 'store_true', 'default': False, 'help': 'sets log level to ERROR'}],
+    [('--mainnet',), {'action': 'store_true', 'default': True, 'help': f'use {config.BTC_NAME} mainet addresses and block numbers'}],
     [('--testnet',), {'action': 'store_true', 'default': False, 'help': f'use {config.BTC_NAME} testnet addresses and block numbers'}],
     [('--testcoin',), {'action': 'store_true', 'default': False, 'help': f'use the test {config.XCP_NAME} network on every blockchain'}],
     [('--regtest',), {'action': 'store_true', 'default': False, 'help': f'use {config.BTC_NAME} regtest addresses and block numbers'}],
@@ -55,6 +56,7 @@
     [('--log-file',), {'nargs': '?', 'const': None, 'default': False, 'help': 'log to the specified file'}],
     [('--api-log-file',), {'nargs': '?', 'const': None, 'default': False, 'help': 'log API requests to the specified file'}],
     [('--no-log-files',), {'action': 'store_true', 'default': False, 'help': 'Don\'t write log files'}],
+    [('--json-log',), {'action': 'store_true', 'default': False, 'help': 'Log events in JSON format'}],
 
     [('--utxo-locks-max-addresses',), {'type': int, 'default': config.DEFAULT_UTXO_LOCKS_MAX_ADDRESSES, 'help': 'max number of addresses for which to track UTXO locks'}],
     [('--utxo-locks-max-age',), {'type': int, 'default': config.DEFAULT_UTXO_LOCKS_MAX_AGE, 'help': 'how long to keep a lock on a UTXO being tracked'}],
@@ -103,7 +105,7 @@ def main():
         util_windows.fix_win32_unicode()
 
     # Post installation tasks
-    server_configfile = generate_config_files()
+    server_configfile, _ = generate_config_files()
 
     # Parse command-line arguments.
     parser = argparse.ArgumentParser(
@@ -189,6 +191,7 @@ def main():
         verbose=args.verbose, quiet=args.quiet,
         log_file=args.log_file, api_log_file=args.api_log_file, no_log_files=args.no_log_files,
         testnet=args.testnet, testcoin=args.testcoin, regtest=args.regtest,
+        json_log=args.json_log
     )
 
     # set up logging

counterparty-cli/counterpartycli/setup.py

@@ -152,11 +152,12 @@ def generate_config_files():
         server_known_config = get_server_known_config()
         generate_config_file(server_configfile, SERVER_CONFIG_ARGS, server_known_config)
 
-        client_configfile = os.path.join(configdir, 'client.conf')
-        if not os.path.exists(client_configfile):
-            client_known_config = server_to_client_config(server_known_config)
-            generate_config_file(client_configfile, CLIENT_CONFIG_ARGS, client_known_config)
-    return server_configfile
+    client_configfile = os.path.join(configdir, 'client.conf')
+    if not os.path.exists(client_configfile):
+        client_known_config = server_to_client_config(server_known_config)
+        generate_config_file(client_configfile, CLIENT_CONFIG_ARGS, client_known_config)
+
+    return server_configfile, client_configfile
 
 def zip_folder(folder_path, zip_path):
     zip_file = zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED)

counterparty-cli/requirements.txt

@@ -5,4 +5,4 @@ colorlog==6.8.0
 python-dateutil==2.8.2
 requests==2.31.0
 termcolor==2.4.0
-counterparty-lib==10.0.0-beta.1
+counterparty-lib==10.0.0-rc.1

counterparty-lib/counterpartylib/lib/api.py

@@ -24,6 +24,7 @@
 import struct
 import apsw
 import flask
+from flask import request
 from flask_httpauth import HTTPBasicAuth
 import jsonrpc
 from jsonrpc import dispatcher
@@ -1010,6 +1011,35 @@ def _set_cors_headers(response):
                 response.headers['Access-Control-Allow-Methods'] = 'GET, POST, OPTIONS'
                 response.headers['Access-Control-Allow-Headers'] = 'DNT,X-Mx-ReqToken,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization'
 
+        ##### REST ROUTES #####
+
+        @app.route('/addresses/<address>/balances', methods=['GET'])
+        def handle_address_balances(address):
+            return ledger.get_address_balances(self.db, address)
+
+        @app.route('/assets/<asset>/balances', methods=['GET'])
+        def handle_asset_balances(asset):
+            return ledger.get_asset_balances(self.db, asset)
+
+        @app.route('/assets/<asset>/', methods=['GET'])
+        def handle_asset_info(asset):
+            return get_asset_info(asset=asset)
+
+        @app.route('/assets/<asset>/orders', methods=['GET'])
+        def handle_asset_orders(asset):
+            status = request.args.get('status', 'open')
+            return ledger.get_orders_by_asset(self.db, asset, status)
+
+        @app.route('/orders/<tx_hash>', methods=['GET'])
+        def handle_order_info(tx_hash):
+            return ledger.get_order(self.db, tx_hash)
+
+        @app.route('/orders/<tx_hash>/matches', methods=['GET'])
+        def handle_order_matches(tx_hash):
+            status = request.args.get('status', 'pending')
+            return ledger.get_order_matches_by_order(self.db, tx_hash, status)
+
+
         @app.route('/healthz', methods=['GET'])
         def handle_healthz():
             msg, code = 'Healthy', 200

counterparty-lib/counterpartylib/lib/backend/addrindexrs.py

@@ -564,8 +564,14 @@ def stop():
 # No locking thread.
 # Assume only one instance of this class is used at a time and not concurrently.
 # This class does not handle most of the errors, it's up to the caller to do so.
-# This class does not check ID in the response.
-# This class assumes response are always not longer than READ_BUF_SIZE (65536 bytes).
+# This class assumes responses are always not longer than READ_BUF_SIZE (65536 bytes).
+# This class assumes responses are always valid JSON.
+
+ADDRINDEXRS_CLIENT_TIMEOUT = 20.0
+
+class AddrindexrsSocketError(Exception):
+    pass
+
 class AddrindexrsSocket:
 
     def __init__(self):
@@ -574,10 +580,10 @@ def __init__(self):
 
     def connect(self):
         self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        self.sock.settimeout(SOCKET_TIMEOUT)
+        self.sock.settimeout(ADDRINDEXRS_CLIENT_TIMEOUT)
         self.sock.connect((config.INDEXD_CONNECT, config.INDEXD_PORT))
 
-    def _send(self, query, timeout=SOCKET_TIMEOUT):
+    def _send(self, query, timeout=ADDRINDEXRS_CLIENT_TIMEOUT):
         query["id"] = self.next_message_id
 
         message = (json.dumps(query) + "\n").encode('utf8')
@@ -591,25 +597,26 @@ def _send(self, query, timeout=SOCKET_TIMEOUT):
             try:
                 data = self.sock.recv(READ_BUF_SIZE)
             except (TimeoutError, ConnectionResetError) as e:
-                return {}
+                raise AddrindexrsSocketError("Timeout or connection reset. Please retry.") from e
             if data:
-                try:
-                    response = json.loads(data.decode('utf-8'))
-                except json.decoder.JSONDecodeError:
-                    return {}
-                if not response:
-                    return {}
+                response = json.loads(data.decode('utf-8')) # assume valid JSON
+                if "id" not in response:
+                    raise AddrindexrsSocketError("No ID in response")
+                if response["id"] != query["id"]:
+                    raise AddrindexrsSocketError("ID mismatch in response")
                 if "error" in response:
-                    return {}
+                    if response["error"] == 'no txs for address':
+                        return {}
+                    raise AddrindexrsSocketError(response["error"])
                 if "result" not in response:
-                    return {}
+                    raise AddrindexrsSocketError("No error and no result in response")
                 return response["result"]
 
             duration = time.time() - start_time
             if duration > timeout:
-                return {}
+                raise AddrindexrsSocketError("Timeout. Please retry.")
 
-    def send(self, query, timeout=SOCKET_TIMEOUT, retry=0):
+    def send(self, query, timeout=ADDRINDEXRS_CLIENT_TIMEOUT, retry=0):
         try:
             return self._send(query, timeout=timeout)
         except BrokenPipeError:
@@ -619,7 +626,7 @@ def send(self, query, timeout=SOCKET_TIMEOUT, retry=0):
             self.connect()
             return self.send(query, timeout=timeout, retry=retry + 1)
 
-    def get_oldest_tx(self, address, timeout=SOCKET_TIMEOUT, block_index=None):
+    def get_oldest_tx(self, address, timeout=ADDRINDEXRS_CLIENT_TIMEOUT, block_index=None):
         hsh = _address_to_hash(address)
         query = {
             "method": "blockchain.scripthash.get_oldest_tx",