Merge pull request #832 from CounterpartyXCP/develop

9.53.0

.gitignore

@@ -1,4 +1,5 @@
 env
+venv
 docs/_build/*
 
 profile.txt
@@ -23,6 +24,9 @@ test/fixtures/scenarios/*.new.*
 /*.egg-info
 
 # Virtualenv folders
-bin/
-lib/
-include/
+/bin/
+/lib/
+/include/
+
+# Cache folders
+.cache

.travis.yml

@@ -11,3 +11,5 @@ notifications:
     secure: cl0G5fWZNnIK5VQ6BPZ4RtwVO2/nfvX/zjhljfF6cQ20OKyvyiJHq+e67fl2pkCKDlqlEqSWaQJ6G52YzUpDQmf+o7qpH6YnkFxSqYp5h4YF1hJw4sCHJ7bVSLrjogWxU8QaNvH7YpL4YkGe+WOau2FgoIZOcMt6hhKrvWclfuc=
 after_success:
 - coveralls
+
+sudo: required

ChangeLog.md

@@ -1,4 +1,18 @@
-## Client Versions ##
+## Library Versions ##
+* v9.53.0 (2016-01-18)
+    * Remove `messages` table from being covered by "undolog" functionality added in `v9.52.0`.
+    * Add `min_message_index` to `get_blocks` API call.
+    * Retry more than once with `getrawtransaction_batch` if a specific txhash is not found in `bitcoind`'s addrindex.
+    * Update `setup.py` to properly utilize (newer) egg-style install. Previously the "old" style install was invoked when it shouldn't have been.
+    * Update backend mempool caching code to keep full mempool, instead of just XCP transactions (from @rubensayshi).
+    * Increase max `OP_RETURN` size used from 40 bytes to 80 bytes (from @rubensayshi).
+    * Add ModuleLoggingFilter for (NodeJS-style) module-level log filtering (from @rubensayshi).
+    * Fixed `backend.get_unspent_outputs` from raising exception when a transaction contains an output with garbage data (from @rubensayshi).
+    * `base58_check_decode` padding byte used should be `x00`, not the `version` (from @rubensayshi).
+    * Require `sudo` for non-container `travis` builds, due to our `serpent` dependency (from @rubensayshi).
+    * Made `getrawtransaction_batch` deal better with missing raw transactions from `bitcoind`.
+    * A number of other small bug fixes, logging tweaks, etc.
+    * NOTE: This versions mhash (message hash) will be different than that of nodes running `9.52.0`, but the other hashes should continue to match.
 * v9.52.0 (2015-10-31)
     * Added "undolog" functionality to avoid full reparses when encountering a blockchain reorganisation
     * Removed use of `tornado` library in the API module. We use `flask`'s threaded server instead

counterpartylib/lib/api.py

@@ -26,6 +26,7 @@
 from flask.ext.httpauth import HTTPBasicAuth
 import jsonrpc
 from jsonrpc import dispatcher
+from jsonrpc.exceptions import JSONRPCDispatchException
 import inspect
 from xmltodict import unparse as serialize_to_xml
 
@@ -71,6 +72,7 @@
 
 API_MAX_LOG_SIZE = 10 * 1024 * 1024 #max log size of 20 MB before rotation (make configurable later)
 API_MAX_LOG_COUNT = 10
+JSON_RPC_ERROR_API_COMPOSE = -32001 #code to use for error composing transaction result
 
 current_api_status_code = None #is updated by the APIStatusPoller
 current_api_status_response_json = None #is updated by the APIStatusPoller
@@ -298,7 +300,6 @@ def compose_transaction(db, name, params,
     for param in missing_params:
         params[param] = None
 
-    # try:  # NOTE: For debugging, e.g. with `Invalid Params` error.
     tx_info = compose_method(db, **params)
     return transaction.construct(db, tx_info, encoding=encoding,
                                         fee_per_kb=fee_per_kb,
@@ -310,9 +311,6 @@ def compose_transaction(db, name, params,
                                         exact_fee=fee,
                                         fee_provided=fee_provided,
                                         unspent_tx_hash=unspent_tx_hash, custom_inputs=custom_inputs)
-    # except:
-        # import traceback
-        # traceback.print_exc()
 
 def conditional_decorator(decorator, condition):
     """Checks the condition and if True applies specified decorator."""
@@ -447,9 +445,13 @@ def create_method(**kwargs):
                 try:
                     transaction_args, common_args, private_key_wif = split_params(**kwargs)
                     return compose_transaction(db, name=tx, params=transaction_args, **common_args)
-                except TypeError as e:          #TODO: generalise for all API methods
+                except TypeError as e:
                     raise APIError(str(e))
-
+                except (script.AddressError, exceptions.ComposeError, exceptions.TransactionError, exceptions.BalanceError) as error:
+                    error_msg = "Error composing {} transaction via API: {}".format(tx, str(error))
+                    logging.warning(error_msg)
+                    raise JSONRPCDispatchException(code=JSON_RPC_ERROR_API_COMPOSE, message=error_msg)
+
             return create_method
 
         for tx in API_TRANSACTIONS:
@@ -564,8 +566,12 @@ def get_block_info(block_index):
             return block
 
         @dispatcher.add_method
-        def get_blocks(block_indexes):
-            """fetches block info and messages for the specified block indexes"""
+        def get_blocks(block_indexes, min_message_index=None):
+            """fetches block info and messages for the specified block indexes
+            @param min_message_index: Retrieve blocks from the message feed on or after this specific message index
+              (useful since blocks may appear in the message feed more than once, if a reorg occurred). Note that
+              if this parameter is not specified, the messages for the first block will be returned.
+            """
             if not isinstance(block_indexes, (list, tuple)):
                 raise APIError("block_indexes must be a list of integers.")
             if len(block_indexes) >= 250:
@@ -574,20 +580,26 @@ def get_blocks(block_indexes):
             block_indexes_str = ','.join([str(x) for x in block_indexes])
             cursor = db.cursor()
 
+            # The blocks table gets rolled back from undolog, so min_message_index doesn't matter for this query
             cursor.execute('SELECT * FROM blocks WHERE block_index IN (%s) ORDER BY block_index ASC'
                 % (block_indexes_str,))
             blocks = cursor.fetchall()
 
-            cursor.execute('SELECT * FROM messages WHERE block_index IN (%s) ORDER BY block_index ASC, message_index ASC'
+            cursor.execute('SELECT * FROM messages WHERE block_index IN (%s) ORDER BY message_index ASC'
                 % (block_indexes_str,))
             messages = collections.deque(cursor.fetchall())
+
+            # Discard any messages less than min_message_index
+            if min_message_index:
+                while len(messages) and messages[0]['message_index'] < min_message_index:
+                    messages.popleft()
 
+            # Packages messages into their appropriate block in the data structure to be returned
             for block in blocks:
-                # messages_in_block = []
                 block['_messages'] = []
                 while len(messages) and messages[0]['block_index'] == block['block_index']:
                     block['_messages'].append(messages.popleft())
-            assert not len(messages) #should have been cleared out
+            #NOTE: if len(messages), then we're only returning the messages for the first set of blocks before the reorg
 
             cursor.close()
             return blocks
@@ -669,16 +681,16 @@ def search_raw_transactions(address, unconfirmed=True):
             return backend.searchrawtransactions(address, unconfirmed=unconfirmed)
 
         @dispatcher.add_method
-        def get_unspent_txouts(address, unconfirmed=False):
-            return backend.get_unspent_txouts(address, unconfirmed=unconfirmed, multisig_inputs=False)
+        def get_unspent_txouts(address, unconfirmed=False, unspent_tx_hash=None):
+            return backend.get_unspent_txouts(address, unconfirmed=unconfirmed, multisig_inputs=False, unspent_tx_hash=unspent_tx_hash)
 
         @dispatcher.add_method
-        def getrawtransaction(tx_hash, verbose=False):
-            return backend.getrawtransaction(tx_hash, verbose=verbose)
+        def getrawtransaction(tx_hash, verbose=False, skip_missing=False):
+            return backend.getrawtransaction(tx_hash, verbose=verbose, skip_missing=skip_missing)
 
         @dispatcher.add_method
-        def getrawtransaction_batch(txhash_list, verbose=False):
-            return backend.getrawtransaction_batch(txhash_list, verbose=verbose)
+        def getrawtransaction_batch(txhash_list, verbose=False, skip_missing=False):
+            return backend.getrawtransaction_batch(txhash_list, verbose=verbose, skip_missing=skip_missing)
 
         @dispatcher.add_method
         def get_tx_info(tx_hex, block_index=None):
@@ -839,7 +851,8 @@ def handle_rest(path_args, flask_request):
                 try:
                     query_data = compose_transaction(db, name=query_type, params=transaction_args, **common_args)
                 except (script.AddressError, exceptions.ComposeError, exceptions.TransactionError, exceptions.BalanceError) as error:
-                    error_msg = str(error.__class__.__name__) + ': ' + str(error)
+                    error_msg = logging.warning("{} -- error composing {} transaction via API: {}".format(
+                        str(error.__class__.__name__), query_type, str(error)))
                     return flask.Response(error_msg, 400, mimetype='application/json')                        
             else:
                 # Need to de-generate extra_args to pass it through.

counterpartylib/lib/backend/__init__.py

@@ -14,6 +14,7 @@
 from counterpartylib.lib import util
 from counterpartylib.lib import script
 from counterpartylib.lib import config
+from counterpartylib.lib import exceptions
 
 from counterpartylib.lib.backend import addrindex, btcd
 
@@ -50,11 +51,11 @@ def getblock(block_hash):
 def searchrawtransactions(address, unconfirmed=False):
     return BACKEND().searchrawtransactions(address, unconfirmed=unconfirmed)
 
-def getrawtransaction(tx_hash, verbose=False):
-    return BACKEND().getrawtransaction(tx_hash, verbose=verbose)
+def getrawtransaction(tx_hash, verbose=False, skip_missing=False):
+    return BACKEND().getrawtransaction(tx_hash, verbose=verbose, skip_missing=skip_missing)
 
-def getrawtransaction_batch(txhash_list, verbose=False):
-    return BACKEND().getrawtransaction_batch(txhash_list, verbose=verbose)
+def getrawtransaction_batch(txhash_list, verbose=False, skip_missing=False):
+    return BACKEND().getrawtransaction_batch(txhash_list, verbose=verbose, skip_missing=skip_missing)
 
 def sendrawtransaction(tx_hex):
     return BACKEND().sendrawtransaction(tx_hex)
@@ -123,7 +124,12 @@ def get_btc_supply(normalize=False):
 
 def is_scriptpubkey_spendable(scriptpubkey_hex, source, multisig_inputs=False):
     c_scriptpubkey = bitcoinlib.core.CScript(bitcoinlib.core.x(scriptpubkey_hex))
-    vout_address = script.scriptpubkey_to_address(c_scriptpubkey)
+
+    try:
+        vout_address = script.scriptpubkey_to_address(c_scriptpubkey)
+    except exceptions.DecodeError:
+        return False
+
     if not vout_address:
         return False