Merge pull request #802 from CounterpartyXCP/develop

9.52.0

ChangeLog.md

@@ -1,4 +1,10 @@
 ## Client Versions ##
+* v9.52.0 (2015-10-31)
+    * Added "undolog" functionality to avoid full reparses when encountering a blockchain reorganisation
+    * Removed use of `tornado` library in the API module. We use `flask`'s threaded server instead
+    * Added `getrawtransaction` and `getrawtransaction_batch` methods to the API
+    * Added optional `custom_inputs` parameter to API calls, which allows for controlling the exact UTXOs to use in transactions (thanks, tokenly)
+    * Added `message_hash`, derived from changes to the `counterparty-lib` database. Displayed on each new block, makes checking for DB-level consensus easier
 * v9.51.4 (2015-09-26)
 	* Significant performance and caching optimizations around bitcoind addrindex interaction
    	* Fixed issues around responsiveness with larger mempool sizes

README.md

@@ -3,7 +3,7 @@
 [![Coverage Status](https://coveralls.io/repos/CounterpartyXCP/counterparty-lib/badge.png?branch=develop)](https://coveralls.io/r/CounterpartyXCP/counterparty-lib?branch=develop)
 [![Latest Version](https://pypip.in/version/counterparty-lib/badge.svg)](https://pypi.python.org/pypi/counterparty-lib/)
 [![License](https://pypip.in/license/counterparty-lib/badge.svg)](https://pypi.python.org/pypi/counterparty-lib/)
-[![Gitter chat](https://badges.gitter.im/gitterHQ/gitter.png)](https://gitter.im/CounterpartyXCP/General)
+[![Slack Status](http://slack.counterparty.io/badge.svg)](http://slack.counterparty.io)
 
 
 # Description

counterpartylib/lib/api.py

@@ -24,10 +24,6 @@
 import apsw
 import flask
 from flask.ext.httpauth import HTTPBasicAuth
-import tornado
-from tornado.wsgi import WSGIContainer
-from tornado.httpserver import HTTPServer
-from tornado.ioloop import IOLoop
 import jsonrpc
 from jsonrpc import dispatcher
 import inspect
@@ -71,7 +67,7 @@
 COMMONS_ARGS = ['encoding', 'fee_per_kb', 'regular_dust_size',
                 'multisig_dust_size', 'op_return_value', 'pubkey',
                 'allow_unconfirmed_inputs', 'fee', 'fee_provided',
-                'unspent_tx_hash']
+                'unspent_tx_hash','custom_inputs']
 
 API_MAX_LOG_SIZE = 10 * 1024 * 1024 #max log size of 20 MB before rotation (make configurable later)
 API_MAX_LOG_COUNT = 10
@@ -270,7 +266,7 @@ def compose_transaction(db, name, params,
                         allow_unconfirmed_inputs=False,
                         fee=None,
                         fee_provided=0,
-                        unspent_tx_hash=None):
+                        unspent_tx_hash=None, custom_inputs=None):
     """Create and return a transaction."""
 
     # Get provided pubkeys.
@@ -313,7 +309,7 @@ def compose_transaction(db, name, params,
                                         allow_unconfirmed_inputs=allow_unconfirmed_inputs,
                                         exact_fee=fee,
                                         fee_provided=fee_provided,
-                                        unspent_tx_hash=unspent_tx_hash)
+                                        unspent_tx_hash=unspent_tx_hash, custom_inputs=custom_inputs)
     # except:
         # import traceback
         # traceback.print_exc()
@@ -326,17 +322,20 @@ def gen_decorator(f):
         return decorator(f)
     return gen_decorator
 
-def init_api_access_log():
+def init_api_access_log(app):
     """Initialize API logger."""
+    loggers = (logging.getLogger('werkzeug'), app.logger)
+
+    # Disable console logging...
+    for l in loggers:
+        l.setLevel(logging.INFO)
+        l.propagate = False
+
+    # Log to file, if configured...    
     if config.API_LOG:
-        access_logger = logging.getLogger("tornado.access")
-        access_logger.setLevel(logging.INFO)
-        access_logger.propagate = False
-
         handler = logging_handlers.RotatingFileHandler(config.API_LOG, 'a', API_MAX_LOG_SIZE, API_MAX_LOG_COUNT)
-        formatter = tornado.log.LogFormatter(datefmt='%Y-%m-%d-T%H:%M:%S%z')    # Default date format is nuts.
-        handler.setFormatter(formatter)
-        access_logger.addHandler(handler)
+        for l in loggers:
+            l.addHandler(handler)
 
 class APIStatusPoller(threading.Thread):
     """Perform regular checks on the state of the backend and the database."""
@@ -384,10 +383,8 @@ def __init__(self):
         self.is_ready = False
         threading.Thread.__init__(self)
         self.stop_event = threading.Event()
-        self.ioloop = IOLoop.instance()
 
     def stop(self):
-        self.ioloop.stop()
         self.join()
         self.stop_event.set()
 
@@ -675,6 +672,14 @@ def search_raw_transactions(address, unconfirmed=True):
         def get_unspent_txouts(address, unconfirmed=False):
             return backend.get_unspent_txouts(address, unconfirmed=unconfirmed, multisig_inputs=False)
 
+        @dispatcher.add_method
+        def getrawtransaction(tx_hash, verbose=False):
+            return backend.getrawtransaction(tx_hash, verbose=verbose)
+
+        @dispatcher.add_method
+        def getrawtransaction_batch(txhash_list, verbose=False):
+            return backend.getrawtransaction_batch(txhash_list, verbose=verbose)
+
         @dispatcher.add_method
         def get_tx_info(tx_hex, block_index=None):
             # block_index mandatory for transactions before block 335000
@@ -866,19 +871,13 @@ def handle_rest(path_args, flask_request):
             return response
 
         # Init the HTTP Server.
-        init_api_access_log()
+        init_api_access_log(app)
 
-        http_server = HTTPServer(WSGIContainer(app), xheaders=True)
-        try:
-            http_server.listen(config.RPC_PORT, address=config.RPC_HOST)
-            self.is_ready = True
-            self.ioloop.start()
-        except OSError:
-            raise APIError("Cannot start the API subsystem. Is server already running, or is something else listening on port {}?".format(config.RPC_PORT))
-
+        # Run app server (blocking)
+        self.is_ready = True
+        app.run(host=config.RPC_HOST, port=config.RPC_PORT, threaded=True)
+
         db.close()
-        http_server.stop()
-        self.ioloop.close()
         return
 
 # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

counterpartylib/lib/backend/__init__.py

@@ -37,18 +37,19 @@ def getit(adict):
 def BACKEND():
     return sys.modules['counterpartylib.lib.backend.{}'.format(config.BACKEND_NAME)] 
 
-# TODO: Generate this block of code dynamically?
-
 def getblockcount():
     return BACKEND().getblockcount()
+
 def getblockhash(blockcount):
     return BACKEND().getblockhash(blockcount)
+
 def getblock(block_hash):
     block_hex = BACKEND().getblock(block_hash)
     return CBlock.deserialize(util.unhexlify(block_hex))
 
 def searchrawtransactions(address, unconfirmed=False):
     return BACKEND().searchrawtransactions(address, unconfirmed=unconfirmed)
+
 def getrawtransaction(tx_hash, verbose=False):
     return BACKEND().getrawtransaction(tx_hash, verbose=verbose)
 
@@ -69,10 +70,10 @@ def refresh_unconfirmed_transactions_cache(mempool_txhash_list):
 
 def deserialize(tx_hex):
     return bitcoinlib.core.CTransaction.deserialize(binascii.unhexlify(tx_hex))
+
 def serialize(ctx):
     return bitcoinlib.core.CTransaction.serialize(ctx)
 
-
 def is_valid(address):
     try:
         script.validate(address)

counterpartylib/lib/backend/addrindex.py

@@ -209,15 +209,13 @@ def getrawtransaction_batch(txhash_list, verbose=False, _recursing=False):
         len(txhash_list), len(raw_transactions_cache), len(payload)))
 
     # populate cache
-    added_entries_to_cache = []
     if len(payload) > 0:
         batch_responses = rpc_batch(payload)
         for response in batch_responses:
             if 'error' not in response or response['error'] is None:
                 tx_hex = response['result']
                 tx_hash = tx_hash_call_id[response['id']]
                 raw_transactions_cache[tx_hash] = tx_hex
-                added_entries_to_cache.append(tx_hash) #for debugging
             else:
                 #TODO: this seems to happen for bogus transactions? Maybe handle it more gracefully than just erroring out?
                 raise BackendRPCError('{} (txhash:: {})'.format(response['error'], tx_hash_call_id.get(response.get('id', '??'), '??')))
@@ -230,7 +228,7 @@ def getrawtransaction_batch(txhash_list, verbose=False, _recursing=False):
                 result[tx_hash] = raw_transactions_cache[tx_hash]
             else:
                 result[tx_hash] = raw_transactions_cache[tx_hash]['hex']
-        except KeyError: #shows up most likely due to finickyness with addrindex not always returning results that we need...
+        except KeyError as e: #shows up most likely due to finickyness with addrindex not always returning results that we need...
             logger.debug("tx missing in rawtx cache: {} -- txhash_list size: {}, hash: {} / raw_transactions_cache size: {} / # rpc_batch calls: {} / txhash in noncached_txhashes: {} / txhash in txhash_list: {} -- list {}".format(
                 e, len(txhash_list), hashlib.md5(json.dumps(list(txhash_list)).encode()).hexdigest(), len(raw_transactions_cache), len(payload),
                 tx_hash in noncached_txhashes, tx_hash in txhash_list, list(txhash_list.difference(noncached_txhashes)) ))