Merge pull request #945 from CounterpartyXCP/sqlite-maxint-crash

fix for mainnet int overflow crash

ChangeLog.md

@@ -1,4 +1,6 @@
 ## Library Versions ##
+* v9.55.1 (2016-12-02)
+    * Hotfix for integer overflow bug that caused a crash on mainnet block #441563
 * v9.55.0 (2016-07-11)
     * P2SH support for source / destination of addresses (protocol change: 423888)
     * Moved check for invalid broadcast to better place to prevent broadcasting a cancel on a locked feed (protocol change: 423888)

counterpartylib/lib/config.py

@@ -7,7 +7,7 @@
 # Versions
 VERSION_MAJOR = 9
 VERSION_MINOR = 55
-VERSION_REVISION = 0
+VERSION_REVISION = 1
 VERSION_STRING = str(VERSION_MAJOR) + '.' + str(VERSION_MINOR) + '.' + str(VERSION_REVISION)
 
 

counterpartylib/lib/messages/bet.py

@@ -14,6 +14,7 @@
 
 import struct
 import decimal
+import json
 D = decimal.Decimal
 import time
 import logging
@@ -231,6 +232,11 @@ def validate (db, source, feed_address, bet_type, deadline, wager_quantity,
 
     if leverage is None: leverage = 5040
 
+    # For SQLite3
+    if wager_quantity > config.MAX_INT or counterwager_quantity > config.MAX_INT or bet_type > config.MAX_INT \
+            or deadline > config.MAX_INT or leverage > config.MAX_INT or block_index + expiration > config.MAX_INT:
+        problems.append('integer overflow')
+
     # Look at feed to be bet on.
     cursor = db.cursor()
     broadcasts = list(cursor.execute('''SELECT * FROM broadcasts WHERE (status = ? AND source = ?) ORDER BY tx_index ASC''', ('valid', feed_address)))
@@ -281,10 +287,6 @@ def validate (db, source, feed_address, bet_type, deadline, wager_quantity,
     if expiration > config.MAX_EXPIRATION:
         problems.append('expiration overflow')
 
-    # For SQLite3
-    if wager_quantity > config.MAX_INT or counterwager_quantity > config.MAX_INT or bet_type > config.MAX_INT or deadline > config.MAX_INT or leverage > config.MAX_INT:
-        problems.append('integer overflow')
-
     return problems, leverage
 
 def compose (db, source, feed_address, bet_type, deadline, wager_quantity,
@@ -372,8 +374,12 @@ def parse (db, tx, message):
         'fee_fraction_int': fee_fraction * 1e8,
         'status': status,
     }
-    sql='insert into bets values(:tx_index, :tx_hash, :block_index, :source, :feed_address, :bet_type, :deadline, :wager_quantity, :wager_remaining, :counterwager_quantity, :counterwager_remaining, :target_value, :leverage, :expiration, :expire_index, :fee_fraction_int, :status)'
-    bet_parse_cursor.execute(sql, bindings)
+    if "integer overflow" not in status:
+        sql = 'insert into bets values(:tx_index, :tx_hash, :block_index, :source, :feed_address, :bet_type, :deadline, :wager_quantity, :wager_remaining, :counterwager_quantity, :counterwager_remaining, :target_value, :leverage, :expiration, :expire_index, :fee_fraction_int, :status)'
+        bet_parse_cursor.execute(sql, bindings)
+    else:
+        logger.warn("Not storing [bet] tx [%s]: %s" % (tx['tx_hash'], status))
+        logger.debug("Bindings: %s" % (json.dumps(bindings), ))
 
     # Match.
     if status == 'open' and tx['block_index'] != config.MEMPOOL_BLOCK_INDEX:

counterpartylib/lib/messages/broadcast.py

@@ -20,12 +20,12 @@
 because it is stored as a four‐byte integer, it may not be greater than about
 42.
 """
-
 import struct
 import decimal
 
 D = decimal.Decimal
 from fractions import Fraction
+import json
 import logging
 logger = logging.getLogger(__name__)
 
@@ -73,6 +73,10 @@ def initialise(db):
 def validate (db, source, timestamp, value, fee_fraction_int, text, block_index):
     problems = []
 
+    # For SQLite3
+    if timestamp > config.MAX_INT or value > config.MAX_INT or fee_fraction_int > config.MAX_INT:
+        problems.append('integer overflow')
+
     if util.enabled('max_fee_fraction'):
         if fee_fraction_int >= config.UNIT:
             problems.append('fee fraction greater than or equal to 1')
@@ -182,8 +186,12 @@ def parse (db, tx, message):
         'locked': lock,
         'status': status,
     }
-    sql = 'insert into broadcasts values(:tx_index, :tx_hash, :block_index, :source, :timestamp, :value, :fee_fraction_int, :text, :locked, :status)'
-    cursor.execute(sql, bindings)
+    if "integer overflow" not in status:
+        sql = 'insert into broadcasts values(:tx_index, :tx_hash, :block_index, :source, :timestamp, :value, :fee_fraction_int, :text, :locked, :status)'
+        cursor.execute(sql, bindings)
+    else:
+        logger.warn("Not storing [broadcast] tx [%s]: %s" % (tx['tx_hash'], status))
+        logger.debug("Bindings: %s" % (json.dumps(bindings), ))
 
     # stop processing if broadcast is invalid for any reason
     if util.enabled('broadcast_invalid_check') and status != 'valid':

counterpartylib/lib/messages/btcpay.py

@@ -1,6 +1,7 @@
 #! /usr/bin/python3
 
 import binascii
+import json
 import pprint
 import struct
 import logging
@@ -152,8 +153,12 @@ def parse (db, tx, message):
         'order_match_id': order_match_id,
         'status': status,
     }
-    sql='insert into btcpays values(:tx_index, :tx_hash, :block_index, :source, :destination, :btc_amount, :order_match_id, :status)'
-    cursor.execute(sql, bindings)
+    if "integer overflow" not in status:
+        sql = 'insert into btcpays values(:tx_index, :tx_hash, :block_index, :source, :destination, :btc_amount, :order_match_id, :status)'
+        cursor.execute(sql, bindings)
+    else:
+        logger.warn("Not storing [btcpay] tx [%s]: %s" % (tx['tx_hash'], status))
+        logger.debug("Bindings: %s" % (json.dumps(bindings), ))
 
 
     cursor.close()

counterpartylib/lib/messages/burn.py

@@ -1,7 +1,10 @@
 #! /usr/bin/python3
-
+import json
 import struct
 import decimal
+import logging
+logger = logging.getLogger(__name__)
+
 D = decimal.Decimal
 from fractions import Fraction
 
@@ -71,6 +74,7 @@ def parse (db, tx, MAINNET_BURNS, message=None):
     burn_parse_cursor = db.cursor()
 
     if config.TESTNET:
+        problems = []
         status = 'valid'
 
         if status == 'valid':
@@ -138,8 +142,12 @@ def parse (db, tx, MAINNET_BURNS, message=None):
         'earned': earned,
         'status': status,
     }
-    sql='insert into burns values(:tx_index, :tx_hash, :block_index, :source, :burned, :earned, :status)'
-    burn_parse_cursor.execute(sql, bindings)
+    if "integer overflow" not in status:
+        sql = 'insert into burns values(:tx_index, :tx_hash, :block_index, :source, :burned, :earned, :status)'
+        burn_parse_cursor.execute(sql, bindings)
+    else:
+        logger.warn("Not storing [burn] tx [%s]: %s" % (tx['tx_hash'], status))
+        logger.debug("Bindings: %s" % (json.dumps(bindings), ))
 
     burn_parse_cursor.close()
 

counterpartylib/lib/messages/cancel.py

@@ -6,6 +6,9 @@
 
 import binascii
 import struct
+import json
+import logging
+logger = logging.getLogger(__name__)
 
 from counterpartylib.lib import (config, exceptions, util)
 from . import (order, bet, rps)
@@ -116,8 +119,12 @@ def parse (db, tx, message):
         'offer_hash': offer_hash,
         'status': status,
     }
-    sql='INSERT INTO cancels VALUES (:tx_index, :tx_hash, :block_index, :source, :offer_hash, :status)'
-    cursor.execute(sql, bindings)
+    if "integer overflow" not in status:
+        sql='INSERT INTO cancels VALUES (:tx_index, :tx_hash, :block_index, :source, :offer_hash, :status)'
+        cursor.execute(sql, bindings)
+    else:
+        logger.warn("Not storing [cancel] tx [%s]: %s" % (tx['tx_hash'], status))
+        logger.debug("Bindings: %s" % (json.dumps(bindings), ))
 
     cursor.close()
 

counterpartylib/lib/messages/destroy.py

@@ -3,6 +3,9 @@
 """Destroy a quantity of an asset."""
 
 import struct
+import json
+import logging
+logger = logging.getLogger(__name__)
 
 from counterpartylib.lib import util
 from counterpartylib.lib import config
@@ -78,7 +81,7 @@ def validate (db, source, destination, asset, quantity):
         raise ValidateError('quantity not integer')
 
     if quantity > config.MAX_INT:
-        raise ValidateError('quantity too large')
+        raise ValidateError('integer overflow, quantity too large')
 
     if quantity < 0:
         raise ValidateError('quantity negative')
@@ -124,9 +127,13 @@ def parse (db, tx, message):
                 'tag': tag,
                 'status': status,
                }
-    sql = 'insert into destructions values(:tx_index, :tx_hash, :block_index, :source, :asset, :quantity, :tag, :status)'
-    cursor = db.cursor()
-    cursor.execute(sql, bindings)
+    if "integer overflow" not in status:
+        sql = 'insert into destructions values(:tx_index, :tx_hash, :block_index, :source, :asset, :quantity, :tag, :status)'
+        cursor = db.cursor()
+        cursor.execute(sql, bindings)
+    else:
+        logger.warn("Not storing [destroy] tx [%s]: %s" % (tx['tx_hash'], status))
+        logger.debug("Bindings: %s" % (json.dumps(bindings), ))
 
 
 # vim: tabstop=8 expandtab shiftwidth=4 softtabstop=4

counterpartylib/lib/messages/dividend.py

@@ -1,7 +1,7 @@
 #! /usr/bin/python3
 
 """Pay out dividends."""
-
+import json
 import struct
 import decimal
 D = decimal.Decimal
@@ -50,7 +50,12 @@ def validate (db, source, quantity_per_unit, asset, dividend_asset, block_index)
         if (not block_index >= 317500) or block_index >= 320000 or config.TESTNET:   # Protocol change.
             problems.append('cannot pay dividends to holders of {}'.format(config.XCP))
 
-    if quantity_per_unit <= 0: problems.append('non‐positive quantity per unit')
+    if quantity_per_unit <= 0:
+        problems.append('non‐positive quantity per unit')
+
+    # For SQLite3
+    if quantity_per_unit > config.MAX_INT:
+        problems.append('integer overflow')
 
     # Examine asset.
     issuances = list(cursor.execute('''SELECT * FROM issuances WHERE (status = ? AND asset = ?) ORDER BY tx_index ASC''', ('valid', asset)))
@@ -121,6 +126,10 @@ def validate (db, source, quantity_per_unit, asset, dividend_asset, block_index)
         if not dividend_balances or dividend_balances[0]['quantity'] < total_cost:
             problems.append('insufficient funds ({})'.format(dividend_asset))
 
+    # For SQLite3
+    if fee > config.MAX_INT or dividend_total > config.MAX_INT:
+        problems.append('integer overflow')
+
     cursor.close()
     return dividend_total, outputs, problems, fee
 
@@ -193,8 +202,12 @@ def parse (db, tx, message):
         'status': status,
     }
 
-    sql='insert into dividends values(:tx_index, :tx_hash, :block_index, :source, :asset, :dividend_asset, :quantity_per_unit, :fee_paid, :status)'
-    dividend_parse_cursor.execute(sql, bindings)
+    if "integer overflow" not in status:
+        sql = 'insert into dividends values(:tx_index, :tx_hash, :block_index, :source, :asset, :dividend_asset, :quantity_per_unit, :fee_paid, :status)'
+        dividend_parse_cursor.execute(sql, bindings)
+    else:
+        logger.warn("Not storing [dividend] tx [%s]: %s" % (tx['tx_hash'], status))
+        logger.debug("Bindings: %s" % (json.dumps(bindings), ))
 
     dividend_parse_cursor.close()
 

counterpartylib/lib/messages/execute.py

@@ -127,7 +127,7 @@ def parse (db, tx, message):
         curr_format = FORMAT + '{}s'.format(len(message) - LENGTH)
         try:
             contract_id, gasprice, startgas, value, payload = struct.unpack(curr_format, message)
-            if gasprice > config.MAX_INT or startgas > config.MAX_INT: # TODO: define max for gasprice and startgas
+            if gasprice > config.MAX_INT or startgas > config.MAX_INT or value > config.MAX_INT: # TODO: define max for gasprice and startgas
                 raise exceptions.UnpackError()
         except (struct.error) as e:
             raise exceptions.UnpackError()

counterpartylib/lib/messages/issuance.py

@@ -6,6 +6,9 @@
 
 import struct
 import decimal
+import json
+import logging
+logger = logging.getLogger(__name__)
 D = decimal.Decimal
 
 from counterpartylib.lib import (config, util, exceptions, util)
@@ -157,6 +160,10 @@ def validate (db, source, destination, asset, quantity, divisible, callable_, ca
     if destination and quantity:
         problems.append('cannot issue and transfer simultaneously')
 
+    # For SQLite3
+    if util.enabled('integer_overflow_fix', block_index=block_index) and (fee > config.MAX_INT or quantity > config.MAX_INT):
+        problems.append('integer overflow')
+
     return call_date, call_price, problems, fee, description, divisible, reissuance
 
 def compose (db, source, transfer_destination, asset, quantity, divisible, description):
@@ -232,7 +239,7 @@ def parse (db, tx, message):
     if status == 'valid':
         call_date, call_price, problems, fee, description, divisible, reissuance = validate(db, tx['source'], tx['destination'], asset, quantity, divisible, callable_, call_date, call_price, description, block_index=tx['block_index'])
         if problems: status = 'invalid: ' + '; '.join(problems)
-        if 'total quantity overflow' in problems:
+        if not util.enabled('integer_overflow_fix', block_index=tx['block_index']) and 'total quantity overflow' in problems:
             quantity = 0
 
     if tx['destination']:
@@ -289,8 +296,12 @@ def parse (db, tx, message):
         'locked': lock,
         'status': status,
     }
-    sql='insert into issuances values(:tx_index, :tx_hash, :block_index, :asset, :quantity, :divisible, :source, :issuer, :transfer, :callable, :call_date, :call_price, :description, :fee_paid, :locked, :status)'
-    issuance_parse_cursor.execute(sql, bindings)
+    if "integer overflow" not in status:
+        sql='insert into issuances values(:tx_index, :tx_hash, :block_index, :asset, :quantity, :divisible, :source, :issuer, :transfer, :callable, :call_date, :call_price, :description, :fee_paid, :locked, :status)'
+        issuance_parse_cursor.execute(sql, bindings)
+    else:
+        logger.warn("Not storing [issuance] tx [%s]: %s" % (tx['tx_hash'], status))
+        logger.debug("Bindings: %s" % (json.dumps(bindings), ))
 
     # Credit.
     if status == 'valid' and quantity:

counterpartylib/lib/messages/order.py

@@ -2,7 +2,7 @@
 
 # Filled orders may not be re‐opened, so only orders not involving BTC (and so
 # which cannot have expired order matches) may be filled.
-
+import json
 import struct
 import decimal
 D = decimal.Decimal
@@ -328,6 +328,10 @@ def validate (db, source, give_asset, give_quantity, get_asset, get_quantity, ex
     problems = []
     cursor = db.cursor()
 
+    # For SQLite3
+    if give_quantity > config.MAX_INT or get_quantity > config.MAX_INT or fee_required > config.MAX_INT or block_index + expiration > config.MAX_INT:
+        problems.append('integer overflow')
+
     if give_asset == config.BTC and get_asset == config.BTC:
         problems.append('cannot trade {} for itself'.format(config.BTC))
 
@@ -362,10 +366,6 @@ def validate (db, source, give_asset, give_quantity, get_asset, get_quantity, ex
     if expiration > config.MAX_EXPIRATION:
         problems.append('expiration overflow')
 
-    # For SQLite3
-    if give_quantity > config.MAX_INT or get_quantity > config.MAX_INT or fee_required > config.MAX_INT:
-        problems.append('integer overflow')
-
     cursor.close()
     return problems
 
@@ -452,8 +452,12 @@ def parse (db, tx, message):
         'fee_provided_remaining': tx['fee'],
         'status': status,
     }
-    sql='insert into orders values(:tx_index, :tx_hash, :block_index, :source, :give_asset, :give_quantity, :give_remaining, :get_asset, :get_quantity, :get_remaining, :expiration, :expire_index, :fee_required, :fee_required_remaining, :fee_provided, :fee_provided_remaining, :status)'
-    order_parse_cursor.execute(sql, bindings)
+    if "integer overflow" not in status:
+        sql = 'insert into orders values(:tx_index, :tx_hash, :block_index, :source, :give_asset, :give_quantity, :give_remaining, :get_asset, :get_quantity, :get_remaining, :expiration, :expire_index, :fee_required, :fee_required_remaining, :fee_provided, :fee_provided_remaining, :status)'
+        order_parse_cursor.execute(sql, bindings)
+    else:
+        logger.warn("Not storing [order] tx [%s]: %s" % (tx['tx_hash'], status))
+        logger.debug("Bindings: %s" % (json.dumps(bindings), ))
 
     # Match.
     if status == 'open' and tx['block_index'] != config.MEMPOOL_BLOCK_INDEX: