From 18db7c7c7ed4760499e2cc3a7575792fb3c0b7f5 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 28 Feb 2022 16:43:11 +0100 Subject: [PATCH 01/15] Added rpc socket connector class --- Connector/rpcutils/rpcsocketconnector.py | 69 ++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 Connector/rpcutils/rpcsocketconnector.py diff --git a/Connector/rpcutils/rpcsocketconnector.py b/Connector/rpcutils/rpcsocketconnector.py new file mode 100644 index 00000000..a1b4b817 --- /dev/null +++ b/Connector/rpcutils/rpcsocketconnector.py @@ -0,0 +1,69 @@ +#!/usr/bin/python +import json +import socket +from logger import logger +from . import error +from .constants import * + + +class RPCSocketConnector(): + + @staticmethod + def request(hostname, port, id, method, params): + + try: + + payload = { + ID: id, + METHOD: method, + PARAMS: params, + JSON_RPC: JSON_RPC_VERSION + } + + logger.printInfo(f"Making RPC socket request to {hostname}:{port}. Payload: {payload}") + + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.connect((hostname, port)) + + jsonText = {"jsonrpc": JSON_RPC_VERSION, "method": method, "params": params, "id": id} + + s.send(json.dumps(jsonText).encode()) + s.send('\n'.encode()) + + chunks = [] + while True: + chunk = s.recv(2048) + chunks.append(chunk) + if chunk[-1:] == b'\n': + break + + s.close() + + responseStr = b''.join(chunks).decode('ascii') + + except Exception as e: + logger.printError(f"Request to client could not be completed: {str(e)}") + raise error.RpcBadRequestError( + id=id, + message=f"Request to client could not be completed: {str(e)}" + ) + + try: + response = json.loads(responseStr) + except Exception as e: + logger.printError(f"Json in client response is not supported: {str(e)}") + raise error.RpcInternalServerError( + id=id, + message=f"Json in client response is not supported: {str(e)}" + ) + + logger.printInfo(f"Response received from {hostname}:{port}: {response}") + + if ERROR in response and response[ERROR] is not None: + logger.printError(f"Exception occured in server: {response[ERROR]}") + raise error.RpcBadRequestError( + id=id, + message=f"Exception occured in server: {response[ERROR]}" + ) + + return response[RESULT] From 099116020dadcfce2d070b7de6344d709af92a68 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 28 Feb 2022 16:52:53 +0100 Subject: [PATCH 02/15] electrumx host and port configuration properties --- Connector/btc/config.py | 24 +++++++++++++++++++++++- Connector/btc/defaultConf.json | 4 +++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/Connector/btc/config.py b/Connector/btc/config.py index 1e130249..de8b1510 100644 --- a/Connector/btc/config.py +++ b/Connector/btc/config.py @@ -24,6 +24,8 @@ def __init__(self, coin, networkName): self._electrumPassword = "" self._bitcoincoreCallbackProtocol = "" self._bitcoincoreCallbackHost = "" + self._electrumxHost = "" + self._electrumxPort = 0 def loadConfig(self, config): @@ -57,6 +59,8 @@ def loadConfig(self, config): else defaultConfig["bitcoincoreCallbackProtocol"] self.bitcoincoreCallbackHost = config["bitcoincoreCallbackHost"] if "bitcoincoreCallbackHost" in config \ else defaultConfig["bitcoincoreCallbackHost"] + self.electrumxHost = config["electrumxHost"] if "electrumxHost" in config else defaultConfig["electrumxHost"] + self.electrumxPort = config["electrumxPort"] if "electrumxPort" in config else defaultConfig["electrumxPort"] return True, None @@ -184,6 +188,22 @@ def bitcoincoreCallbackHost(self): def bitcoincoreCallbackHost(self, value): self._bitcoincoreCallbackHost = value + @property + def electrumxHost(self): + return self._electrumxHost + + @electrumxHost.setter + def electrumxHost(self, value): + self._electrumxHost = value + + @property + def electrumxPort(self): + return self._electrumxPort + + @electrumxPort.setter + def electrumxPort(self, value): + self._electrumxPort = value + @property def bitcoincoreRpcEndpoint(self): return f"{self.bitcoincoreProtocol}://" \ @@ -226,5 +246,7 @@ def encode(self, o): "electrumHost": o.electrumHost, "electrumPort": o.electrumPort, "electrumUser": o.electrumUser, - "electrumPassword": o.electrumPassword + "electrumPassword": o.electrumPassword, + "electrumxHost": o.electrumxHost, + "electrumxPort": o.electrumxPort } diff --git a/Connector/btc/defaultConf.json b/Connector/btc/defaultConf.json index 25641a0b..858d947f 100644 --- a/Connector/btc/defaultConf.json +++ b/Connector/btc/defaultConf.json @@ -12,5 +12,7 @@ "electrumUser": "swapper", "electrumPassword": "swapper", "bitcoincoreCallbackProtocol": "http", - "bitcoincoreCallbackHost": "connector" + "bitcoincoreCallbackHost": "connector", + "electrumxHost": "electrs", + "electrumxPort": 60001 } \ No newline at end of file From ad65ece222cc8645a4e04de3f6505bf9633ec8d8 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 28 Feb 2022 18:39:50 +0100 Subject: [PATCH 03/15] Added methods form getTransaction --- Connector/btc/apirpc.py | 73 +++++++++++++------------- Connector/btc/utils.py | 113 +++++++++++++++++++++++++--------------- 2 files changed, 105 insertions(+), 81 deletions(-) diff --git a/Connector/btc/apirpc.py b/Connector/btc/apirpc.py index 40600557..aacecce0 100644 --- a/Connector/btc/apirpc.py +++ b/Connector/btc/apirpc.py @@ -3,7 +3,7 @@ from rpcutils import rpcmethod from logger import logger from rpcutils import error -from rpcutils.rpcconnector import RPCConnector +from rpcutils.rpcconnector import RPCConnector, RPCSocketConnector from . import utils from .constants import * @@ -393,53 +393,50 @@ def getTransaction(id, params, config): raise error.RpcBadRequestError(err.message) try: - # Parameters: TransactionId, include_watchonly, verbose - transaction = RPCConnector.request( - endpoint=config.bitcoincoreRpcEndpoint, + transactionRaw = RPCSocketConnector.request( + hostname=config.electrumxHost, + port=config.electrumxPort, id=id, - method=GET_TRANSACTION_METHOD, + method="blockchain.transaction.get", params=[ - params["txHash"], - True, - True + params["txHash"] ] ) - vinAddressBalances = {} - transactionAmount = 0 - - if "generated" not in transaction: + transactionDecoded = RPCConnector.request( + endpoint=config.bitcoincoreRpcEndpoint, + id=id, + method="decoderawtransaction", + params=[transactionRaw] + ) - for vin in transaction["decoded"]["vin"]: - inputTransaction = RPCConnector.request( - endpoint=config.bitcoincoreRpcEndpoint, - id=id, - method=GET_TRANSACTION_METHOD, - params=[ - vin["txid"], - True, - True - ] - ) + # We pick a random script hash (first one) and after that, we find the transaction that matches + # out transaction to find the height. + # This is ugly but has to be done because of Electrum protocol, that it is (apparently) unable + # of give the transaction height in 'blockchain.transaction.get' method + txWorkaround = RPCSocketConnector.request( + hostname=config.electrumxHost, + port=config.electrumxPort, + id=id, + method="blockchain.scripthash.get_history", + params=[ + utils.getWorkaroundScriptHash(transactionDecoded) + ] + ) - transactionAmount += inputTransaction["decoded"]["vout"][vin["vout"]]["value"] - address = inputTransaction["decoded"]["vout"][vin["vout"]]["scriptPubKey"]["addresses"][0] - value = inputTransaction["decoded"]["vout"][vin["vout"]]["value"] - vinAddressBalances[address] = value + txHeight = None + for transaction in txWorkaround: + if transaction["tx_hash"] == transactionDecoded["txid"]: + txHeight = transaction["height"] + break response = { "transaction": { - "txHash": params["txHash"], - "blockhash": transaction["blockhash"] if transaction["confirmations"] >= 1 else None, - "blockNumber": str(transaction["blockheight"]) if transaction["confirmations"] >= 1 else None, - "fee": str(utils.convertToSatoshi(-transaction["fee"])) if "generated" not in transaction else "0", - "transfers": utils.parseBalancesToTransfers( - vinAddressBalances, - transaction["details"], - -transaction["fee"] if "generated" not in transaction else 0, - transactionAmount - ), - "data": transaction["decoded"] + "txHash": transactionDecoded["txHash"], + "blockNumber": str(txHeight) if txHeight is not None else None, + "fee": str(utils.convertToSatoshi(-transactionDecoded["fee"])) if "generated" not in transactionDecoded else "0", + "transfers": utils.getTransactionTransfers(transactionDecoded, config.bitcoincoreRpcEndpoint, config.electrumxHost, config.electrumxPort), + "data": transactionDecoded["decoded"] } } diff --git a/Connector/btc/utils.py b/Connector/btc/utils.py index 0d52780e..92289e32 100644 --- a/Connector/btc/utils.py +++ b/Connector/btc/utils.py @@ -1,10 +1,14 @@ #!/usr/bin/python3 +import binascii +import hashlib +import math from decimal import Decimal import random import sys from logger import logger from rpcutils import error as rpcerrorhandler from wsutils import topics +from rpcutils.rpcconnector import RPCConnector, RPCSocketConnector from .constants import * from . import apirpc @@ -64,49 +68,72 @@ def closeAddrBalanceTopic(topicName): raise rpcerrorhandler.BadRequestError(f"Can not unsubscribe {topicName} to node") -def parseBalancesToTransfers(vin, vout, fee, amount): - - transfers = [] - diff = 0 - - for utxo in vout: - - if utxo["category"] == "send": - - for address in list(vin.keys()): - - voutAmount = -utxo["amount"] - vinAmount = vin[address] - - if vinAmount <= (voutAmount + diff): - transfer = { - "from": address, - "to": utxo["address"], - "amount": str(convertToSatoshi(vinAmount)), - "fee": str(convertToSatoshi(round(vinAmount * fee / amount, BTC_PRECISION))) - } - del vin[address] - else: - transfer = { - "from": address, - "to": utxo["address"], - "amount": str(convertToSatoshi(voutAmount)), - "fee": str(convertToSatoshi(round(voutAmount * fee / amount, BTC_PRECISION))) - } - - diff = diff + voutAmount - vinAmount - transfers.append(transfer) - - if utxo["category"] in ["generate", "immature", "orphan"]: - transfers.append( - { - "to": utxo["address"], - "fee": "0", - "amount": str(convertToSatoshi(utxo["maount"])) - } - ) - - return transfers +def getWorkaroundScriptHash(txDecoded): + script = txDecoded["vout"][0]["scriptPubKey"]["hex"] + scriptUnex = binascii.unhexlify(script) + hash = hashlib.sha256(scriptUnex).digest()[::-1].hex() + return hash + + +def getTransactionTransfers(txDecoded, bitcoincoreRpcEndpoint, electrumxHost, electrumxPort): + outputs = [] + for output in txDecoded["vout"]: + if "addresses" in output["scriptPubKey"] and len(output["scriptPubKey"]["addresses"]) == 1: + outputs.append( + {"amount": math.trunc(output["value"] * 100000000), "address": output["scriptPubKey"]["addresses"][0]}) + else: + outputs.append({"amount": math.trunc(output["value"] * 100000000), "address": None}) + + sumOutputs = 0 + for output in outputs: + sumOutputs += output["amount"] + + inputs = [] + for txInput in txDecoded["vin"]: + + if "coinbase" in txInput: # This is a coinbase transaction and thus it have one only input of 'sumOutputs' amount + inputs.append({"amount": sumOutputs, "address": None}) + break + + txInRaw = RPCSocketConnector.sendRequest(electrumxHost, electrumxPort, 0, "blockchain.transaction.get", + [txInput["txid"]]) + txInDecoded = RPCConnector.request(bitcoincoreRpcEndpoint, "decoderawtransaction", + [txInRaw]) + + for txOutput in txInDecoded["vout"]: + if txOutput["n"] == txInput["vout"] and "addresses" in txOutput["scriptPubKey"] and len( + txOutput["scriptPubKey"]["addresses"]) == 1: + inputs.append({"amount": math.trunc(txOutput["value"] * 100000000), + "address": txOutput["scriptPubKey"]["addresses"][0]}) + elif len(txOutput["scriptPubKey"]["addresses"]) != 1: + inputs.append({"amount": math.trunc(txOutput["value"] * 100000000), "address": None}) + + sumInputs = 0 + for txInput in inputs: + sumInputs += txInput["amount"] + + sumOutputs = 0 + for txOutput in outputs: + sumOutputs += txOutput["amount"] + + fee = sumInputs - sumOutputs + + transfers = {} + for inputTx in inputs: + inputTotalProvide = inputTx["amount"] - (fee / len(inputs)) + for txOutput in outputs: + outputPercentage = txOutput["amount"] / sumOutputs + + if (inputTx["address"], txOutput["address"]) in transfers: + transfers[(inputTx["address"], txOutput["address"])] = transfers[( + inputTx["address"], txOutput["address"])] + (inputTotalProvide * outputPercentage) + else: + transfers[(inputTx["address"], txOutput["address"])] = inputTotalProvide * outputPercentage + + transfersResult = [] + for key in transfers.keys(): + transfersResult.append({"from": key[0], "to": key[1], "amount": transfers[key]}) + return transfersResult def sortUnspentOutputs(outputs): From caad11ca8849556f372f03d87b3b8fec14d397fe Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 28 Feb 2022 19:09:19 +0100 Subject: [PATCH 04/15] Bugfixes --- Connector/btc/apirpc.py | 8 +++++--- Connector/btc/utils.py | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Connector/btc/apirpc.py b/Connector/btc/apirpc.py index aacecce0..8bb13646 100644 --- a/Connector/btc/apirpc.py +++ b/Connector/btc/apirpc.py @@ -3,7 +3,8 @@ from rpcutils import rpcmethod from logger import logger from rpcutils import error -from rpcutils.rpcconnector import RPCConnector, RPCSocketConnector +from rpcutils.rpcconnector import RPCConnector +from rpcutils.rpcsocketconnector import RPCSocketConnector from . import utils from .constants import * @@ -432,11 +433,12 @@ def getTransaction(id, params, config): response = { "transaction": { - "txHash": transactionDecoded["txHash"], + "txId": transactionDecoded["id"], + "txHash": transactionDecoded["hash"], "blockNumber": str(txHeight) if txHeight is not None else None, "fee": str(utils.convertToSatoshi(-transactionDecoded["fee"])) if "generated" not in transactionDecoded else "0", "transfers": utils.getTransactionTransfers(transactionDecoded, config.bitcoincoreRpcEndpoint, config.electrumxHost, config.electrumxPort), - "data": transactionDecoded["decoded"] + "data": transactionDecoded } } diff --git a/Connector/btc/utils.py b/Connector/btc/utils.py index 92289e32..a0b87c86 100644 --- a/Connector/btc/utils.py +++ b/Connector/btc/utils.py @@ -8,7 +8,8 @@ from logger import logger from rpcutils import error as rpcerrorhandler from wsutils import topics -from rpcutils.rpcconnector import RPCConnector, RPCSocketConnector +from rpcutils.rpcconnector import RPCConnector +from rpcutils.rpcsocketconnector import RPCSocketConnector from .constants import * from . import apirpc From f20a1add008bb551f6d95f1b3d13060a4a2adfac Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 28 Feb 2022 19:24:15 +0100 Subject: [PATCH 05/15] Bugfixes --- Connector/btc/apirpc.py | 13 ++++++++----- Connector/btc/utils.py | 30 +++++++++++++++++++++--------- Connector/server.py | 2 +- 3 files changed, 30 insertions(+), 15 deletions(-) diff --git a/Connector/btc/apirpc.py b/Connector/btc/apirpc.py index 8bb13646..f3ec0a9e 100644 --- a/Connector/btc/apirpc.py +++ b/Connector/btc/apirpc.py @@ -384,6 +384,7 @@ def getTransactionHex(id, params, config): @rpcmethod.rpcMethod(coin=COIN_SYMBOL) @httpmethod.postHttpMethod(coin=COIN_SYMBOL) def getTransaction(id, params, config): + #TODO: Test unconfirmed tx logger.printInfo(f"Executing RPC method getTransaction with id {id} and params {params}") @@ -425,19 +426,21 @@ def getTransaction(id, params, config): ] ) - txHeight = None + transactionHeight = None for transaction in txWorkaround: if transaction["tx_hash"] == transactionDecoded["txid"]: - txHeight = transaction["height"] + transactionHeight = transaction["height"] break + transactionDetails = utils.decodeTransactionDetails(transactionDecoded, config.bitcoincoreRpcEndpoint, config.electrumxHost, config.electrumxPort) + response = { "transaction": { "txId": transactionDecoded["id"], "txHash": transactionDecoded["hash"], - "blockNumber": str(txHeight) if txHeight is not None else None, - "fee": str(utils.convertToSatoshi(-transactionDecoded["fee"])) if "generated" not in transactionDecoded else "0", - "transfers": utils.getTransactionTransfers(transactionDecoded, config.bitcoincoreRpcEndpoint, config.electrumxHost, config.electrumxPort), + "blockNumber": str(transactionHeight) if transactionHeight is not None else None, + "fee": transactionDetails["fee"], + "transfers": transactionDecoded["transfers"], "data": transactionDecoded } } diff --git a/Connector/btc/utils.py b/Connector/btc/utils.py index a0b87c86..368d21c0 100644 --- a/Connector/btc/utils.py +++ b/Connector/btc/utils.py @@ -76,14 +76,14 @@ def getWorkaroundScriptHash(txDecoded): return hash -def getTransactionTransfers(txDecoded, bitcoincoreRpcEndpoint, electrumxHost, electrumxPort): +def decodeTransactionDetails(txDecoded, bitcoincoreRpcEndpoint, electrumxHost, electrumxPort): outputs = [] for output in txDecoded["vout"]: if "addresses" in output["scriptPubKey"] and len(output["scriptPubKey"]["addresses"]) == 1: outputs.append( - {"amount": math.trunc(output["value"] * 100000000), "address": output["scriptPubKey"]["addresses"][0]}) + {"amount": math.trunc(output["value"] * 100000000), "address": output["scriptPubKey"]["addresses"][0]}) # TODO: Convert to satoshi test else: - outputs.append({"amount": math.trunc(output["value"] * 100000000), "address": None}) + outputs.append({"amount": math.trunc(output["value"] * 100000000), "address": None}) # TODO: Convert to satoshi test sumOutputs = 0 for output in outputs: @@ -96,17 +96,28 @@ def getTransactionTransfers(txDecoded, bitcoincoreRpcEndpoint, electrumxHost, el inputs.append({"amount": sumOutputs, "address": None}) break - txInRaw = RPCSocketConnector.sendRequest(electrumxHost, electrumxPort, 0, "blockchain.transaction.get", - [txInput["txid"]]) - txInDecoded = RPCConnector.request(bitcoincoreRpcEndpoint, "decoderawtransaction", - [txInRaw]) + txInRaw = RPCSocketConnector.request( + hostname=electrumxHost, + port=electrumxPort, + id=0, + method="blockchain.transaction.get", + params=[ + txInput["txid"] + ] + ) + txInDecoded = RPCConnector.request( + endpoint=bitcoincoreRpcEndpoint, + id=0, + method="decoderawtransaction", + params=[txInRaw] + ) for txOutput in txInDecoded["vout"]: if txOutput["n"] == txInput["vout"] and "addresses" in txOutput["scriptPubKey"] and len( txOutput["scriptPubKey"]["addresses"]) == 1: inputs.append({"amount": math.trunc(txOutput["value"] * 100000000), "address": txOutput["scriptPubKey"]["addresses"][0]}) - elif len(txOutput["scriptPubKey"]["addresses"]) != 1: + elif "addresses" not in txOutput["scriptPubKey"] or len(txOutput["scriptPubKey"]["addresses"]) != 1: inputs.append({"amount": math.trunc(txOutput["value"] * 100000000), "address": None}) sumInputs = 0 @@ -134,7 +145,8 @@ def getTransactionTransfers(txDecoded, bitcoincoreRpcEndpoint, electrumxHost, el transfersResult = [] for key in transfers.keys(): transfersResult.append({"from": key[0], "to": key[1], "amount": transfers[key]}) - return transfersResult + + return {"transfers": transfersResult, "fee": fee} def sortUnspentOutputs(outputs): diff --git a/Connector/server.py b/Connector/server.py index 12dcf772..ff2afc1c 100644 --- a/Connector/server.py +++ b/Connector/server.py @@ -75,7 +75,7 @@ def runServer(): logger.printInfo("Starting connector") # TODO: Do not hardcode port - web.run_app(mainApp, port=80) + web.run_app(mainApp, port=8080) if __name__ == '__main__': From 3c65feb579b46a040ba416b202fe67d3c003d26a Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 28 Feb 2022 19:29:41 +0100 Subject: [PATCH 06/15] Bugfixes --- Connector/btc/apirpc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Connector/btc/apirpc.py b/Connector/btc/apirpc.py index f3ec0a9e..fc97c983 100644 --- a/Connector/btc/apirpc.py +++ b/Connector/btc/apirpc.py @@ -436,11 +436,11 @@ def getTransaction(id, params, config): response = { "transaction": { - "txId": transactionDecoded["id"], + "txId": transactionDecoded["txid"], "txHash": transactionDecoded["hash"], "blockNumber": str(transactionHeight) if transactionHeight is not None else None, - "fee": transactionDetails["fee"], - "transfers": transactionDecoded["transfers"], + "fee": str(transactionDetails["fee"]), + "transfers": transactionDetails["transfers"], "data": transactionDecoded } } From 34d37adfd39deaffd3ad26d54a938726039e648d Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 28 Feb 2022 22:06:46 +0100 Subject: [PATCH 07/15] Bugfixes --- Connector/btc/apirpc.py | 12 ++++++-- .../rpcschemas/gettransaction_response.json | 27 +++++++++++++----- Connector/btc/utils.py | 28 ++++--------------- 3 files changed, 35 insertions(+), 32 deletions(-) diff --git a/Connector/btc/apirpc.py b/Connector/btc/apirpc.py index fc97c983..24edd92f 100644 --- a/Connector/btc/apirpc.py +++ b/Connector/btc/apirpc.py @@ -434,13 +434,21 @@ def getTransaction(id, params, config): transactionDetails = utils.decodeTransactionDetails(transactionDecoded, config.bitcoincoreRpcEndpoint, config.electrumxHost, config.electrumxPort) + # Converting all transaction details to str + transactionDetails["fee"] = str(transactionDetails["fee"]) + for input in transactionDetails["inputs"]: + input["amount"] = str(input["amount"]) + for output in transactionDetails["outputs"]: + output["amount"] = str(output["amount"]) + response = { "transaction": { "txId": transactionDecoded["txid"], "txHash": transactionDecoded["hash"], "blockNumber": str(transactionHeight) if transactionHeight is not None else None, - "fee": str(transactionDetails["fee"]), - "transfers": transactionDetails["transfers"], + "fee": transactionDetails["fee"], + "inputs": transactionDetails["inputs"], + "outputs": transactionDetails["outputs"], "data": transactionDecoded } } diff --git a/Connector/btc/rpcschemas/gettransaction_response.json b/Connector/btc/rpcschemas/gettransaction_response.json index 25f862b0..f74999f8 100644 --- a/Connector/btc/rpcschemas/gettransaction_response.json +++ b/Connector/btc/rpcschemas/gettransaction_response.json @@ -28,18 +28,31 @@ "fee": { "type": "string" }, - "transfers": { + "inputs": { "type": "array", "items": { "properties": { - "from": { - "type": "string" - }, - "to": { - "type": "string" + "address": { + "type": [ + "string", + "null" + ] }, - "fee": { + "amount": { "type": "string" + } + } + } + }, + "outputs": { + "type": "array", + "items": { + "properties": { + "address": { + "type": [ + "string", + "null" + ] }, "amount": { "type": "string" diff --git a/Connector/btc/utils.py b/Connector/btc/utils.py index 368d21c0..93dc0301 100644 --- a/Connector/btc/utils.py +++ b/Connector/btc/utils.py @@ -81,9 +81,9 @@ def decodeTransactionDetails(txDecoded, bitcoincoreRpcEndpoint, electrumxHost, e for output in txDecoded["vout"]: if "addresses" in output["scriptPubKey"] and len(output["scriptPubKey"]["addresses"]) == 1: outputs.append( - {"amount": math.trunc(output["value"] * 100000000), "address": output["scriptPubKey"]["addresses"][0]}) # TODO: Convert to satoshi test + {"amount": math.trunc(output["value"] * 100000000), "address": output["scriptPubKey"]["addresses"][0]}) else: - outputs.append({"amount": math.trunc(output["value"] * 100000000), "address": None}) # TODO: Convert to satoshi test + outputs.append({"amount": math.trunc(output["value"] * 100000000), "address": None}) sumOutputs = 0 for output in outputs: @@ -92,7 +92,7 @@ def decodeTransactionDetails(txDecoded, bitcoincoreRpcEndpoint, electrumxHost, e inputs = [] for txInput in txDecoded["vin"]: - if "coinbase" in txInput: # This is a coinbase transaction and thus it have one only input of 'sumOutputs' amount + if "coinbase" in txInput: # This is a coinbase transaction and thus it have one only input of 'sumOutputs' inputs.append({"amount": sumOutputs, "address": None}) break @@ -124,29 +124,11 @@ def decodeTransactionDetails(txDecoded, bitcoincoreRpcEndpoint, electrumxHost, e for txInput in inputs: sumInputs += txInput["amount"] - sumOutputs = 0 - for txOutput in outputs: - sumOutputs += txOutput["amount"] - fee = sumInputs - sumOutputs - transfers = {} - for inputTx in inputs: - inputTotalProvide = inputTx["amount"] - (fee / len(inputs)) - for txOutput in outputs: - outputPercentage = txOutput["amount"] / sumOutputs - - if (inputTx["address"], txOutput["address"]) in transfers: - transfers[(inputTx["address"], txOutput["address"])] = transfers[( - inputTx["address"], txOutput["address"])] + (inputTotalProvide * outputPercentage) - else: - transfers[(inputTx["address"], txOutput["address"])] = inputTotalProvide * outputPercentage - - transfersResult = [] - for key in transfers.keys(): - transfersResult.append({"from": key[0], "to": key[1], "amount": transfers[key]}) + transactionsDetails = {"fee": fee, "inputs": inputs, "outputs": outputs} - return {"transfers": transfersResult, "fee": fee} + return transactionsDetails def sortUnspentOutputs(outputs): From 96ad332e424d4dec31b567a44f47abe6a739b0c5 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 28 Feb 2022 22:12:53 +0100 Subject: [PATCH 08/15] Ethereum rpc adapted to new api standard --- Connector/eth/apirpc.py | 14 ++++++---- .../rpcschemas/gettransaction_response.json | 26 +++++++++++++------ 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/Connector/eth/apirpc.py b/Connector/eth/apirpc.py index 089fb994..ecf12e11 100644 --- a/Connector/eth/apirpc.py +++ b/Connector/eth/apirpc.py @@ -222,12 +222,16 @@ def getTransaction(id, params, config): "blockHash": transaction["blockHash"], "blockNumber": str(int(transaction["blockNumber"], 16)) if transaction["blockNumber"] is not None else None, "data": transaction, - "transfers": [ + "inputs": [ { - "from": transaction["from"], - "to": transaction["to"], - "amount": str(utils.toWei(transaction["value"])), - "fee": str(utils.toWei(transaction["gasPrice"]) * utils.toWei(transaction["gas"])) + "address": transaction["from"], + "amount": str(utils.toWei(transaction["value"])) + } + ], + "outputs": [ + { + "address": transaction["to"], + "amount": str(utils.toWei(transaction["value"])) } ] } diff --git a/Connector/eth/rpcschemas/gettransaction_response.json b/Connector/eth/rpcschemas/gettransaction_response.json index f75f3b57..e369333b 100644 --- a/Connector/eth/rpcschemas/gettransaction_response.json +++ b/Connector/eth/rpcschemas/gettransaction_response.json @@ -25,21 +25,31 @@ "fee": { "type": "string" }, - "transfers": { + "inputs": { "type": "array", "items": { - "type": "object", "properties": { - "from": { - "type": "string" - }, - "to": { - "type": "string" + "address": { + "type": [ + "string" + ] }, "amount": { "type": "string" + } + } + } + }, + "outputs": { + "type": "array", + "items": { + "properties": { + "address": { + "type": [ + "string" + ] }, - "fee": { + "amount": { "type": "string" } } From 2bd03445cfe91f5616d0d5b0174e63f2c25ac570 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 28 Feb 2022 22:16:49 +0100 Subject: [PATCH 09/15] Bugfixes --- Connector/btc/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Connector/btc/utils.py b/Connector/btc/utils.py index 93dc0301..5f1470b1 100644 --- a/Connector/btc/utils.py +++ b/Connector/btc/utils.py @@ -70,7 +70,7 @@ def closeAddrBalanceTopic(topicName): def getWorkaroundScriptHash(txDecoded): - script = txDecoded["vout"][0]["scriptPubKey"]["hex"] + script = txDecoded["vout"][1]["scriptPubKey"]["hex"] scriptUnex = binascii.unhexlify(script) hash = hashlib.sha256(scriptUnex).digest()[::-1].hex() return hash From 32ecfc0b14cc3f8676e8a9e799b95e0bc0cf230b Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 28 Feb 2022 22:32:54 +0100 Subject: [PATCH 10/15] getTransaction not working anymore with electrumx --- Connector/btc/apirpc.py | 46 ++++++++++++----------------------------- Connector/btc/utils.py | 27 +++++++----------------- 2 files changed, 20 insertions(+), 53 deletions(-) diff --git a/Connector/btc/apirpc.py b/Connector/btc/apirpc.py index 24edd92f..a3f7519b 100644 --- a/Connector/btc/apirpc.py +++ b/Connector/btc/apirpc.py @@ -395,44 +395,24 @@ def getTransaction(id, params, config): raise error.RpcBadRequestError(err.message) try: - transactionRaw = RPCSocketConnector.request( - hostname=config.electrumxHost, - port=config.electrumxPort, + transaction = RPCConnector.request( + endpoint=config.bitcoincoreRpcEndpoint, id=id, - method="blockchain.transaction.get", + method="getrawtransaction", params=[ - params["txHash"] + params["txHash"], + True ] ) - transactionDecoded = RPCConnector.request( + transactionBlock = RPCConnector.request( endpoint=config.bitcoincoreRpcEndpoint, id=id, - method="decoderawtransaction", - params=[transactionRaw] + method="getblock", + params=[transaction["blockhash"], 1] ) - # We pick a random script hash (first one) and after that, we find the transaction that matches - # out transaction to find the height. - # This is ugly but has to be done because of Electrum protocol, that it is (apparently) unable - # of give the transaction height in 'blockchain.transaction.get' method - txWorkaround = RPCSocketConnector.request( - hostname=config.electrumxHost, - port=config.electrumxPort, - id=id, - method="blockchain.scripthash.get_history", - params=[ - utils.getWorkaroundScriptHash(transactionDecoded) - ] - ) - - transactionHeight = None - for transaction in txWorkaround: - if transaction["tx_hash"] == transactionDecoded["txid"]: - transactionHeight = transaction["height"] - break - - transactionDetails = utils.decodeTransactionDetails(transactionDecoded, config.bitcoincoreRpcEndpoint, config.electrumxHost, config.electrumxPort) + transactionDetails = utils.decodeTransactionDetails(transaction, config.bitcoincoreRpcEndpoint) # Converting all transaction details to str transactionDetails["fee"] = str(transactionDetails["fee"]) @@ -443,13 +423,13 @@ def getTransaction(id, params, config): response = { "transaction": { - "txId": transactionDecoded["txid"], - "txHash": transactionDecoded["hash"], - "blockNumber": str(transactionHeight) if transactionHeight is not None else None, + "txId": transaction["txid"], + "txHash": transaction["hash"], + "blockNumber": str(transactionBlock["height"]), "fee": transactionDetails["fee"], "inputs": transactionDetails["inputs"], "outputs": transactionDetails["outputs"], - "data": transactionDecoded + "data": transaction } } diff --git a/Connector/btc/utils.py b/Connector/btc/utils.py index 5f1470b1..87eea8ca 100644 --- a/Connector/btc/utils.py +++ b/Connector/btc/utils.py @@ -69,14 +69,7 @@ def closeAddrBalanceTopic(topicName): raise rpcerrorhandler.BadRequestError(f"Can not unsubscribe {topicName} to node") -def getWorkaroundScriptHash(txDecoded): - script = txDecoded["vout"][1]["scriptPubKey"]["hex"] - scriptUnex = binascii.unhexlify(script) - hash = hashlib.sha256(scriptUnex).digest()[::-1].hex() - return hash - - -def decodeTransactionDetails(txDecoded, bitcoincoreRpcEndpoint, electrumxHost, electrumxPort): +def decodeTransactionDetails(txDecoded, bitcoincoreRpcEndpoint): outputs = [] for output in txDecoded["vout"]: if "addresses" in output["scriptPubKey"] and len(output["scriptPubKey"]["addresses"]) == 1: @@ -96,23 +89,17 @@ def decodeTransactionDetails(txDecoded, bitcoincoreRpcEndpoint, electrumxHost, e inputs.append({"amount": sumOutputs, "address": None}) break - txInRaw = RPCSocketConnector.request( - hostname=electrumxHost, - port=electrumxPort, + transaction = RPCConnector.request( + endpoint=bitcoincoreRpcEndpoint, id=0, - method="blockchain.transaction.get", + method="getrawtransaction", params=[ - txInput["txid"] + txInput["txid"], + True ] ) - txInDecoded = RPCConnector.request( - endpoint=bitcoincoreRpcEndpoint, - id=0, - method="decoderawtransaction", - params=[txInRaw] - ) - for txOutput in txInDecoded["vout"]: + for txOutput in transaction["vout"]: if txOutput["n"] == txInput["vout"] and "addresses" in txOutput["scriptPubKey"] and len( txOutput["scriptPubKey"]["addresses"]) == 1: inputs.append({"amount": math.trunc(txOutput["value"] * 100000000), From d0f567e9663138fe93aeb59192f016e1d5d65ab0 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 28 Feb 2022 22:40:06 +0100 Subject: [PATCH 11/15] Unconfirmed transactions fix --- Connector/btc/apirpc.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/Connector/btc/apirpc.py b/Connector/btc/apirpc.py index a3f7519b..f2413d0b 100644 --- a/Connector/btc/apirpc.py +++ b/Connector/btc/apirpc.py @@ -405,12 +405,19 @@ def getTransaction(id, params, config): ] ) - transactionBlock = RPCConnector.request( - endpoint=config.bitcoincoreRpcEndpoint, - id=id, - method="getblock", - params=[transaction["blockhash"], 1] - ) + isConfirmed = "blockhash" in transaction + + if isConfirmed: + transactionBlock = RPCConnector.request( + endpoint=config.bitcoincoreRpcEndpoint, + id=id, + method="getblock", + params=[transaction["blockhash"], 1] + ) + blockNumber = transactionBlock["height"] + else: + blockNumber = None + transactionDetails = utils.decodeTransactionDetails(transaction, config.bitcoincoreRpcEndpoint) @@ -425,7 +432,7 @@ def getTransaction(id, params, config): "transaction": { "txId": transaction["txid"], "txHash": transaction["hash"], - "blockNumber": str(transactionBlock["height"]), + "blockNumber": str(blockNumber) if blockNumber is not None else blockNumber, "fee": transactionDetails["fee"], "inputs": transactionDetails["inputs"], "outputs": transactionDetails["outputs"], From 66cb7c3f39b62347f4844a69d160fdcca92ba393 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 28 Feb 2022 22:50:35 +0100 Subject: [PATCH 12/15] Removed todo --- Connector/btc/apirpc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Connector/btc/apirpc.py b/Connector/btc/apirpc.py index f2413d0b..65d34c67 100644 --- a/Connector/btc/apirpc.py +++ b/Connector/btc/apirpc.py @@ -384,8 +384,6 @@ def getTransactionHex(id, params, config): @rpcmethod.rpcMethod(coin=COIN_SYMBOL) @httpmethod.postHttpMethod(coin=COIN_SYMBOL) def getTransaction(id, params, config): - #TODO: Test unconfirmed tx - logger.printInfo(f"Executing RPC method getTransaction with id {id} and params {params}") requestSchema, responseSchema = utils.getMethodSchemas(GET_TRANSACTION) From 004505259fd17f10cb545c2cd9f46922a035e016 Mon Sep 17 00:00:00 2001 From: Your Name Date: Mon, 28 Feb 2022 23:13:50 +0100 Subject: [PATCH 13/15] Changed server port back to 80 --- Connector/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Connector/server.py b/Connector/server.py index ff2afc1c..12dcf772 100644 --- a/Connector/server.py +++ b/Connector/server.py @@ -75,7 +75,7 @@ def runServer(): logger.printInfo("Starting connector") # TODO: Do not hardcode port - web.run_app(mainApp, port=8080) + web.run_app(mainApp, port=80) if __name__ == '__main__': From 78b679db8c9cea89babbfa4d5c9a4e9349436fb0 Mon Sep 17 00:00:00 2001 From: dragon Date: Wed, 2 Mar 2022 13:00:33 +0100 Subject: [PATCH 14/15] Fixed requested things in PR --- Connector/btc/apirpc.py | 9 ++++----- Connector/btc/constants.py | 1 + 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Connector/btc/apirpc.py b/Connector/btc/apirpc.py index 65d34c67..5c1eae8d 100644 --- a/Connector/btc/apirpc.py +++ b/Connector/btc/apirpc.py @@ -396,20 +396,19 @@ def getTransaction(id, params, config): transaction = RPCConnector.request( endpoint=config.bitcoincoreRpcEndpoint, id=id, - method="getrawtransaction", + method=GET_TRANSACTION_METHOD, params=[ params["txHash"], True ] ) - isConfirmed = "blockhash" in transaction - - if isConfirmed: + # Check if transaction is confirmed, and obtain block number + if "blockhash" in transaction: transactionBlock = RPCConnector.request( endpoint=config.bitcoincoreRpcEndpoint, id=id, - method="getblock", + method=GET_BLOCK, params=[transaction["blockhash"], 1] ) blockNumber = transactionBlock["height"] diff --git a/Connector/btc/constants.py b/Connector/btc/constants.py index 6a459e48..fd826630 100644 --- a/Connector/btc/constants.py +++ b/Connector/btc/constants.py @@ -100,6 +100,7 @@ NOTIFY = "notify" NEW_HASH_BLOCK_ZMQ_TOPIC = "hashblock" NEW_RAW_BLOCK_ZMQ_TOPIC = "rawblock" +GET_BLOCK = "getblock" SUBSCRIBE_ADDRESS_BALANCE = "subscribetoaddressbalance" UNSUBSCRIBE_ADDRESS_BALANCE = "unsubscribefromaddressbalance" From b1605969eeb8226dc5ccfa6c07deaa93775a69b2 Mon Sep 17 00:00:00 2001 From: dragon Date: Wed, 2 Mar 2022 13:16:04 +0100 Subject: [PATCH 15/15] Get transaction method changed for getrawtransaction --- Connector/btc/apirpc.py | 4 ++-- Connector/btc/constants.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Connector/btc/apirpc.py b/Connector/btc/apirpc.py index 5c1eae8d..d1c18bc9 100644 --- a/Connector/btc/apirpc.py +++ b/Connector/btc/apirpc.py @@ -368,7 +368,7 @@ def getTransactionHex(id, params, config): rawTransaction = RPCConnector.request( endpoint=config.bitcoincoreRpcEndpoint, id=id, - method=GET_TRANSACTION_METHOD, + method=GET_RAW_TRANSACTION_METHOD, params=[params["txHash"]] ) @@ -396,7 +396,7 @@ def getTransaction(id, params, config): transaction = RPCConnector.request( endpoint=config.bitcoincoreRpcEndpoint, id=id, - method=GET_TRANSACTION_METHOD, + method=GET_RAW_TRANSACTION_METHOD, params=[ params["txHash"], True diff --git a/Connector/btc/constants.py b/Connector/btc/constants.py index fd826630..68d2db3f 100644 --- a/Connector/btc/constants.py +++ b/Connector/btc/constants.py @@ -8,7 +8,7 @@ GET_BLOCK_HASH_METHOD = "getblockhash" GET_BLOCK_COUNT_METHOD = "getblockcount" ESTIMATE_SMART_FEE_METHOD = "estimatesmartfee" -GET_TRANSACTION_METHOD = "gettransaction" +GET_RAW_TRANSACTION_METHOD = "getrawtransaction" DECODE_RAW_TRANSACTION_METHOD = "decoderawtransaction" SEND_RAW_TRANSACTION_METHOD = "sendrawtransaction" NOTIFY_METHOD = "notify"