electrum-personal-server

Maximally lightweight electrum server for a single user
git clone https://git.parazyd.org/electrum-personal-server
Log | Files | Refs | README

commit 5b71d929988cf99a6ff2d5595218e4419c9cc620
parent a057f49693a05187b0a898cc348176513ffcee37
Author: chris-belcher <chris-belcher@users.noreply.github.com>
Date:   Tue, 26 Jun 2018 23:31:44 +0100

Expand tests to include reorgs and many many txes

Several new tests are added to test the reorganization-checking code.
Also added is a test which simulates building a history with 1100
transactions, and where 130 transactions arrive afterwards.
Some more debug print statements are added where they are useful.

Diffstat:
Melectrumpersonalserver/transactionmonitor.py | 30+++++++++++++++---------------
Mtest/test_parse_mpks.py | 6+++++-
Mtest/test_transactionmonitor.py | 218++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------
3 files changed, 204 insertions(+), 50 deletions(-)

diff --git a/electrumpersonalserver/transactionmonitor.py b/electrumpersonalserver/transactionmonitor.py @@ -1,6 +1,7 @@ import time, pprint, math, sys from decimal import Decimal +from collections import defaultdict from electrumpersonalserver.jsonrpc import JsonRpcError import electrumpersonalserver.hashes as hashes @@ -155,14 +156,11 @@ class TransactionMonitor(object): new_history_element["height"], sh_to_add)) count += 1 - unconfirmed_txes = {} + unconfirmed_txes = defaultdict(list) for scrhash, his in address_history.items(): uctx = self.sort_address_history_list(his) for u in uctx: - if u["tx_hash"] in unconfirmed_txes: - unconfirmed_txes[u["tx_hash"]].append(scrhash) - else: - unconfirmed_txes[u["tx_hash"]] = [scrhash] + unconfirmed_txes[u["tx_hash"]].append(scrhash) self.debug("unconfirmed_txes = " + str(unconfirmed_txes)) self.debug("reorganizable_txes = " + str(self.reorganizable_txes)) if len(ret) > 0: @@ -292,10 +290,7 @@ class TransactionMonitor(object): #transaction became unconfirmed in a reorg self.log("A transaction was reorg'd out: " + txid) elements_removed.append(reorgable_tx) - if txid in self.unconfirmed_txes: - self.unconfirmed_txes[txid].extend(scrhashes) - else: - self.unconfirmed_txes[txid] = list(scrhashes) + self.unconfirmed_txes[txid].extend(scrhashes) #add to history as unconfirmed txd = self.rpc.call("decoderawtransaction", [tx["hex"]]) @@ -312,11 +307,13 @@ class TransactionMonitor(object): elif tx["blockhash"] != blockhash: block = self.rpc.call("getblockheader", [tx["blockhash"]]) if block["height"] == height: #reorg but height is the same + self.log("A transaction was reorg'd but still confirmed " + + "at same height: " + txid) continue #reorged but still confirmed at a different height updated_scrhashes.update(scrhashes) - self.log("A transaction was reorg'd but still confirmed at " + - "same height: " + txid) + self.log("A transaction was reorg'd but still confirmed to " + + "a new block and different height: " + txid) #update history with the new height for scrhash in scrhashes: for h in self.address_history[scrhash]["history"]: @@ -383,6 +380,12 @@ class TransactionMonitor(object): for i in range(max_attempts): self.debug("listtransactions tx_request_count=" + str(tx_request_count)) + ##how listtransactions works + ##skip and count parameters take most-recent txes first + ## so skip=0 count=1 will return the most recent tx + ##and skip=0 count=3 will return the 3 most recent txes + ##but the actual list returned has the REVERSED order + ##skip=0 count=3 will return a list with the most recent tx LAST ret = self.rpc.call("listtransactions", ["*", tx_request_count, 0, True]) ret = ret[::-1] @@ -459,10 +462,7 @@ class TransactionMonitor(object): self.address_history[scrhash]["history"].append( new_history_element) if new_history_element["height"] == 0: - if tx["txid"] in self.unconfirmed_txes: - self.unconfirmed_txes[tx["txid"]].append(scrhash) - else: - self.unconfirmed_txes[tx["txid"]] = [scrhash] + self.unconfirmed_txes[tx["txid"]].append(scrhash) if tx["confirmations"] > 0: self.reorganizable_txes.append((tx["txid"], tx["blockhash"], new_history_element["height"], matching_scripthashes)) diff --git a/test/test_parse_mpks.py b/test/test_parse_mpks.py @@ -16,7 +16,11 @@ from electrumpersonalserver import parse_electrum_master_public_key "2 tpubD6NzVbkrYhZ4YVMVzC7wZeRfz3bhqcHvV8M3UiULCfzFtLtp5nwvi6LnBQegrkx" + "YGPkSzXUEvcPEHcKdda8W1YShVBkhFBGkLxjSQ1Nx3cJ Vpub5fAqpSRkLmvXwqbuR61M" + "aKMSwj5z5xUBwanaz3qnJ5MgaBDpFSLUvKTiNK9zHpdvrg2LHHXkKxSXBHNWNpZz9b1Vq" + - "ADjmcCs3arSoxN3F3r" #inconsistent magic + "ADjmcCs3arSoxN3F3r", #inconsistent magic + "e9d4b7866dd1e91c862aebf62a49548c7dbf7bcc6e4b7b8c9da820c7737968df9c09d" + + "5a3e271dc814a29981f81b3faaf2737b551ef5dcc6189cf0f8252c442", #wrong length + "e9d4b7866dd1e91c862aebf62a49548c7dbf7bcc6e4b7b8c9da820c7737968df9c09d" + + "5a3e271dc814a29981f81b3faaf2737b551ef5dcc6189cf0f8252c442ZZ" #not hex ] ) diff --git a/test/test_transactionmonitor.py b/test/test_transactionmonitor.py @@ -5,6 +5,11 @@ from electrumpersonalserver import (DeterministicWallet, TransactionMonitor, JsonRpcError, script_to_scripthash) class DummyJsonRpc(object): + """ + Electrum Personal Server gets all its information about the bitcoin network + from the json-rpc interface. This dummy interface is used for simulating + events in bitcoin + """ def __init__(self, txlist, utxoset, block_heights): self.txlist = txlist self.utxoset = utxoset @@ -15,7 +20,7 @@ class DummyJsonRpc(object): if method == "listtransactions": count = int(params[1]) skip = int(params[2]) - return self.txlist[skip:skip + count] + return self.txlist[skip:skip + count][::-1] elif method == "gettransaction": for t in self.txlist: if t["txid"] == params[0]: @@ -25,16 +30,19 @@ class DummyJsonRpc(object): for t in self.txlist: if t["hex"] == params[0]: return t + debugf(params[0]) assert 0 elif method == "gettxout": for u in self.utxoset: if u["txid"] == params[0] and u["vout"] == params[1]: return u + debugf("txid = " + params[0] + " vout = " + str(params[1])) assert 0 elif method == "getblockheader": - if params[0] not in self.block_heights: - assert 0 - return {"height": self.block_heights[params[0]]} + if params[0] in self.block_heights: + return {"height": self.block_heights[params[0]]} + debugf(params[0]) + assert 0 elif method == "decodescript": return {"addresses": [dummy_spk_to_address(params[0])]} elif method == "importaddress": @@ -43,7 +51,7 @@ class DummyJsonRpc(object): raise ValueError("unknown method in dummy jsonrpc") def add_transaction(self, tx): - self.txlist.append(tx) + self.txlist = [tx] + self.txlist def get_imported_addresses(self): return self.imported_addresses @@ -62,6 +70,7 @@ class DummyDeterministicWallet(DeterministicWallet): def dummy_spk_to_address(spk): + ##spk is short for scriptPubKey return spk + "-address" debugf = lambda x: print("[DEBUG] " + x) @@ -97,6 +106,7 @@ def create_dummy_funding_tx(confirmations=1, output_spk=None, "blockhash": dummy_containing_block, "hex": "placeholder-test-txhex" + str(dummy_id) } + debugf("created dummy tx: " + str(dummy_tx)) return dummy_spk, containing_block_height, dummy_tx def assert_address_history_tx(address_history, spk, height, txid, subscribed): @@ -139,6 +149,42 @@ def test_two_txes(): height=containing_block_height2, txid=dummy_tx2["txid"], subscribed=False) +def test_many_txes(): + ##many txes in wallet and many more added,, intended to test the loop + ## in build_addr_history and check_for_new_txes() + input_spk, input_block_height1, input_tx = create_dummy_funding_tx() + dummy_spk, containing_block_height, dummy_tx = create_dummy_funding_tx( + confirmations=0, input_txid=input_tx["vin"][0]) + sh = script_to_scripthash(dummy_spk) + + #batch size is 1000 + INITIAL_TX_COUNT = 1100 + txes = [dummy_tx] + #0confirm to avoid having to obtain block hash + txes.extend( (create_dummy_funding_tx(output_spk=dummy_spk, + input_txid=input_tx["vin"][0], confirmations=0)[2] + for i in range(INITIAL_TX_COUNT-1)) ) + assert len(txes) == INITIAL_TX_COUNT + + rpc = DummyJsonRpc(txes, [dummy_tx["vin"][0]], {}) + txmonitor = TransactionMonitor(rpc, deterministic_wallets, debugf, logf) + assert txmonitor.build_address_history([dummy_spk]) + assert len(txmonitor.address_history) == 1 + assert len(list(txmonitor.check_for_updated_txes())) == 0 + assert len(txmonitor.address_history[sh]["history"]) == INITIAL_TX_COUNT + + ADDED_TX_COUNT = 130 + new_txes = [] + new_txes.extend( (create_dummy_funding_tx(output_spk=dummy_spk, + input_txid=input_tx["vin"][0], confirmations=0)[2] + for i in range(ADDED_TX_COUNT)) ) + + for tx in new_txes: + rpc.add_transaction(tx) + assert len(list(txmonitor.check_for_updated_txes())) == 0 + assert len(txmonitor.address_history[sh]["history"]) == (INITIAL_TX_COUNT + + ADDED_TX_COUNT) + def test_non_subscribed_confirmation(): ###one unconfirmed tx in wallet belonging to us, with confirmed inputs, ### addr history built, then tx confirms, not subscribed to address @@ -199,6 +245,28 @@ def test_unrelated_tx(): assert len(txmonitor.get_electrum_history(script_to_scripthash( our_dummy_spk))) == 0 +def test_duplicate_txid(): + ###two txes with the same txid, built history + dummy_spk, containing_block_height1, dummy_tx1 = create_dummy_funding_tx() + dummy_spk, containing_block_height2, dummy_tx2 = create_dummy_funding_tx( + output_spk=dummy_spk) + dummy_spk, containing_block_height3, dummy_tx3 = create_dummy_funding_tx( + output_spk=dummy_spk) + dummy_tx2["txid"] = dummy_tx1["txid"] + dummy_tx3["txid"] = dummy_tx1["txid"] + sh = script_to_scripthash(dummy_spk) + rpc = DummyJsonRpc([dummy_tx1, dummy_tx2], [], {dummy_tx1["blockhash"]: + containing_block_height1, dummy_tx2["blockhash"]: containing_block_height2, dummy_tx3["blockhash"]: containing_block_height3}) + txmonitor = TransactionMonitor(rpc, deterministic_wallets, debugf, logf) + assert txmonitor.build_address_history([dummy_spk]) + assert len(txmonitor.get_electrum_history(sh)) == 1 + txmonitor.subscribe_address(sh) + assert txmonitor.get_electrum_history(sh)[0]["tx_hash"] == dummy_tx1["txid"] + rpc.add_transaction(dummy_tx3) + assert len(list(txmonitor.check_for_updated_txes())) == 1 + assert len(txmonitor.get_electrum_history(sh)) == 1 + assert txmonitor.get_electrum_history(sh)[0]["tx_hash"] == dummy_tx1["txid"] + def test_address_reuse(): ###transaction which arrives to an address which already has a tx on it dummy_spk1, containing_block_height1, dummy_tx1 = create_dummy_funding_tx() @@ -284,63 +352,145 @@ def test_conflicted_tx(): ###conflicted transaction should get rejected dummy_spk, containing_block_height, dummy_tx = create_dummy_funding_tx( confirmations=-1) - rpc = DummyJsonRpc([dummy_tx], [], {}) txmonitor = TransactionMonitor(rpc, deterministic_wallets, debugf, logf) + sh = script_to_scripthash(dummy_spk) + assert txmonitor.build_address_history([dummy_spk]) assert len(txmonitor.address_history) == 1 - assert len(txmonitor.get_electrum_history(script_to_scripthash( - dummy_spk))) == 0 #shouldnt show up after build history b/c conflicted + #shouldnt show up after build history because conflicted + assert len(txmonitor.get_electrum_history(sh)) == 0 + + dummy_spk, containing_block_height, dummy_tx = create_dummy_funding_tx( + confirmations=-1, output_spk=dummy_spk) rpc.add_transaction(dummy_tx) assert len(list(txmonitor.check_for_updated_txes())) == 0 - assert len(txmonitor.get_electrum_history(script_to_scripthash( - dummy_spk))) == 0 #incoming tx is not added too + #incoming tx is not added either + assert len(txmonitor.get_electrum_history(sh)) == 0 -def test_double_spend(): +def test_reorg_finney_attack(): ###an unconfirmed tx being broadcast, another conflicting tx being ### confirmed, the first tx gets conflicted status - dummy_spk, containing_block_height1, dummy_tx1 = create_dummy_funding_tx( + dummy_spk1, containing_block_height1, dummy_tx1 = create_dummy_funding_tx( confirmations=0) - dummy_spk_, containing_block_height2, dummy_tx2 = create_dummy_funding_tx( - confirmations=0, input_txid=dummy_tx1["vin"][0], output_spk=dummy_spk) + dummy_spk2, containing_block_height2, dummy_tx2 = create_dummy_funding_tx( + confirmations=0, input_txid=dummy_tx1["vin"][0]) #two unconfirmed txes spending the same input, so they are in conflict rpc = DummyJsonRpc([dummy_tx1], [dummy_tx1["vin"][0]], {dummy_tx1["blockhash"]: containing_block_height1, dummy_tx2["blockhash"]: containing_block_height2}) txmonitor = TransactionMonitor(rpc, deterministic_wallets, debugf, logf) - assert txmonitor.build_address_history([dummy_spk]) - assert len(txmonitor.address_history) == 1 - sh = script_to_scripthash(dummy_spk) - assert len(txmonitor.get_electrum_history(sh)) == 1 - assert_address_history_tx(txmonitor.address_history, spk=dummy_spk, + assert txmonitor.build_address_history([dummy_spk1, dummy_spk2]) + assert len(txmonitor.address_history) == 2 + sh1 = script_to_scripthash(dummy_spk1) + sh2 = script_to_scripthash(dummy_spk2) + assert len(txmonitor.get_electrum_history(sh1)) == 1 + assert len(txmonitor.get_electrum_history(sh2)) == 0 + assert_address_history_tx(txmonitor.address_history, spk=dummy_spk1, height=0, txid=dummy_tx1["txid"], subscribed=False) # a conflicting transaction confirms rpc.add_transaction(dummy_tx2) dummy_tx1["confirmations"] = -1 dummy_tx2["confirmations"] = 1 assert len(list(txmonitor.check_for_updated_txes())) == 0 - assert len(txmonitor.get_electrum_history(sh)) == 1 - assert_address_history_tx(txmonitor.address_history, spk=dummy_spk, + assert len(txmonitor.get_electrum_history(sh1)) == 0 + assert len(txmonitor.get_electrum_history(sh2)) == 1 + assert_address_history_tx(txmonitor.address_history, spk=dummy_spk2, + height=containing_block_height2, txid=dummy_tx2["txid"], + subscribed=False) + +def test_reorg_race_attack(): + #a tx is confirmed, a chain reorganization happens and that tx is replaced + # by another tx spending the same input, the original tx is now conflicted + dummy_spk1, containing_block_height1, dummy_tx1 = create_dummy_funding_tx() + dummy_spk2, containing_block_height2, dummy_tx2 = create_dummy_funding_tx( + input_txid=dummy_tx1["vin"][0]) + + rpc = DummyJsonRpc([dummy_tx1], [], + {dummy_tx1["blockhash"]: containing_block_height1, + dummy_tx2["blockhash"]: containing_block_height2}) + txmonitor = TransactionMonitor(rpc, deterministic_wallets, debugf, logf) + assert txmonitor.build_address_history([dummy_spk1, dummy_spk2]) + assert len(txmonitor.address_history) == 2 + sh1 = script_to_scripthash(dummy_spk1) + sh2 = script_to_scripthash(dummy_spk2) + assert len(txmonitor.get_electrum_history(sh1)) == 1 + assert len(txmonitor.get_electrum_history(sh2)) == 0 + assert_address_history_tx(txmonitor.address_history, spk=dummy_spk1, + height=containing_block_height1, txid=dummy_tx1["txid"], subscribed=False) + #race attack happens + #dummy_tx1 goes to -1 confirmations, dummy_tx2 gets confirmed + rpc.add_transaction(dummy_tx2) + dummy_tx1["confirmations"] = -1 + dummy_tx2["confirmations"] = 1 + assert len(list(txmonitor.check_for_updated_txes())) == 0 + assert len(txmonitor.get_electrum_history(sh1)) == 0 + assert len(txmonitor.get_electrum_history(sh2)) == 1 + assert_address_history_tx(txmonitor.address_history, spk=dummy_spk2, height=containing_block_height2, txid=dummy_tx2["txid"], subscribed=False) +def test_reorg_censor_tx(): + #confirmed tx gets reorgd out and becomes unconfirmed + dummy_spk1, containing_block_height1, dummy_tx1 = create_dummy_funding_tx() + + rpc = DummyJsonRpc([dummy_tx1], [dummy_tx1["vin"][0]], + {dummy_tx1["blockhash"]: containing_block_height1}) + txmonitor = TransactionMonitor(rpc, deterministic_wallets, debugf, logf) + assert txmonitor.build_address_history([dummy_spk1]) + assert len(txmonitor.address_history) == 1 + sh = script_to_scripthash(dummy_spk1) + assert len(txmonitor.get_electrum_history(sh)) == 1 + assert_address_history_tx(txmonitor.address_history, spk=dummy_spk1, + height=containing_block_height1, txid=dummy_tx1["txid"], subscribed=False) + #blocks appear which reorg out the tx, making it unconfirmed + dummy_tx1["confirmations"] = 0 + assert len(list(txmonitor.check_for_updated_txes())) == 0 + assert len(txmonitor.get_electrum_history(sh)) == 1 + assert_address_history_tx(txmonitor.address_history, spk=dummy_spk1, + height=0, txid=dummy_tx1["txid"], subscribed=False) + +def test_reorg_different_block(): + #confirmed tx gets reorged into another block with a different height + dummy_spk1, containing_block_height1, dummy_tx1 = create_dummy_funding_tx() + dummy_spk2, containing_block_height2, dummy_tx2 = create_dummy_funding_tx() + + rpc = DummyJsonRpc([dummy_tx1], [], + {dummy_tx1["blockhash"]: containing_block_height1, + dummy_tx2["blockhash"]: containing_block_height2}) + txmonitor = TransactionMonitor(rpc, deterministic_wallets, debugf, logf) + assert txmonitor.build_address_history([dummy_spk1]) + assert len(txmonitor.address_history) == 1 + sh = script_to_scripthash(dummy_spk1) + assert len(txmonitor.get_electrum_history(sh)) == 1 + assert_address_history_tx(txmonitor.address_history, spk=dummy_spk1, + height=containing_block_height1, txid=dummy_tx1["txid"], subscribed=False) + + #tx gets reorged into another block (so still confirmed) + dummy_tx1["blockhash"] = dummy_tx2["blockhash"] + assert len(list(txmonitor.check_for_updated_txes())) == 0 + assert len(txmonitor.get_electrum_history(sh)) == 1 + assert_address_history_tx(txmonitor.address_history, spk=dummy_spk1, + height=containing_block_height2, txid=dummy_tx1["txid"], + subscribed=False) + +def test_tx_safe_from_reorg(): + ##tx confirmed with 1 confirmation, then confirmations goes to 100 + ## test that the reorganizable_txes list length goes down + dummy_spk1, containing_block_height1, dummy_tx1 = create_dummy_funding_tx() + rpc = DummyJsonRpc([dummy_tx1], [], + {dummy_tx1["blockhash"]: containing_block_height1}) + txmonitor = TransactionMonitor(rpc, deterministic_wallets, debugf, logf) + assert txmonitor.build_address_history([dummy_spk1]) + assert len(list(txmonitor.check_for_updated_txes())) == 0 + assert len(txmonitor.reorganizable_txes) == 1 + dummy_tx1["confirmations"] = 2000 + assert len(list(txmonitor.check_for_updated_txes())) == 0 + assert len(txmonitor.reorganizable_txes) == 0 #other possible stuff to test: #finding confirmed and unconfirmed tx, in that order, then both confirm #finding unconfirmed and confirmed tx, in that order, then both confirm -#tests about conflicts: -#build address history where reorgable txes are found -#an unconfirmed tx arrives, gets confirmed, reaches the safe threshold -# and gets removed from list -#a confirmed tx arrives, reaches safe threshold and gets removed -#an unconfirmed tx arrives, confirms, gets reorgd out, returns to -# unconfirmed -#an unconfirmed tx arrives, confirms, gets reorgd out and conflicted -#an unconfirmed tx arrives, confirms, gets reorgd out and confirmed at -# a different height -#an unconfirmed tx arrives, confirms, gets reorgd out and confirmed in -# the same height -