commit 3e52ed9e11aab176fa2d8e8c619a9cae568424d4
parent db78fb9e2d819f80b7501c43d56d96b8c22f4237
Author: chris-belcher <chris-belcher@users.noreply.github.com>
Date: Tue, 5 May 2020 21:46:45 +0100
Use RPC deriveaddresses to generate addresses
Output Descriptors and the RPC call deriveaddresses can generate
addresses much faster than the previously-used pure python routines.
This functionality is only fully supported in Bitcoin Core 0.20.0 so
the code checks for that version.
Diffstat:
4 files changed, 154 insertions(+), 162 deletions(-)
diff --git a/electrumpersonalserver/bitcoin/deterministic.py b/electrumpersonalserver/bitcoin/deterministic.py
@@ -40,7 +40,7 @@ def raw_bip32_ckd(rawtuple, i):
if i >= 2**31:
if vbytes in PUBLIC:
- raise Exception("Can't do private derivation on public key!")
+ raise ValueError("Can't do private derivation on public key!")
I = hmac.new(chaincode, b'\x00' + priv[:32] + encode(i, 256, 4),
hashlib.sha512).digest()
else:
@@ -70,7 +70,7 @@ def bip32_serialize(rawtuple):
def bip32_deserialize(data):
dbin = changebase(data, 58, 256)
if bin_dbl_sha256(dbin[:-4])[:4] != dbin[-4:]:
- raise Exception("Invalid checksum")
+ raise ValueError("Invalid checksum")
vbytes = dbin[0:4]
depth = from_byte_to_int(dbin[4])
fingerprint = dbin[5:9]
@@ -118,7 +118,7 @@ def raw_crack_bip32_privkey(parent_pub, priv):
i = int(i)
if i >= 2**31:
- raise Exception("Can't crack private derivation!")
+ raise ValueError("Can't crack private derivation!")
I = hmac.new(pchaincode, pkey + encode(i, 256, 4), hashlib.sha512).digest()
diff --git a/electrumpersonalserver/server/common.py b/electrumpersonalserver/server/common.py
@@ -174,34 +174,39 @@ def get_scriptpubkeys_to_monitor(rpc, config):
deterministic_wallets = []
for key in config.options("master-public-keys"):
- wal = deterministicwallet.parse_electrum_master_public_key(
- config.get("master-public-keys", key),
- int(config.get("bitcoin-rpc", "gap_limit")))
+ mpk = config.get("master-public-keys", key)
+ gaplimit = int(config.get("bitcoin-rpc", "gap_limit"))
+ chain = rpc.call("getblockchaininfo", [])["chain"]
+ try:
+ wal = deterministicwallet.parse_electrum_master_public_key(mpk,
+ gaplimit, rpc, chain)
+ except ValueError:
+ raise ValueError("Bad master public key format. Get it from " +
+ "Electrum menu `Wallet` -> `Information`")
deterministic_wallets.append(wal)
#check whether these deterministic wallets have already been imported
import_needed = False
wallets_imported = 0
- spks_to_import = []
+ addresses_to_import = []
TEST_ADDR_COUNT = 3
logger.info("Displaying first " + str(TEST_ADDR_COUNT) + " addresses of " +
"each master public key:")
for config_mpk_key, wal in zip(config.options("master-public-keys"),
deterministic_wallets):
- first_spks = wal.get_scriptpubkeys(change=0, from_index=0,
+ first_addrs, first_spk = wal.get_addresses(change=0, from_index=0,
count=TEST_ADDR_COUNT)
- first_addrs = [hashes.script_to_address(s, rpc) for s in first_spks]
logger.info("\n" + config_mpk_key + " =>\n\t" + "\n\t".join(
first_addrs))
- last_spk = wal.get_scriptpubkeys(0, int(config.get("bitcoin-rpc",
- "initial_import_count")) - 1, 1)
- last_addr = [hashes.script_to_address(last_spk[0], rpc)]
+ last_addr, last_spk = wal.get_addresses(change=0, from_index=int(
+ config.get("bitcoin-rpc", "initial_import_count")) - 1, count=1)
if not set(first_addrs + last_addr).issubset(imported_addresses):
import_needed = True
wallets_imported += 1
for change in [0, 1]:
- spks_to_import.extend(wal.get_scriptpubkeys(change, 0,
- int(config.get("bitcoin-rpc", "initial_import_count"))))
+ addrs, spks = wal.get_addresses(change, 0,
+ int(config.get("bitcoin-rpc", "initial_import_count")))
+ addresses_to_import.extend(addrs)
logger.info("Obtaining bitcoin addresses to monitor . . .")
#check whether watch-only addresses have been imported
watch_only_addresses = []
@@ -223,8 +228,6 @@ def get_scriptpubkeys_to_monitor(rpc, config):
#if addresses need to be imported then return them
if import_needed:
- addresses_to_import = [hashes.script_to_address(spk, rpc)
- for spk in spks_to_import]
#TODO minus imported_addresses
logger.info("Importing " + str(wallets_imported) + " wallets and " +
str(len(watch_only_addresses_to_import)) + " watch-only " +
@@ -242,15 +245,15 @@ def get_scriptpubkeys_to_monitor(rpc, config):
spks_to_monitor = []
for wal in deterministic_wallets:
for change in [0, 1]:
- spks_to_monitor.extend(wal.get_scriptpubkeys(change, 0,
- int(config.get("bitcoin-rpc", "initial_import_count"))))
+ addrs, spks = wal.get_addresses(change, 0,
+ int(config.get("bitcoin-rpc", "initial_import_count")))
+ spks_to_monitor.extend(spks)
#loop until one address found that isnt imported
while True:
- spk = wal.get_new_scriptpubkeys(change, count=1)[0]
- spks_to_monitor.append(spk)
- if hashes.script_to_address(spk, rpc) not in imported_addresses:
+ addrs, spks = wal.get_new_addresses(change, count=1)
+ if addrs[0] not in imported_addresses:
break
- spks_to_monitor.pop()
+ spks_to_monitor.append(spks[0])
wal.rewind_one(change)
spks_to_monitor.extend([hashes.address_to_script(addr, rpc)
@@ -389,6 +392,22 @@ def main():
logger.error("Wallet related RPC call failed, possibly the " +
"bitcoin node was compiled with the disable wallet flag")
return
+
+ test_keydata = (
+ "2 tpubD6NzVbkrYhZ4YVMVzC7wZeRfz3bhqcHvV8M3UiULCfzFtLtp5nwvi6LnBQegrkx" +
+ "YGPkSzXUEvcPEHcKdda8W1YShVBkhFBGkLxjSQ1Nx3cJ tpubD6NzVbkrYhZ4WjgNYq2nF" +
+ "TbiSLW2SZAzs4g5JHLqwQ3AmR3tCWpqsZJJEoZuP5HAEBNxgYQhtWMezszoaeTCg6FWGQB" +
+ "T74sszGaxaf64o5s")
+ chain = rpc.call("getblockchaininfo", [])["chain"]
+ try:
+ gaplimit = 5
+ deterministicwallet.parse_electrum_master_public_key(test_keydata,
+ gaplimit, rpc, chain)
+ except ValueError as e:
+ logger.error(repr(e))
+ logger.error("Descriptor related RPC call failed. Bitcoin Core 0.20.0"
+ + " or higher required. Exiting..")
+ return
if opts.rescan:
rescan_script(logger, rpc, opts.rescan_date)
return
diff --git a/electrumpersonalserver/server/deterministicwallet.py b/electrumpersonalserver/server/deterministicwallet.py
@@ -1,21 +1,8 @@
import electrumpersonalserver.bitcoin as btc
-from electrumpersonalserver.server.hashes import bh2u, hash_160, bfh, sha256
-
-# the class hierarchy for deterministic wallets in this file:
-# subclasses are written towards the right
-# each class knows how to create the scriptPubKeys of that wallet
-#
-# |-- SingleSigOldMnemonicWallet
-# |-- SingleSigP2PKHWallet
-# |-- SingleSigP2WPKHWallet
-# SingleSigWallet --|
-# / |-- SingleSigP2WPKH_P2SHWallet
-# DeterministicWallet
-# \ |-- MultisigP2SHWallet
-# MultisigWallet --|
-# |-- MultisigP2WSHWallet
-# |-- MultisigP2WSH_P2SHWallet
+from electrumpersonalserver.server.hashes import bh2u, hash_160, bfh, sha256,\
+ address_to_script, script_to_address
+from electrumpersonalserver.server.jsonrpc import JsonRpcError
#the wallet types are here
#https://github.com/spesmilo/electrum/blob/3.0.6/RELEASE-NOTES
@@ -29,13 +16,28 @@ def is_string_parsable_as_hex_int(s):
except:
return False
-def parse_electrum_master_public_key(keydata, gaplimit):
+def parse_electrum_master_public_key(keydata, gaplimit, rpc, chain):
+ if chain == "main":
+ xpub_vbytes = b"\x04\x88\xb2\x1e"
+ elif chain == "test" or chain == "regtest":
+ xpub_vbytes = b"\x04\x35\x87\xcf"
+ else:
+ assert False
+
+ #https://github.com/bitcoin/bitcoin/blob/master/doc/descriptors.md
+
+ descriptor_template = None
if keydata[:4] in ("xpub", "tpub"):
- wallet = SingleSigP2PKHWallet(keydata)
+ descriptor_template = "pkh({xpub}/{change}/*)"
elif keydata[:4] in ("zpub", "vpub"):
- wallet = SingleSigP2WPKHWallet(keydata)
+ descriptor_template = "wpkh({xpub}/{change}/*)"
elif keydata[:4] in ("ypub", "upub"):
- wallet = SingleSigP2WPKH_P2SHWallet(keydata)
+ descriptor_template = "sh(wpkh({xpub}/{change}/*))"
+
+ if descriptor_template != None:
+ wallet = SingleSigWallet(rpc, xpub_vbytes, keydata, descriptor_template)
+ elif is_string_parsable_as_hex_int(keydata) and len(keydata) == 128:
+ wallet = SingleSigOldMnemonicWallet(rpc, keydata)
elif keydata.find(" ") != -1: #multiple keys = multisig
chunks = keydata.split(" ")
try:
@@ -47,32 +49,40 @@ def parse_electrum_master_public_key(keydata, gaplimit):
if not all([pubkeys[0][:4] == pub[:4] for pub in pubkeys[1:]]):
raise ValueError("Inconsistent master public key types")
if pubkeys[0][:4] in ("xpub", "tpub"):
- wallet = MultisigP2SHWallet(m, pubkeys)
+ descriptor_script = "sh(sortedmulti("
elif pubkeys[0][:4] in ("Zpub", "Vpub"):
- wallet = MultisigP2WSHWallet(m, pubkeys)
+ descriptor_script = "wsh(sortedmulti("
elif pubkeys[0][:4] in ("Ypub", "Upub"):
- wallet = MultisigP2WSH_P2SHWallet(m, pubkeys)
- elif is_string_parsable_as_hex_int(keydata) and len(keydata) == 128:
- wallet = SingleSigOldMnemonicWallet(keydata)
+ descriptor_script = "sh(wsh(sortedmulti("
+ wallet = MultisigWallet(rpc, xpub_vbytes, m, pubkeys, descriptor_script)
else:
raise ValueError("Unrecognized electrum mpk format: " + keydata[:4])
wallet.gaplimit = gaplimit
return wallet
class DeterministicWallet(object):
- def __init__(self):
+ def __init__(self, rpc):
self.gaplimit = 0
self.next_index = [0, 0]
self.scriptpubkey_index = {}
+ self.rpc = rpc
- def get_new_scriptpubkeys(self, change, count):
- """Returns newly-generated addresses from this deterministic wallet"""
- return self.get_scriptpubkeys(change, self.next_index[change],
- count)
+ def _derive_addresses(self, change, from_index, count):
+ raise RuntimeError()
- def get_scriptpubkeys(self, change, from_index, count):
+ def get_addresses(self, change, from_index, count):
"""Returns addresses from this deterministic wallet"""
- pass
+ addrs = self._derive_addresses(change, from_index, count)
+ spks = [address_to_script(a, self.rpc) for a in addrs]
+ for index, spk in enumerate(spks):
+ self.scriptpubkey_index[spk] = (change, from_index + index)
+ self.next_index[change] = max(self.next_index[change], from_index+count)
+ return addrs, spks
+
+ def get_new_addresses(self, change, count):
+ """Returns newly-generated addresses from this deterministic wallet"""
+ addrs, spks = self.get_addresses(change, self.next_index[change], count)
+ return addrs, spks
#called in check_for_new_txes() when a new tx of ours arrives
#to see if we need to import more addresses
@@ -102,120 +112,84 @@ class DeterministicWallet(object):
"""Go back one pubkey in a branch"""
self.next_index[change] -= 1
-class SingleSigWallet(DeterministicWallet):
- def __init__(self, mpk):
- super(SingleSigWallet, self).__init__()
- try:
- self.branches = (btc.bip32_ckd(mpk, 0), btc.bip32_ckd(mpk, 1))
- except Exception:
- raise ValueError("Bad master public key format. Get it from " +
- "Electrum menu `Wallet` -> `Information`")
- #m/change/i
-
- def pubkey_to_scriptpubkey(self, pubkey):
- raise RuntimeError()
-
- def get_pubkey(self, change, index):
- return btc.bip32_extract_key(btc.bip32_ckd(self.branches[change],
- index))
+class DescriptorDeterministicWallet(DeterministicWallet):
+ def __init__(self, rpc, xpub_vbytes, *args):
+ super(DescriptorDeterministicWallet, self).__init__(rpc)
+ self.xpub_vbytes = xpub_vbytes
- def get_scriptpubkeys(self, change, from_index, count):
- result = []
- for index in range(from_index, from_index + count):
- pubkey = self.get_pubkey(change, index)
- scriptpubkey = self.pubkey_to_scriptpubkey(pubkey)
- self.scriptpubkey_index[scriptpubkey] = (change, index)
- result.append(scriptpubkey)
- self.next_index[change] = max(self.next_index[change], from_index+count)
- return result
+ descriptors_without_checksum = \
+ self.obtain_descriptors_without_checksum(args)
-class SingleSigP2PKHWallet(SingleSigWallet):
- def pubkey_to_scriptpubkey(self, pubkey):
- pkh = bh2u(hash_160(bfh(pubkey)))
- #op_dup op_hash_160 length hash160 op_equalverify op_checksig
- return "76a914" + pkh + "88ac"
+ try:
+ self.descriptors = []
+ for desc in descriptors_without_checksum:
+ self.descriptors.append(self.rpc.call("getdescriptorinfo",
+ [desc])["descriptor"])
+ except JsonRpcError as e:
+ raise ValueError(repr(e))
+
+ def obtain_descriptors_without_checksum(self, *args):
+ raise RuntimeError()
-class SingleSigP2WPKHWallet(SingleSigWallet):
- def pubkey_to_scriptpubkey(self, pubkey):
- pkh = bh2u(hash_160(bfh(pubkey)))
- #witness-version length hash160
- #witness version is always 0, length is always 0x14
- return "0014" + pkh
-
-class SingleSigP2WPKH_P2SHWallet(SingleSigWallet):
- def pubkey_to_scriptpubkey(self, pubkey):
- #witness-version length pubkeyhash
- #witness version is always 0, length is always 0x14
- redeem_script = '0014' + bh2u(hash_160(bfh(pubkey)))
- sh = bh2u(hash_160(bfh(redeem_script)))
- return "a914" + sh + "87"
-
-class SingleSigOldMnemonicWallet(SingleSigWallet):
- def __init__(self, mpk):
- super(SingleSigWallet, self).__init__()
+ def _derive_addresses(self, change, from_index, count):
+ return self.rpc.call("deriveaddresses", [self.descriptors[change], [
+ from_index, from_index + count - 1]])
+ ##the minus 1 is because deriveaddresses uses inclusive range
+ ##e.g. to get just the first address you use [0, 0]
+
+ def _convert_to_standard_xpub(self, mpk):
+ return btc.bip32_serialize((self.xpub_vbytes, *btc.bip32_deserialize(
+ mpk)[1:]))
+
+class SingleSigWallet(DescriptorDeterministicWallet):
+ def __init__(self, rpc, xpub_vbytes, xpub, descriptor_template):
+ super(SingleSigWallet, self).__init__(rpc, xpub_vbytes, xpub,
+ descriptor_template)
+
+ def obtain_descriptors_without_checksum(self, args):
+ ##example descriptor_template:
+ #"pkh({xpub}/{change}/*)"
+ xpub, descriptor_template = args
+
+ descriptors_without_checksum = []
+ xpub = self._convert_to_standard_xpub(xpub)
+ for change in [0, 1]:
+ descriptors_without_checksum.append(descriptor_template.format(
+ change=change, xpub=xpub))
+ return descriptors_without_checksum
+
+class MultisigWallet(DescriptorDeterministicWallet):
+ def __init__(self, rpc, xpub_vbytes, m, xpub_list, descriptor_script):
+ super(MultisigWallet, self).__init__(rpc, xpub_vbytes, m, xpub_list,
+ descriptor_script)
+
+ def obtain_descriptors_without_checksum(self, args):
+ ##example descriptor_script:
+ #"sh(sortedmulti("
+ m, xpub_list, descriptor_script = args
+
+ descriptors_without_checksum = []
+ xpub_list = [self._convert_to_standard_xpub(xpub) for xpub in xpub_list]
+ for change in [0, 1]:
+ descriptors_without_checksum.append(descriptor_script + str(m) +\
+ "," + ",".join([xpub + "/" + str(change) + "/*"
+ for xpub in xpub_list]) + ")"*descriptor_script.count("("))
+ return descriptors_without_checksum
+
+class SingleSigOldMnemonicWallet(DeterministicWallet):
+ def __init__(self, rpc, mpk):
+ super(SingleSigOldMnemonicWallet, self).__init__(rpc)
self.mpk = mpk
- def get_pubkey(self, change, index):
- return btc.electrum_pubkey(self.mpk, index, change)
-
- def pubkey_to_scriptpubkey(self, pubkey):
+ def _pubkey_to_scriptpubkey(self, pubkey):
pkh = bh2u(hash_160(bfh(pubkey)))
#op_dup op_hash_160 length hash160 op_equalverify op_checksig
return "76a914" + pkh + "88ac"
-class MultisigWallet(DeterministicWallet):
- def __init__(self, m, mpk_list):
- super(MultisigWallet, self).__init__()
- self.m = m
- try:
- self.pubkey_branches = [(btc.bip32_ckd(mpk, 0), btc.bip32_ckd(mpk,
- 1)) for mpk in mpk_list]
- except Exception:
- raise ValueError("Bad master public key format. Get it from " +
- "Electrum menu `Wallet` -> `Information`")
- #derivation path for pubkeys is m/change/index
-
- def redeem_script_to_scriptpubkey(self, redeem_script):
- raise RuntimeError()
-
- def get_scriptpubkeys(self, change, from_index, count):
+ def _derive_addresses(self, change, from_index, count):
result = []
for index in range(from_index, from_index + count):
- pubkeys = [btc.bip32_extract_key(btc.bip32_ckd(branch[change],
- index)) for branch in self.pubkey_branches]
- pubkeys = sorted(pubkeys)
- redeemScript = ""
- redeemScript += "%x"%(0x50 + self.m) #op_m
- for p in pubkeys:
- redeemScript += "21" #length
- redeemScript += p
- redeemScript += "%x"%(0x50 + len(pubkeys)) #op_n
- redeemScript += "ae" # op_checkmultisig
- scriptpubkey = self.redeem_script_to_scriptpubkey(redeemScript)
- self.scriptpubkey_index[scriptpubkey] = (change, index)
- result.append(scriptpubkey)
- self.next_index[change] = max(self.next_index[change], from_index+count)
+ pubkey = btc.electrum_pubkey(self.mpk, index, change)
+ scriptpubkey = self._pubkey_to_scriptpubkey(pubkey)
+ result.append(script_to_address(scriptpubkey, self.rpc))
return result
-
-class MultisigP2SHWallet(MultisigWallet):
- def redeem_script_to_scriptpubkey(self, redeem_script):
- sh = bh2u(hash_160(bfh(redeem_script)))
- #op_hash160 length hash160 op_equal
- return "a914" + sh + "87"
-
-class MultisigP2WSHWallet(MultisigWallet):
- def redeem_script_to_scriptpubkey(self, redeem_script):
- sh = bh2u(sha256(bfh(redeem_script)))
- #witness-version length sha256
- #witness version is always 0, length is always 0x20
- return "0020" + sh
-
-class MultisigP2WSH_P2SHWallet(MultisigWallet):
- def redeem_script_to_scriptpubkey(self, redeem_script):
- #witness-version length sha256
- #witness version is always 0, length is always 0x20
- nested_redeemScript = "0020" + bh2u(sha256(bfh(redeem_script)))
- sh = bh2u(hash_160(bfh(nested_redeemScript)))
- #op_hash160 length hash160 op_equal
- return "a914" + sh + "87"
-
diff --git a/electrumpersonalserver/server/transactionmonitor.py b/electrumpersonalserver/server/transactionmonitor.py
@@ -500,12 +500,11 @@ class TransactionMonitor(object):
output_scriptpubkeys)
if overrun_depths != None:
for change, import_count in overrun_depths.items():
- spks = wal.get_new_scriptpubkeys(change, import_count)
+ new_addrs, spks = wal.get_new_addresses(change,
+ import_count)
for spk in spks:
self.address_history[script_to_scripthash(
spk)] = {'history': [], 'subscribed': False}
- new_addrs = [script_to_address(s, self.rpc)
- for s in spks]
logger.debug("importing " + str(len(spks)) +
" into change=" + str(change))
import_addresses(self.rpc, new_addrs, logger)
