electrum-personal-server

Maximally lightweight electrum server for a single user
git clone https://git.parazyd.org/electrum-personal-server
Log | Files | Refs | README

commit dcada7385f5e47a593fdece3ef045d03b16b971c
parent 96b81af376581e3ce32317c8feb8d9e41357a185
Author: chris-belcher <chris-belcher@users.noreply.github.com>
Date:   Tue, 20 Mar 2018 15:52:30 +0000

restored jsonrpc as persistent version wasnt working, renamed util.py to hashes.py, added to README

Diffstat:
MREADME.md | 43+++++++++++++++++++++++++++++++++++--------
Mconfig.cfg_sample | 2+-
Mdeterministicwallet.py | 21++++++++++-----------
Mjsonrpc.py | 125++++++++++++++++++-------------------------------------------------------------
Mmerkleproof.py | 5++---
Mrescan-script.py | 4++--
Mserver.py | 29+++++++++++++++--------------
7 files changed, 93 insertions(+), 136 deletions(-)

diff --git a/README.md b/README.md @@ -1,7 +1,7 @@ # Electrum Personal Server Electrum Personal Server is an implementation of the Electrum server protocol -which fulfills the specific need of using the Electrum UI with full node +which fulfills the specific need of using the Electrum wallet with full node verification and privacy, but without the heavyweight server backend, for a single user. It allows the user to benefit from all of Bitcoin Core's resource-saving features like @@ -13,13 +13,27 @@ txindex. All of Electrum's feature-richness like hardware wallet integration, [mnemonic recovery phrases](https://en.bitcoin.it/wiki/Mnemonic_phrase) and so on can still be used, but backed by the user's own full node. +Full node wallets are important in bitcoin because they are an big part of what +makes the system be trustless. No longer do people have to trust a financial +institution like a bank or paypal, they can run software on their own +computers. If bitcoin is digital gold, then a full node wallet is your own +personal goldsmith who checks for you that received payments are genuine. You +wouldn't accept large amounts of cash or gold coins without checking they are +actually genuine, the same applies for bitcoin. + +Full node wallets are also important for privacy. Using Electrum under default +configuration requires it to send all your bitcoin addresses to some server. +That server can then easily spy on you. Full nodes download the entire +blockchain and scan it for the user's own addresses, and therefore don't reveal +to anyone else which bitcoin addresses they are interested in. + Using Electrum with Electrum Personal Server is probably the most resource-efficient way right now to use a hardware wallet connected to your -own full node. +own full node. -For a longer explaination of this project and why it's important, see the +For a longer explaination of this project, see the [mailing list email](https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2018-February/015707.html) -and [bitcointalk thread](https://bitcointalk.org/index.php?topic=2664747.msg27179198). +and [bitcointalk thread](https://bitcointalk.org/index.php?topic=2664747.msg27179198). See also the Bitcoin Wiki [pages](https://en.bitcoin.it/wiki/Clearing_Up_Misconceptions_About_Full_Nodes) on [full nodes](https://en.bitcoin.it/wiki/Full_node). See also the Electrum bitcoin wallet [website](https://electrum.org/). @@ -50,13 +64,26 @@ Electrum can be started in testnet mode with the command line flag `--testnet`. #### Exposure to the Internet -You really don't want other people connecting to your server. They won't be +Other people should not be connecting to your server. They won't be able to synchronize their wallet, and they could potentially learn all your wallet addresses. -By default the server will bind to and accept connections only from `localhost` -so you should either run Electrum wallet from the same computer or use a SSH -tunnel. +By default the server will accept connections only from `localhost` so you +should either run Electrum wallet from the same computer or use a SSH tunnel to +another computer. + +#### How is this different from other Electrum servers ? + +They are different approaches with different tradeoffs. Electrum Personal +Server is compatible with pruning, blocksonly and txindex=0, uses less CPU and +RAM and doesn't require an index of every bitcoin address ever used; the +tradeoff is when recovering an old wallet, you must to import your wallet into +it first and you may need to rescan, so it loses the "instant on" feature of +Electrum wallet. Other Electrum server implementations will be able to sync +your wallet immediately even if you have historical transactions, and they can +serve multiple Electrum wallets at once. + +Definitely check out implementations like [ElectrumX](https://github.com/kyuupichan/electrumx/) if you're interested in this sort of thing. ## Project Readiness diff --git a/config.cfg_sample b/config.cfg_sample @@ -32,7 +32,7 @@ poll_interval_connected = 5 # Parameters for dealing with deterministic wallets # how many addresses to import first time, should be big because if you import too little you may have to rescan again -initial_import_count = 1000 +initial_import_count = 100 # number of unused addresses kept at the head of the wallet gap_limit = 25 diff --git a/deterministicwallet.py b/deterministicwallet.py @@ -1,6 +1,6 @@ import bitcoin as btc -import util +from hashes import bh2u, hash_160, bfh, sha256 # the class hierarchy for deterministic wallets in this file: # subclasses are written towards the right @@ -127,13 +127,13 @@ class SingleSigWallet(DeterministicWallet): class SingleSigP2PKHWallet(SingleSigWallet): def pubkey_to_scriptpubkey(self, pubkey): - pkh = util.bh2u(util.hash_160(util.bfh(pubkey))) + pkh = bh2u(hash_160(bfh(pubkey))) #op_dup op_hash_160 length hash160 op_equalverify op_checksig return "76a914" + pkh + "88ac" class SingleSigP2WPKHWallet(SingleSigWallet): def pubkey_to_scriptpubkey(self, pubkey): - pkh = util.bh2u(util.hash_160(util.bfh(pubkey))) + pkh = bh2u(hash_160(bfh(pubkey))) #witness-version length hash160 #witness version is always 0, length is always 0x14 return "0014" + pkh @@ -142,8 +142,8 @@ class SingleSigP2WPKH_P2SHWallet(SingleSigWallet): def pubkey_to_scriptpubkey(self, pubkey): #witness-version length pubkeyhash #witness version is always 0, length is always 0x14 - redeem_script = '0014' + util.bh2u(util.hash_160(util.bfh(pubkey))) - sh = util.bh2u(util.hash_160(util.bfh(redeem_script))) + redeem_script = '0014' + bh2u(hash_160(bfh(pubkey))) + sh = bh2u(hash_160(bfh(redeem_script))) return "a914" + sh + "87" class SingleSigOldMnemonicWallet(SingleSigWallet): @@ -155,7 +155,7 @@ class SingleSigOldMnemonicWallet(SingleSigWallet): return btc.electrum_pubkey(self.mpk, index, change) def pubkey_to_scriptpubkey(self, pubkey): - pkh = util.bh2u(util.hash_160(util.bfh(pubkey))) + pkh = bh2u(hash_160(bfh(pubkey))) #op_dup op_hash_160 length hash160 op_equalverify op_checksig return "76a914" + pkh + "88ac" @@ -191,13 +191,13 @@ class MultisigWallet(DeterministicWallet): class MultisigP2SHWallet(MultisigWallet): def redeem_script_to_scriptpubkey(self, redeem_script): - sh = util.bh2u(util.hash_160(util.bfh(redeem_script))) + sh = bh2u(hash_160(bfh(redeem_script))) #op_hash160 length hash160 op_equal return "a914" + sh + "87" class MultisigP2WSHWallet(MultisigWallet): def redeem_script_to_scriptpubkey(self, redeem_script): - sh = util.bh2u(util.sha256(util.bfh(redeem_script))) + sh = bh2u(sha256(bfh(redeem_script))) #witness-version length sha256 #witness version is always 0, length is always 0x20 return "0020" + sh @@ -206,9 +206,8 @@ class MultisigP2WSH_P2SHWallet(MultisigWallet): def redeem_script_to_scriptpubkey(self, redeem_script): #witness-version length sha256 #witness version is always 0, length is always 0x20 - nested_redeemScript = "0020" + util.bh2u(util.sha256( - util.bfh(redeem_script))) - sh = util.bh2u(util.hash_160(util.bfh(nested_redeemScript))) + nested_redeemScript = "0020" + bh2u(sha256(bfh(redeem_script))) + sh = bh2u(hash_160(bfh(nested_redeemScript))) #op_hash160 length hash160 op_equal return "a914" + sh + "87" diff --git a/jsonrpc.py b/jsonrpc.py @@ -1,128 +1,59 @@ -# from https://github.com/JoinMarket-Org/joinmarket-clientserver/blob/py3/jmclient/jmclient/jsonrpc.py - -# Copyright (C) 2013,2015 by Daniel Kraft <d@domob.eu> -# Copyright (C) 2014 by phelix / blockchained.com -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. +#jsonrpc.py from https://github.com/JoinMarket-Org/joinmarket/blob/master/joinmarket/jsonrpc.py +#copyright # Copyright (C) 2013,2015 by Daniel Kraft <d@domob.eu> and phelix / blockchained.com import base64 -try: - import http.client as httplib -except ImportError: - import httplib +import http.client import json - class JsonRpcError(Exception): - """ - The called method returned an error in the JSON-RPC response. - """ def __init__(self, obj): self.code = obj["code"] self.message = obj["message"] - -class JsonRpcConnectionError(Exception): - """ - Error thrown when the RPC connection itself failed. This means - that the server is either down or the connection settings - are wrong. - """ - pass - +class JsonRpcConnectionError(JsonRpcError): pass class JsonRpc(object): - """ - Simple implementation of a JSON-RPC client that is used - to connect to Bitcoin. - """ def __init__(self, host, port, user, password): self.host = host self.port = port - self.conn = httplib.HTTPConnection(self.host, self.port) - self.authstr = bytes("%s:%s" % (user, password), "utf-8") + self.authstr = "%s:%s" % (user, password) self.queryId = 1 def queryHTTP(self, obj): - """ - Send an appropriate HTTP query to the server. The JSON-RPC - request should be (as object) in 'obj'. If the call succeeds, - the resulting JSON object is returned. In case of an error - with the connection (not JSON-RPC itself), an exception is raised. - """ - headers = {"User-Agent": "joinmarket", + headers = {"User-Agent": "electrum-personal-server", "Content-Type": "application/json", "Accept": "application/json"} headers["Authorization"] = "Basic %s" % base64.b64encode( - self.authstr).decode("utf-8") + self.authstr.encode()).decode() body = json.dumps(obj) - while True: - try: - self.conn.request("POST", "", body, headers) - response = self.conn.getresponse() - if response.status == 401: - self.conn.close() - raise JsonRpcConnectionError( - "authentication for JSON-RPC failed") - # All of below codes are 'fine' from a JSON-RPC point of view. - if response.status not in [200, 404, 500]: - self.conn.close() - raise JsonRpcConnectionError("unknown error in JSON-RPC") - data = response.read() - return json.loads(data.decode("utf-8")) - except JsonRpcConnectionError as exc: - raise exc - except httplib.BadStatusLine: - return "CONNFAILURE" - except Exception as exc: - if str(exc) == "Connection reset by peer": - self.conn.connect() - continue - else: - raise JsonRpcConnectionError("JSON-RPC connection failed" + - ". Err:" + repr(exc)) - break + try: + conn = http.client.HTTPConnection(self.host, self.port) + conn.request("POST", "", body, headers) + response = conn.getresponse() + if response.status == 401: + conn.close() + raise JsonRpcConnectionError( + "authentication for JSON-RPC failed") + # All of the codes below are 'fine' from a JSON-RPC point of view. + if response.status not in [200, 404, 500]: + conn.close() + raise JsonRpcConnectionError("unknown error in JSON-RPC") + data = response.read() + conn.close() + return json.loads(data.decode()) + except JsonRpcConnectionError as exc: + raise exc + except Exception as exc: + raise JsonRpcConnectionError("JSON-RPC connection failed. Err:" + + repr(exc)) def call(self, method, params): - """ - Call a method over JSON-RPC. - """ currentId = self.queryId self.queryId += 1 request = {"method": method, "params": params, "id": currentId} - #query can fail from keepalive timeout; keep retrying if it does, up - #to a reasonable limit, then raise (failure to access blockchain - #is a critical failure). Note that a real failure to connect (e.g. - #wrong port) is raised in queryHTTP directly. - response_received = False - for i in range(100): - response = self.queryHTTP(request) - if response != "CONNFAILURE": - response_received = True - break - #Failure means keepalive timed out, just make a new one - self.conn = httplib.HTTPConnection(self.host, self.port) - if not response_received: - raise JsonRpcConnectionError("Unable to connect over RPC") + response = self.queryHTTP(request) if response["id"] != currentId: raise JsonRpcConnectionError("invalid id returned by query") if response["error"] is not None: raise JsonRpcError(response["error"]) return response["result"] - diff --git a/merkleproof.py b/merkleproof.py @@ -3,8 +3,7 @@ import bitcoin as btc import binascii from math import ceil, log -import util -from util import hash_encode, hash_decode, Hash +from hashes import hash_encode, hash_decode, Hash, hash_merkle_root #lots of ideas and code taken from bitcoin core and breadwallet #https://github.com/bitcoin/bitcoin/blob/master/src/merkleblock.h @@ -248,7 +247,7 @@ def test(): try: electrum_proof = convert_core_to_electrum_merkle_proof(proof) #print(electrum_proof) - implied_merkle_root = util.hash_merkle_root( + implied_merkle_root = hash_merkle_root( electrum_proof["merkle"], electrum_proof["txid"], electrum_proof["pos"]) assert implied_merkle_root == electrum_proof["merkleroot"] diff --git a/rescan-script.py b/rescan-script.py @@ -35,9 +35,9 @@ def main(): try: config = ConfigParser() config.read(["config.cfg"]) - config.options("electrum-master-public-keys") + config.options("master-public-keys") except NoSectionError: - log("Non-existant configuration file `config.cfg`") + print("Non-existant configuration file `config.cfg`") return rpc = JsonRpc(host = config.get("bitcoin-rpc", "host"), port = int(config.get("bitcoin-rpc", "port")), diff --git a/server.py b/server.py @@ -8,7 +8,7 @@ from configparser import ConfigParser, NoSectionError from decimal import Decimal from jsonrpc import JsonRpc, JsonRpcError -import util, merkleproof, deterministicwallet +import hashes, merkleproof, deterministicwallet ADDRESSES_LABEL = "electrum-watchonly-addresses" @@ -76,7 +76,7 @@ def on_heartbeat_connected(sock, rpc, address_history, unconfirmed_txes, for scrhash in updated_scripthashes: if not address_history[scrhash]["subscribed"]: continue - history_hash = util.get_status_electrum( ((h["tx_hash"], h["height"]) + history_hash = hashes.get_status_electrum( ((h["tx_hash"], h["height"]) for h in address_history[scrhash]["history"]) ) update = {"method": "blockchain.scripthash.subscribe", "params": [scrhash, history_hash]} @@ -107,7 +107,7 @@ def handle_query(sock, line, rpc, address_history, deterministic_wallets): core_proof = rpc.call("gettxoutproof", [[txid], tx["blockhash"]]) electrum_proof = merkleproof.convert_core_to_electrum_merkle_proof( core_proof) - implied_merkle_root = util.hash_merkle_root( + implied_merkle_root = hashes.hash_merkle_root( electrum_proof["merkle"], txid, electrum_proof["pos"]) if implied_merkle_root != electrum_proof["merkleroot"]: raise ValueError @@ -125,12 +125,12 @@ def handle_query(sock, line, rpc, address_history, deterministic_wallets): scrhash = query["params"][0] if scrhash in address_history: address_history[scrhash]["subscribed"] = True - history_hash = util.get_status_electrum(( + history_hash = hashes.get_status_electrum(( (h["tx_hash"], h["height"]) for h in address_history[scrhash]["history"])) else: log("WARNING: address scripthash not known to us: " + scrhash) - history_hash = util.get_status_electrum([]) + history_hash = hashes.get_status_electrum([]) send_response(sock, query, history_hash) elif method == "blockchain.scripthash.get_history": scrhash = query["params"][0] @@ -459,7 +459,7 @@ def check_for_new_txes(rpc, address_history, unconfirmed_txes, get_input_and_output_scriptpubkeys(rpc, tx["txid"]) matching_scripthashes = [] for spk in (output_scriptpubkeys + input_scriptpubkeys): - scripthash = util.script_to_scripthash(spk) + scripthash = hashes.script_to_scripthash(spk) if scripthash in address_history: matching_scripthashes.append(scripthash) if len(matching_scripthashes) == 0: @@ -471,7 +471,7 @@ def check_for_new_txes(rpc, address_history, unconfirmed_txes, if overrun_depths != None: for change, import_count in overrun_depths.items(): spks = wal.get_new_scriptpubkeys(change, import_count) - new_addrs = [util.script_to_address(s, rpc) for s in spks] + new_addrs = [hashes.script_to_address(s, rpc) for s in spks] debug("Importing " + str(len(spks)) + " into change=" + str(change)) import_addresses(rpc, new_addrs) @@ -495,7 +495,7 @@ def build_address_history(rpc, monitored_scriptpubkeys, deterministic_wallets): st = time.time() address_history = {} for spk in monitored_scriptpubkeys: - address_history[util.script_to_scripthash(spk)] = {'history': [], + address_history[hashes.script_to_scripthash(spk)] = {'history': [], 'subscribed': False} wallet_addr_scripthashes = set(address_history.keys()) #populate history @@ -525,11 +525,11 @@ def build_address_history(rpc, monitored_scriptpubkeys, deterministic_wallets): #obtain all the addresses this transaction is involved with output_scriptpubkeys, input_scriptpubkeys, txd = \ get_input_and_output_scriptpubkeys(rpc, tx["txid"]) - output_scripthashes = [util.script_to_scripthash(sc) + output_scripthashes = [hashes.script_to_scripthash(sc) for sc in output_scriptpubkeys] sh_to_add = wallet_addr_scripthashes.intersection(set( output_scripthashes)) - input_scripthashes = [util.script_to_scripthash(sc) + input_scripthashes = [hashes.script_to_scripthash(sc) for sc in input_scriptpubkeys] sh_to_add |= wallet_addr_scripthashes.intersection(set( input_scripthashes)) @@ -593,7 +593,7 @@ def get_scriptpubkeys_to_monitor(rpc, config): wallets_imported = 0 spks_to_import = [] for wal in deterministic_wallets: - first_addr = util.script_to_address(wal.get_scriptpubkeys(change=0, + first_addr = hashes.script_to_address(wal.get_scriptpubkeys(change=0, from_index=0, count=1)[0], rpc) if first_addr not in imported_addresses: import_needed = True @@ -613,7 +613,7 @@ def get_scriptpubkeys_to_monitor(rpc, config): watch_only_addresses_to_import = wallet_addresses - imported_addresses if import_needed: - addresses_to_import = [util.script_to_address(spk, rpc) + addresses_to_import = [hashes.script_to_address(spk, rpc) for spk in spks_to_import] #TODO minus imported_addresses log("Importing " + str(wallets_imported) + " wallets and " + @@ -638,12 +638,12 @@ def get_scriptpubkeys_to_monitor(rpc, config): while True: spk = wal.get_new_scriptpubkeys(change, count=1)[0] spks_to_monitor.append(spk) - if util.script_to_address(spk, rpc) not in imported_addresses: + if hashes.script_to_address(spk, rpc) not in imported_addresses: break spks_to_monitor.pop() wal.rewind_one(change) - spks_to_monitor.extend([util.address_to_script(addr, rpc) + spks_to_monitor.extend([hashes.address_to_script(addr, rpc) for addr in watch_only_addresses]) return False, spks_to_monitor, deterministic_wallets @@ -683,6 +683,7 @@ def main(): printed_error_msg = True time.sleep(5) + log("Starting Electrum Personal Server") import_needed, relevant_spks_addrs, deterministic_wallets = \ get_scriptpubkeys_to_monitor(rpc, config) if import_needed: