common.py (23598B)
1 import socket 2 import time 3 from datetime import datetime 4 import ssl 5 import os 6 import os.path 7 import logging 8 import tempfile 9 import platform 10 import json 11 import traceback 12 from json.decoder import JSONDecodeError 13 from configparser import RawConfigParser, NoSectionError, NoOptionError 14 from ipaddress import ip_network, ip_address 15 16 from electrumpersonalserver.server.jsonrpc import JsonRpc, JsonRpcError 17 import electrumpersonalserver.server.hashes as hashes 18 import electrumpersonalserver.server.deterministicwallet as deterministicwallet 19 import electrumpersonalserver.server.transactionmonitor as transactionmonitor 20 from electrumpersonalserver.server.electrumprotocol import ( 21 SERVER_VERSION_NUMBER, 22 UnknownScripthashError, 23 ElectrumProtocol, 24 get_block_header, 25 get_current_header, 26 get_block_headers_hex, 27 DONATION_ADDR, 28 ) 29 from electrumpersonalserver.server.mempoolhistogram import ( 30 MempoolSync, 31 PollIntervalChange 32 ) 33 34 ##python has demented rules for variable scope, so these 35 ## global variables are actually mutable lists 36 bestblockhash = [None] 37 38 last_heartbeat_listening = [datetime.now()] 39 last_heartbeat_connected = [datetime.now()] 40 41 def on_heartbeat_listening(poll_interval_listening, txmonitor): 42 if ((datetime.now() - last_heartbeat_listening[0]).total_seconds() 43 < poll_interval_listening): 44 return True 45 last_heartbeat_listening[0] = datetime.now() 46 logger = logging.getLogger('ELECTRUMPERSONALSERVER') 47 try: 48 txmonitor.check_for_updated_txes() 49 is_node_reachable = True 50 except JsonRpcError: 51 is_node_reachable = False 52 return is_node_reachable 53 54 def on_heartbeat_connected(poll_interval_connected, rpc, txmonitor, protocol): 55 if ((datetime.now() - last_heartbeat_connected[0]).total_seconds() 56 < poll_interval_connected): 57 return 58 last_heartbeat_connected[0] = datetime.now() 59 logger = logging.getLogger('ELECTRUMPERSONALSERVER') 60 is_tip_updated, header = check_for_new_blockchain_tip(rpc, 61 protocol.are_headers_raw) 62 if is_tip_updated: 63 logger.debug("Blockchain tip updated " + (str(header["height"]) if 64 "height" in header else "")) 65 protocol.on_blockchain_tip_updated(header) 66 updated_scripthashes = txmonitor.check_for_updated_txes() 67 protocol.on_updated_scripthashes(updated_scripthashes) 68 69 def check_for_new_blockchain_tip(rpc, raw): 70 new_bestblockhash, header = get_current_header(rpc, raw) 71 is_tip_new = bestblockhash[0] != new_bestblockhash 72 bestblockhash[0] = new_bestblockhash 73 return is_tip_new, header 74 75 def create_server_socket(hostport): 76 logger = logging.getLogger('ELECTRUMPERSONALSERVER') 77 server_sock = socket.socket() 78 server_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 79 server_sock.bind(hostport) 80 server_sock.listen(1) 81 logger.info("Listening for Electrum Wallet on " + str(hostport) + "\n\n" 82 + "If this project is valuable to you please consider donating:\n\t" 83 + DONATION_ADDR) 84 return server_sock 85 86 def run_electrum_server(rpc, txmonitor, config): 87 logger = logging.getLogger('ELECTRUMPERSONALSERVER') 88 logger.debug("Starting electrum server") 89 90 hostport = (config.get("electrum-server", "host"), 91 int(config.get("electrum-server", "port"))) 92 ip_whitelist = [] 93 for ip in config.get("electrum-server", "ip_whitelist").split(" "): 94 if ip == "*": 95 #matches everything 96 ip_whitelist.append(ip_network("0.0.0.0/0")) 97 ip_whitelist.append(ip_network("::0/0")) 98 else: 99 ip_whitelist.append(ip_network(ip, strict=False)) 100 poll_interval_listening = int(config.get("bitcoin-rpc", 101 "poll_interval_listening")) 102 poll_interval_connected = int(config.get("bitcoin-rpc", 103 "poll_interval_connected")) 104 certfile, keyfile = get_certs(config) 105 logger.debug('using cert: {}, key: {}'.format(certfile, keyfile)) 106 disable_mempool_fee_histogram = config.getboolean("electrum-server", 107 "disable_mempool_fee_histogram", fallback=False) 108 mempool_update_interval = int(config.get("bitcoin-rpc", 109 "mempool_update_interval", fallback=60)) 110 broadcast_method = config.get("electrum-server", "broadcast_method", 111 fallback="own-node") 112 tor_host = config.get("electrum-server", "tor_host", fallback="localhost") 113 tor_port = int(config.get("electrum-server", "tor_port", fallback="9050")) 114 tor_hostport = (tor_host, tor_port) 115 116 mempool_sync = MempoolSync(rpc, 117 disable_mempool_fee_histogram, mempool_update_interval) 118 mempool_sync.initial_sync(logger) 119 120 protocol = ElectrumProtocol(rpc, txmonitor, logger, broadcast_method, 121 tor_hostport, mempool_sync) 122 123 normal_listening_timeout = min(poll_interval_listening, 124 mempool_update_interval) 125 fast_listening_timeout = 0.5 126 server_sock = create_server_socket(hostport) 127 server_sock.settimeout(normal_listening_timeout) 128 accepting_clients = True 129 while True: 130 # main server loop, runs forever 131 sock = None 132 while sock == None: 133 # loop waiting for a successful connection from client 134 try: 135 sock, addr = server_sock.accept() 136 if not accepting_clients: 137 logger.debug("Refusing connection from client because" 138 + " Bitcoin node isnt reachable") 139 raise ConnectionRefusedError() 140 if not any([ip_address(addr[0]) in ipnet 141 for ipnet in ip_whitelist]): 142 logger.debug(addr[0] + " not in whitelist, closing") 143 raise ConnectionRefusedError() 144 sock = ssl.wrap_socket(sock, server_side=True, 145 certfile=certfile, keyfile=keyfile, 146 ssl_version=ssl.PROTOCOL_SSLv23) 147 except socket.timeout: 148 poll_interval_change = mempool_sync.poll_update(1) 149 if poll_interval_change == PollIntervalChange.FAST_POLLING: 150 server_sock.settimeout(fast_listening_timeout) 151 elif poll_interval_change == PollIntervalChange.NORMAL_POLLING: 152 server_sock.settimeout(normal_listening_timeout) 153 154 is_node_reachable = on_heartbeat_listening( 155 poll_interval_listening, txmonitor) 156 accepting_clients = is_node_reachable 157 except (ConnectionRefusedError, ssl.SSLError, IOError): 158 sock.close() 159 sock = None 160 logger.debug('Electrum connected from ' + str(addr[0])) 161 162 def send_reply_fun(reply): 163 line = json.dumps(reply) 164 sock.sendall(line.encode('utf-8') + b'\n') 165 logger.debug('<= ' + line) 166 protocol.set_send_reply_fun(send_reply_fun) 167 168 try: 169 normal_connected_timeout = min(poll_interval_connected, 170 mempool_update_interval) 171 fast_connected_timeout = 0.5 172 sock.settimeout(normal_connected_timeout) 173 recv_buffer = bytearray() 174 while True: 175 # loop for replying to client queries 176 try: 177 recv_data = sock.recv(4096) 178 if not recv_data or len(recv_data) == 0: 179 raise EOFError() 180 recv_buffer.extend(recv_data) 181 lb = recv_buffer.find(b'\n') 182 if lb == -1: 183 continue 184 while lb != -1: 185 line = recv_buffer[:lb].rstrip() 186 recv_buffer = recv_buffer[lb + 1:] 187 lb = recv_buffer.find(b'\n') 188 try: 189 line = line.decode("utf-8") 190 query = json.loads(line) 191 except (UnicodeDecodeError, JSONDecodeError) as e: 192 raise IOError(repr(e)) 193 logger.debug("=> " + line) 194 protocol.handle_query(query) 195 except socket.timeout: 196 poll_interval_change = mempool_sync.poll_update(1) 197 if poll_interval_change == PollIntervalChange.FAST_POLLING: 198 sock.settimeout(fast_connected_timeout) 199 elif (poll_interval_change 200 == PollIntervalChange.NORMAL_POLLING): 201 sock.settimeout(normal_connected_timeout) 202 203 on_heartbeat_connected(poll_interval_connected, rpc, 204 txmonitor, protocol) 205 except JsonRpcError as e: 206 logger.debug("Error with node connection, e = " + repr(e) 207 + "\ntraceback = " + str(traceback.format_exc())) 208 accepting_clients = False 209 except UnknownScripthashError as e: 210 logger.debug("Disconnecting client due to misconfiguration. User" 211 + " must correctly configure master public key(s)") 212 except (IOError, EOFError) as e: 213 if isinstance(e, (EOFError, ConnectionRefusedError)): 214 logger.debug("Electrum wallet disconnected") 215 else: 216 logger.debug("IOError: " + repr(e)) 217 try: 218 if sock != None: 219 sock.close() 220 except IOError: 221 pass 222 protocol.on_disconnect() 223 time.sleep(0.2) 224 225 def is_address_imported(rpc, address): 226 return rpc.call("getaddressinfo", [address])["iswatchonly"] 227 228 def get_scriptpubkeys_to_monitor(rpc, config): 229 logger = logging.getLogger('ELECTRUMPERSONALSERVER') 230 st = time.time() 231 232 deterministic_wallets = [] 233 for key in config.options("master-public-keys"): 234 mpk = config.get("master-public-keys", key) 235 gaplimit = int(config.get("bitcoin-rpc", "gap_limit")) 236 chain = rpc.call("getblockchaininfo", [])["chain"] 237 try: 238 wal = deterministicwallet.parse_electrum_master_public_key(mpk, 239 gaplimit, rpc, chain) 240 except ValueError: 241 raise ValueError("Bad master public key format. Get it from " + 242 "Electrum menu `Wallet` -> `Information`") 243 deterministic_wallets.append(wal) 244 #check whether these deterministic wallets have already been imported 245 import_needed = False 246 wallets_to_import = [] 247 TEST_ADDR_COUNT = 3 248 logger.info("Displaying first " + str(TEST_ADDR_COUNT) + " addresses of " + 249 "each master public key:") 250 for config_mpk_key, wal in zip(config.options("master-public-keys"), 251 deterministic_wallets): 252 first_addrs, first_spk = wal.get_addresses(change=0, from_index=0, 253 count=TEST_ADDR_COUNT) 254 logger.info("\n" + config_mpk_key + " =>\n\t" + "\n\t".join( 255 first_addrs)) 256 last_addr, last_spk = wal.get_addresses(change=0, from_index=int( 257 config.get("bitcoin-rpc", "initial_import_count")) - 1, count=1) 258 if not all((is_address_imported(rpc, a) for a in (first_addrs 259 + last_addr))): 260 import_needed = True 261 wallets_to_import.append(wal) 262 logger.info("Obtaining bitcoin addresses to monitor . . .") 263 #check whether watch-only addresses have been imported 264 watch_only_addresses = [] 265 for key in config.options("watch-only-addresses"): 266 watch_only_addresses.extend(config.get("watch-only-addresses", 267 key).split(' ')) 268 watch_only_addresses_to_import = [a for a in watch_only_addresses 269 if not is_address_imported(rpc, a)] 270 if len(watch_only_addresses_to_import) > 0: 271 import_needed = True 272 273 if len(deterministic_wallets) == 0 and len(watch_only_addresses) == 0: 274 logger.error("No master public keys or watch-only addresses have " + 275 "been configured at all. Exiting..") 276 #import = true and none other params means exit 277 return (True, None, None) 278 279 #if addresses need to be imported then return them 280 if import_needed: 281 logger.info("Importing " + str(len(wallets_to_import)) 282 + " wallets and " + str(len(watch_only_addresses_to_import)) 283 + " watch-only addresses into the Bitcoin node") 284 time.sleep(5) 285 return True, watch_only_addresses_to_import, wallets_to_import 286 287 #test 288 # importing one det wallet and no addrs, two det wallets and no addrs 289 # no det wallets and some addrs, some det wallets and some addrs 290 291 #at this point we know we dont need to import any addresses 292 #find which index the deterministic wallets are up to 293 spks_to_monitor = [] 294 for wal in deterministic_wallets: 295 for change in [0, 1]: 296 addrs, spks = wal.get_addresses(change, 0, 297 int(config.get("bitcoin-rpc", "initial_import_count"))) 298 spks_to_monitor.extend(spks) 299 #loop until one address found that isnt imported 300 while True: 301 addrs, spks = wal.get_new_addresses(change, count=1) 302 if not is_address_imported(rpc, addrs[0]): 303 break 304 spks_to_monitor.append(spks[0]) 305 wal.rewind_one(change) 306 307 spks_to_monitor.extend([hashes.address_to_script(addr, rpc) 308 for addr in watch_only_addresses]) 309 et = time.time() 310 logger.info("Obtained list of addresses to monitor in " + str(et - st) 311 + "sec") 312 return False, spks_to_monitor, deterministic_wallets 313 314 def get_certs(config): 315 from pkg_resources import resource_filename 316 from electrumpersonalserver import __certfile__, __keyfile__ 317 318 logger = logging.getLogger('ELECTRUMPERSONALSERVER') 319 certfile = config.get('electrum-server', 'certfile', fallback=None) 320 keyfile = config.get('electrum-server', 'keyfile', fallback=None) 321 if (certfile and keyfile) and \ 322 (os.path.exists(certfile) and os.path.exists(keyfile)): 323 return certfile, keyfile 324 else: 325 certfile = resource_filename('electrumpersonalserver', __certfile__) 326 keyfile = resource_filename('electrumpersonalserver', __keyfile__) 327 if os.path.exists(certfile) and os.path.exists(keyfile): 328 return certfile, keyfile 329 else: 330 raise ValueError('invalid cert: {}, key: {}'.format( 331 certfile, keyfile)) 332 333 def obtain_cookie_file_path(datadir): 334 logger = logging.getLogger('ELECTRUMPERSONALSERVER') 335 if len(datadir.strip()) == 0: 336 logger.debug("no datadir configuration, checking in default location") 337 systemname = platform.system() 338 #paths from https://en.bitcoin.it/wiki/Data_directory 339 if systemname == "Linux": 340 datadir = os.path.expanduser("~/.bitcoin") 341 elif systemname == "Windows": 342 datadir = os.path.expandvars("%APPDATA%\Bitcoin") 343 elif systemname == "Darwin": #mac os 344 datadir = os.path.expanduser( 345 "~/Library/Application Support/Bitcoin/") 346 cookie_path = os.path.join(datadir, ".cookie") 347 if not os.path.exists(cookie_path): 348 logger.warning("Unable to find .cookie file, try setting `datadir`" + 349 " config") 350 return None 351 return cookie_path 352 353 def parse_args(): 354 from argparse import ArgumentParser 355 356 parser = ArgumentParser(description='Electrum Personal Server daemon') 357 parser.add_argument('config_file', 358 help='configuration file (mandatory)') 359 parser.add_argument("--rescan", action="store_true", help="Start the " + 360 " rescan script instead") 361 parser.add_argument("--rescan-date", action="store", dest="rescan_date", 362 default=None, help="Earliest wallet creation date (DD/MM/YYYY) or " 363 + "block height to rescan from") 364 parser.add_argument("-v", "--version", action="version", version= 365 "%(prog)s " + SERVER_VERSION_NUMBER) 366 return parser.parse_args() 367 368 #log for checking up/seeing your wallet, debug for when something has gone wrong 369 def logger_config(logger, config): 370 formatter = logging.Formatter(config.get("logging", "log_format", 371 fallback="%(levelname)s:%(asctime)s: %(message)s")) 372 logstream = logging.StreamHandler() 373 logstream.setFormatter(formatter) 374 logstream.setLevel(config.get("logging", "log_level_stdout", fallback= 375 "INFO")) 376 logger.addHandler(logstream) 377 filename = config.get("logging", "log_file_location", fallback="") 378 if len(filename.strip()) == 0: 379 filename= tempfile.gettempdir() + "/electrumpersonalserver.log" 380 logfile = logging.FileHandler(filename, mode=('a' if 381 config.get("logging", "append_log", fallback="false") else 'w')) 382 logfile.setFormatter(formatter) 383 logfile.setLevel(logging.DEBUG) 384 logger.addHandler(logfile) 385 logger.setLevel(logging.DEBUG) 386 return logger, filename 387 388 # returns non-zero status code on failure 389 def main(): 390 opts = parse_args() 391 392 try: 393 config = RawConfigParser() 394 config.read(opts.config_file) 395 config.options("master-public-keys") 396 except NoSectionError: 397 print("ERROR: Non-existant configuration file {}".format( 398 opts.config_file)) 399 return 1 400 logger = logging.getLogger('ELECTRUMPERSONALSERVER') 401 logger, logfilename = logger_config(logger, config) 402 logger.info('Starting Electrum Personal Server ' + str( 403 SERVER_VERSION_NUMBER)) 404 logger.info('Logging to ' + logfilename) 405 logger.debug("Process ID (PID) = " + str(os.getpid())) 406 rpc_u = None 407 rpc_p = None 408 cookie_path = None 409 try: 410 rpc_u = config.get("bitcoin-rpc", "rpc_user") 411 rpc_p = config.get("bitcoin-rpc", "rpc_password") 412 logger.debug("obtaining auth from rpc_user/pass") 413 except NoOptionError: 414 cookie_path = obtain_cookie_file_path(config.get( 415 "bitcoin-rpc", "datadir")) 416 logger.debug("obtaining auth from .cookie") 417 if rpc_u == None and cookie_path == None: 418 return 1 419 rpc = JsonRpc(host = config.get("bitcoin-rpc", "host"), 420 port = int(config.get("bitcoin-rpc", "port")), 421 user = rpc_u, password = rpc_p, cookie_path = cookie_path, 422 wallet_filename=config.get("bitcoin-rpc", "wallet_filename").strip(), 423 logger=logger) 424 425 #TODO somewhere here loop until rpc works and fully sync'd, to allow 426 # people to run this script without waiting for their node to fully 427 # catch up sync'd when getblockchaininfo blocks == headers, or use 428 # verificationprogress 429 printed_error_msg = False 430 while bestblockhash[0] == None: 431 try: 432 bestblockhash[0] = rpc.call("getbestblockhash", []) 433 except JsonRpcError as e: 434 if not printed_error_msg: 435 logger.error("Error with bitcoin json-rpc: " + repr(e)) 436 printed_error_msg = True 437 time.sleep(5) 438 try: 439 rpc.call("listunspent", []) 440 except JsonRpcError as e: 441 logger.error(repr(e)) 442 logger.error("Wallet related RPC call failed, possibly the " + 443 "bitcoin node was compiled with the disable wallet flag") 444 return 1 445 446 test_keydata = ( 447 "2 tpubD6NzVbkrYhZ4YVMVzC7wZeRfz3bhqcHvV8M3UiULCfzFtLtp5nwvi6LnBQegrkx" + 448 "YGPkSzXUEvcPEHcKdda8W1YShVBkhFBGkLxjSQ1Nx3cJ tpubD6NzVbkrYhZ4WjgNYq2nF" + 449 "TbiSLW2SZAzs4g5JHLqwQ3AmR3tCWpqsZJJEoZuP5HAEBNxgYQhtWMezszoaeTCg6FWGQB" + 450 "T74sszGaxaf64o5s") 451 chain = rpc.call("getblockchaininfo", [])["chain"] 452 try: 453 gaplimit = 5 454 deterministicwallet.parse_electrum_master_public_key(test_keydata, 455 gaplimit, rpc, chain) 456 except ValueError as e: 457 logger.error(repr(e)) 458 logger.error("Descriptor related RPC call failed. Bitcoin Core 0.20.0" 459 + " or higher required. Exiting..") 460 return 1 461 if opts.rescan: 462 rescan_script(logger, rpc, opts.rescan_date) 463 return 0 464 while True: 465 logger.debug("Checking whether rescan is in progress") 466 walletinfo = rpc.call("getwalletinfo", []) 467 if "scanning" in walletinfo and walletinfo["scanning"]: 468 logger.debug("Waiting for Core wallet rescan to finish") 469 time.sleep(300) 470 continue 471 break 472 import_needed, relevant_spks_addrs, deterministic_wallets = \ 473 get_scriptpubkeys_to_monitor(rpc, config) 474 if import_needed: 475 if not relevant_spks_addrs and not deterministic_wallets: 476 #import = true and no addresses means exit 477 return 0 478 deterministicwallet.import_addresses(rpc, relevant_spks_addrs, 479 deterministic_wallets, change_param=-1, 480 count=int(config.get("bitcoin-rpc", "initial_import_count"))) 481 logger.info("Done.\nIf recovering a wallet which already has existing" + 482 " transactions, then\nrun the rescan script. If you're confident" + 483 " that the wallets are new\nand empty then there's no need to" + 484 " rescan, just restart this script") 485 else: 486 txmonitor = transactionmonitor.TransactionMonitor(rpc, 487 deterministic_wallets, logger) 488 if not txmonitor.build_address_history(relevant_spks_addrs): 489 return 1 490 try: 491 run_electrum_server(rpc, txmonitor, config) 492 except KeyboardInterrupt: 493 logger.info('Received KeyboardInterrupt, quitting') 494 return 1 495 return 0 496 497 def search_for_block_height_of_date(datestr, rpc): 498 logger = logging.getLogger('ELECTRUMPERSONALSERVER') 499 target_time = datetime.strptime(datestr, "%d/%m/%Y") 500 bestblockhash = rpc.call("getbestblockhash", []) 501 best_head = rpc.call("getblockheader", [bestblockhash]) 502 if target_time > datetime.fromtimestamp(best_head["time"]): 503 logger.error("date in the future") 504 return -1 505 genesis_block = rpc.call("getblockheader", [rpc.call("getblockhash", [0])]) 506 if target_time < datetime.fromtimestamp(genesis_block["time"]): 507 logger.warning("date is before the creation of bitcoin") 508 return 0 509 first_height = 0 510 last_height = best_head["height"] 511 while True: 512 m = (first_height + last_height) // 2 513 m_header = rpc.call("getblockheader", [rpc.call("getblockhash", [m])]) 514 m_header_time = datetime.fromtimestamp(m_header["time"]) 515 m_time_diff = (m_header_time - target_time).total_seconds() 516 if abs(m_time_diff) < 60*60*2: #2 hours 517 return m_header["height"] 518 elif m_time_diff < 0: 519 first_height = m 520 elif m_time_diff > 0: 521 last_height = m 522 else: 523 return -1 524 525 def rescan_script(logger, rpc, rescan_date): 526 if rescan_date: 527 user_input = rescan_date 528 else: 529 user_input = input("Enter earliest wallet creation date (DD/MM/YYYY) " 530 "or block height to rescan from: ") 531 try: 532 height = int(user_input) 533 except ValueError: 534 height = search_for_block_height_of_date(user_input, rpc) 535 if height == -1: 536 return 537 height -= 2016 #go back two weeks for safety 538 539 if not rescan_date: 540 if input("Rescan from block height " + str(height) + " ? (y/n):") \ 541 != 'y': 542 return 543 logger.info("Rescanning. . . for progress indicator see the bitcoin node's" 544 + " debug.log file") 545 rpc.call("rescanblockchain", [height]) 546 logger.info("end") 547 548 if __name__ == "__main__": 549 #entry point for pyinstaller executable 550 try: 551 res = main() 552 except: 553 res = 1 554 555 # only relevant for pyinstaller executables (on Windows): 556 if os.name == 'nt': 557 os.system("pause") 558 559 sys.exit(res) 560