wallet.py (136312B)
1 # Electrum - lightweight Bitcoin client 2 # Copyright (C) 2015 Thomas Voegtlin 3 # 4 # Permission is hereby granted, free of charge, to any person 5 # obtaining a copy of this software and associated documentation files 6 # (the "Software"), to deal in the Software without restriction, 7 # including without limitation the rights to use, copy, modify, merge, 8 # publish, distribute, sublicense, and/or sell copies of the Software, 9 # and to permit persons to whom the Software is furnished to do so, 10 # subject to the following conditions: 11 # 12 # The above copyright notice and this permission notice shall be 13 # included in all copies or substantial portions of the Software. 14 # 15 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 17 # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 19 # BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 20 # ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 21 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 # SOFTWARE. 23 24 # Wallet classes: 25 # - Imported_Wallet: imported addresses or single keys, 0 or 1 keystore 26 # - Standard_Wallet: one HD keystore, P2PKH-like scripts 27 # - Multisig_Wallet: several HD keystores, M-of-N OP_CHECKMULTISIG scripts 28 29 import os 30 import sys 31 import random 32 import time 33 import json 34 import copy 35 import errno 36 import traceback 37 import operator 38 import math 39 from functools import partial 40 from collections import defaultdict 41 from numbers import Number 42 from decimal import Decimal 43 from typing import TYPE_CHECKING, List, Optional, Tuple, Union, NamedTuple, Sequence, Dict, Any, Set 44 from abc import ABC, abstractmethod 45 import itertools 46 import threading 47 import enum 48 49 from aiorpcx import TaskGroup, timeout_after, TaskTimeout, ignore_after 50 51 from .i18n import _ 52 from .bip32 import BIP32Node, convert_bip32_intpath_to_strpath, convert_bip32_path_to_list_of_uint32 53 from .crypto import sha256 54 from . import util 55 from .util import (NotEnoughFunds, UserCancelled, profiler, 56 format_satoshis, format_fee_satoshis, NoDynamicFeeEstimates, 57 WalletFileException, BitcoinException, MultipleSpendMaxTxOutputs, 58 InvalidPassword, format_time, timestamp_to_datetime, Satoshis, 59 Fiat, bfh, bh2u, TxMinedInfo, quantize_feerate, create_bip21_uri, OrderedDictWithIndex) 60 from .util import get_backup_dir 61 from .simple_config import SimpleConfig, FEE_RATIO_HIGH_WARNING, FEERATE_WARNING_HIGH_FEE 62 from .bitcoin import COIN, TYPE_ADDRESS 63 from .bitcoin import is_address, address_to_script, is_minikey, relayfee, dust_threshold 64 from .crypto import sha256d 65 from . import keystore 66 from .keystore import (load_keystore, Hardware_KeyStore, KeyStore, KeyStoreWithMPK, 67 AddressIndexGeneric, CannotDerivePubkey) 68 from .util import multisig_type 69 from .storage import StorageEncryptionVersion, WalletStorage 70 from .wallet_db import WalletDB 71 from . import transaction, bitcoin, coinchooser, paymentrequest, ecc, bip32 72 from .transaction import (Transaction, TxInput, UnknownTxinType, TxOutput, 73 PartialTransaction, PartialTxInput, PartialTxOutput, TxOutpoint) 74 from .plugin import run_hook 75 from .address_synchronizer import (AddressSynchronizer, TX_HEIGHT_LOCAL, 76 TX_HEIGHT_UNCONF_PARENT, TX_HEIGHT_UNCONFIRMED, TX_HEIGHT_FUTURE) 77 from .invoices import Invoice, OnchainInvoice, LNInvoice 78 from .invoices import PR_PAID, PR_UNPAID, PR_UNKNOWN, PR_EXPIRED, PR_UNCONFIRMED, PR_TYPE_ONCHAIN, PR_TYPE_LN 79 from .contacts import Contacts 80 from .interface import NetworkException 81 from .mnemonic import Mnemonic 82 from .logging import get_logger 83 from .lnworker import LNWallet 84 from .paymentrequest import PaymentRequest 85 from .util import read_json_file, write_json_file, UserFacingException 86 87 if TYPE_CHECKING: 88 from .network import Network 89 from .exchange_rate import FxThread 90 91 92 _logger = get_logger(__name__) 93 94 TX_STATUS = [ 95 _('Unconfirmed'), 96 _('Unconfirmed parent'), 97 _('Not Verified'), 98 _('Local'), 99 ] 100 101 102 class BumpFeeStrategy(enum.Enum): 103 COINCHOOSER = enum.auto() 104 DECREASE_CHANGE = enum.auto() 105 DECREASE_PAYMENT = enum.auto() 106 107 108 async def _append_utxos_to_inputs(*, inputs: List[PartialTxInput], network: 'Network', 109 pubkey: str, txin_type: str, imax: int) -> None: 110 if txin_type in ('p2pkh', 'p2wpkh', 'p2wpkh-p2sh'): 111 address = bitcoin.pubkey_to_address(txin_type, pubkey) 112 scripthash = bitcoin.address_to_scripthash(address) 113 elif txin_type == 'p2pk': 114 script = bitcoin.public_key_to_p2pk_script(pubkey) 115 scripthash = bitcoin.script_to_scripthash(script) 116 else: 117 raise Exception(f'unexpected txin_type to sweep: {txin_type}') 118 119 async def append_single_utxo(item): 120 prev_tx_raw = await network.get_transaction(item['tx_hash']) 121 prev_tx = Transaction(prev_tx_raw) 122 prev_txout = prev_tx.outputs()[item['tx_pos']] 123 if scripthash != bitcoin.script_to_scripthash(prev_txout.scriptpubkey.hex()): 124 raise Exception('scripthash mismatch when sweeping') 125 prevout_str = item['tx_hash'] + ':%d' % item['tx_pos'] 126 prevout = TxOutpoint.from_str(prevout_str) 127 txin = PartialTxInput(prevout=prevout) 128 txin.utxo = prev_tx 129 txin.block_height = int(item['height']) 130 txin.script_type = txin_type 131 txin.pubkeys = [bfh(pubkey)] 132 txin.num_sig = 1 133 if txin_type == 'p2wpkh-p2sh': 134 txin.redeem_script = bfh(bitcoin.p2wpkh_nested_script(pubkey)) 135 inputs.append(txin) 136 137 u = await network.listunspent_for_scripthash(scripthash) 138 async with TaskGroup() as group: 139 for item in u: 140 if len(inputs) >= imax: 141 break 142 await group.spawn(append_single_utxo(item)) 143 144 145 async def sweep_preparations(privkeys, network: 'Network', imax=100): 146 147 async def find_utxos_for_privkey(txin_type, privkey, compressed): 148 pubkey = ecc.ECPrivkey(privkey).get_public_key_hex(compressed=compressed) 149 await _append_utxos_to_inputs( 150 inputs=inputs, 151 network=network, 152 pubkey=pubkey, 153 txin_type=txin_type, 154 imax=imax) 155 keypairs[pubkey] = privkey, compressed 156 157 inputs = [] # type: List[PartialTxInput] 158 keypairs = {} 159 async with TaskGroup() as group: 160 for sec in privkeys: 161 txin_type, privkey, compressed = bitcoin.deserialize_privkey(sec) 162 await group.spawn(find_utxos_for_privkey(txin_type, privkey, compressed)) 163 # do other lookups to increase support coverage 164 if is_minikey(sec): 165 # minikeys don't have a compressed byte 166 # we lookup both compressed and uncompressed pubkeys 167 await group.spawn(find_utxos_for_privkey(txin_type, privkey, not compressed)) 168 elif txin_type == 'p2pkh': 169 # WIF serialization does not distinguish p2pkh and p2pk 170 # we also search for pay-to-pubkey outputs 171 await group.spawn(find_utxos_for_privkey('p2pk', privkey, compressed)) 172 if not inputs: 173 raise UserFacingException(_('No inputs found.')) 174 return inputs, keypairs 175 176 177 async def sweep( 178 privkeys, 179 *, 180 network: 'Network', 181 config: 'SimpleConfig', 182 to_address: str, 183 fee: int = None, 184 imax=100, 185 locktime=None, 186 tx_version=None 187 ) -> PartialTransaction: 188 inputs, keypairs = await sweep_preparations(privkeys, network, imax) 189 total = sum(txin.value_sats() for txin in inputs) 190 if fee is None: 191 outputs = [PartialTxOutput(scriptpubkey=bfh(bitcoin.address_to_script(to_address)), 192 value=total)] 193 tx = PartialTransaction.from_io(inputs, outputs) 194 fee = config.estimate_fee(tx.estimated_size()) 195 if total - fee < 0: 196 raise Exception(_('Not enough funds on address.') + '\nTotal: %d satoshis\nFee: %d'%(total, fee)) 197 if total - fee < dust_threshold(network): 198 raise Exception(_('Not enough funds on address.') + '\nTotal: %d satoshis\nFee: %d\nDust Threshold: %d'%(total, fee, dust_threshold(network))) 199 200 outputs = [PartialTxOutput(scriptpubkey=bfh(bitcoin.address_to_script(to_address)), 201 value=total - fee)] 202 if locktime is None: 203 locktime = get_locktime_for_new_transaction(network) 204 205 tx = PartialTransaction.from_io(inputs, outputs, locktime=locktime, version=tx_version) 206 rbf = bool(config.get('use_rbf', True)) 207 tx.set_rbf(rbf) 208 tx.sign(keypairs) 209 return tx 210 211 212 def get_locktime_for_new_transaction(network: 'Network') -> int: 213 # if no network or not up to date, just set locktime to zero 214 if not network: 215 return 0 216 chain = network.blockchain() 217 if chain.is_tip_stale(): 218 return 0 219 # discourage "fee sniping" 220 locktime = chain.height() 221 # sometimes pick locktime a bit further back, to help privacy 222 # of setups that need more time (offline/multisig/coinjoin/...) 223 if random.randint(0, 9) == 0: 224 locktime = max(0, locktime - random.randint(0, 99)) 225 return locktime 226 227 228 229 class CannotBumpFee(Exception): 230 def __str__(self): 231 return _('Cannot bump fee') + ':\n\n' + Exception.__str__(self) 232 233 class CannotDoubleSpendTx(Exception): 234 def __str__(self): 235 return _('Cannot cancel transaction') + ':\n\n' + Exception.__str__(self) 236 237 class CannotCPFP(Exception): 238 def __str__(self): 239 return _('Cannot create child transaction') + ':\n\n' + Exception.__str__(self) 240 241 class InternalAddressCorruption(Exception): 242 def __str__(self): 243 return _("Wallet file corruption detected. " 244 "Please restore your wallet from seed, and compare the addresses in both files") 245 246 247 class TxWalletDetails(NamedTuple): 248 txid: Optional[str] 249 status: str 250 label: str 251 can_broadcast: bool 252 can_bump: bool 253 can_cpfp: bool 254 can_dscancel: bool # whether user can double-spend to self 255 can_save_as_local: bool 256 amount: Optional[int] 257 fee: Optional[int] 258 tx_mined_status: TxMinedInfo 259 mempool_depth_bytes: Optional[int] 260 can_remove: bool # whether user should be allowed to delete tx 261 is_lightning_funding_tx: bool 262 263 264 class Abstract_Wallet(AddressSynchronizer, ABC): 265 """ 266 Wallet classes are created to handle various address generation methods. 267 Completion states (watching-only, single account, no seed, etc) are handled inside classes. 268 """ 269 270 LOGGING_SHORTCUT = 'w' 271 max_change_outputs = 3 272 gap_limit_for_change = 10 273 274 txin_type: str 275 wallet_type: str 276 lnworker: Optional['LNWallet'] 277 278 def __init__(self, db: WalletDB, storage: Optional[WalletStorage], *, config: SimpleConfig): 279 if not db.is_ready_to_be_used_by_wallet(): 280 raise Exception("storage not ready to be used by Abstract_Wallet") 281 282 self.config = config 283 assert self.config is not None, "config must not be None" 284 self.db = db 285 self.storage = storage 286 # load addresses needs to be called before constructor for sanity checks 287 db.load_addresses(self.wallet_type) 288 self.keystore = None # type: Optional[KeyStore] # will be set by load_keystore 289 AddressSynchronizer.__init__(self, db) 290 291 # saved fields 292 self.use_change = db.get('use_change', True) 293 self.multiple_change = db.get('multiple_change', False) 294 self._labels = db.get_dict('labels') 295 self._frozen_addresses = set(db.get('frozen_addresses', [])) 296 self._frozen_coins = db.get_dict('frozen_coins') # type: Dict[str, bool] 297 self.fiat_value = db.get_dict('fiat_value') 298 self.receive_requests = db.get_dict('payment_requests') # type: Dict[str, Invoice] 299 self.invoices = db.get_dict('invoices') # type: Dict[str, Invoice] 300 self._reserved_addresses = set(db.get('reserved_addresses', [])) 301 302 self._freeze_lock = threading.Lock() # for mutating/iterating frozen_{addresses,coins} 303 304 self._prepare_onchain_invoice_paid_detection() 305 self.calc_unused_change_addresses() 306 # save wallet type the first time 307 if self.db.get('wallet_type') is None: 308 self.db.put('wallet_type', self.wallet_type) 309 self.contacts = Contacts(self.db) 310 self._coin_price_cache = {} 311 312 self.lnworker = None 313 314 def save_db(self): 315 if self.storage: 316 self.db.write(self.storage) 317 318 def save_backup(self): 319 backup_dir = get_backup_dir(self.config) 320 if backup_dir is None: 321 return 322 new_db = WalletDB(self.db.dump(), manual_upgrades=False) 323 324 if self.lnworker: 325 channel_backups = new_db.get_dict('channel_backups') 326 for chan_id, chan in self.lnworker.channels.items(): 327 channel_backups[chan_id.hex()] = self.lnworker.create_channel_backup(chan_id) 328 new_db.put('channels', None) 329 new_db.put('lightning_privkey2', None) 330 331 new_path = os.path.join(backup_dir, self.basename() + '.backup') 332 new_storage = WalletStorage(new_path) 333 new_storage._encryption_version = self.storage._encryption_version 334 new_storage.pubkey = self.storage.pubkey 335 new_db.set_modified(True) 336 new_db.write(new_storage) 337 return new_path 338 339 def has_lightning(self): 340 return bool(self.lnworker) 341 342 def can_have_lightning(self): 343 # we want static_remotekey to be a wallet address 344 return self.txin_type == 'p2wpkh' 345 346 def init_lightning(self): 347 assert self.can_have_lightning() 348 if self.db.get('lightning_privkey2'): 349 return 350 # TODO derive this deterministically from wallet.keystore at keystore generation time 351 # probably along a hardened path ( lnd-equivalent would be m/1017'/coinType'/ ) 352 seed = os.urandom(32) 353 node = BIP32Node.from_rootseed(seed, xtype='standard') 354 ln_xprv = node.to_xprv() 355 self.db.put('lightning_privkey2', ln_xprv) 356 357 async def stop(self): 358 """Stop all networking and save DB to disk.""" 359 try: 360 async with ignore_after(5): 361 await super().stop() 362 if self.network: 363 if self.lnworker: 364 await self.lnworker.stop() 365 self.lnworker = None 366 finally: # even if we get cancelled 367 if any([ks.is_requesting_to_be_rewritten_to_wallet_file for ks in self.get_keystores()]): 368 self.save_keystore() 369 self.save_db() 370 371 def set_up_to_date(self, b): 372 super().set_up_to_date(b) 373 if b: self.save_db() 374 375 def clear_history(self): 376 super().clear_history() 377 self.save_db() 378 379 def start_network(self, network): 380 AddressSynchronizer.start_network(self, network) 381 if network: 382 if self.lnworker: 383 self.lnworker.start_network(network) 384 # only start gossiping when we already have channels 385 if self.db.get('channels'): 386 self.network.start_gossip() 387 388 def load_and_cleanup(self): 389 self.load_keystore() 390 self.test_addresses_sanity() 391 super().load_and_cleanup() 392 393 @abstractmethod 394 def load_keystore(self) -> None: 395 pass 396 397 def diagnostic_name(self): 398 return self.basename() 399 400 def __str__(self): 401 return self.basename() 402 403 def get_master_public_key(self): 404 return None 405 406 def get_master_public_keys(self): 407 return [] 408 409 def basename(self) -> str: 410 return self.storage.basename() if self.storage else 'no name' 411 412 def test_addresses_sanity(self) -> None: 413 addrs = self.get_receiving_addresses() 414 if len(addrs) > 0: 415 addr = str(addrs[0]) 416 if not bitcoin.is_address(addr): 417 neutered_addr = addr[:5] + '..' + addr[-2:] 418 raise WalletFileException(f'The addresses in this wallet are not bitcoin addresses.\n' 419 f'e.g. {neutered_addr} (length: {len(addr)})') 420 421 def check_returned_address_for_corruption(func): 422 def wrapper(self, *args, **kwargs): 423 addr = func(self, *args, **kwargs) 424 self.check_address_for_corruption(addr) 425 return addr 426 return wrapper 427 428 def calc_unused_change_addresses(self) -> Sequence[str]: 429 """Returns a list of change addresses to choose from, for usage in e.g. new transactions. 430 The caller should give priority to earlier ones in the list. 431 """ 432 with self.lock: 433 # We want a list of unused change addresses. 434 # As a performance optimisation, to avoid checking all addresses every time, 435 # we maintain a list of "not old" addresses ("old" addresses have deeply confirmed history), 436 # and only check those. 437 if not hasattr(self, '_not_old_change_addresses'): 438 self._not_old_change_addresses = self.get_change_addresses() 439 self._not_old_change_addresses = [addr for addr in self._not_old_change_addresses 440 if not self.address_is_old(addr)] 441 unused_addrs = [addr for addr in self._not_old_change_addresses 442 if not self.is_used(addr) and not self.is_address_reserved(addr)] 443 return unused_addrs 444 445 def is_deterministic(self) -> bool: 446 return self.keystore.is_deterministic() 447 448 def _set_label(self, key: str, value: Optional[str]) -> None: 449 with self.lock: 450 if value is None: 451 self._labels.pop(key, None) 452 else: 453 self._labels[key] = value 454 455 def set_label(self, name: str, text: str = None) -> bool: 456 if not name: 457 return False 458 changed = False 459 with self.lock: 460 old_text = self._labels.get(name) 461 if text: 462 text = text.replace("\n", " ") 463 if old_text != text: 464 self._labels[name] = text 465 changed = True 466 else: 467 if old_text is not None: 468 self._labels.pop(name) 469 changed = True 470 if changed: 471 run_hook('set_label', self, name, text) 472 return changed 473 474 def import_labels(self, path): 475 data = read_json_file(path) 476 for key, value in data.items(): 477 self.set_label(key, value) 478 479 def export_labels(self, path): 480 write_json_file(path, self.get_all_labels()) 481 482 def set_fiat_value(self, txid, ccy, text, fx, value_sat): 483 if not self.db.get_transaction(txid): 484 return 485 # since fx is inserting the thousands separator, 486 # and not util, also have fx remove it 487 text = fx.remove_thousands_separator(text) 488 def_fiat = self.default_fiat_value(txid, fx, value_sat) 489 formatted = fx.ccy_amount_str(def_fiat, commas=False) 490 def_fiat_rounded = Decimal(formatted) 491 reset = not text 492 if not reset: 493 try: 494 text_dec = Decimal(text) 495 text_dec_rounded = Decimal(fx.ccy_amount_str(text_dec, commas=False)) 496 reset = text_dec_rounded == def_fiat_rounded 497 except: 498 # garbage. not resetting, but not saving either 499 return False 500 if reset: 501 d = self.fiat_value.get(ccy, {}) 502 if d and txid in d: 503 d.pop(txid) 504 else: 505 # avoid saving empty dict 506 return True 507 else: 508 if ccy not in self.fiat_value: 509 self.fiat_value[ccy] = {} 510 self.fiat_value[ccy][txid] = text 511 return reset 512 513 def get_fiat_value(self, txid, ccy): 514 fiat_value = self.fiat_value.get(ccy, {}).get(txid) 515 try: 516 return Decimal(fiat_value) 517 except: 518 return 519 520 def is_mine(self, address) -> bool: 521 if not address: return False 522 return bool(self.get_address_index(address)) 523 524 def is_change(self, address) -> bool: 525 if not self.is_mine(address): 526 return False 527 return self.get_address_index(address)[0] == 1 528 529 @abstractmethod 530 def get_address_index(self, address: str) -> Optional[AddressIndexGeneric]: 531 pass 532 533 @abstractmethod 534 def get_address_path_str(self, address: str) -> Optional[str]: 535 """Returns derivation path str such as "m/0/5" to address, 536 or None if not applicable. 537 """ 538 pass 539 540 @abstractmethod 541 def get_redeem_script(self, address: str) -> Optional[str]: 542 pass 543 544 @abstractmethod 545 def get_witness_script(self, address: str) -> Optional[str]: 546 pass 547 548 @abstractmethod 549 def get_txin_type(self, address: str) -> str: 550 """Return script type of wallet address.""" 551 pass 552 553 def export_private_key(self, address: str, password: Optional[str]) -> str: 554 if self.is_watching_only(): 555 raise Exception(_("This is a watching-only wallet")) 556 if not is_address(address): 557 raise Exception(f"Invalid bitcoin address: {address}") 558 if not self.is_mine(address): 559 raise Exception(_('Address not in wallet.') + f' {address}') 560 index = self.get_address_index(address) 561 pk, compressed = self.keystore.get_private_key(index, password) 562 txin_type = self.get_txin_type(address) 563 serialized_privkey = bitcoin.serialize_privkey(pk, compressed, txin_type) 564 return serialized_privkey 565 566 def export_private_key_for_path(self, path: Union[Sequence[int], str], password: Optional[str]) -> str: 567 raise Exception("this wallet is not deterministic") 568 569 @abstractmethod 570 def get_public_keys(self, address: str) -> Sequence[str]: 571 pass 572 573 def get_public_keys_with_deriv_info(self, address: str) -> Dict[bytes, Tuple[KeyStoreWithMPK, Sequence[int]]]: 574 """Returns a map: pubkey -> (keystore, derivation_suffix)""" 575 return {} 576 577 def get_tx_info(self, tx: Transaction) -> TxWalletDetails: 578 tx_wallet_delta = self.get_wallet_delta(tx) 579 is_relevant = tx_wallet_delta.is_relevant 580 is_any_input_ismine = tx_wallet_delta.is_any_input_ismine 581 fee = tx_wallet_delta.fee 582 exp_n = None 583 can_broadcast = False 584 can_bump = False 585 can_cpfp = False 586 tx_hash = tx.txid() # note: txid can be None! e.g. when called from GUI tx dialog 587 is_lightning_funding_tx = False 588 if self.has_lightning() and tx_hash is not None: 589 is_lightning_funding_tx = any([chan.funding_outpoint.txid == tx_hash 590 for chan in self.lnworker.channels.values()]) 591 tx_we_already_have_in_db = self.db.get_transaction(tx_hash) 592 can_save_as_local = (is_relevant and tx.txid() is not None 593 and (tx_we_already_have_in_db is None or not tx_we_already_have_in_db.is_complete())) 594 label = '' 595 tx_mined_status = self.get_tx_height(tx_hash) 596 can_remove = ((tx_mined_status.height in [TX_HEIGHT_FUTURE, TX_HEIGHT_LOCAL]) 597 # otherwise 'height' is unreliable (typically LOCAL): 598 and is_relevant 599 # don't offer during common signing flow, e.g. when watch-only wallet starts creating a tx: 600 and bool(tx_we_already_have_in_db)) 601 can_dscancel = False 602 if tx.is_complete(): 603 if tx_we_already_have_in_db: 604 label = self.get_label_for_txid(tx_hash) 605 if tx_mined_status.height > 0: 606 if tx_mined_status.conf: 607 status = _("{} confirmations").format(tx_mined_status.conf) 608 else: 609 status = _('Not verified') 610 elif tx_mined_status.height in (TX_HEIGHT_UNCONF_PARENT, TX_HEIGHT_UNCONFIRMED): 611 status = _('Unconfirmed') 612 if fee is None: 613 fee = self.get_tx_fee(tx_hash) 614 if fee and self.network and self.config.has_fee_mempool(): 615 size = tx.estimated_size() 616 fee_per_byte = fee / size 617 exp_n = self.config.fee_to_depth(fee_per_byte) 618 can_bump = is_any_input_ismine and not tx.is_final() 619 can_dscancel = (is_any_input_ismine and not tx.is_final() 620 and not all([self.is_mine(txout.address) for txout in tx.outputs()])) 621 try: 622 self.cpfp(tx, 0) 623 can_cpfp = True 624 except: 625 can_cpfp = False 626 else: 627 status = _('Local') 628 can_broadcast = self.network is not None 629 can_bump = is_any_input_ismine and not tx.is_final() 630 else: 631 status = _("Signed") 632 can_broadcast = self.network is not None 633 else: 634 assert isinstance(tx, PartialTransaction) 635 s, r = tx.signature_count() 636 status = _("Unsigned") if s == 0 else _('Partially signed') + ' (%d/%d)'%(s,r) 637 638 if is_relevant: 639 if tx_wallet_delta.is_all_input_ismine: 640 assert fee is not None 641 amount = tx_wallet_delta.delta + fee 642 else: 643 amount = tx_wallet_delta.delta 644 else: 645 amount = None 646 647 if is_lightning_funding_tx: 648 can_bump = False # would change txid 649 650 return TxWalletDetails( 651 txid=tx_hash, 652 status=status, 653 label=label, 654 can_broadcast=can_broadcast, 655 can_bump=can_bump, 656 can_cpfp=can_cpfp, 657 can_dscancel=can_dscancel, 658 can_save_as_local=can_save_as_local, 659 amount=amount, 660 fee=fee, 661 tx_mined_status=tx_mined_status, 662 mempool_depth_bytes=exp_n, 663 can_remove=can_remove, 664 is_lightning_funding_tx=is_lightning_funding_tx, 665 ) 666 667 def get_spendable_coins(self, domain, *, nonlocal_only=False) -> Sequence[PartialTxInput]: 668 confirmed_only = self.config.get('confirmed_only', False) 669 with self._freeze_lock: 670 frozen_addresses = self._frozen_addresses.copy() 671 utxos = self.get_utxos(domain, 672 excluded_addresses=frozen_addresses, 673 mature_only=True, 674 confirmed_only=confirmed_only, 675 nonlocal_only=nonlocal_only) 676 utxos = [utxo for utxo in utxos if not self.is_frozen_coin(utxo)] 677 return utxos 678 679 @abstractmethod 680 def get_receiving_addresses(self, *, slice_start=None, slice_stop=None) -> Sequence[str]: 681 pass 682 683 @abstractmethod 684 def get_change_addresses(self, *, slice_start=None, slice_stop=None) -> Sequence[str]: 685 pass 686 687 def dummy_address(self): 688 # first receiving address 689 return self.get_receiving_addresses(slice_start=0, slice_stop=1)[0] 690 691 def get_frozen_balance(self): 692 with self._freeze_lock: 693 frozen_addresses = self._frozen_addresses.copy() 694 # note: for coins, use is_frozen_coin instead of _frozen_coins, 695 # as latter only contains *manually* frozen ones 696 frozen_coins = {utxo.prevout.to_str() for utxo in self.get_utxos() 697 if self.is_frozen_coin(utxo)} 698 if not frozen_coins: # shortcut 699 return self.get_balance(frozen_addresses) 700 c1, u1, x1 = self.get_balance() 701 c2, u2, x2 = self.get_balance( 702 excluded_addresses=frozen_addresses, 703 excluded_coins=frozen_coins, 704 ) 705 return c1-c2, u1-u2, x1-x2 706 707 def balance_at_timestamp(self, domain, target_timestamp): 708 # we assume that get_history returns items ordered by block height 709 # we also assume that block timestamps are monotonic (which is false...!) 710 h = self.get_history(domain=domain) 711 balance = 0 712 for hist_item in h: 713 balance = hist_item.balance 714 if hist_item.tx_mined_status.timestamp is None or hist_item.tx_mined_status.timestamp > target_timestamp: 715 return balance - hist_item.delta 716 # return last balance 717 return balance 718 719 def get_onchain_history(self, *, domain=None): 720 monotonic_timestamp = 0 721 for hist_item in self.get_history(domain=domain): 722 monotonic_timestamp = max(monotonic_timestamp, (hist_item.tx_mined_status.timestamp or 999_999_999_999)) 723 yield { 724 'txid': hist_item.txid, 725 'fee_sat': hist_item.fee, 726 'height': hist_item.tx_mined_status.height, 727 'confirmations': hist_item.tx_mined_status.conf, 728 'timestamp': hist_item.tx_mined_status.timestamp, 729 'monotonic_timestamp': monotonic_timestamp, 730 'incoming': True if hist_item.delta>0 else False, 731 'bc_value': Satoshis(hist_item.delta), 732 'bc_balance': Satoshis(hist_item.balance), 733 'date': timestamp_to_datetime(hist_item.tx_mined_status.timestamp), 734 'label': self.get_label_for_txid(hist_item.txid), 735 'txpos_in_block': hist_item.tx_mined_status.txpos, 736 } 737 738 def create_invoice(self, *, outputs: List[PartialTxOutput], message, pr, URI) -> Invoice: 739 height=self.get_local_height() 740 if pr: 741 return OnchainInvoice.from_bip70_payreq(pr, height) 742 if '!' in (x.value for x in outputs): 743 amount = '!' 744 else: 745 amount = sum(x.value for x in outputs) 746 timestamp = None 747 exp = None 748 if URI: 749 timestamp = URI.get('time') 750 exp = URI.get('exp') 751 timestamp = timestamp or int(time.time()) 752 exp = exp or 0 753 _id = bh2u(sha256d(repr(outputs) + "%d"%timestamp))[0:10] 754 invoice = OnchainInvoice( 755 type=PR_TYPE_ONCHAIN, 756 amount_sat=amount, 757 outputs=outputs, 758 message=message, 759 id=_id, 760 time=timestamp, 761 exp=exp, 762 bip70=None, 763 requestor=None, 764 height=height, 765 ) 766 return invoice 767 768 def save_invoice(self, invoice: Invoice) -> None: 769 key = self.get_key_for_outgoing_invoice(invoice) 770 if not invoice.is_lightning(): 771 assert isinstance(invoice, OnchainInvoice) 772 if self.is_onchain_invoice_paid(invoice, 0): 773 self.logger.info("saving invoice... but it is already paid!") 774 with self.transaction_lock: 775 for txout in invoice.outputs: 776 self._invoices_from_scriptpubkey_map[txout.scriptpubkey].add(key) 777 self.invoices[key] = invoice 778 self.save_db() 779 780 def clear_invoices(self): 781 self.invoices = {} 782 self.save_db() 783 784 def clear_requests(self): 785 self.receive_requests = {} 786 self.save_db() 787 788 def get_invoices(self): 789 out = list(self.invoices.values()) 790 out.sort(key=lambda x:x.time) 791 return out 792 793 def get_unpaid_invoices(self): 794 invoices = self.get_invoices() 795 return [x for x in invoices if self.get_invoice_status(x) != PR_PAID] 796 797 def get_invoice(self, key): 798 return self.invoices.get(key) 799 800 def import_requests(self, path): 801 data = read_json_file(path) 802 for x in data: 803 req = Invoice.from_json(x) 804 self.add_payment_request(req) 805 806 def export_requests(self, path): 807 write_json_file(path, list(self.receive_requests.values())) 808 809 def import_invoices(self, path): 810 data = read_json_file(path) 811 for x in data: 812 invoice = Invoice.from_json(x) 813 self.save_invoice(invoice) 814 815 def export_invoices(self, path): 816 write_json_file(path, list(self.invoices.values())) 817 818 def _get_relevant_invoice_keys_for_tx(self, tx: Transaction) -> Set[str]: 819 relevant_invoice_keys = set() 820 with self.transaction_lock: 821 for txout in tx.outputs(): 822 for invoice_key in self._invoices_from_scriptpubkey_map.get(txout.scriptpubkey, set()): 823 # note: the invoice might have been deleted since, so check now: 824 if invoice_key in self.invoices: 825 relevant_invoice_keys.add(invoice_key) 826 return relevant_invoice_keys 827 828 def get_relevant_invoices_for_tx(self, tx: Transaction) -> Sequence[OnchainInvoice]: 829 invoice_keys = self._get_relevant_invoice_keys_for_tx(tx) 830 invoices = [self.get_invoice(key) for key in invoice_keys] 831 invoices = [inv for inv in invoices if inv] # filter out None 832 for inv in invoices: 833 assert isinstance(inv, OnchainInvoice), f"unexpected type {type(inv)}" 834 return invoices 835 836 def _prepare_onchain_invoice_paid_detection(self): 837 # scriptpubkey -> list(invoice_keys) 838 self._invoices_from_scriptpubkey_map = defaultdict(set) # type: Dict[bytes, Set[str]] 839 for invoice_key, invoice in self.invoices.items(): 840 if invoice.type == PR_TYPE_ONCHAIN: 841 assert isinstance(invoice, OnchainInvoice) 842 for txout in invoice.outputs: 843 self._invoices_from_scriptpubkey_map[txout.scriptpubkey].add(invoice_key) 844 845 def _is_onchain_invoice_paid(self, invoice: Invoice, conf: int) -> Tuple[bool, Sequence[str]]: 846 """Returns whether on-chain invoice is satisfied, and list of relevant TXIDs.""" 847 assert invoice.type == PR_TYPE_ONCHAIN 848 assert isinstance(invoice, OnchainInvoice) 849 invoice_amounts = defaultdict(int) # type: Dict[bytes, int] # scriptpubkey -> value_sats 850 for txo in invoice.outputs: # type: PartialTxOutput 851 invoice_amounts[txo.scriptpubkey] += 1 if txo.value == '!' else txo.value 852 relevant_txs = [] 853 with self.transaction_lock: 854 for invoice_scriptpubkey, invoice_amt in invoice_amounts.items(): 855 scripthash = bitcoin.script_to_scripthash(invoice_scriptpubkey.hex()) 856 prevouts_and_values = self.db.get_prevouts_by_scripthash(scripthash) 857 total_received = 0 858 for prevout, v in prevouts_and_values: 859 tx_height = self.get_tx_height(prevout.txid.hex()) 860 if tx_height.height > 0 and tx_height.height <= invoice.height: 861 continue 862 if tx_height.conf < conf: 863 continue 864 total_received += v 865 relevant_txs.append(prevout.txid.hex()) 866 # check that there is at least one TXO, and that they pay enough. 867 # note: "at least one TXO" check is needed for zero amount invoice (e.g. OP_RETURN) 868 if len(prevouts_and_values) == 0: 869 return False, [] 870 if total_received < invoice_amt: 871 return False, [] 872 return True, relevant_txs 873 874 def is_onchain_invoice_paid(self, invoice: Invoice, conf: int) -> bool: 875 return self._is_onchain_invoice_paid(invoice, conf)[0] 876 877 def _maybe_set_tx_label_based_on_invoices(self, tx: Transaction) -> bool: 878 # note: this is not done in 'get_default_label' as that would require deserializing each tx 879 tx_hash = tx.txid() 880 labels = [] 881 for invoice in self.get_relevant_invoices_for_tx(tx): 882 if invoice.message: 883 labels.append(invoice.message) 884 if labels and not self._labels.get(tx_hash, ''): 885 self.set_label(tx_hash, "; ".join(labels)) 886 return bool(labels) 887 888 def add_transaction(self, tx, *, allow_unrelated=False): 889 tx_was_added = super().add_transaction(tx, allow_unrelated=allow_unrelated) 890 891 if tx_was_added: 892 self._maybe_set_tx_label_based_on_invoices(tx) 893 return tx_was_added 894 895 @profiler 896 def get_full_history(self, fx=None, *, onchain_domain=None, include_lightning=True): 897 transactions_tmp = OrderedDictWithIndex() 898 # add on-chain txns 899 onchain_history = self.get_onchain_history(domain=onchain_domain) 900 for tx_item in onchain_history: 901 txid = tx_item['txid'] 902 transactions_tmp[txid] = tx_item 903 # add lnworker onchain transactions 904 lnworker_history = self.lnworker.get_onchain_history() if self.lnworker and include_lightning else {} 905 for txid, item in lnworker_history.items(): 906 if txid in transactions_tmp: 907 tx_item = transactions_tmp[txid] 908 tx_item['group_id'] = item.get('group_id') # for swaps 909 tx_item['label'] = item['label'] 910 tx_item['type'] = item['type'] 911 ln_value = Decimal(item['amount_msat']) / 1000 # for channel open/close tx 912 tx_item['ln_value'] = Satoshis(ln_value) 913 else: 914 transactions_tmp[txid] = item 915 ln_value = Decimal(item['amount_msat']) / 1000 # for channel open/close tx 916 item['ln_value'] = Satoshis(ln_value) 917 # add lightning_transactions 918 lightning_history = self.lnworker.get_lightning_history() if self.lnworker and include_lightning else {} 919 for tx_item in lightning_history.values(): 920 txid = tx_item.get('txid') 921 ln_value = Decimal(tx_item['amount_msat']) / 1000 922 tx_item['lightning'] = True 923 tx_item['ln_value'] = Satoshis(ln_value) 924 key = tx_item.get('txid') or tx_item['payment_hash'] 925 transactions_tmp[key] = tx_item 926 # sort on-chain and LN stuff into new dict, by timestamp 927 # (we rely on this being a *stable* sort) 928 transactions = OrderedDictWithIndex() 929 for k, v in sorted(list(transactions_tmp.items()), 930 key=lambda x: x[1].get('monotonic_timestamp') or x[1].get('timestamp') or float('inf')): 931 transactions[k] = v 932 now = time.time() 933 balance = 0 934 for item in transactions.values(): 935 # add on-chain and lightning values 936 value = Decimal(0) 937 if item.get('bc_value'): 938 value += item['bc_value'].value 939 if item.get('ln_value'): 940 value += item.get('ln_value').value 941 # note: 'value' and 'balance' has msat precision (as LN has msat precision) 942 item['value'] = Satoshis(value) 943 balance += value 944 item['balance'] = Satoshis(balance) 945 if fx and fx.is_enabled() and fx.get_history_config(): 946 txid = item.get('txid') 947 if not item.get('lightning') and txid: 948 fiat_fields = self.get_tx_item_fiat(tx_hash=txid, amount_sat=value, fx=fx, tx_fee=item['fee_sat']) 949 item.update(fiat_fields) 950 else: 951 timestamp = item['timestamp'] or now 952 fiat_value = value / Decimal(bitcoin.COIN) * fx.timestamp_rate(timestamp) 953 item['fiat_value'] = Fiat(fiat_value, fx.ccy) 954 item['fiat_default'] = True 955 return transactions 956 957 @profiler 958 def get_detailed_history(self, from_timestamp=None, to_timestamp=None, 959 fx=None, show_addresses=False, from_height=None, to_height=None): 960 # History with capital gains, using utxo pricing 961 # FIXME: Lightning capital gains would requires FIFO 962 if (from_timestamp is not None or to_timestamp is not None) \ 963 and (from_height is not None or to_height is not None): 964 raise Exception('timestamp and block height based filtering cannot be used together') 965 out = [] 966 income = 0 967 expenditures = 0 968 capital_gains = Decimal(0) 969 fiat_income = Decimal(0) 970 fiat_expenditures = Decimal(0) 971 now = time.time() 972 for item in self.get_onchain_history(): 973 timestamp = item['timestamp'] 974 if from_timestamp and (timestamp or now) < from_timestamp: 975 continue 976 if to_timestamp and (timestamp or now) >= to_timestamp: 977 continue 978 height = item['height'] 979 if from_height is not None and from_height > height > 0: 980 continue 981 if to_height is not None and (height >= to_height or height <= 0): 982 continue 983 tx_hash = item['txid'] 984 tx = self.db.get_transaction(tx_hash) 985 tx_fee = item['fee_sat'] 986 item['fee'] = Satoshis(tx_fee) if tx_fee is not None else None 987 if show_addresses: 988 item['inputs'] = list(map(lambda x: x.to_json(), tx.inputs())) 989 item['outputs'] = list(map(lambda x: {'address': x.get_ui_address_str(), 'value': Satoshis(x.value)}, 990 tx.outputs())) 991 # fixme: use in and out values 992 value = item['bc_value'].value 993 if value < 0: 994 expenditures += -value 995 else: 996 income += value 997 # fiat computations 998 if fx and fx.is_enabled() and fx.get_history_config(): 999 fiat_fields = self.get_tx_item_fiat(tx_hash=tx_hash, amount_sat=value, fx=fx, tx_fee=tx_fee) 1000 fiat_value = fiat_fields['fiat_value'].value 1001 item.update(fiat_fields) 1002 if value < 0: 1003 capital_gains += fiat_fields['capital_gain'].value 1004 fiat_expenditures += -fiat_value 1005 else: 1006 fiat_income += fiat_value 1007 out.append(item) 1008 # add summary 1009 if out: 1010 b, v = out[0]['bc_balance'].value, out[0]['bc_value'].value 1011 start_balance = None if b is None or v is None else b - v 1012 end_balance = out[-1]['bc_balance'].value 1013 if from_timestamp is not None and to_timestamp is not None: 1014 start_date = timestamp_to_datetime(from_timestamp) 1015 end_date = timestamp_to_datetime(to_timestamp) 1016 else: 1017 start_date = None 1018 end_date = None 1019 summary = { 1020 'start_date': start_date, 1021 'end_date': end_date, 1022 'from_height': from_height, 1023 'to_height': to_height, 1024 'start_balance': Satoshis(start_balance), 1025 'end_balance': Satoshis(end_balance), 1026 'incoming': Satoshis(income), 1027 'outgoing': Satoshis(expenditures) 1028 } 1029 if fx and fx.is_enabled() and fx.get_history_config(): 1030 unrealized = self.unrealized_gains(None, fx.timestamp_rate, fx.ccy) 1031 summary['fiat_currency'] = fx.ccy 1032 summary['fiat_capital_gains'] = Fiat(capital_gains, fx.ccy) 1033 summary['fiat_incoming'] = Fiat(fiat_income, fx.ccy) 1034 summary['fiat_outgoing'] = Fiat(fiat_expenditures, fx.ccy) 1035 summary['fiat_unrealized_gains'] = Fiat(unrealized, fx.ccy) 1036 summary['fiat_start_balance'] = Fiat(fx.historical_value(start_balance, start_date), fx.ccy) 1037 summary['fiat_end_balance'] = Fiat(fx.historical_value(end_balance, end_date), fx.ccy) 1038 summary['fiat_start_value'] = Fiat(fx.historical_value(COIN, start_date), fx.ccy) 1039 summary['fiat_end_value'] = Fiat(fx.historical_value(COIN, end_date), fx.ccy) 1040 else: 1041 summary = {} 1042 return { 1043 'transactions': out, 1044 'summary': summary 1045 } 1046 1047 def default_fiat_value(self, tx_hash, fx, value_sat): 1048 return value_sat / Decimal(COIN) * self.price_at_timestamp(tx_hash, fx.timestamp_rate) 1049 1050 def get_tx_item_fiat( 1051 self, 1052 *, 1053 tx_hash: str, 1054 amount_sat: int, 1055 fx: 'FxThread', 1056 tx_fee: Optional[int], 1057 ) -> Dict[str, Any]: 1058 item = {} 1059 fiat_value = self.get_fiat_value(tx_hash, fx.ccy) 1060 fiat_default = fiat_value is None 1061 fiat_rate = self.price_at_timestamp(tx_hash, fx.timestamp_rate) 1062 fiat_value = fiat_value if fiat_value is not None else self.default_fiat_value(tx_hash, fx, amount_sat) 1063 fiat_fee = tx_fee / Decimal(COIN) * fiat_rate if tx_fee is not None else None 1064 item['fiat_currency'] = fx.ccy 1065 item['fiat_rate'] = Fiat(fiat_rate, fx.ccy) 1066 item['fiat_value'] = Fiat(fiat_value, fx.ccy) 1067 item['fiat_fee'] = Fiat(fiat_fee, fx.ccy) if fiat_fee else None 1068 item['fiat_default'] = fiat_default 1069 if amount_sat < 0: 1070 acquisition_price = - amount_sat / Decimal(COIN) * self.average_price(tx_hash, fx.timestamp_rate, fx.ccy) 1071 liquidation_price = - fiat_value 1072 item['acquisition_price'] = Fiat(acquisition_price, fx.ccy) 1073 cg = liquidation_price - acquisition_price 1074 item['capital_gain'] = Fiat(cg, fx.ccy) 1075 return item 1076 1077 def get_label(self, key: str) -> str: 1078 # key is typically: address / txid / LN-payment-hash-hex 1079 return self._labels.get(key) or '' 1080 1081 def get_label_for_txid(self, tx_hash: str) -> str: 1082 return self._labels.get(tx_hash) or self._get_default_label_for_txid(tx_hash) 1083 1084 def _get_default_label_for_txid(self, tx_hash: str) -> str: 1085 # if no inputs are ismine, concat labels of output addresses 1086 if not self.db.get_txi_addresses(tx_hash): 1087 labels = [] 1088 for addr in self.db.get_txo_addresses(tx_hash): 1089 label = self._labels.get(addr) 1090 if label: 1091 labels.append(label) 1092 return ', '.join(labels) 1093 return '' 1094 1095 def get_all_labels(self) -> Dict[str, str]: 1096 with self.lock: 1097 return copy.copy(self._labels) 1098 1099 def get_tx_status(self, tx_hash, tx_mined_info: TxMinedInfo): 1100 extra = [] 1101 height = tx_mined_info.height 1102 conf = tx_mined_info.conf 1103 timestamp = tx_mined_info.timestamp 1104 if height == TX_HEIGHT_FUTURE: 1105 assert conf < 0, conf 1106 num_blocks_remainining = -conf 1107 return 2, f'in {num_blocks_remainining} blocks' 1108 if conf == 0: 1109 tx = self.db.get_transaction(tx_hash) 1110 if not tx: 1111 return 2, 'unknown' 1112 is_final = tx and tx.is_final() 1113 if not is_final: 1114 extra.append('rbf') 1115 fee = self.get_tx_fee(tx_hash) 1116 if fee is not None: 1117 size = tx.estimated_size() 1118 fee_per_byte = fee / size 1119 extra.append(format_fee_satoshis(fee_per_byte) + ' sat/b') 1120 if fee is not None and height in (TX_HEIGHT_UNCONF_PARENT, TX_HEIGHT_UNCONFIRMED) \ 1121 and self.config.has_fee_mempool(): 1122 exp_n = self.config.fee_to_depth(fee_per_byte) 1123 if exp_n is not None: 1124 extra.append('%.2f MB'%(exp_n/1000000)) 1125 if height == TX_HEIGHT_LOCAL: 1126 status = 3 1127 elif height == TX_HEIGHT_UNCONF_PARENT: 1128 status = 1 1129 elif height == TX_HEIGHT_UNCONFIRMED: 1130 status = 0 1131 else: 1132 status = 2 # not SPV verified 1133 else: 1134 status = 3 + min(conf, 6) 1135 time_str = format_time(timestamp) if timestamp else _("unknown") 1136 status_str = TX_STATUS[status] if status < 4 else time_str 1137 if extra: 1138 status_str += ' [%s]'%(', '.join(extra)) 1139 return status, status_str 1140 1141 def relayfee(self): 1142 return relayfee(self.network) 1143 1144 def dust_threshold(self): 1145 return dust_threshold(self.network) 1146 1147 def get_unconfirmed_base_tx_for_batching(self) -> Optional[Transaction]: 1148 candidate = None 1149 for hist_item in self.get_history(): 1150 # tx should not be mined yet 1151 if hist_item.tx_mined_status.conf > 0: continue 1152 # conservative future proofing of code: only allow known unconfirmed types 1153 if hist_item.tx_mined_status.height not in (TX_HEIGHT_UNCONFIRMED, 1154 TX_HEIGHT_UNCONF_PARENT, 1155 TX_HEIGHT_LOCAL): 1156 continue 1157 # tx should be "outgoing" from wallet 1158 if hist_item.delta >= 0: 1159 continue 1160 tx = self.db.get_transaction(hist_item.txid) 1161 if not tx: 1162 continue 1163 # is_mine outputs should not be spent yet 1164 # to avoid cancelling our own dependent transactions 1165 txid = tx.txid() 1166 if any([self.is_mine(o.address) and self.db.get_spent_outpoint(txid, output_idx) 1167 for output_idx, o in enumerate(tx.outputs())]): 1168 continue 1169 # all inputs should be is_mine 1170 if not all([self.is_mine(self.get_txin_address(txin)) for txin in tx.inputs()]): 1171 continue 1172 # prefer txns already in mempool (vs local) 1173 if hist_item.tx_mined_status.height == TX_HEIGHT_LOCAL: 1174 candidate = tx 1175 continue 1176 # tx must have opted-in for RBF 1177 if tx.is_final(): continue 1178 return tx 1179 return candidate 1180 1181 def get_change_addresses_for_new_transaction( 1182 self, preferred_change_addr=None, *, allow_reuse: bool = True, 1183 ) -> List[str]: 1184 change_addrs = [] 1185 if preferred_change_addr: 1186 if isinstance(preferred_change_addr, (list, tuple)): 1187 change_addrs = list(preferred_change_addr) 1188 else: 1189 change_addrs = [preferred_change_addr] 1190 elif self.use_change: 1191 # Recalc and get unused change addresses 1192 addrs = self.calc_unused_change_addresses() 1193 # New change addresses are created only after a few 1194 # confirmations. 1195 if addrs: 1196 # if there are any unused, select all 1197 change_addrs = addrs 1198 else: 1199 # if there are none, take one randomly from the last few 1200 if not allow_reuse: 1201 return [] 1202 addrs = self.get_change_addresses(slice_start=-self.gap_limit_for_change) 1203 change_addrs = [random.choice(addrs)] if addrs else [] 1204 for addr in change_addrs: 1205 assert is_address(addr), f"not valid bitcoin address: {addr}" 1206 # note that change addresses are not necessarily ismine 1207 # in which case this is a no-op 1208 self.check_address_for_corruption(addr) 1209 max_change = self.max_change_outputs if self.multiple_change else 1 1210 return change_addrs[:max_change] 1211 1212 def get_single_change_address_for_new_transaction( 1213 self, preferred_change_addr=None, *, allow_reuse: bool = True, 1214 ) -> Optional[str]: 1215 addrs = self.get_change_addresses_for_new_transaction( 1216 preferred_change_addr=preferred_change_addr, 1217 allow_reuse=allow_reuse, 1218 ) 1219 if addrs: 1220 return addrs[0] 1221 return None 1222 1223 @check_returned_address_for_corruption 1224 def get_new_sweep_address_for_channel(self) -> str: 1225 # Recalc and get unused change addresses 1226 addrs = self.calc_unused_change_addresses() 1227 if addrs: 1228 selected_addr = addrs[0] 1229 else: 1230 # if there are none, take one randomly from the last few 1231 addrs = self.get_change_addresses(slice_start=-self.gap_limit_for_change) 1232 if addrs: 1233 selected_addr = random.choice(addrs) 1234 else: # fallback for e.g. imported wallets 1235 selected_addr = self.get_receiving_address() 1236 assert is_address(selected_addr), f"not valid bitcoin address: {selected_addr}" 1237 return selected_addr 1238 1239 def make_unsigned_transaction(self, *, coins: Sequence[PartialTxInput], 1240 outputs: List[PartialTxOutput], fee=None, 1241 change_addr: str = None, is_sweep=False) -> PartialTransaction: 1242 1243 if any([c.already_has_some_signatures() for c in coins]): 1244 raise Exception("Some inputs already contain signatures!") 1245 1246 # prevent side-effect with '!' 1247 outputs = copy.deepcopy(outputs) 1248 1249 # check outputs 1250 i_max = None 1251 for i, o in enumerate(outputs): 1252 if o.value == '!': 1253 if i_max is not None: 1254 raise MultipleSpendMaxTxOutputs() 1255 i_max = i 1256 1257 if fee is None and self.config.fee_per_kb() is None: 1258 raise NoDynamicFeeEstimates() 1259 1260 for item in coins: 1261 self.add_input_info(item) 1262 1263 # Fee estimator 1264 if fee is None: 1265 fee_estimator = self.config.estimate_fee 1266 elif isinstance(fee, Number): 1267 fee_estimator = lambda size: fee 1268 elif callable(fee): 1269 fee_estimator = fee 1270 else: 1271 raise Exception(f'Invalid argument fee: {fee}') 1272 1273 if i_max is None: 1274 # Let the coin chooser select the coins to spend 1275 coin_chooser = coinchooser.get_coin_chooser(self.config) 1276 # If there is an unconfirmed RBF tx, merge with it 1277 base_tx = self.get_unconfirmed_base_tx_for_batching() 1278 if self.config.get('batch_rbf', False) and base_tx: 1279 # make sure we don't try to spend change from the tx-to-be-replaced: 1280 coins = [c for c in coins if c.prevout.txid.hex() != base_tx.txid()] 1281 is_local = self.get_tx_height(base_tx.txid()).height == TX_HEIGHT_LOCAL 1282 base_tx = PartialTransaction.from_tx(base_tx) 1283 base_tx.add_info_from_wallet(self) 1284 base_tx_fee = base_tx.get_fee() 1285 relayfeerate = Decimal(self.relayfee()) / 1000 1286 original_fee_estimator = fee_estimator 1287 def fee_estimator(size: Union[int, float, Decimal]) -> int: 1288 size = Decimal(size) 1289 lower_bound = base_tx_fee + round(size * relayfeerate) 1290 lower_bound = lower_bound if not is_local else 0 1291 return int(max(lower_bound, original_fee_estimator(size))) 1292 txi = base_tx.inputs() 1293 txo = list(filter(lambda o: not self.is_change(o.address), base_tx.outputs())) 1294 old_change_addrs = [o.address for o in base_tx.outputs() if self.is_change(o.address)] 1295 else: 1296 txi = [] 1297 txo = [] 1298 old_change_addrs = [] 1299 # change address. if empty, coin_chooser will set it 1300 change_addrs = self.get_change_addresses_for_new_transaction(change_addr or old_change_addrs) 1301 tx = coin_chooser.make_tx(coins=coins, 1302 inputs=txi, 1303 outputs=list(outputs) + txo, 1304 change_addrs=change_addrs, 1305 fee_estimator_vb=fee_estimator, 1306 dust_threshold=self.dust_threshold()) 1307 else: 1308 # "spend max" branch 1309 # note: This *will* spend inputs with negative effective value (if there are any). 1310 # Given as the user is spending "max", and so might be abandoning the wallet, 1311 # try to include all UTXOs, otherwise leftover might remain in the UTXO set 1312 # forever. see #5433 1313 # note: Actually it might be the case that not all UTXOs from the wallet are 1314 # being spent if the user manually selected UTXOs. 1315 sendable = sum(map(lambda c: c.value_sats(), coins)) 1316 outputs[i_max].value = 0 1317 tx = PartialTransaction.from_io(list(coins), list(outputs)) 1318 fee = fee_estimator(tx.estimated_size()) 1319 amount = sendable - tx.output_value() - fee 1320 if amount < 0: 1321 raise NotEnoughFunds() 1322 outputs[i_max].value = amount 1323 tx = PartialTransaction.from_io(list(coins), list(outputs)) 1324 1325 # Timelock tx to current height. 1326 tx.locktime = get_locktime_for_new_transaction(self.network) 1327 1328 tx.set_rbf(False) # caller can set RBF manually later 1329 tx.add_info_from_wallet(self) 1330 run_hook('make_unsigned_transaction', self, tx) 1331 return tx 1332 1333 def mktx(self, *, outputs: List[PartialTxOutput], password=None, fee=None, change_addr=None, 1334 domain=None, rbf=False, nonlocal_only=False, tx_version=None, sign=True) -> PartialTransaction: 1335 coins = self.get_spendable_coins(domain, nonlocal_only=nonlocal_only) 1336 tx = self.make_unsigned_transaction(coins=coins, 1337 outputs=outputs, 1338 fee=fee, 1339 change_addr=change_addr) 1340 tx.set_rbf(rbf) 1341 if tx_version is not None: 1342 tx.version = tx_version 1343 if sign: 1344 self.sign_transaction(tx, password) 1345 return tx 1346 1347 def is_frozen_address(self, addr: str) -> bool: 1348 return addr in self._frozen_addresses 1349 1350 def is_frozen_coin(self, utxo: PartialTxInput) -> bool: 1351 prevout_str = utxo.prevout.to_str() 1352 frozen = self._frozen_coins.get(prevout_str, None) 1353 # note: there are three possible states for 'frozen': 1354 # True/False if the user explicitly set it, 1355 # None otherwise 1356 if frozen is None: 1357 return self._is_coin_small_and_unconfirmed(utxo) 1358 return bool(frozen) 1359 1360 def _is_coin_small_and_unconfirmed(self, utxo: PartialTxInput) -> bool: 1361 """If true, the coin should not be spent. 1362 The idea here is that an attacker might send us a UTXO in a 1363 large low-fee unconfirmed tx that will ~never confirm. If we 1364 spend it as part of a tx ourselves, that too will not confirm 1365 (unless we use a high fee but that might not be worth it for 1366 a small value UTXO). 1367 In particular, this test triggers for large "dusting transactions" 1368 that are used for advertising purposes by some entities. 1369 see #6960 1370 """ 1371 # confirmed UTXOs are fine; check this first for performance: 1372 block_height = utxo.block_height 1373 assert block_height is not None 1374 if block_height > 0: 1375 return False 1376 # exempt large value UTXOs 1377 value_sats = utxo.value_sats() 1378 assert value_sats is not None 1379 threshold = self.config.get('unconf_utxo_freeze_threshold', 5_000) 1380 if value_sats >= threshold: 1381 return False 1382 # if funding tx has any is_mine input, then UTXO is fine 1383 funding_tx = self.db.get_transaction(utxo.prevout.txid.hex()) 1384 if funding_tx is None: 1385 # we should typically have the funding tx available; 1386 # might not have it e.g. while not up_to_date 1387 return True 1388 if any(self.is_mine(self.get_txin_address(txin)) 1389 for txin in funding_tx.inputs()): 1390 return False 1391 return True 1392 1393 def set_frozen_state_of_addresses(self, addrs: Sequence[str], freeze: bool) -> bool: 1394 """Set frozen state of the addresses to FREEZE, True or False""" 1395 if all(self.is_mine(addr) for addr in addrs): 1396 with self._freeze_lock: 1397 if freeze: 1398 self._frozen_addresses |= set(addrs) 1399 else: 1400 self._frozen_addresses -= set(addrs) 1401 self.db.put('frozen_addresses', list(self._frozen_addresses)) 1402 return True 1403 return False 1404 1405 def set_frozen_state_of_coins(self, utxos: Sequence[str], freeze: bool) -> None: 1406 """Set frozen state of the utxos to FREEZE, True or False""" 1407 # basic sanity check that input is not garbage: (see if raises) 1408 [TxOutpoint.from_str(utxo) for utxo in utxos] 1409 with self._freeze_lock: 1410 for utxo in utxos: 1411 self._frozen_coins[utxo] = bool(freeze) 1412 1413 def is_address_reserved(self, addr: str) -> bool: 1414 # note: atm 'reserved' status is only taken into consideration for 'change addresses' 1415 return addr in self._reserved_addresses 1416 1417 def set_reserved_state_of_address(self, addr: str, *, reserved: bool) -> None: 1418 if not self.is_mine(addr): 1419 return 1420 with self.lock: 1421 if reserved: 1422 self._reserved_addresses.add(addr) 1423 else: 1424 self._reserved_addresses.discard(addr) 1425 self.db.put('reserved_addresses', list(self._reserved_addresses)) 1426 1427 def can_export(self): 1428 return not self.is_watching_only() and hasattr(self.keystore, 'get_private_key') 1429 1430 def address_is_old(self, address: str, *, req_conf: int = 3) -> bool: 1431 """Returns whether address has any history that is deeply confirmed. 1432 Used for reorg-safe(ish) gap limit roll-forward. 1433 """ 1434 max_conf = -1 1435 h = self.db.get_addr_history(address) 1436 needs_spv_check = not self.config.get("skipmerklecheck", False) 1437 for tx_hash, tx_height in h: 1438 if needs_spv_check: 1439 tx_age = self.get_tx_height(tx_hash).conf 1440 else: 1441 if tx_height <= 0: 1442 tx_age = 0 1443 else: 1444 tx_age = self.get_local_height() - tx_height + 1 1445 max_conf = max(max_conf, tx_age) 1446 return max_conf >= req_conf 1447 1448 def bump_fee( 1449 self, 1450 *, 1451 tx: Transaction, 1452 txid: str = None, 1453 new_fee_rate: Union[int, float, Decimal], 1454 coins: Sequence[PartialTxInput] = None, 1455 strategies: Sequence[BumpFeeStrategy] = None, 1456 ) -> PartialTransaction: 1457 """Increase the miner fee of 'tx'. 1458 'new_fee_rate' is the target min rate in sat/vbyte 1459 'coins' is a list of UTXOs we can choose from as potential new inputs to be added 1460 """ 1461 txid = txid or tx.txid() 1462 assert txid 1463 assert tx.txid() in (None, txid) 1464 if not isinstance(tx, PartialTransaction): 1465 tx = PartialTransaction.from_tx(tx) 1466 assert isinstance(tx, PartialTransaction) 1467 tx.remove_signatures() 1468 if tx.is_final(): 1469 raise CannotBumpFee(_('Transaction is final')) 1470 new_fee_rate = quantize_feerate(new_fee_rate) # strip excess precision 1471 try: 1472 # note: this might download input utxos over network 1473 tx.add_info_from_wallet(self, ignore_network_issues=False) 1474 except NetworkException as e: 1475 raise CannotBumpFee(repr(e)) 1476 old_tx_size = tx.estimated_size() 1477 old_fee = tx.get_fee() 1478 assert old_fee is not None 1479 old_fee_rate = old_fee / old_tx_size # sat/vbyte 1480 if new_fee_rate <= old_fee_rate: 1481 raise CannotBumpFee(_("The new fee rate needs to be higher than the old fee rate.")) 1482 1483 if not strategies: 1484 strategies = [BumpFeeStrategy.COINCHOOSER, BumpFeeStrategy.DECREASE_CHANGE] 1485 tx_new = None 1486 exc = None 1487 for strat in strategies: 1488 try: 1489 if strat == BumpFeeStrategy.COINCHOOSER: 1490 tx_new = self._bump_fee_through_coinchooser( 1491 tx=tx, 1492 txid=txid, 1493 new_fee_rate=new_fee_rate, 1494 coins=coins, 1495 ) 1496 elif strat == BumpFeeStrategy.DECREASE_CHANGE: 1497 tx_new = self._bump_fee_through_decreasing_change( 1498 tx=tx, new_fee_rate=new_fee_rate) 1499 elif strat == BumpFeeStrategy.DECREASE_PAYMENT: 1500 tx_new = self._bump_fee_through_decreasing_payment( 1501 tx=tx, new_fee_rate=new_fee_rate) 1502 else: 1503 raise NotImplementedError(f"unexpected strategy: {strat}") 1504 except CannotBumpFee as e: 1505 exc = e 1506 else: 1507 strat_used = strat 1508 break 1509 if tx_new is None: 1510 assert exc 1511 raise exc # all strategies failed, re-raise last exception 1512 1513 target_min_fee = new_fee_rate * tx_new.estimated_size() 1514 actual_fee = tx_new.get_fee() 1515 if actual_fee + 1 < target_min_fee: 1516 raise CannotBumpFee( 1517 f"bump_fee fee target was not met (strategy: {strat_used}). " 1518 f"got {actual_fee}, expected >={target_min_fee}. " 1519 f"target rate was {new_fee_rate}") 1520 tx_new.locktime = get_locktime_for_new_transaction(self.network) 1521 tx_new.set_rbf(True) 1522 tx_new.add_info_from_wallet(self) 1523 return tx_new 1524 1525 def _bump_fee_through_coinchooser( 1526 self, 1527 *, 1528 tx: PartialTransaction, 1529 txid: str, 1530 new_fee_rate: Union[int, Decimal], 1531 coins: Sequence[PartialTxInput] = None, 1532 ) -> PartialTransaction: 1533 """Increase the miner fee of 'tx'. 1534 1535 - keeps all inputs 1536 - keeps all not is_mine outputs, 1537 - allows adding new inputs 1538 """ 1539 assert txid 1540 tx = copy.deepcopy(tx) 1541 tx.add_info_from_wallet(self) 1542 assert tx.get_fee() is not None 1543 old_inputs = list(tx.inputs()) 1544 old_outputs = list(tx.outputs()) 1545 # change address 1546 old_change_addrs = [o.address for o in old_outputs if self.is_change(o.address)] 1547 change_addrs = self.get_change_addresses_for_new_transaction(old_change_addrs) 1548 # which outputs to keep? 1549 if old_change_addrs: 1550 fixed_outputs = list(filter(lambda o: not self.is_change(o.address), old_outputs)) 1551 else: 1552 if all(self.is_mine(o.address) for o in old_outputs): 1553 # all outputs are is_mine and none of them are change. 1554 # we bail out as it's unclear what the user would want! 1555 # the coinchooser bump fee method is probably not a good idea in this case 1556 raise CannotBumpFee(_('All outputs are non-change is_mine')) 1557 old_not_is_mine = list(filter(lambda o: not self.is_mine(o.address), old_outputs)) 1558 if old_not_is_mine: 1559 fixed_outputs = old_not_is_mine 1560 else: 1561 fixed_outputs = old_outputs 1562 if not fixed_outputs: 1563 raise CannotBumpFee(_('Could not figure out which outputs to keep')) 1564 1565 if coins is None: 1566 coins = self.get_spendable_coins(None) 1567 # make sure we don't try to spend output from the tx-to-be-replaced: 1568 coins = [c for c in coins if c.prevout.txid.hex() != txid] 1569 for item in coins: 1570 self.add_input_info(item) 1571 def fee_estimator(size): 1572 return self.config.estimate_fee_for_feerate(fee_per_kb=new_fee_rate*1000, size=size) 1573 coin_chooser = coinchooser.get_coin_chooser(self.config) 1574 try: 1575 return coin_chooser.make_tx( 1576 coins=coins, 1577 inputs=old_inputs, 1578 outputs=fixed_outputs, 1579 change_addrs=change_addrs, 1580 fee_estimator_vb=fee_estimator, 1581 dust_threshold=self.dust_threshold()) 1582 except NotEnoughFunds as e: 1583 raise CannotBumpFee(e) 1584 1585 def _bump_fee_through_decreasing_change( 1586 self, 1587 *, 1588 tx: PartialTransaction, 1589 new_fee_rate: Union[int, Decimal], 1590 ) -> PartialTransaction: 1591 """Increase the miner fee of 'tx'. 1592 1593 - keeps all inputs 1594 - no new inputs are added 1595 - allows decreasing and removing outputs (change is decreased first) 1596 This is less "safe" than "coinchooser" method as it might end up decreasing 1597 e.g. a payment to a merchant; but e.g. if the user has sent "Max" previously, 1598 this is the only way to RBF. 1599 """ 1600 tx = copy.deepcopy(tx) 1601 tx.add_info_from_wallet(self) 1602 assert tx.get_fee() is not None 1603 inputs = tx.inputs() 1604 outputs = tx._outputs # note: we will mutate this directly 1605 1606 # use own outputs 1607 s = list(filter(lambda o: self.is_mine(o.address), outputs)) 1608 # ... unless there is none 1609 if not s: 1610 s = outputs 1611 x_fee = run_hook('get_tx_extra_fee', self, tx) 1612 if x_fee: 1613 x_fee_address, x_fee_amount = x_fee 1614 s = list(filter(lambda o: o.address != x_fee_address, s)) 1615 if not s: 1616 raise CannotBumpFee('No outputs at all??') 1617 1618 # prioritize low value outputs, to get rid of dust 1619 s = sorted(s, key=lambda o: o.value) 1620 for o in s: 1621 target_fee = int(math.ceil(tx.estimated_size() * new_fee_rate)) 1622 delta = target_fee - tx.get_fee() 1623 i = outputs.index(o) 1624 if o.value - delta >= self.dust_threshold(): 1625 new_output_value = o.value - delta 1626 assert isinstance(new_output_value, int) 1627 outputs[i].value = new_output_value 1628 delta = 0 1629 break 1630 else: 1631 del outputs[i] 1632 # note: we mutated the outputs of tx, which will affect 1633 # tx.estimated_size() in the next iteration 1634 if delta > 0: 1635 raise CannotBumpFee(_('Could not find suitable outputs')) 1636 1637 return PartialTransaction.from_io(inputs, outputs) 1638 1639 def _bump_fee_through_decreasing_payment( 1640 self, 1641 *, 1642 tx: PartialTransaction, 1643 new_fee_rate: Union[int, Decimal], 1644 ) -> PartialTransaction: 1645 """Increase the miner fee of 'tx'. 1646 1647 - keeps all inputs 1648 - no new inputs are added 1649 - decreases payment outputs (not change!). Each non-ismine output is decreased 1650 proportionally to their byte-size. 1651 """ 1652 tx = copy.deepcopy(tx) 1653 tx.add_info_from_wallet(self) 1654 assert tx.get_fee() is not None 1655 inputs = tx.inputs() 1656 outputs = tx.outputs() 1657 1658 # select non-ismine outputs 1659 s = [(idx, out) for (idx, out) in enumerate(outputs) 1660 if not self.is_mine(out.address)] 1661 # exempt 2fa fee output if present 1662 x_fee = run_hook('get_tx_extra_fee', self, tx) 1663 if x_fee: 1664 x_fee_address, x_fee_amount = x_fee 1665 s = [(idx, out) for (idx, out) in s if out.address != x_fee_address] 1666 if not s: 1667 raise CannotBumpFee("Cannot find payment output") 1668 1669 del_out_idxs = set() 1670 tx_size = tx.estimated_size() 1671 cur_fee = tx.get_fee() 1672 # Main loop. Each iteration decreases value of all selected outputs. 1673 # The number of iterations is bounded by len(s) as only the final iteration 1674 # can *not remove* any output. 1675 for __ in range(len(s) + 1): 1676 target_fee = int(math.ceil(tx_size * new_fee_rate)) 1677 delta_total = target_fee - cur_fee 1678 if delta_total <= 0: 1679 break 1680 out_size_total = sum(Transaction.estimated_output_size_for_script(out.scriptpubkey.hex()) 1681 for (idx, out) in s if idx not in del_out_idxs) 1682 for idx, out in s: 1683 out_size = Transaction.estimated_output_size_for_script(out.scriptpubkey.hex()) 1684 delta = int(math.ceil(delta_total * out_size / out_size_total)) 1685 if out.value - delta >= self.dust_threshold(): 1686 new_output_value = out.value - delta 1687 assert isinstance(new_output_value, int) 1688 outputs[idx].value = new_output_value 1689 cur_fee += delta 1690 else: # remove output 1691 tx_size -= out_size 1692 cur_fee += out.value 1693 del_out_idxs.add(idx) 1694 if delta_total > 0: 1695 raise CannotBumpFee(_('Could not find suitable outputs')) 1696 1697 outputs = [out for (idx, out) in enumerate(outputs) if idx not in del_out_idxs] 1698 return PartialTransaction.from_io(inputs, outputs) 1699 1700 def cpfp(self, tx: Transaction, fee: int) -> Optional[PartialTransaction]: 1701 txid = tx.txid() 1702 for i, o in enumerate(tx.outputs()): 1703 address, value = o.address, o.value 1704 if self.is_mine(address): 1705 break 1706 else: 1707 raise CannotCPFP(_("Could not find suitable output")) 1708 coins = self.get_addr_utxo(address) 1709 item = coins.get(TxOutpoint.from_str(txid+':%d'%i)) 1710 if not item: 1711 raise CannotCPFP(_("Could not find coins for output")) 1712 inputs = [item] 1713 out_address = (self.get_single_change_address_for_new_transaction(allow_reuse=False) 1714 or self.get_unused_address() 1715 or address) 1716 output_value = value - fee 1717 if output_value < self.dust_threshold(): 1718 raise CannotCPFP(_("The output value remaining after fee is too low.")) 1719 outputs = [PartialTxOutput.from_address_and_value(out_address, output_value)] 1720 locktime = get_locktime_for_new_transaction(self.network) 1721 tx_new = PartialTransaction.from_io(inputs, outputs, locktime=locktime) 1722 tx_new.set_rbf(True) 1723 tx_new.add_info_from_wallet(self) 1724 return tx_new 1725 1726 def dscancel( 1727 self, *, tx: Transaction, new_fee_rate: Union[int, float, Decimal] 1728 ) -> PartialTransaction: 1729 """Double-Spend-Cancel: cancel an unconfirmed tx by double-spending 1730 its inputs, paying ourselves. 1731 'new_fee_rate' is the target min rate in sat/vbyte 1732 """ 1733 if not isinstance(tx, PartialTransaction): 1734 tx = PartialTransaction.from_tx(tx) 1735 assert isinstance(tx, PartialTransaction) 1736 tx.remove_signatures() 1737 1738 if tx.is_final(): 1739 raise CannotDoubleSpendTx(_('Transaction is final')) 1740 new_fee_rate = quantize_feerate(new_fee_rate) # strip excess precision 1741 try: 1742 # note: this might download input utxos over network 1743 tx.add_info_from_wallet(self, ignore_network_issues=False) 1744 except NetworkException as e: 1745 raise CannotDoubleSpendTx(repr(e)) 1746 old_tx_size = tx.estimated_size() 1747 old_fee = tx.get_fee() 1748 assert old_fee is not None 1749 old_fee_rate = old_fee / old_tx_size # sat/vbyte 1750 if new_fee_rate <= old_fee_rate: 1751 raise CannotDoubleSpendTx(_("The new fee rate needs to be higher than the old fee rate.")) 1752 # grab all ismine inputs 1753 inputs = [txin for txin in tx.inputs() 1754 if self.is_mine(self.get_txin_address(txin))] 1755 value = sum([txin.value_sats() for txin in inputs]) 1756 # figure out output address 1757 old_change_addrs = [o.address for o in tx.outputs() if self.is_mine(o.address)] 1758 out_address = (self.get_single_change_address_for_new_transaction(old_change_addrs) 1759 or self.get_receiving_address()) 1760 locktime = get_locktime_for_new_transaction(self.network) 1761 outputs = [PartialTxOutput.from_address_and_value(out_address, value)] 1762 tx_new = PartialTransaction.from_io(inputs, outputs, locktime=locktime) 1763 new_tx_size = tx_new.estimated_size() 1764 new_fee = max( 1765 new_fee_rate * new_tx_size, 1766 old_fee + self.relayfee() * new_tx_size / Decimal(1000), # BIP-125 rules 3 and 4 1767 ) 1768 new_fee = int(math.ceil(new_fee)) 1769 output_value = value - new_fee 1770 if output_value < self.dust_threshold(): 1771 raise CannotDoubleSpendTx(_("The output value remaining after fee is too low.")) 1772 outputs = [PartialTxOutput.from_address_and_value(out_address, value - new_fee)] 1773 tx_new = PartialTransaction.from_io(inputs, outputs, locktime=locktime) 1774 tx_new.set_rbf(True) 1775 tx_new.add_info_from_wallet(self) 1776 return tx_new 1777 1778 @abstractmethod 1779 def _add_input_sig_info(self, txin: PartialTxInput, address: str, *, only_der_suffix: bool) -> None: 1780 pass 1781 1782 def _add_txinout_derivation_info(self, txinout: Union[PartialTxInput, PartialTxOutput], 1783 address: str, *, only_der_suffix: bool) -> None: 1784 pass # implemented by subclasses 1785 1786 def _add_input_utxo_info( 1787 self, 1788 txin: PartialTxInput, 1789 *, 1790 address: str = None, 1791 ignore_network_issues: bool = True, 1792 ) -> None: 1793 # We prefer to include UTXO (full tx) for every input. 1794 # We cannot include UTXO if the prev tx is not signed yet though (chain of unsigned txs), 1795 # in which case we might include a WITNESS_UTXO. 1796 address = address or txin.address 1797 if txin.witness_utxo is None and txin.is_segwit() and address: 1798 received, spent = self.get_addr_io(address) 1799 item = received.get(txin.prevout.to_str()) 1800 if item: 1801 txin_value = item[1] 1802 txin.witness_utxo = TxOutput.from_address_and_value(address, txin_value) 1803 if txin.utxo is None: 1804 txin.utxo = self.get_input_tx(txin.prevout.txid.hex(), ignore_network_issues=ignore_network_issues) 1805 txin.ensure_there_is_only_one_utxo() 1806 1807 def _learn_derivation_path_for_address_from_txinout(self, txinout: Union[PartialTxInput, PartialTxOutput], 1808 address: str) -> bool: 1809 """Tries to learn the derivation path for an address (potentially beyond gap limit) 1810 using data available in given txin/txout. 1811 Returns whether the address was found to be is_mine. 1812 """ 1813 return False # implemented by subclasses 1814 1815 def add_input_info( 1816 self, 1817 txin: PartialTxInput, 1818 *, 1819 only_der_suffix: bool = False, 1820 ignore_network_issues: bool = True, 1821 ) -> None: 1822 address = self.get_txin_address(txin) 1823 # note: we add input utxos regardless of is_mine 1824 self._add_input_utxo_info(txin, ignore_network_issues=ignore_network_issues, address=address) 1825 if not self.is_mine(address): 1826 is_mine = self._learn_derivation_path_for_address_from_txinout(txin, address) 1827 if not is_mine: 1828 return 1829 # set script_type first, as later checks might rely on it: 1830 txin.script_type = self.get_txin_type(address) 1831 txin.num_sig = self.m if isinstance(self, Multisig_Wallet) else 1 1832 if txin.redeem_script is None: 1833 try: 1834 redeem_script_hex = self.get_redeem_script(address) 1835 txin.redeem_script = bfh(redeem_script_hex) if redeem_script_hex else None 1836 except UnknownTxinType: 1837 pass 1838 if txin.witness_script is None: 1839 try: 1840 witness_script_hex = self.get_witness_script(address) 1841 txin.witness_script = bfh(witness_script_hex) if witness_script_hex else None 1842 except UnknownTxinType: 1843 pass 1844 self._add_input_sig_info(txin, address, only_der_suffix=only_der_suffix) 1845 1846 def can_sign(self, tx: Transaction) -> bool: 1847 if not isinstance(tx, PartialTransaction): 1848 return False 1849 if tx.is_complete(): 1850 return False 1851 # add info to inputs if we can; otherwise we might return a false negative: 1852 tx.add_info_from_wallet(self) 1853 for txin in tx.inputs(): 1854 # note: is_mine check needed to avoid false positives. 1855 # just because keystore could sign, txin does not necessarily belong to wallet. 1856 # Example: we have p2pkh-like addresses and txin is a multisig that involves our pubkey. 1857 if not self.is_mine(txin.address): 1858 continue 1859 for k in self.get_keystores(): 1860 if k.can_sign_txin(txin): 1861 return True 1862 return False 1863 1864 def get_input_tx(self, tx_hash: str, *, ignore_network_issues=False) -> Optional[Transaction]: 1865 # First look up an input transaction in the wallet where it 1866 # will likely be. If co-signing a transaction it may not have 1867 # all the input txs, in which case we ask the network. 1868 tx = self.db.get_transaction(tx_hash) 1869 if not tx and self.network and self.network.has_internet_connection(): 1870 try: 1871 raw_tx = self.network.run_from_another_thread( 1872 self.network.get_transaction(tx_hash, timeout=10)) 1873 except NetworkException as e: 1874 self.logger.info(f'got network error getting input txn. err: {repr(e)}. txid: {tx_hash}. ' 1875 f'if you are intentionally offline, consider using the --offline flag') 1876 if not ignore_network_issues: 1877 raise e 1878 else: 1879 tx = Transaction(raw_tx) 1880 if not tx and not ignore_network_issues: 1881 raise NetworkException('failed to get prev tx from network') 1882 return tx 1883 1884 def add_output_info(self, txout: PartialTxOutput, *, only_der_suffix: bool = False) -> None: 1885 address = txout.address 1886 if not self.is_mine(address): 1887 is_mine = self._learn_derivation_path_for_address_from_txinout(txout, address) 1888 if not is_mine: 1889 return 1890 txout.script_type = self.get_txin_type(address) 1891 txout.is_mine = True 1892 txout.is_change = self.is_change(address) 1893 if isinstance(self, Multisig_Wallet): 1894 txout.num_sig = self.m 1895 self._add_txinout_derivation_info(txout, address, only_der_suffix=only_der_suffix) 1896 if txout.redeem_script is None: 1897 try: 1898 redeem_script_hex = self.get_redeem_script(address) 1899 txout.redeem_script = bfh(redeem_script_hex) if redeem_script_hex else None 1900 except UnknownTxinType: 1901 pass 1902 if txout.witness_script is None: 1903 try: 1904 witness_script_hex = self.get_witness_script(address) 1905 txout.witness_script = bfh(witness_script_hex) if witness_script_hex else None 1906 except UnknownTxinType: 1907 pass 1908 1909 def sign_transaction(self, tx: Transaction, password) -> Optional[PartialTransaction]: 1910 if self.is_watching_only(): 1911 return 1912 if not isinstance(tx, PartialTransaction): 1913 return 1914 # add info to a temporary tx copy; including xpubs 1915 # and full derivation paths as hw keystores might want them 1916 tmp_tx = copy.deepcopy(tx) 1917 tmp_tx.add_info_from_wallet(self, include_xpubs=True) 1918 # sign. start with ready keystores. 1919 for k in sorted(self.get_keystores(), key=lambda ks: ks.ready_to_sign(), reverse=True): 1920 try: 1921 if k.can_sign(tmp_tx): 1922 k.sign_transaction(tmp_tx, password) 1923 except UserCancelled: 1924 continue 1925 # remove sensitive info; then copy back details from temporary tx 1926 tmp_tx.remove_xpubs_and_bip32_paths() 1927 tx.combine_with_other_psbt(tmp_tx) 1928 tx.add_info_from_wallet(self, include_xpubs=False) 1929 return tx 1930 1931 def try_detecting_internal_addresses_corruption(self) -> None: 1932 pass 1933 1934 def check_address_for_corruption(self, addr: str) -> None: 1935 pass 1936 1937 def get_unused_addresses(self) -> Sequence[str]: 1938 domain = self.get_receiving_addresses() 1939 # TODO we should index receive_requests by id 1940 in_use_by_request = [k for k in self.receive_requests.keys() 1941 if self.get_request_status(k) != PR_EXPIRED] 1942 in_use_by_request = set(in_use_by_request) 1943 return [addr for addr in domain if not self.is_used(addr) 1944 and addr not in in_use_by_request] 1945 1946 @check_returned_address_for_corruption 1947 def get_unused_address(self) -> Optional[str]: 1948 """Get an unused receiving address, if there is one. 1949 Note: there might NOT be one available! 1950 """ 1951 addrs = self.get_unused_addresses() 1952 if addrs: 1953 return addrs[0] 1954 1955 @check_returned_address_for_corruption 1956 def get_receiving_address(self) -> str: 1957 """Get a receiving address. Guaranteed to always return an address.""" 1958 unused_addr = self.get_unused_address() 1959 if unused_addr: 1960 return unused_addr 1961 domain = self.get_receiving_addresses() 1962 if not domain: 1963 raise Exception("no receiving addresses in wallet?!") 1964 choice = domain[0] 1965 for addr in domain: 1966 if not self.is_used(addr): 1967 if addr not in self.receive_requests.keys(): 1968 return addr 1969 else: 1970 choice = addr 1971 return choice 1972 1973 def create_new_address(self, for_change: bool = False): 1974 raise Exception("this wallet cannot generate new addresses") 1975 1976 def import_address(self, address: str) -> str: 1977 raise Exception("this wallet cannot import addresses") 1978 1979 def import_addresses(self, addresses: List[str], *, 1980 write_to_disk=True) -> Tuple[List[str], List[Tuple[str, str]]]: 1981 raise Exception("this wallet cannot import addresses") 1982 1983 def delete_address(self, address: str) -> None: 1984 raise Exception("this wallet cannot delete addresses") 1985 1986 def get_onchain_request_status(self, r): 1987 address = r.get_address() 1988 amount = r.get_amount_sat() 1989 received, sent = self.get_addr_io(address) 1990 l = [] 1991 for txo, x in received.items(): 1992 h, v, is_cb = x 1993 txid, n = txo.split(':') 1994 tx_height = self.get_tx_height(txid) 1995 height = tx_height.height 1996 if height > 0 and height <= r.height: 1997 continue 1998 conf = tx_height.conf 1999 l.append((conf, v)) 2000 vsum = 0 2001 for conf, v in reversed(sorted(l)): 2002 vsum += v 2003 if vsum >= amount: 2004 return True, conf 2005 return False, None 2006 2007 def get_request_URI(self, req: OnchainInvoice) -> str: 2008 addr = req.get_address() 2009 message = self.get_label(addr) 2010 amount = req.amount_sat 2011 extra_query_params = {} 2012 if req.time: 2013 extra_query_params['time'] = str(int(req.time)) 2014 if req.exp: 2015 extra_query_params['exp'] = str(int(req.exp)) 2016 #if req.get('name') and req.get('sig'): 2017 # sig = bfh(req.get('sig')) 2018 # sig = bitcoin.base_encode(sig, base=58) 2019 # extra_query_params['name'] = req['name'] 2020 # extra_query_params['sig'] = sig 2021 uri = create_bip21_uri(addr, amount, message, extra_query_params=extra_query_params) 2022 return str(uri) 2023 2024 def check_expired_status(self, r: Invoice, status): 2025 if r.is_lightning() and r.exp == 0: 2026 status = PR_EXPIRED # for BOLT-11 invoices, exp==0 means 0 seconds 2027 if status == PR_UNPAID and r.exp > 0 and r.time + r.exp < time.time(): 2028 status = PR_EXPIRED 2029 return status 2030 2031 def get_invoice_status(self, invoice: Invoice): 2032 if invoice.is_lightning(): 2033 status = self.lnworker.get_invoice_status(invoice) if self.lnworker else PR_UNKNOWN 2034 else: 2035 if self.is_onchain_invoice_paid(invoice, 1): 2036 status =PR_PAID 2037 elif self.is_onchain_invoice_paid(invoice, 0): 2038 status = PR_UNCONFIRMED 2039 else: 2040 status = PR_UNPAID 2041 return self.check_expired_status(invoice, status) 2042 2043 def get_request_status(self, key): 2044 r = self.get_request(key) 2045 if r is None: 2046 return PR_UNKNOWN 2047 if r.is_lightning(): 2048 assert isinstance(r, LNInvoice) 2049 status = self.lnworker.get_payment_status(bfh(r.rhash)) if self.lnworker else PR_UNKNOWN 2050 else: 2051 assert isinstance(r, OnchainInvoice) 2052 paid, conf = self.get_onchain_request_status(r) 2053 if not paid: 2054 status = PR_UNPAID 2055 elif conf == 0: 2056 status = PR_UNCONFIRMED 2057 else: 2058 status = PR_PAID 2059 return self.check_expired_status(r, status) 2060 2061 def get_request(self, key): 2062 return self.receive_requests.get(key) 2063 2064 def get_formatted_request(self, key): 2065 x = self.receive_requests.get(key) 2066 if x: 2067 return self.export_request(x) 2068 2069 def export_request(self, x: Invoice) -> Dict[str, Any]: 2070 key = self.get_key_for_receive_request(x) 2071 status = self.get_request_status(key) 2072 status_str = x.get_status_str(status) 2073 is_lightning = x.is_lightning() 2074 d = { 2075 'is_lightning': is_lightning, 2076 'amount_BTC': format_satoshis(x.get_amount_sat()), 2077 'message': x.message, 2078 'timestamp': x.time, 2079 'expiration': x.exp, 2080 'status': status, 2081 'status_str': status_str, 2082 } 2083 if is_lightning: 2084 assert isinstance(x, LNInvoice) 2085 d['rhash'] = x.rhash 2086 d['invoice'] = x.invoice 2087 d['amount_msat'] = x.get_amount_msat() 2088 if self.lnworker and status == PR_UNPAID: 2089 d['can_receive'] = self.lnworker.can_receive_invoice(x) 2090 else: 2091 assert isinstance(x, OnchainInvoice) 2092 paid, conf = self.get_onchain_request_status(x) 2093 d['amount_sat'] = x.get_amount_sat() 2094 d['address'] = x.get_address() 2095 d['URI'] = self.get_request_URI(x) 2096 if conf is not None: 2097 d['confirmations'] = conf 2098 # add URL if we are running a payserver 2099 payserver = self.config.get_netaddress('payserver_address') 2100 if payserver: 2101 root = self.config.get('payserver_root', '/r') 2102 use_ssl = bool(self.config.get('ssl_keyfile')) 2103 protocol = 'https' if use_ssl else 'http' 2104 base = '%s://%s:%d'%(protocol, payserver.host, payserver.port) 2105 d['view_url'] = base + root + '/pay?id=' + key 2106 if use_ssl and 'URI' in d: 2107 request_url = base + '/bip70/' + key + '.bip70' 2108 d['bip70_url'] = request_url 2109 return d 2110 2111 def export_invoice(self, x: Invoice) -> Dict[str, Any]: 2112 status = self.get_invoice_status(x) 2113 status_str = x.get_status_str(status) 2114 is_lightning = x.is_lightning() 2115 d = { 2116 'is_lightning': is_lightning, 2117 'amount_BTC': format_satoshis(x.get_amount_sat()), 2118 'message': x.message, 2119 'timestamp': x.time, 2120 'expiration': x.exp, 2121 'status': status, 2122 'status_str': status_str, 2123 } 2124 if is_lightning: 2125 assert isinstance(x, LNInvoice) 2126 d['invoice'] = x.invoice 2127 d['amount_msat'] = x.get_amount_msat() 2128 if self.lnworker and status == PR_UNPAID: 2129 d['can_pay'] = self.lnworker.can_pay_invoice(x) 2130 else: 2131 assert isinstance(x, OnchainInvoice) 2132 amount_sat = x.get_amount_sat() 2133 assert isinstance(amount_sat, (int, str, type(None))) 2134 d['amount_sat'] = amount_sat 2135 d['outputs'] = [y.to_legacy_tuple() for y in x.outputs] 2136 if x.bip70: 2137 d['bip70'] = x.bip70 2138 d['requestor'] = x.requestor 2139 return d 2140 2141 def receive_tx_callback(self, tx_hash, tx, tx_height): 2142 super().receive_tx_callback(tx_hash, tx, tx_height) 2143 for txo in tx.outputs(): 2144 addr = self.get_txout_address(txo) 2145 if addr in self.receive_requests: 2146 status = self.get_request_status(addr) 2147 util.trigger_callback('request_status', self, addr, status) 2148 2149 def make_payment_request(self, address, amount_sat, message, expiration): 2150 # TODO maybe merge with wallet.create_invoice()... 2151 # note that they use incompatible "id" 2152 amount_sat = amount_sat or 0 2153 timestamp = int(time.time()) 2154 _id = bh2u(sha256d(address + "%d"%timestamp))[0:10] 2155 expiration = expiration or 0 2156 return OnchainInvoice( 2157 type=PR_TYPE_ONCHAIN, 2158 outputs=[(TYPE_ADDRESS, address, amount_sat)], 2159 message=message, 2160 time=timestamp, 2161 amount_sat=amount_sat, 2162 exp=expiration, 2163 id=_id, 2164 bip70=None, 2165 requestor=None, 2166 height=self.get_local_height(), 2167 ) 2168 2169 def sign_payment_request(self, key, alias, alias_addr, password): # FIXME this is broken 2170 req = self.receive_requests.get(key) 2171 assert isinstance(req, OnchainInvoice) 2172 alias_privkey = self.export_private_key(alias_addr, password) 2173 pr = paymentrequest.make_unsigned_request(req) 2174 paymentrequest.sign_request_with_alias(pr, alias, alias_privkey) 2175 req.bip70 = pr.raw.hex() 2176 req['name'] = pr.pki_data 2177 req['sig'] = bh2u(pr.signature) 2178 self.receive_requests[key] = req 2179 2180 @classmethod 2181 def get_key_for_outgoing_invoice(cls, invoice: Invoice) -> str: 2182 """Return the key to use for this invoice in self.invoices.""" 2183 if invoice.is_lightning(): 2184 assert isinstance(invoice, LNInvoice) 2185 key = invoice.rhash 2186 else: 2187 assert isinstance(invoice, OnchainInvoice) 2188 key = invoice.id 2189 return key 2190 2191 def get_key_for_receive_request(self, req: Invoice, *, sanity_checks: bool = False) -> str: 2192 """Return the key to use for this invoice in self.receive_requests.""" 2193 if not req.is_lightning(): 2194 assert isinstance(req, OnchainInvoice) 2195 addr = req.get_address() 2196 if sanity_checks: 2197 if not bitcoin.is_address(addr): 2198 raise Exception(_('Invalid Bitcoin address.')) 2199 if not self.is_mine(addr): 2200 raise Exception(_('Address not in wallet.')) 2201 key = addr 2202 else: 2203 assert isinstance(req, LNInvoice) 2204 key = req.rhash 2205 return key 2206 2207 def add_payment_request(self, req: Invoice): 2208 key = self.get_key_for_receive_request(req, sanity_checks=True) 2209 message = req.message 2210 self.receive_requests[key] = req 2211 self.set_label(key, message) # should be a default label 2212 return req 2213 2214 def delete_request(self, key): 2215 """ lightning or on-chain """ 2216 if key in self.receive_requests: 2217 self.remove_payment_request(key) 2218 elif self.lnworker: 2219 self.lnworker.delete_payment(key) 2220 2221 def delete_invoice(self, key): 2222 """ lightning or on-chain """ 2223 if key in self.invoices: 2224 self.invoices.pop(key) 2225 elif self.lnworker: 2226 self.lnworker.delete_payment(key) 2227 2228 def remove_payment_request(self, addr): 2229 if addr not in self.receive_requests: 2230 return False 2231 self.receive_requests.pop(addr) 2232 return True 2233 2234 def get_sorted_requests(self) -> List[Invoice]: 2235 """ sorted by timestamp """ 2236 out = [self.get_request(x) for x in self.receive_requests.keys()] 2237 out = [x for x in out if x is not None] 2238 out.sort(key=lambda x: x.time) 2239 return out 2240 2241 def get_unpaid_requests(self): 2242 out = [self.get_request(x) for x in self.receive_requests.keys() if self.get_request_status(x) != PR_PAID] 2243 out = [x for x in out if x is not None] 2244 out.sort(key=lambda x: x.time) 2245 return out 2246 2247 @abstractmethod 2248 def get_fingerprint(self) -> str: 2249 """Returns a string that can be used to identify this wallet. 2250 Used e.g. by Labels plugin, and LN channel backups. 2251 Returns empty string "" for wallets that don't have an ID. 2252 """ 2253 pass 2254 2255 def can_import_privkey(self): 2256 return False 2257 2258 def can_import_address(self): 2259 return False 2260 2261 def can_delete_address(self): 2262 return False 2263 2264 def has_password(self): 2265 return self.has_keystore_encryption() or self.has_storage_encryption() 2266 2267 def can_have_keystore_encryption(self): 2268 return self.keystore and self.keystore.may_have_password() 2269 2270 def get_available_storage_encryption_version(self) -> StorageEncryptionVersion: 2271 """Returns the type of storage encryption offered to the user. 2272 2273 A wallet file (storage) is either encrypted with this version 2274 or is stored in plaintext. 2275 """ 2276 if isinstance(self.keystore, Hardware_KeyStore): 2277 return StorageEncryptionVersion.XPUB_PASSWORD 2278 else: 2279 return StorageEncryptionVersion.USER_PASSWORD 2280 2281 def has_keystore_encryption(self): 2282 """Returns whether encryption is enabled for the keystore. 2283 2284 If True, e.g. signing a transaction will require a password. 2285 """ 2286 if self.can_have_keystore_encryption(): 2287 return self.db.get('use_encryption', False) 2288 return False 2289 2290 def has_storage_encryption(self): 2291 """Returns whether encryption is enabled for the wallet file on disk.""" 2292 return self.storage and self.storage.is_encrypted() 2293 2294 @classmethod 2295 def may_have_password(cls): 2296 return True 2297 2298 def check_password(self, password): 2299 if self.has_keystore_encryption(): 2300 self.keystore.check_password(password) 2301 if self.has_storage_encryption(): 2302 self.storage.check_password(password) 2303 2304 def update_password(self, old_pw, new_pw, *, encrypt_storage: bool = True): 2305 if old_pw is None and self.has_password(): 2306 raise InvalidPassword() 2307 self.check_password(old_pw) 2308 if self.storage: 2309 if encrypt_storage: 2310 enc_version = self.get_available_storage_encryption_version() 2311 else: 2312 enc_version = StorageEncryptionVersion.PLAINTEXT 2313 self.storage.set_password(new_pw, enc_version) 2314 # make sure next storage.write() saves changes 2315 self.db.set_modified(True) 2316 2317 # note: Encrypting storage with a hw device is currently only 2318 # allowed for non-multisig wallets. Further, 2319 # Hardware_KeyStore.may_have_password() == False. 2320 # If these were not the case, 2321 # extra care would need to be taken when encrypting keystores. 2322 self._update_password_for_keystore(old_pw, new_pw) 2323 encrypt_keystore = self.can_have_keystore_encryption() 2324 self.db.set_keystore_encryption(bool(new_pw) and encrypt_keystore) 2325 self.save_db() 2326 2327 @abstractmethod 2328 def _update_password_for_keystore(self, old_pw: Optional[str], new_pw: Optional[str]) -> None: 2329 pass 2330 2331 def sign_message(self, address, message, password): 2332 index = self.get_address_index(address) 2333 return self.keystore.sign_message(index, message, password) 2334 2335 def decrypt_message(self, pubkey: str, message, password) -> bytes: 2336 addr = self.pubkeys_to_address([pubkey]) 2337 index = self.get_address_index(addr) 2338 return self.keystore.decrypt_message(index, message, password) 2339 2340 @abstractmethod 2341 def pubkeys_to_address(self, pubkeys: Sequence[str]) -> Optional[str]: 2342 pass 2343 2344 def price_at_timestamp(self, txid, price_func): 2345 """Returns fiat price of bitcoin at the time tx got confirmed.""" 2346 timestamp = self.get_tx_height(txid).timestamp 2347 return price_func(timestamp if timestamp else time.time()) 2348 2349 def unrealized_gains(self, domain, price_func, ccy): 2350 coins = self.get_utxos(domain) 2351 now = time.time() 2352 p = price_func(now) 2353 ap = sum(self.coin_price(coin.prevout.txid.hex(), price_func, ccy, self.get_txin_value(coin)) for coin in coins) 2354 lp = sum([coin.value_sats() for coin in coins]) * p / Decimal(COIN) 2355 return lp - ap 2356 2357 def average_price(self, txid, price_func, ccy) -> Decimal: 2358 """ Average acquisition price of the inputs of a transaction """ 2359 input_value = 0 2360 total_price = 0 2361 txi_addresses = self.db.get_txi_addresses(txid) 2362 if not txi_addresses: 2363 return Decimal('NaN') 2364 for addr in txi_addresses: 2365 d = self.db.get_txi_addr(txid, addr) 2366 for ser, v in d: 2367 input_value += v 2368 total_price += self.coin_price(ser.split(':')[0], price_func, ccy, v) 2369 return total_price / (input_value/Decimal(COIN)) 2370 2371 def clear_coin_price_cache(self): 2372 self._coin_price_cache = {} 2373 2374 def coin_price(self, txid, price_func, ccy, txin_value) -> Decimal: 2375 """ 2376 Acquisition price of a coin. 2377 This assumes that either all inputs are mine, or no input is mine. 2378 """ 2379 if txin_value is None: 2380 return Decimal('NaN') 2381 cache_key = "{}:{}:{}".format(str(txid), str(ccy), str(txin_value)) 2382 result = self._coin_price_cache.get(cache_key, None) 2383 if result is not None: 2384 return result 2385 if self.db.get_txi_addresses(txid): 2386 result = self.average_price(txid, price_func, ccy) * txin_value/Decimal(COIN) 2387 self._coin_price_cache[cache_key] = result 2388 return result 2389 else: 2390 fiat_value = self.get_fiat_value(txid, ccy) 2391 if fiat_value is not None: 2392 return fiat_value 2393 else: 2394 p = self.price_at_timestamp(txid, price_func) 2395 return p * txin_value/Decimal(COIN) 2396 2397 def is_billing_address(self, addr): 2398 # overridden for TrustedCoin wallets 2399 return False 2400 2401 @abstractmethod 2402 def is_watching_only(self) -> bool: 2403 pass 2404 2405 def get_keystore(self) -> Optional[KeyStore]: 2406 return self.keystore 2407 2408 def get_keystores(self) -> Sequence[KeyStore]: 2409 return [self.keystore] if self.keystore else [] 2410 2411 @abstractmethod 2412 def save_keystore(self): 2413 pass 2414 2415 @abstractmethod 2416 def has_seed(self) -> bool: 2417 pass 2418 2419 @abstractmethod 2420 def get_all_known_addresses_beyond_gap_limit(self) -> Set[str]: 2421 pass 2422 2423 def create_transaction(self, outputs, *, fee=None, feerate=None, change_addr=None, domain_addr=None, domain_coins=None, 2424 unsigned=False, rbf=None, password=None, locktime=None): 2425 if fee is not None and feerate is not None: 2426 raise Exception("Cannot specify both 'fee' and 'feerate' at the same time!") 2427 coins = self.get_spendable_coins(domain_addr) 2428 if domain_coins is not None: 2429 coins = [coin for coin in coins if (coin.prevout.to_str() in domain_coins)] 2430 if feerate is not None: 2431 fee_per_kb = 1000 * Decimal(feerate) 2432 fee_estimator = partial(SimpleConfig.estimate_fee_for_feerate, fee_per_kb) 2433 else: 2434 fee_estimator = fee 2435 tx = self.make_unsigned_transaction( 2436 coins=coins, 2437 outputs=outputs, 2438 fee=fee_estimator, 2439 change_addr=change_addr) 2440 if locktime is not None: 2441 tx.locktime = locktime 2442 if rbf is None: 2443 rbf = bool(self.config.get('use_rbf', True)) 2444 tx.set_rbf(rbf) 2445 if not unsigned: 2446 self.sign_transaction(tx, password) 2447 return tx 2448 2449 def get_warning_for_risk_of_burning_coins_as_fees(self, tx: 'PartialTransaction') -> Optional[str]: 2450 """Returns a warning message if there is risk of burning coins as fees if we sign. 2451 Note that if not all inputs are ismine, e.g. coinjoin, the risk is not just about fees. 2452 2453 Note: 2454 - legacy sighash does not commit to any input amounts 2455 - BIP-0143 sighash only commits to the *corresponding* input amount 2456 - BIP-taproot sighash commits to *all* input amounts 2457 """ 2458 assert isinstance(tx, PartialTransaction) 2459 # if we have all full previous txs, we *know* all the input amounts -> fine 2460 if all([txin.utxo for txin in tx.inputs()]): 2461 return None 2462 # a single segwit input -> fine 2463 if len(tx.inputs()) == 1 and tx.inputs()[0].is_segwit() and tx.inputs()[0].witness_utxo: 2464 return None 2465 # coinjoin or similar 2466 if any([not self.is_mine(txin.address) for txin in tx.inputs()]): 2467 return (_("Warning") + ": " 2468 + _("The input amounts could not be verified as the previous transactions are missing.\n" 2469 "The amount of money being spent CANNOT be verified.")) 2470 # some inputs are legacy 2471 if any([not txin.is_segwit() for txin in tx.inputs()]): 2472 return (_("Warning") + ": " 2473 + _("The fee could not be verified. Signing non-segwit inputs is risky:\n" 2474 "if this transaction was maliciously modified before you sign,\n" 2475 "you might end up paying a higher mining fee than displayed.")) 2476 # all inputs are segwit 2477 # https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2017-August/014843.html 2478 return (_("Warning") + ": " 2479 + _("If you received this transaction from an untrusted device, " 2480 "do not accept to sign it more than once,\n" 2481 "otherwise you could end up paying a different fee.")) 2482 2483 def get_tx_fee_warning( 2484 self, 2485 *, 2486 invoice_amt: int, 2487 tx_size: int, 2488 fee: int, 2489 ) -> Optional[Tuple[bool, str, str]]: 2490 feerate = Decimal(fee) / tx_size # sat/byte 2491 fee_ratio = Decimal(fee) / invoice_amt if invoice_amt else 1 2492 long_warning = None 2493 short_warning = None 2494 allow_send = True 2495 if feerate < self.relayfee() / 1000: 2496 long_warning = ( 2497 _("This transaction requires a higher fee, or it will not be propagated by your current server") + "\n" 2498 + _("Try to raise your transaction fee, or use a server with a lower relay fee.") 2499 ) 2500 short_warning = _("below relay fee") + "!" 2501 allow_send = False 2502 elif fee_ratio >= FEE_RATIO_HIGH_WARNING: 2503 long_warning = ( 2504 _('Warning') + ': ' + _("The fee for this transaction seems unusually high.") 2505 + f'\n({fee_ratio*100:.2f}% of amount)') 2506 short_warning = _("high fee ratio") + "!" 2507 elif feerate > FEERATE_WARNING_HIGH_FEE / 1000: 2508 long_warning = ( 2509 _('Warning') + ': ' + _("The fee for this transaction seems unusually high.") 2510 + f'\n(feerate: {feerate:.2f} sat/byte)') 2511 short_warning = _("high fee rate") + "!" 2512 if long_warning is None: 2513 return None 2514 else: 2515 return allow_send, long_warning, short_warning 2516 2517 2518 class Simple_Wallet(Abstract_Wallet): 2519 # wallet with a single keystore 2520 2521 def is_watching_only(self): 2522 return self.keystore.is_watching_only() 2523 2524 def _update_password_for_keystore(self, old_pw, new_pw): 2525 if self.keystore and self.keystore.may_have_password(): 2526 self.keystore.update_password(old_pw, new_pw) 2527 self.save_keystore() 2528 2529 def save_keystore(self): 2530 self.db.put('keystore', self.keystore.dump()) 2531 2532 @abstractmethod 2533 def get_public_key(self, address: str) -> Optional[str]: 2534 pass 2535 2536 def get_public_keys(self, address: str) -> Sequence[str]: 2537 return [self.get_public_key(address)] 2538 2539 def get_redeem_script(self, address: str) -> Optional[str]: 2540 txin_type = self.get_txin_type(address) 2541 if txin_type in ('p2pkh', 'p2wpkh', 'p2pk'): 2542 return None 2543 if txin_type == 'p2wpkh-p2sh': 2544 pubkey = self.get_public_key(address) 2545 return bitcoin.p2wpkh_nested_script(pubkey) 2546 if txin_type == 'address': 2547 return None 2548 raise UnknownTxinType(f'unexpected txin_type {txin_type}') 2549 2550 def get_witness_script(self, address: str) -> Optional[str]: 2551 return None 2552 2553 2554 class Imported_Wallet(Simple_Wallet): 2555 # wallet made of imported addresses 2556 2557 wallet_type = 'imported' 2558 txin_type = 'address' 2559 2560 def __init__(self, db, storage, *, config): 2561 Abstract_Wallet.__init__(self, db, storage, config=config) 2562 2563 def is_watching_only(self): 2564 return self.keystore is None 2565 2566 def can_import_privkey(self): 2567 return bool(self.keystore) 2568 2569 def load_keystore(self): 2570 self.keystore = load_keystore(self.db, 'keystore') if self.db.get('keystore') else None 2571 2572 def save_keystore(self): 2573 self.db.put('keystore', self.keystore.dump()) 2574 2575 def can_import_address(self): 2576 return self.is_watching_only() 2577 2578 def can_delete_address(self): 2579 return True 2580 2581 def has_seed(self): 2582 return False 2583 2584 def is_deterministic(self): 2585 return False 2586 2587 def is_change(self, address): 2588 return False 2589 2590 def get_all_known_addresses_beyond_gap_limit(self) -> Set[str]: 2591 return set() 2592 2593 def get_fingerprint(self): 2594 return '' 2595 2596 def get_addresses(self): 2597 # note: overridden so that the history can be cleared 2598 return self.db.get_imported_addresses() 2599 2600 def get_receiving_addresses(self, **kwargs): 2601 return self.get_addresses() 2602 2603 def get_change_addresses(self, **kwargs): 2604 return [] 2605 2606 def import_addresses(self, addresses: List[str], *, 2607 write_to_disk=True) -> Tuple[List[str], List[Tuple[str, str]]]: 2608 good_addr = [] # type: List[str] 2609 bad_addr = [] # type: List[Tuple[str, str]] 2610 for address in addresses: 2611 if not bitcoin.is_address(address): 2612 bad_addr.append((address, _('invalid address'))) 2613 continue 2614 if self.db.has_imported_address(address): 2615 bad_addr.append((address, _('address already in wallet'))) 2616 continue 2617 good_addr.append(address) 2618 self.db.add_imported_address(address, {}) 2619 self.add_address(address) 2620 if write_to_disk: 2621 self.save_db() 2622 return good_addr, bad_addr 2623 2624 def import_address(self, address: str) -> str: 2625 good_addr, bad_addr = self.import_addresses([address]) 2626 if good_addr and good_addr[0] == address: 2627 return address 2628 else: 2629 raise BitcoinException(str(bad_addr[0][1])) 2630 2631 def delete_address(self, address: str) -> None: 2632 if not self.db.has_imported_address(address): 2633 return 2634 if len(self.get_addresses()) <= 1: 2635 raise UserFacingException("cannot delete last remaining address from wallet") 2636 transactions_to_remove = set() # only referred to by this address 2637 transactions_new = set() # txs that are not only referred to by address 2638 with self.lock: 2639 for addr in self.db.get_history(): 2640 details = self.get_address_history(addr) 2641 if addr == address: 2642 for tx_hash, height in details: 2643 transactions_to_remove.add(tx_hash) 2644 else: 2645 for tx_hash, height in details: 2646 transactions_new.add(tx_hash) 2647 transactions_to_remove -= transactions_new 2648 self.db.remove_addr_history(address) 2649 for tx_hash in transactions_to_remove: 2650 self.remove_transaction(tx_hash) 2651 self.set_label(address, None) 2652 self.remove_payment_request(address) 2653 self.set_frozen_state_of_addresses([address], False) 2654 pubkey = self.get_public_key(address) 2655 self.db.remove_imported_address(address) 2656 if pubkey: 2657 # delete key iff no other address uses it (e.g. p2pkh and p2wpkh for same key) 2658 for txin_type in bitcoin.WIF_SCRIPT_TYPES.keys(): 2659 try: 2660 addr2 = bitcoin.pubkey_to_address(txin_type, pubkey) 2661 except NotImplementedError: 2662 pass 2663 else: 2664 if self.db.has_imported_address(addr2): 2665 break 2666 else: 2667 self.keystore.delete_imported_key(pubkey) 2668 self.save_keystore() 2669 self.save_db() 2670 2671 def is_mine(self, address) -> bool: 2672 if not address: return False 2673 return self.db.has_imported_address(address) 2674 2675 def get_address_index(self, address) -> Optional[str]: 2676 # returns None if address is not mine 2677 return self.get_public_key(address) 2678 2679 def get_address_path_str(self, address): 2680 return None 2681 2682 def get_public_key(self, address) -> Optional[str]: 2683 x = self.db.get_imported_address(address) 2684 return x.get('pubkey') if x else None 2685 2686 def import_private_keys(self, keys: List[str], password: Optional[str], *, 2687 write_to_disk=True) -> Tuple[List[str], List[Tuple[str, str]]]: 2688 good_addr = [] # type: List[str] 2689 bad_keys = [] # type: List[Tuple[str, str]] 2690 for key in keys: 2691 try: 2692 txin_type, pubkey = self.keystore.import_privkey(key, password) 2693 except Exception as e: 2694 bad_keys.append((key, _('invalid private key') + f': {e}')) 2695 continue 2696 if txin_type not in ('p2pkh', 'p2wpkh', 'p2wpkh-p2sh'): 2697 bad_keys.append((key, _('not implemented type') + f': {txin_type}')) 2698 continue 2699 addr = bitcoin.pubkey_to_address(txin_type, pubkey) 2700 good_addr.append(addr) 2701 self.db.add_imported_address(addr, {'type':txin_type, 'pubkey':pubkey}) 2702 self.add_address(addr) 2703 self.save_keystore() 2704 if write_to_disk: 2705 self.save_db() 2706 return good_addr, bad_keys 2707 2708 def import_private_key(self, key: str, password: Optional[str]) -> str: 2709 good_addr, bad_keys = self.import_private_keys([key], password=password) 2710 if good_addr: 2711 return good_addr[0] 2712 else: 2713 raise BitcoinException(str(bad_keys[0][1])) 2714 2715 def get_txin_type(self, address): 2716 return self.db.get_imported_address(address).get('type', 'address') 2717 2718 def _add_input_sig_info(self, txin, address, *, only_der_suffix): 2719 if not self.is_mine(address): 2720 return 2721 if txin.script_type in ('unknown', 'address'): 2722 return 2723 elif txin.script_type in ('p2pkh', 'p2wpkh', 'p2wpkh-p2sh'): 2724 pubkey = self.get_public_key(address) 2725 if not pubkey: 2726 return 2727 txin.pubkeys = [bfh(pubkey)] 2728 else: 2729 raise Exception(f'Unexpected script type: {txin.script_type}. ' 2730 f'Imported wallets are not implemented to handle this.') 2731 2732 def pubkeys_to_address(self, pubkeys): 2733 pubkey = pubkeys[0] 2734 for addr in self.db.get_imported_addresses(): # FIXME slow... 2735 if self.db.get_imported_address(addr)['pubkey'] == pubkey: 2736 return addr 2737 return None 2738 2739 def decrypt_message(self, pubkey: str, message, password) -> bytes: 2740 # this is significantly faster than the implementation in the superclass 2741 return self.keystore.decrypt_message(pubkey, message, password) 2742 2743 2744 class Deterministic_Wallet(Abstract_Wallet): 2745 2746 def __init__(self, db, storage, *, config): 2747 self._ephemeral_addr_to_addr_index = {} # type: Dict[str, Sequence[int]] 2748 Abstract_Wallet.__init__(self, db, storage, config=config) 2749 self.gap_limit = db.get('gap_limit', 20) 2750 # generate addresses now. note that without libsecp this might block 2751 # for a few seconds! 2752 self.synchronize() 2753 2754 # create lightning keys 2755 if self.can_have_lightning(): 2756 self.init_lightning() 2757 ln_xprv = self.db.get('lightning_privkey2') 2758 # lnworker can only be initialized once receiving addresses are available 2759 # therefore we instantiate lnworker in DeterministicWallet 2760 self.lnworker = LNWallet(self, ln_xprv) if ln_xprv else None 2761 2762 def has_seed(self): 2763 return self.keystore.has_seed() 2764 2765 def get_addresses(self): 2766 # note: overridden so that the history can be cleared. 2767 # addresses are ordered based on derivation 2768 out = self.get_receiving_addresses() 2769 out += self.get_change_addresses() 2770 return out 2771 2772 def get_receiving_addresses(self, *, slice_start=None, slice_stop=None): 2773 return self.db.get_receiving_addresses(slice_start=slice_start, slice_stop=slice_stop) 2774 2775 def get_change_addresses(self, *, slice_start=None, slice_stop=None): 2776 return self.db.get_change_addresses(slice_start=slice_start, slice_stop=slice_stop) 2777 2778 @profiler 2779 def try_detecting_internal_addresses_corruption(self): 2780 addresses_all = self.get_addresses() 2781 # sample 1: first few 2782 addresses_sample1 = addresses_all[:10] 2783 # sample2: a few more randomly selected 2784 addresses_rand = addresses_all[10:] 2785 addresses_sample2 = random.sample(addresses_rand, min(len(addresses_rand), 10)) 2786 for addr_found in itertools.chain(addresses_sample1, addresses_sample2): 2787 self.check_address_for_corruption(addr_found) 2788 2789 def check_address_for_corruption(self, addr): 2790 if addr and self.is_mine(addr): 2791 if addr != self.derive_address(*self.get_address_index(addr)): 2792 raise InternalAddressCorruption() 2793 2794 def get_seed(self, password): 2795 return self.keystore.get_seed(password) 2796 2797 def change_gap_limit(self, value): 2798 '''This method is not called in the code, it is kept for console use''' 2799 value = int(value) 2800 if value >= self.min_acceptable_gap(): 2801 self.gap_limit = value 2802 self.db.put('gap_limit', self.gap_limit) 2803 self.save_db() 2804 return True 2805 else: 2806 return False 2807 2808 def num_unused_trailing_addresses(self, addresses): 2809 k = 0 2810 for addr in addresses[::-1]: 2811 if self.db.get_addr_history(addr): 2812 break 2813 k += 1 2814 return k 2815 2816 def min_acceptable_gap(self) -> int: 2817 # fixme: this assumes wallet is synchronized 2818 n = 0 2819 nmax = 0 2820 addresses = self.get_receiving_addresses() 2821 k = self.num_unused_trailing_addresses(addresses) 2822 for addr in addresses[0:-k]: 2823 if self.address_is_old(addr): 2824 n = 0 2825 else: 2826 n += 1 2827 nmax = max(nmax, n) 2828 return nmax + 1 2829 2830 @abstractmethod 2831 def derive_pubkeys(self, c: int, i: int) -> Sequence[str]: 2832 pass 2833 2834 def derive_address(self, for_change: int, n: int) -> str: 2835 for_change = int(for_change) 2836 pubkeys = self.derive_pubkeys(for_change, n) 2837 return self.pubkeys_to_address(pubkeys) 2838 2839 def export_private_key_for_path(self, path: Union[Sequence[int], str], password: Optional[str]) -> str: 2840 if isinstance(path, str): 2841 path = convert_bip32_path_to_list_of_uint32(path) 2842 pk, compressed = self.keystore.get_private_key(path, password) 2843 txin_type = self.get_txin_type() # assumes no mixed-scripts in wallet 2844 return bitcoin.serialize_privkey(pk, compressed, txin_type) 2845 2846 def get_public_keys_with_deriv_info(self, address: str): 2847 der_suffix = self.get_address_index(address) 2848 der_suffix = [int(x) for x in der_suffix] 2849 return {k.derive_pubkey(*der_suffix): (k, der_suffix) 2850 for k in self.get_keystores()} 2851 2852 def _add_input_sig_info(self, txin, address, *, only_der_suffix): 2853 self._add_txinout_derivation_info(txin, address, only_der_suffix=only_der_suffix) 2854 2855 def _add_txinout_derivation_info(self, txinout, address, *, only_der_suffix): 2856 if not self.is_mine(address): 2857 return 2858 pubkey_deriv_info = self.get_public_keys_with_deriv_info(address) 2859 txinout.pubkeys = sorted([pk for pk in list(pubkey_deriv_info)]) 2860 for pubkey in pubkey_deriv_info: 2861 ks, der_suffix = pubkey_deriv_info[pubkey] 2862 fp_bytes, der_full = ks.get_fp_and_derivation_to_be_used_in_partial_tx(der_suffix, 2863 only_der_suffix=only_der_suffix) 2864 txinout.bip32_paths[pubkey] = (fp_bytes, der_full) 2865 2866 def create_new_address(self, for_change: bool = False): 2867 assert type(for_change) is bool 2868 with self.lock: 2869 n = self.db.num_change_addresses() if for_change else self.db.num_receiving_addresses() 2870 address = self.derive_address(int(for_change), n) 2871 self.db.add_change_address(address) if for_change else self.db.add_receiving_address(address) 2872 self.add_address(address) 2873 if for_change: 2874 # note: if it's actually "old", it will get filtered later 2875 self._not_old_change_addresses.append(address) 2876 return address 2877 2878 def synchronize_sequence(self, for_change): 2879 limit = self.gap_limit_for_change if for_change else self.gap_limit 2880 while True: 2881 num_addr = self.db.num_change_addresses() if for_change else self.db.num_receiving_addresses() 2882 if num_addr < limit: 2883 self.create_new_address(for_change) 2884 continue 2885 if for_change: 2886 last_few_addresses = self.get_change_addresses(slice_start=-limit) 2887 else: 2888 last_few_addresses = self.get_receiving_addresses(slice_start=-limit) 2889 if any(map(self.address_is_old, last_few_addresses)): 2890 self.create_new_address(for_change) 2891 else: 2892 break 2893 2894 @AddressSynchronizer.with_local_height_cached 2895 def synchronize(self): 2896 with self.lock: 2897 self.synchronize_sequence(False) 2898 self.synchronize_sequence(True) 2899 2900 def get_all_known_addresses_beyond_gap_limit(self): 2901 # note that we don't stop at first large gap 2902 found = set() 2903 2904 def process_addresses(addrs, gap_limit): 2905 rolling_num_unused = 0 2906 for addr in addrs: 2907 if self.db.get_addr_history(addr): 2908 rolling_num_unused = 0 2909 else: 2910 if rolling_num_unused >= gap_limit: 2911 found.add(addr) 2912 rolling_num_unused += 1 2913 2914 process_addresses(self.get_receiving_addresses(), self.gap_limit) 2915 process_addresses(self.get_change_addresses(), self.gap_limit_for_change) 2916 return found 2917 2918 def get_address_index(self, address) -> Optional[Sequence[int]]: 2919 return self.db.get_address_index(address) or self._ephemeral_addr_to_addr_index.get(address) 2920 2921 def get_address_path_str(self, address): 2922 intpath = self.get_address_index(address) 2923 if intpath is None: 2924 return None 2925 return convert_bip32_intpath_to_strpath(intpath) 2926 2927 def _learn_derivation_path_for_address_from_txinout(self, txinout, address): 2928 for ks in self.get_keystores(): 2929 pubkey, der_suffix = ks.find_my_pubkey_in_txinout(txinout, only_der_suffix=True) 2930 if der_suffix is not None: 2931 # note: we already know the pubkey belongs to the keystore, 2932 # but the script template might be different 2933 if len(der_suffix) != 2: continue 2934 try: 2935 my_address = self.derive_address(*der_suffix) 2936 except CannotDerivePubkey: 2937 my_address = None 2938 if my_address == address: 2939 self._ephemeral_addr_to_addr_index[address] = list(der_suffix) 2940 return True 2941 return False 2942 2943 def get_master_public_keys(self): 2944 return [self.get_master_public_key()] 2945 2946 def get_fingerprint(self): 2947 return self.get_master_public_key() 2948 2949 def get_txin_type(self, address=None): 2950 return self.txin_type 2951 2952 2953 class Simple_Deterministic_Wallet(Simple_Wallet, Deterministic_Wallet): 2954 2955 """ Deterministic Wallet with a single pubkey per address """ 2956 2957 def __init__(self, db, storage, *, config): 2958 Deterministic_Wallet.__init__(self, db, storage, config=config) 2959 2960 def get_public_key(self, address): 2961 sequence = self.get_address_index(address) 2962 pubkeys = self.derive_pubkeys(*sequence) 2963 return pubkeys[0] 2964 2965 def load_keystore(self): 2966 self.keystore = load_keystore(self.db, 'keystore') 2967 try: 2968 xtype = bip32.xpub_type(self.keystore.xpub) 2969 except: 2970 xtype = 'standard' 2971 self.txin_type = 'p2pkh' if xtype == 'standard' else xtype 2972 2973 def get_master_public_key(self): 2974 return self.keystore.get_master_public_key() 2975 2976 def derive_pubkeys(self, c, i): 2977 return [self.keystore.derive_pubkey(c, i).hex()] 2978 2979 2980 2981 2982 2983 2984 class Standard_Wallet(Simple_Deterministic_Wallet): 2985 wallet_type = 'standard' 2986 2987 def pubkeys_to_address(self, pubkeys): 2988 pubkey = pubkeys[0] 2989 return bitcoin.pubkey_to_address(self.txin_type, pubkey) 2990 2991 2992 class Multisig_Wallet(Deterministic_Wallet): 2993 # generic m of n 2994 2995 def __init__(self, db, storage, *, config): 2996 self.wallet_type = db.get('wallet_type') 2997 self.m, self.n = multisig_type(self.wallet_type) 2998 Deterministic_Wallet.__init__(self, db, storage, config=config) 2999 3000 def get_public_keys(self, address): 3001 return [pk.hex() for pk in self.get_public_keys_with_deriv_info(address)] 3002 3003 def pubkeys_to_address(self, pubkeys): 3004 redeem_script = self.pubkeys_to_scriptcode(pubkeys) 3005 return bitcoin.redeem_script_to_address(self.txin_type, redeem_script) 3006 3007 def pubkeys_to_scriptcode(self, pubkeys: Sequence[str]) -> str: 3008 return transaction.multisig_script(sorted(pubkeys), self.m) 3009 3010 def get_redeem_script(self, address): 3011 txin_type = self.get_txin_type(address) 3012 pubkeys = self.get_public_keys(address) 3013 scriptcode = self.pubkeys_to_scriptcode(pubkeys) 3014 if txin_type == 'p2sh': 3015 return scriptcode 3016 elif txin_type == 'p2wsh-p2sh': 3017 return bitcoin.p2wsh_nested_script(scriptcode) 3018 elif txin_type == 'p2wsh': 3019 return None 3020 raise UnknownTxinType(f'unexpected txin_type {txin_type}') 3021 3022 def get_witness_script(self, address): 3023 txin_type = self.get_txin_type(address) 3024 pubkeys = self.get_public_keys(address) 3025 scriptcode = self.pubkeys_to_scriptcode(pubkeys) 3026 if txin_type == 'p2sh': 3027 return None 3028 elif txin_type in ('p2wsh-p2sh', 'p2wsh'): 3029 return scriptcode 3030 raise UnknownTxinType(f'unexpected txin_type {txin_type}') 3031 3032 def derive_pubkeys(self, c, i): 3033 return [k.derive_pubkey(c, i).hex() for k in self.get_keystores()] 3034 3035 def load_keystore(self): 3036 self.keystores = {} 3037 for i in range(self.n): 3038 name = 'x%d/'%(i+1) 3039 self.keystores[name] = load_keystore(self.db, name) 3040 self.keystore = self.keystores['x1/'] 3041 xtype = bip32.xpub_type(self.keystore.xpub) 3042 self.txin_type = 'p2sh' if xtype == 'standard' else xtype 3043 3044 def save_keystore(self): 3045 for name, k in self.keystores.items(): 3046 self.db.put(name, k.dump()) 3047 3048 def get_keystore(self): 3049 return self.keystores.get('x1/') 3050 3051 def get_keystores(self): 3052 return [self.keystores[i] for i in sorted(self.keystores.keys())] 3053 3054 def can_have_keystore_encryption(self): 3055 return any([k.may_have_password() for k in self.get_keystores()]) 3056 3057 def _update_password_for_keystore(self, old_pw, new_pw): 3058 for name, keystore in self.keystores.items(): 3059 if keystore.may_have_password(): 3060 keystore.update_password(old_pw, new_pw) 3061 self.db.put(name, keystore.dump()) 3062 3063 def check_password(self, password): 3064 for name, keystore in self.keystores.items(): 3065 if keystore.may_have_password(): 3066 keystore.check_password(password) 3067 if self.has_storage_encryption(): 3068 self.storage.check_password(password) 3069 3070 def get_available_storage_encryption_version(self): 3071 # multisig wallets are not offered hw device encryption 3072 return StorageEncryptionVersion.USER_PASSWORD 3073 3074 def has_seed(self): 3075 return self.keystore.has_seed() 3076 3077 def is_watching_only(self): 3078 return all([k.is_watching_only() for k in self.get_keystores()]) 3079 3080 def get_master_public_key(self): 3081 return self.keystore.get_master_public_key() 3082 3083 def get_master_public_keys(self): 3084 return [k.get_master_public_key() for k in self.get_keystores()] 3085 3086 def get_fingerprint(self): 3087 return ''.join(sorted(self.get_master_public_keys())) 3088 3089 3090 wallet_types = ['standard', 'multisig', 'imported'] 3091 3092 def register_wallet_type(category): 3093 wallet_types.append(category) 3094 3095 wallet_constructors = { 3096 'standard': Standard_Wallet, 3097 'old': Standard_Wallet, 3098 'xpub': Standard_Wallet, 3099 'imported': Imported_Wallet 3100 } 3101 3102 def register_constructor(wallet_type, constructor): 3103 wallet_constructors[wallet_type] = constructor 3104 3105 # former WalletFactory 3106 class Wallet(object): 3107 """The main wallet "entry point". 3108 This class is actually a factory that will return a wallet of the correct 3109 type when passed a WalletStorage instance.""" 3110 3111 def __new__(self, db: 'WalletDB', storage: Optional[WalletStorage], *, config: SimpleConfig): 3112 wallet_type = db.get('wallet_type') 3113 WalletClass = Wallet.wallet_class(wallet_type) 3114 wallet = WalletClass(db, storage, config=config) 3115 return wallet 3116 3117 @staticmethod 3118 def wallet_class(wallet_type): 3119 if multisig_type(wallet_type): 3120 return Multisig_Wallet 3121 if wallet_type in wallet_constructors: 3122 return wallet_constructors[wallet_type] 3123 raise WalletFileException("Unknown wallet type: " + str(wallet_type)) 3124 3125 3126 def create_new_wallet(*, path, config: SimpleConfig, passphrase=None, password=None, 3127 encrypt_file=True, seed_type=None, gap_limit=None) -> dict: 3128 """Create a new wallet""" 3129 storage = WalletStorage(path) 3130 if storage.file_exists(): 3131 raise Exception("Remove the existing wallet first!") 3132 db = WalletDB('', manual_upgrades=False) 3133 3134 seed = Mnemonic('en').make_seed(seed_type=seed_type) 3135 k = keystore.from_seed(seed, passphrase) 3136 db.put('keystore', k.dump()) 3137 db.put('wallet_type', 'standard') 3138 if gap_limit is not None: 3139 db.put('gap_limit', gap_limit) 3140 wallet = Wallet(db, storage, config=config) 3141 wallet.update_password(old_pw=None, new_pw=password, encrypt_storage=encrypt_file) 3142 wallet.synchronize() 3143 msg = "Please keep your seed in a safe place; if you lose it, you will not be able to restore your wallet." 3144 wallet.save_db() 3145 return {'seed': seed, 'wallet': wallet, 'msg': msg} 3146 3147 3148 def restore_wallet_from_text(text, *, path, config: SimpleConfig, 3149 passphrase=None, password=None, encrypt_file=True, 3150 gap_limit=None) -> dict: 3151 """Restore a wallet from text. Text can be a seed phrase, a master 3152 public key, a master private key, a list of bitcoin addresses 3153 or bitcoin private keys.""" 3154 storage = WalletStorage(path) 3155 if storage.file_exists(): 3156 raise Exception("Remove the existing wallet first!") 3157 db = WalletDB('', manual_upgrades=False) 3158 text = text.strip() 3159 if keystore.is_address_list(text): 3160 wallet = Imported_Wallet(db, storage, config=config) 3161 addresses = text.split() 3162 good_inputs, bad_inputs = wallet.import_addresses(addresses, write_to_disk=False) 3163 # FIXME tell user about bad_inputs 3164 if not good_inputs: 3165 raise Exception("None of the given addresses can be imported") 3166 elif keystore.is_private_key_list(text, allow_spaces_inside_key=False): 3167 k = keystore.Imported_KeyStore({}) 3168 db.put('keystore', k.dump()) 3169 wallet = Imported_Wallet(db, storage, config=config) 3170 keys = keystore.get_private_keys(text, allow_spaces_inside_key=False) 3171 good_inputs, bad_inputs = wallet.import_private_keys(keys, None, write_to_disk=False) 3172 # FIXME tell user about bad_inputs 3173 if not good_inputs: 3174 raise Exception("None of the given privkeys can be imported") 3175 else: 3176 if keystore.is_master_key(text): 3177 k = keystore.from_master_key(text) 3178 elif keystore.is_seed(text): 3179 k = keystore.from_seed(text, passphrase) 3180 else: 3181 raise Exception("Seed or key not recognized") 3182 db.put('keystore', k.dump()) 3183 db.put('wallet_type', 'standard') 3184 if gap_limit is not None: 3185 db.put('gap_limit', gap_limit) 3186 wallet = Wallet(db, storage, config=config) 3187 assert not storage.file_exists(), "file was created too soon! plaintext keys might have been written to disk" 3188 wallet.update_password(old_pw=None, new_pw=password, encrypt_storage=encrypt_file) 3189 wallet.synchronize() 3190 msg = ("This wallet was restored offline. It may contain more addresses than displayed. " 3191 "Start a daemon and use load_wallet to sync its history.") 3192 wallet.save_db() 3193 return {'wallet': wallet, 'msg': msg} 3194 3195 3196 def check_password_for_directory(config: SimpleConfig, old_password, new_password=None) -> bool: 3197 """Checks password against all wallets and returns True if they can all be updated. 3198 If new_password is not None, update all wallet passwords to new_password. 3199 """ 3200 dirname = os.path.dirname(config.get_wallet_path()) 3201 failed = [] 3202 for filename in os.listdir(dirname): 3203 path = os.path.join(dirname, filename) 3204 if not os.path.isfile(path): 3205 continue 3206 basename = os.path.basename(path) 3207 storage = WalletStorage(path) 3208 if not storage.is_encrypted(): 3209 # it is a bit wasteful load the wallet here, but that is fine 3210 # because we are progressively enforcing storage encryption. 3211 db = WalletDB(storage.read(), manual_upgrades=False) 3212 wallet = Wallet(db, storage, config=config) 3213 if wallet.has_keystore_encryption(): 3214 try: 3215 wallet.check_password(old_password) 3216 except: 3217 failed.append(basename) 3218 continue 3219 if new_password: 3220 wallet.update_password(old_password, new_password) 3221 else: 3222 if new_password: 3223 wallet.update_password(None, new_password) 3224 continue 3225 if not storage.is_encrypted_with_user_pw(): 3226 failed.append(basename) 3227 continue 3228 try: 3229 storage.check_password(old_password) 3230 except: 3231 failed.append(basename) 3232 continue 3233 db = WalletDB(storage.read(), manual_upgrades=False) 3234 wallet = Wallet(db, storage, config=config) 3235 try: 3236 wallet.check_password(old_password) 3237 except: 3238 failed.append(basename) 3239 continue 3240 if new_password: 3241 wallet.update_password(old_password, new_password) 3242 return failed == [] 3243 3244 3245 def update_password_for_directory(config: SimpleConfig, old_password, new_password) -> bool: 3246 assert new_password is not None 3247 assert check_password_for_directory(config, old_password, None) 3248 return check_password_for_directory(config, old_password, new_password)