electrum

Electrum Bitcoin wallet
git clone https://git.parazyd.org/electrum
Log | Files | Refs | Submodules

commit 791e680a962f566ce4dba892fc0dfcb1b5ea2069
parent 7f2083f667caea442eae3184dadba4e29be7c447
Author: ThomasV <thomasv@electrum.org>
Date:   Wed, 20 Feb 2019 14:45:15 +0100

abstract database away from wallet and address_synchronizer

Diffstat:
Melectrum/address_synchronizer.py | 211++++++++++++++++++++++++-------------------------------------------------------
Melectrum/json_db.py | 156++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---
Melectrum/network.py | 2--
Melectrum/synchronizer.py | 7++++---
Melectrum/verifier.py | 4++--
Melectrum/wallet.py | 67+++++++++++++++++++++++++++++++++++++------------------------------
6 files changed, 258 insertions(+), 189 deletions(-)

diff --git a/electrum/address_synchronizer.py b/electrum/address_synchronizer.py @@ -61,6 +61,7 @@ class AddressSynchronizer(PrintError): def __init__(self, storage: 'WalletStorage'): self.storage = storage + self.db = self.storage.db self.network = None # type: Network # verifier (SPV) and synchronizer are started in start_network self.synchronizer = None # type: Synchronizer @@ -68,17 +69,6 @@ class AddressSynchronizer(PrintError): # locks: if you need to take multiple ones, acquire them in the order they are defined here! self.lock = threading.RLock() self.transaction_lock = threading.RLock() - # address -> list(txid, height) - self.history = storage.get('addr_history',{}) - # Verified transactions. txid -> TxMinedInfo. Access with self.lock. - verified_tx = storage.get('verified_tx3', {}) - self.verified_tx = {} # type: Dict[str, TxMinedInfo] - for txid, (height, timestamp, txpos, header_hash) in verified_tx.items(): - self.verified_tx[txid] = TxMinedInfo(height=height, - conf=None, - timestamp=timestamp, - txpos=txpos, - header_hash=header_hash) # Transactions pending verification. txid -> tx_height. Access with self.lock. self.unverified_tx = defaultdict(int) # true when synchronized @@ -95,17 +85,16 @@ class AddressSynchronizer(PrintError): return func_wrapper def load_and_cleanup(self): - self.load_transactions() self.load_local_history() self.check_history() self.load_unverified_transactions() self.remove_local_transactions_we_dont_have() def is_mine(self, address): - return address in self.history + return address in self.db.get_history() def get_addresses(self): - return sorted(self.history.keys()) + return sorted(self.db.get_history()) def get_address_history(self, addr): h = [] @@ -128,8 +117,8 @@ class AddressSynchronizer(PrintError): return addr prevout_hash = txi.get('prevout_hash') prevout_n = txi.get('prevout_n') - dd = self.txo.get(prevout_hash, {}) - for addr, l in dd.items(): + for addr in self.db.get_txo(prevout_hash): + l = self.db.get_txo_addr(prevout_hash, addr) for n, v, is_cb in l: if n == prevout_n: return addr @@ -146,7 +135,8 @@ class AddressSynchronizer(PrintError): def load_unverified_transactions(self): # review transactions that are in the history - for addr, hist in self.history.items(): + for addr in self.db.get_history(): + hist = self.db.get_addr_history(addr) for tx_hash, tx_height in hist: # add it in case it was previously unconfirmed self.add_unverified_tx(tx_hash, tx_height) @@ -167,13 +157,11 @@ class AddressSynchronizer(PrintError): self.verifier = None self.storage.put('stored_height', self.get_local_height()) if write_to_disk: - self.save_transactions() - self.save_verified_tx() self.storage.write() def add_address(self, address): - if address not in self.history: - self.history[address] = [] + if address not in self.db.get_history(): + self.db.history[address] = [] self.set_up_to_date(False) if self.synchronizer: self.synchronizer.add(address) @@ -191,11 +179,11 @@ class AddressSynchronizer(PrintError): continue prevout_hash = txin['prevout_hash'] prevout_n = txin['prevout_n'] - spending_tx_hash = self.spent_outpoints[prevout_hash].get(prevout_n) + spending_tx_hash = self.db.get_spent_outpoint(prevout_hash, prevout_n) if spending_tx_hash is None: continue # this outpoint has already been spent, by spending_tx - assert spending_tx_hash in self.transactions + assert spending_tx_hash in self.db.list_transactions() conflicting_txns |= {spending_tx_hash} if tx_hash in conflicting_txns: # this tx is already in history, so it conflicts with itself @@ -256,49 +244,39 @@ class AddressSynchronizer(PrintError): self.remove_transaction(tx_hash2) # add inputs def add_value_from_prev_output(): - dd = self.txo.get(prevout_hash, {}) # note: this nested loop takes linear time in num is_mine outputs of prev_tx - for addr, outputs in dd.items(): + for addr in self.db.get_txo(prevout_hash): + outputs = self.db.get_txo_addr(prevout_hash, addr) # note: instead of [(n, v, is_cb), ...]; we could store: {n -> (v, is_cb)} for n, v, is_cb in outputs: if n == prevout_n: if addr and self.is_mine(addr): - if d.get(addr) is None: - d[addr] = set() - d[addr].add((ser, v)) + self.db.add_txi_addr(tx_hash, addr, ser, v) return - self.txi[tx_hash] = d = {} for txi in tx.inputs(): if txi['type'] == 'coinbase': continue prevout_hash = txi['prevout_hash'] prevout_n = txi['prevout_n'] ser = prevout_hash + ':%d' % prevout_n - self.spent_outpoints[prevout_hash][prevout_n] = tx_hash + self.db.set_spent_outpoint(prevout_hash, prevout_n, tx_hash) add_value_from_prev_output() # add outputs - self.txo[tx_hash] = d = {} for n, txo in enumerate(tx.outputs()): v = txo[2] ser = tx_hash + ':%d'%n addr = self.get_txout_address(txo) if addr and self.is_mine(addr): - if d.get(addr) is None: - d[addr] = [] - d[addr].append((n, v, is_coinbase)) + self.db.add_txo_addr(tx_hash, addr, n, v, is_coinbase) # give v to txi that spends me - next_tx = self.spent_outpoints[tx_hash].get(n) + next_tx = self.db.get_spent_outpoint(tx_hash, n) if next_tx is not None: - dd = self.txi.get(next_tx, {}) - if dd.get(addr) is None: - dd[addr] = set() - if (ser, v) not in dd[addr]: - dd[addr].add((ser, v)) + self.db.add_txi_addr(next_tx, addr, ser, v) self._add_tx_to_local_history(next_tx) # add to local history self._add_tx_to_local_history(tx_hash) # save - self.transactions[tx_hash] = tx + self.db.add_transaction(tx_hash, tx) return True def remove_transaction(self, tx_hash): @@ -328,17 +306,18 @@ class AddressSynchronizer(PrintError): with self.transaction_lock: self.print_error("removing tx from history", tx_hash) - tx = self.transactions.pop(tx_hash, None) + tx = self.db.remove_transaction(tx_hash) remove_from_spent_outpoints() self._remove_tx_from_local_history(tx_hash) - self.txi.pop(tx_hash, None) - self.txo.pop(tx_hash, None) + self.db.remove_txi(tx_hash) + self.db.remove_txo(tx_hash) def get_depending_transactions(self, tx_hash): """Returns all (grand-)children of tx_hash in this wallet.""" with self.transaction_lock: children = set() - for other_hash in self.spent_outpoints[tx_hash].values(): + for n in self.db.get_spent_outpoints(tx_hash): + other_hash = self.db.get_spent_outpoint(tx_hash, n) children.add(other_hash) children |= self.get_depending_transactions(other_hash) return children @@ -354,129 +333,68 @@ class AddressSynchronizer(PrintError): if (tx_hash, height) not in hist: # make tx local self.unverified_tx.pop(tx_hash, None) - self.verified_tx.pop(tx_hash, None) + self.db.remove_verified_tx(tx_hash) if self.verifier: self.verifier.remove_spv_proof_for_tx(tx_hash) - self.history[addr] = hist + self.db.set_addr_history(addr, hist) for tx_hash, tx_height in hist: # add it in case it was previously unconfirmed self.add_unverified_tx(tx_hash, tx_height) # if addr is new, we have to recompute txi and txo - tx = self.transactions.get(tx_hash) + tx = self.db.get_transaction(tx_hash) if tx is None: continue self.add_transaction(tx_hash, tx, allow_unrelated=True) # Store fees - self.tx_fees.update(tx_fees) + self.db.update_tx_fees(tx_fees) - @profiler - def load_transactions(self): - # load txi, txo, tx_fees - # bookkeeping data of is_mine inputs of transactions - self.txi = self.storage.get('txi', {}) # txid -> address -> (prev_outpoint, value) - for txid, d in list(self.txi.items()): - for addr, lst in d.items(): - self.txi[txid][addr] = set([tuple(x) for x in lst]) - # bookkeeping data of is_mine outputs of transactions - self.txo = self.storage.get('txo', {}) # txid -> address -> (output_index, value, is_coinbase) - self.tx_fees = self.storage.get('tx_fees', {}) - tx_list = self.storage.get('transactions', {}) - # load transactions - self.transactions = {} - for tx_hash, raw in tx_list.items(): - tx = Transaction(raw) - self.transactions[tx_hash] = tx - if self.txi.get(tx_hash) is None and self.txo.get(tx_hash) is None: - self.print_error("removing unreferenced tx", tx_hash) - self.transactions.pop(tx_hash) - # load spent_outpoints - _spent_outpoints = self.storage.get('spent_outpoints', {}) - self.spent_outpoints = defaultdict(dict) - for prevout_hash, d in _spent_outpoints.items(): - for prevout_n_str, spending_txid in d.items(): - prevout_n = int(prevout_n_str) - if spending_txid not in self.transactions: - continue # only care about txns we have - self.spent_outpoints[prevout_hash][prevout_n] = spending_txid @profiler def load_local_history(self): self._history_local = {} # address -> set(txid) self._address_history_changed_events = defaultdict(asyncio.Event) # address -> Event - for txid in itertools.chain(self.txi, self.txo): + for txid in itertools.chain(self.db.get_txi_keys(), self.db.get_txo_keys()): self._add_tx_to_local_history(txid) @profiler def check_history(self): save = False - hist_addrs_mine = list(filter(lambda k: self.is_mine(k), self.history.keys())) - hist_addrs_not_mine = list(filter(lambda k: not self.is_mine(k), self.history.keys())) + hist_addrs_mine = list(filter(lambda k: self.is_mine(k), self.db.get_history())) + hist_addrs_not_mine = list(filter(lambda k: not self.is_mine(k), self.db.get_history())) for addr in hist_addrs_not_mine: - self.history.pop(addr) + self.db.remove_addr_history(addr) save = True for addr in hist_addrs_mine: - hist = self.history[addr] + hist = self.db.get_addr_history(addr) for tx_hash, tx_height in hist: - if self.txi.get(tx_hash) or self.txo.get(tx_hash): + if self.db.get_txi(tx_hash) or self.db.get_txo(tx_hash): continue - tx = self.transactions.get(tx_hash) + tx = self.db.get_transaction(tx_hash) if tx is not None: self.add_transaction(tx_hash, tx, allow_unrelated=True) save = True if save: - self.save_transactions() + self.storage.write() def remove_local_transactions_we_dont_have(self): - txid_set = set(self.txi) | set(self.txo) - for txid in txid_set: + for txid in itertools.chain(list(self.db.get_txi_keys()), list(self.db.get_txo_keys())): tx_height = self.get_tx_height(txid).height - if tx_height == TX_HEIGHT_LOCAL and txid not in self.transactions: + if tx_height == TX_HEIGHT_LOCAL and txid not in self.db.list_transactions(): self.remove_transaction(txid) - @profiler - def save_transactions(self, write=False): - with self.transaction_lock: - tx = {} - for k,v in self.transactions.items(): - tx[k] = str(v) - self.storage.put('transactions', tx) - self.storage.put('txi', self.txi) - self.storage.put('txo', self.txo) - self.storage.put('tx_fees', self.tx_fees) - self.storage.put('addr_history', self.history) - self.storage.put('spent_outpoints', self.spent_outpoints) - if write: - self.storage.write() - - def save_verified_tx(self, write=False): - with self.lock: - verified_tx_to_save = {} - for txid, tx_info in self.verified_tx.items(): - verified_tx_to_save[txid] = (tx_info.height, tx_info.timestamp, - tx_info.txpos, tx_info.header_hash) - self.storage.put('verified_tx3', verified_tx_to_save) - if write: - self.storage.write() - def clear_history(self): with self.lock: with self.transaction_lock: - self.txi = {} - self.txo = {} - self.tx_fees = {} - self.spent_outpoints = defaultdict(dict) - self.history = {} - self.verified_tx = {} - self.transactions = {} # type: Dict[str, Transaction] - self.save_transactions() + self.db.clear_history() + self.storage.write() def get_txpos(self, tx_hash): """Returns (height, txpos) tuple, even if the tx is unverified.""" with self.lock: - if tx_hash in self.verified_tx: - info = self.verified_tx[tx_hash] + if tx_hash in self.db.list_verified_tx(): + info = self.db.get_verified_tx(tx_hash) return info.height, info.txpos elif tx_hash in self.unverified_tx: height = self.unverified_tx[tx_hash] @@ -540,7 +458,7 @@ class AddressSynchronizer(PrintError): def _add_tx_to_local_history(self, txid): with self.transaction_lock: - for addr in itertools.chain(self.txi.get(txid, []), self.txo.get(txid, [])): + for addr in itertools.chain(self.db.get_txi(txid), self.db.get_txo(txid)): cur_hist = self._history_local.get(addr, set()) cur_hist.add(txid) self._history_local[addr] = cur_hist @@ -548,7 +466,7 @@ class AddressSynchronizer(PrintError): def _remove_tx_from_local_history(self, txid): with self.transaction_lock: - for addr in itertools.chain(self.txi.get(txid, []), self.txo.get(txid, [])): + for addr in itertools.chain(self.db.get_txi(txid), self.db.get_txo(txid)): cur_hist = self._history_local.get(addr, set()) try: cur_hist.remove(txid) @@ -573,10 +491,10 @@ class AddressSynchronizer(PrintError): await self._address_history_changed_events[addr].wait() def add_unverified_tx(self, tx_hash, tx_height): - if tx_hash in self.verified_tx: + if tx_hash in self.db.list_verified_tx(): if tx_height in (TX_HEIGHT_UNCONFIRMED, TX_HEIGHT_UNCONF_PARENT): with self.lock: - self.verified_tx.pop(tx_hash) + self.db.remove_verified_tx(tx_hash) if self.verifier: self.verifier.remove_spv_proof_for_tx(tx_hash) else: @@ -594,7 +512,7 @@ class AddressSynchronizer(PrintError): # Remove from the unverified map and add to the verified map with self.lock: self.unverified_tx.pop(tx_hash, None) - self.verified_tx[tx_hash] = info + self.db.add_verified_tx(tx_hash, info) tx_mined_status = self.get_tx_height(tx_hash) self.network.trigger_callback('verified', self, tx_hash, tx_mined_status) @@ -607,12 +525,13 @@ class AddressSynchronizer(PrintError): '''Used by the verifier when a reorg has happened''' txs = set() with self.lock: - for tx_hash, info in list(self.verified_tx.items()): + for tx_hash in list(self.db.list_verified_tx()): + info = self.db.get_verified_tx(tx_hash) tx_height = info.height if tx_height >= height: header = blockchain.read_header(tx_height) if not header or hash_header(header) != info.header_hash: - self.verified_tx.pop(tx_hash, None) + self.db.remove_verified_tx(tx_hash) # NOTE: we should add these txns to self.unverified_tx, # but with what height? # If on the new fork after the reorg, the txn is at the @@ -635,8 +554,8 @@ class AddressSynchronizer(PrintError): def get_tx_height(self, tx_hash: str) -> TxMinedInfo: with self.lock: - if tx_hash in self.verified_tx: - info = self.verified_tx[tx_hash] + if tx_hash in self.db.list_verified_tx(): + info = self.db.get_verified_tx(tx_hash) conf = max(self.get_local_height() - info.height + 1, 0) return info._replace(conf=conf) elif tx_hash in self.unverified_tx: @@ -652,11 +571,7 @@ class AddressSynchronizer(PrintError): if self.network: self.network.notify('status') if up_to_date: - self.save_transactions(write=True) - # if the verifier is also up to date, persist that too; - # otherwise it will persist its results when it finishes - if self.verifier and self.verifier.is_up_to_date(): - self.save_verified_tx(write=True) + self.storage.write() def is_up_to_date(self): with self.lock: return self.up_to_date @@ -666,11 +581,11 @@ class AddressSynchronizer(PrintError): """effect of tx on address""" delta = 0 # substract the value of coins sent from address - d = self.txi.get(tx_hash, {}).get(address, []) + d = self.db.get_txi_addr(tx_hash, address) for n, v in d: delta -= v # add the value of the coins received at address - d = self.txo.get(tx_hash, {}).get(address, []) + d = self.db.get_txo_addr(tx_hash, address) for n, v, cb in d: delta += v return delta @@ -679,10 +594,12 @@ class AddressSynchronizer(PrintError): def get_tx_value(self, txid): """effect of tx on the entire domain""" delta = 0 - for addr, d in self.txi.get(txid, {}).items(): + for addr in self.db.get_txi(txid): + d = self.db.get_txi_addr(txid, addr) for n, v in d: delta -= v - for addr, d in self.txo.get(txid, {}).items(): + for addr in self.db.get_txo(txid): + d = self.db.get_txo_addr(txid, addr) for n, v, cb in d: delta += v return delta @@ -699,7 +616,7 @@ class AddressSynchronizer(PrintError): if self.is_mine(addr): is_mine = True is_relevant = True - d = self.txo.get(txin['prevout_hash'], {}).get(addr, []) + d = self.db.get_txo_addr(txin['prevout_hash'], addr) for n, v, cb in d: if n == txin['prevout_n']: value = v @@ -748,7 +665,7 @@ class AddressSynchronizer(PrintError): is_relevant, is_mine, v, fee = self.get_wallet_delta(tx) if fee is None: txid = tx.txid() - fee = self.tx_fees.get(txid) + fee = self.db.get_tx_fee(txid) # only cache non-None, as None can still change while syncing if fee is not None: tx._cached_fee = fee @@ -760,11 +677,11 @@ class AddressSynchronizer(PrintError): received = {} sent = {} for tx_hash, height in h: - l = self.txo.get(tx_hash, {}).get(address, []) + l = self.db.get_txo_addr(tx_hash, address) for n, v, is_cb in l: received[tx_hash + ':%d'%n] = (height, v, is_cb) for tx_hash, height in h: - l = self.txi.get(tx_hash, {}).get(address, []) + l = self.db.get_txi_addr(tx_hash, address) for txi, v in l: sent[txi] = height return received, sent @@ -849,7 +766,7 @@ class AddressSynchronizer(PrintError): return cc, uu, xx def is_used(self, address): - h = self.history.get(address,[]) + h = self.db.get_addr_history(address) return len(h) != 0 def is_empty(self, address): diff --git a/electrum/json_db.py b/electrum/json_db.py @@ -35,10 +35,10 @@ import zlib from collections import defaultdict from . import util, bitcoin, ecc -from .util import PrintError, profiler, InvalidPassword, WalletFileException, bfh, standardize_path, multisig_type +from .util import PrintError, profiler, InvalidPassword, WalletFileException, bfh, standardize_path, multisig_type, TxMinedInfo from .plugin import run_hook, plugin_loaders from .keystore import bip44_derivation - +from .transaction import Transaction # seed_version is now used for the version of the wallet file @@ -49,9 +49,6 @@ FINAL_SEED_VERSION = 18 # electrum >= 2.7 will set this to prevent - - - class JsonDB(PrintError): def __init__(self, raw, manual_upgrades): @@ -61,6 +58,7 @@ class JsonDB(PrintError): self.load_data(raw) else: self.put('seed_version', FINAL_SEED_VERSION) + self.load_transactions() def get(self, key, default=None): v = self.data.get(key) @@ -498,3 +496,151 @@ class JsonDB(PrintError): msg += "\nPlease open this file with Electrum 1.9.8, and move your coins to a new wallet." raise WalletFileException(msg) + + def get_txi(self, tx_hash): + return self.txi.get(tx_hash, {}).keys() + + def get_txo(self, tx_hash): + return self.txo.get(tx_hash, {}).keys() + + def get_txi_addr(self, tx_hash, address): + return self.txi.get(tx_hash, {}).get(address, []) + + def get_txo_addr(self, tx_hash, address): + return self.txo.get(tx_hash, {}).get(address, []) + + def add_txi_addr(self, tx_hash, addr, ser, v): + if tx_hash not in self.txi: + self.txi[tx_hash] = {} + d = self.txi[tx_hash] + if addr not in d: + d[addr] = set() + d[addr].add((ser, v)) + + def add_txo_addr(self, tx_hash, addr, n, v, is_coinbase): + if tx_hash not in self.txo: + self.txo[tx_hash] = {} + d = self.txo[tx_hash] + if addr not in d: + d[addr] = [] + d[addr].append((n, v, is_coinbase)) + + def get_txi_keys(self): + return self.txi.keys() + + def get_txo_keys(self): + return self.txo.keys() + + def remove_txi(self, tx_hash): + self.txi.pop(tx_hash, None) + + def remove_txo(self, tx_hash): + self.txo.pop(tx_hash, None) + + def get_spent_outpoints(self, prevout_hash): + return self.spent_outpoints.get(prevout_hash, {}).keys() + + def get_spent_outpoint(self, prevout_hash, prevout_n): + return self.spent_outpoints.get(prevout_hash, {}).get(str(prevout_n)) + + def set_spent_outpoint(self, prevout_hash, prevout_n, tx_hash): + if prevout_hash not in self.spent_outpoints: + self.spent_outpoints[prevout_hash] = {} + self.spent_outpoints[prevout_hash][str(prevout_n)] = tx_hash + + def add_transaction(self, tx_hash, tx): + self.transactions[tx_hash] = str(tx) + + def remove_transaction(self, tx_hash): + self.transactions.pop(tx_hash, None) + + def get_transaction(self, tx_hash): + tx = self.transactions.get(tx_hash) + return Transaction(tx) if tx else None + + def list_transactions(self): + return self.transactions.keys() + + def get_history(self): + return self.history.keys() + + def get_addr_history(self, addr): + return self.history.get(addr, []) + + def set_addr_history(self, addr, hist): + self.history[addr] = hist + + def remove_addr_history(self, addr): + self.history.pop(addr, None) + + def list_verified_tx(self): + return self.verified_tx.keys() + + def get_verified_tx(self, txid): + if txid not in self.verified_tx: + return None + height, timestamp, txpos, header_hash = self.verified_tx[txid] + return TxMinedInfo(height=height, + conf=None, + timestamp=timestamp, + txpos=txpos, + header_hash=header_hash) + + def add_verified_tx(self, txid, info): + self.verified_tx[txid] = (info.height, info.timestamp, info.txpos, info.header_hash) + + def remove_verified_tx(self, txid): + self.verified_tx.pop(txid, None) + + def update_tx_fees(self, d): + return self.tx_fees.update(d) + + def get_tx_fee(self, txid): + return self.tx_fees.get(txid) + + def remove_tx_fee(self, txid): + self.tx_fees.pop(txid, None) + + def get_data_ref(self, name): + if name not in self.data: + self.data[name] = {} + return self.data[name] + + @profiler + def load_transactions(self): + # references in self.data + self.txi = self.get_data_ref('txi') # txid -> address -> (prev_outpoint, value) + self.txo = self.get_data_ref('txo') # txid -> address -> (output_index, value, is_coinbase) + self.transactions = self.get_data_ref('transactions') # type: Dict[str, Transaction] + self.spent_outpoints = self.get_data_ref('spent_outpoints') + self.history = self.get_data_ref('history') # address -> list(txid, height) + self.verified_tx = self.get_data_ref('verified_tx3') # txid -> TxMinedInfo. Access with self.lock. + self.tx_fees = self.get_data_ref('tx_fees') + + # tuple to set + for txid, d in list(self.txi.items()): + for addr, lst in d.items(): + self.txi[txid][addr] = set([tuple(x) for x in lst]) + + # remove unreferenced tx + for tx_hash in self.transactions: + if not self.get_txi(tx_hash) and not self.get_txo(tx_hash): + self.print_error("removing unreferenced tx", tx_hash) + self.transactions.pop(tx_hash) + + # remove unreferenced outpoints + for prevout_hash in self.spent_outpoints.keys(): + d = self.spent_outpoints[prevout_hash] + for prevout_n, spending_txid in list(d.items()): + if spending_txid not in self.transactions: + self.print_error("removing unreferenced spent outpoint") + d.pop(prevout_n) + + def clear_history(self): + self.txi.clear() + self.txo.clear() + self.spent_outpoints.clear() + self.transactions.clear() + self.history.clear() + self.verified_tx.clear() + self.tx_fees.clear() diff --git a/electrum/network.py b/electrum/network.py @@ -1184,8 +1184,6 @@ class Network(PrintError): coro = asyncio.run_coroutine_threadsafe(cls._send_http_on_proxy(method, url, **kwargs), loop) return coro.result(5) - - # methods used in scripts async def get_peers(self): while not self.is_connected(): diff --git a/electrum/synchronizer.py b/electrum/synchronizer.py @@ -139,7 +139,7 @@ class Synchronizer(SynchronizerBase): and not self.requested_tx) async def _on_address_status(self, addr, status): - history = self.wallet.history.get(addr, []) + history = self.wallet.db.get_addr_history(addr) if history_status(history) == status: return if addr in self.requested_histories: @@ -175,7 +175,7 @@ class Synchronizer(SynchronizerBase): for tx_hash, tx_height in hist: if tx_hash in self.requested_tx: continue - if tx_hash in self.wallet.transactions: + if tx_hash in self.wallet.db.list_transactions(): continue transaction_hashes.append(tx_hash) self.requested_tx[tx_hash] = tx_height @@ -216,7 +216,8 @@ class Synchronizer(SynchronizerBase): async def main(self): self.wallet.set_up_to_date(False) # request missing txns, if any - for history in self.wallet.history.values(): + for addr in self.wallet.db.get_history(): + history = self.wallet.db.get_addr_history(addr) # Old electrum servers returned ['*'] when all history for the address # was pruned. This no longer happens but may remain in old wallets. if history == ['*']: continue diff --git a/electrum/verifier.py b/electrum/verifier.py @@ -135,8 +135,8 @@ class SPV(NetworkJobOnDefaultServer): txpos=pos, header_hash=header_hash) self.wallet.add_verified_tx(tx_hash, tx_info) - if self.is_up_to_date() and self.wallet.is_up_to_date(): - self.wallet.save_verified_tx(write=True) + #if self.is_up_to_date() and self.wallet.is_up_to_date(): + # self.wallet.save_verified_tx(write=True) @classmethod def hash_merkle_root(cls, merkle_branch: Sequence[str], tx_hash: str, leaf_pos_in_tree: int): diff --git a/electrum/wallet.py b/electrum/wallet.py @@ -276,7 +276,7 @@ class Abstract_Wallet(AddressSynchronizer): return changed def set_fiat_value(self, txid, ccy, text, fx, value_sat): - if txid not in self.transactions: + if txid not in self.db.list_transactions(): return # since fx is inserting the thousands separator, # and not util, also have fx remove it @@ -346,7 +346,8 @@ class Abstract_Wallet(AddressSynchronizer): return [self.get_public_key(address)] def is_found(self): - return self.history.values() != [[]] * len(self.history) + return True + #return self.history.values() != [[]] * len(self.history) def get_tx_info(self, tx): is_relevant, is_mine, v, fee = self.get_wallet_delta(tx) @@ -357,7 +358,7 @@ class Abstract_Wallet(AddressSynchronizer): height = conf = timestamp = None tx_hash = tx.txid() if tx.is_complete(): - if tx_hash in self.transactions.keys(): + if tx_hash in self.db.list_transactions(): label = self.get_label(tx_hash) tx_mined_status = self.get_tx_height(tx_hash) height, conf = tx_mined_status.height, tx_mined_status.conf @@ -369,7 +370,7 @@ class Abstract_Wallet(AddressSynchronizer): elif height in (TX_HEIGHT_UNCONF_PARENT, TX_HEIGHT_UNCONFIRMED): status = _('Unconfirmed') if fee is None: - fee = self.tx_fees.get(tx_hash) + fee = self.db.get_tx_fee(tx_hash) if fee and self.network and self.network.config.has_fee_mempool(): size = tx.estimated_size() fee_per_byte = fee / size @@ -447,7 +448,7 @@ class Abstract_Wallet(AddressSynchronizer): continue if to_height is not None and height >= to_height: continue - tx = self.transactions.get(tx_hash) + tx = self.db.get_transaction(tx_hash) item = { 'txid': tx_hash, 'height': height, @@ -554,10 +555,9 @@ class Abstract_Wallet(AddressSynchronizer): return label def get_default_label(self, tx_hash): - if self.txi.get(tx_hash) == {}: - d = self.txo.get(tx_hash, {}) + if not self.db.get_txi(tx_hash): labels = [] - for addr in d.keys(): + for addr in self.db.get_txo(tx_hash): label = self.labels.get(addr) if label: labels.append(label) @@ -570,7 +570,7 @@ class Abstract_Wallet(AddressSynchronizer): conf = tx_mined_info.conf timestamp = tx_mined_info.timestamp if conf == 0: - tx = self.transactions.get(tx_hash) + tx = self.db.get_transaction(tx_hash) if not tx: return 2, 'unknown' is_final = tx and tx.is_final() @@ -578,7 +578,7 @@ class Abstract_Wallet(AddressSynchronizer): extra.append('rbf') fee = self.get_wallet_delta(tx)[3] if fee is None: - fee = self.tx_fees.get(tx_hash) + fee = self.db.get_tx_fee(tx_hash) if fee is not None: size = tx.estimated_size() fee_per_byte = fee / size @@ -616,9 +616,11 @@ class Abstract_Wallet(AddressSynchronizer): # tx should not be mined yet if tx_mined_status.conf > 0: continue # tx should be "outgoing" from wallet - if delta >= 0: continue - tx = self.transactions.get(tx_hash) - if not tx: continue + if delta >= 0: + continue + tx = self.db.get_transaction(tx_hash) + if not tx: + continue # is_mine outputs should not be spent yet # to avoid cancelling our own dependent transactions txid = tx.txid() @@ -787,7 +789,7 @@ class Abstract_Wallet(AddressSynchronizer): def address_is_old(self, address, age_limit=2): age = -1 - h = self.history.get(address, []) + h = self.db.get_addr_history(address) for tx_hash, tx_height in h: if tx_height <= 0: tx_age = 0 @@ -884,7 +886,7 @@ class Abstract_Wallet(AddressSynchronizer): # First look up an input transaction in the wallet where it # will likely be. If co-signing a transaction it may not have # all the input txs, in which case we ask the network. - tx = self.transactions.get(tx_hash, None) + tx = self.db.get_transaction(tx_hash) if not tx and self.network: try: raw_tx = self.network.run_from_another_thread( @@ -950,7 +952,7 @@ class Abstract_Wallet(AddressSynchronizer): def get_unused_addresses(self): # fixme: use slots from expired requests domain = self.get_receiving_addresses() - return [addr for addr in domain if not self.history.get(addr) + return [addr for addr in domain if not self.db.get_addr_history(addr) and addr not in self.receive_requests.keys()] @check_returned_address @@ -967,7 +969,7 @@ class Abstract_Wallet(AddressSynchronizer): return choice = domain[0] for addr in domain: - if not self.history.get(addr): + if not self.db.get_addr_history(addr): if addr not in self.receive_requests.keys(): return addr else: @@ -981,7 +983,7 @@ class Abstract_Wallet(AddressSynchronizer): for txo, x in received.items(): h, v, is_cb = x txid, n = txo.split(':') - info = self.verified_tx.get(txid) + info = self.db.get_verified_tx(txid) if info: conf = local_height - info.height else: @@ -1214,7 +1216,8 @@ class Abstract_Wallet(AddressSynchronizer): def txin_value(self, txin): txid = txin['prevout_hash'] prev_n = txin['prevout_n'] - for address, d in self.txo.get(txid, {}).items(): + for addr in self.db.get_txo(txid): + d = self.db.get_txo_addr(txid, addr) for n, v, cb in d: if n == prev_n: return v @@ -1238,7 +1241,8 @@ class Abstract_Wallet(AddressSynchronizer): """ Average acquisition price of the inputs of a transaction """ input_value = 0 total_price = 0 - for addr, d in self.txi.get(txid, {}).items(): + for addr in self.db.get_txi(txid): + d = self.db.get_txi_addr(txid, addr) for ser, v in d: input_value += v total_price += self.coin_price(ser.split(':')[0], price_func, ccy, v) @@ -1258,7 +1262,7 @@ class Abstract_Wallet(AddressSynchronizer): result = self._coin_price_cache.get(cache_key, None) if result is not None: return result - if self.txi.get(txid, {}) != {}: + if self.db.get_txi(txid): result = self.average_price(txid, price_func, ccy) * txin_value/Decimal(COIN) self._coin_price_cache[cache_key] = result return result @@ -1398,7 +1402,8 @@ class Imported_Wallet(Simple_Wallet): transactions_to_remove = set() # only referred to by this address transactions_new = set() # txs that are not only referred to by address with self.lock: - for addr, details in self.history.items(): + for addr in self.db.get_history(): + details = self.db.get_addr_history(addr) if addr == address: for tx_hash, height in details: transactions_to_remove.add(tx_hash) @@ -1406,14 +1411,14 @@ class Imported_Wallet(Simple_Wallet): for tx_hash, height in details: transactions_new.add(tx_hash) transactions_to_remove -= transactions_new - self.history.pop(address, None) + self.db.remove_history(address) for tx_hash in transactions_to_remove: self.remove_transaction(tx_hash) - self.tx_fees.pop(tx_hash, None) - self.verified_tx.pop(tx_hash, None) + self.db.remove_tx_fee(tx_hash) + self.db.remove_verified_tx(tx_hash) self.unverified_tx.pop(tx_hash, None) - self.transactions.pop(tx_hash, None) + self.db.remove_transaction(tx_hash) self.save_verified_tx() self.save_transactions() @@ -1465,7 +1470,8 @@ class Imported_Wallet(Simple_Wallet): self.add_address(addr) self.save_keystore() self.save_addresses() - self.save_transactions(write=write_to_disk) + if write_to_disk: + self.storage.write() return good_addr, bad_keys def import_private_key(self, key: str, password: Optional[str]) -> str: @@ -1571,7 +1577,8 @@ class Deterministic_Wallet(Abstract_Wallet): def num_unused_trailing_addresses(self, addresses): k = 0 for a in addresses[::-1]: - if self.history.get(a):break + if a in self.db.get_history(): + break k = k + 1 return k @@ -1582,7 +1589,7 @@ class Deterministic_Wallet(Abstract_Wallet): addresses = self.get_receiving_addresses() k = self.num_unused_trailing_addresses(addresses) for a in addresses[0:-k]: - if self.history.get(a): + if a in self.db.get_history(): n = 0 else: n += 1 @@ -1641,7 +1648,7 @@ class Deterministic_Wallet(Abstract_Wallet): return False prev_addresses = addr_list[max(0, i - limit):max(0, i)] for addr in prev_addresses: - if self.history.get(addr): + if addr in self.db.get_history(): return False return True