from fixtures import * # noqa: F401,F403 from decimal import Decimal from pyln.client import Millisatoshi, RpcError from pyln.testing.db import Sqlite3Db from fixtures import TEST_NETWORK from utils import ( sync_blockheight, wait_for, only_one, first_channel_id, TIMEOUT ) from pathlib import Path import os import pytest import unittest def find_tags(evs, tag): return [e for e in evs if e['tag'] == tag] def find_first_tag(evs, tag): ev = find_tags(evs, tag) assert len(ev) > 0 return ev[0] @unittest.skipIf(TEST_NETWORK != 'regtest', "fixme: broadcast fails, dusty") def test_bookkeeping_closing_trimmed_htlcs(node_factory, bitcoind, executor): l1, l2 = node_factory.line_graph(2) # Send l2 funds via the channel l1.pay(l2, 11000000) l1.rpc.dev_ignore_htlcs(id=l2.info['id'], ignore=True) # This will get stuck due to l3 ignoring htlcs executor.submit(l2.pay, l1, 100001) l1.daemon.wait_for_log('their htlc 0 dev_ignore_htlcs') l1.rpc.dev_fail(l2.info['id']) l1.wait_for_channel_onchain(l2.info['id']) bitcoind.generate_block(1) l1.daemon.wait_for_log(' to ONCHAIN') l2.daemon.wait_for_log(' to ONCHAIN') _, txid, blocks = l1.wait_for_onchaind_tx('OUR_DELAYED_RETURN_TO_WALLET', 'OUR_UNILATERAL/DELAYED_OUTPUT_TO_US') assert blocks == 4 bitcoind.generate_block(4) bitcoind.generate_block(20, wait_for_mempool=txid) sync_blockheight(bitcoind, [l1]) l1.daemon.wait_for_log(r'All outputs resolved.*') evs = l1.rpc.bkpr_listaccountevents()['events'] close = find_first_tag(evs, 'channel_close') delayed_to = find_first_tag(evs, 'delayed_to_us') # find the chain fee entry for the channel close fees = find_tags(evs, 'onchain_fee') close_fee = [e for e in fees if e['txid'] == close['txid']] assert len(close_fee) == 1 assert close_fee[0]['credit_msat'] + delayed_to['credit_msat'] == close['debit_msat'] # l2's fees should equal the trimmed htlc out evs = l2.rpc.bkpr_listaccountevents()['events'] close = find_first_tag(evs, 'channel_close') deposit = find_first_tag(evs, 'deposit') fees = find_tags(evs, 'onchain_fee') close_fee = [e for e in fees if e['txid'] == close['txid']] assert len(close_fee) == 1 # sent htlc was too small, we lose it, rounded up to nearest sat assert close_fee[0]['credit_msat'] == Millisatoshi('101000msat') assert close_fee[0]['credit_msat'] + deposit['credit_msat'] == close['debit_msat'] @unittest.skipIf(TEST_NETWORK != 'regtest', "fixme: broadcast fails, dusty") def test_bookkeeping_closing_subsat_htlcs(node_factory, bitcoind, chainparams): """Test closing balances when HTLCs are: sub 1-satoshi""" l1, l2 = node_factory.line_graph(2) l1.pay(l2, 111) l1.pay(l2, 222) l1.pay(l2, 4000000) # Make sure l2 bookkeeper processes event before we stop it! wait_for(lambda: len([e for e in l2.rpc.bkpr_listaccountevents()['events'] if e['tag'] == 'invoice']) == 3) l2.stop() l1.rpc.close(l2.info['id'], 1) bitcoind.generate_block(1, wait_for_mempool=1) _, txid, blocks = l1.wait_for_onchaind_tx('OUR_DELAYED_RETURN_TO_WALLET', 'OUR_UNILATERAL/DELAYED_OUTPUT_TO_US') assert blocks == 4 bitcoind.generate_block(4) l2.start() bitcoind.generate_block(80, wait_for_mempool=txid) sync_blockheight(bitcoind, [l1, l2]) evs = l1.rpc.bkpr_listaccountevents()['events'] # check that closing equals onchain deposits + fees close = find_first_tag(evs, 'channel_close') delayed_to = find_first_tag(evs, 'delayed_to_us') fees = find_tags(evs, 'onchain_fee') close_fee = [e for e in fees if e['txid'] == close['txid']] assert len(close_fee) == 1 assert close_fee[0]['credit_msat'] + delayed_to['credit_msat'] == close['debit_msat'] evs = l2.rpc.bkpr_listaccountevents()['events'] close = find_first_tag(evs, 'channel_close') deposit = find_first_tag(evs, 'deposit') fees = find_tags(evs, 'onchain_fee') close_fee = [e for e in fees if e['txid'] == close['txid']] assert len(close_fee) == 1 # too small to fit, we lose them as miner fees assert close_fee[0]['credit_msat'] == Millisatoshi('333msat') assert close_fee[0]['credit_msat'] + deposit['credit_msat'] == close['debit_msat'] @unittest.skipIf(TEST_NETWORK != 'regtest', "External wallet support doesn't work with elements yet.") def test_bookkeeping_external_withdraws(node_factory, bitcoind): """ Withdrawals to an external address shouldn't be included in the income statements until confirmed""" l1 = node_factory.get_node() addr = l1.rpc.newaddr()['bech32'] amount = 1111111 amount_msat = Millisatoshi(amount * 1000) bitcoind.rpc.sendtoaddress(addr, amount / 10**8) bitcoind.rpc.sendtoaddress(addr, amount / 10**8) bitcoind.generate_block(1) wait_for(lambda: len(l1.rpc.listfunds()['outputs']) == 2) waddr = l1.bitcoin.rpc.getnewaddress() # Ok, now we send some funds to an external address out = l1.rpc.withdraw(waddr, amount // 2) # Make sure bitcoind received the withdrawal unspent = l1.bitcoin.rpc.listunspent(0) withdrawal = [u for u in unspent if u['txid'] == out['txid']] assert withdrawal[0]['amount'] == Decimal('0.00555555') incomes = l1.rpc.bkpr_listincome()['income_events'] # There are two income events: deposits to wallet # for {amount} assert len(incomes) == 2 for inc in incomes: assert inc['account'] == 'wallet' assert inc['tag'] == 'deposit' assert inc['credit_msat'] == amount_msat # The event should show up in the 'bkpr_listaccountevents' however events = l1.rpc.bkpr_listaccountevents()['events'] assert len(events) == 3 external = [e for e in events if e['account'] == 'external'][0] assert external['credit_msat'] == Millisatoshi(amount // 2 * 1000) btc_balance = only_one(only_one(l1.rpc.bkpr_listbalances()['accounts'])['balances']) assert btc_balance['balance_msat'] == amount_msat * 2 # Restart the node, issues a balance snapshot # If we were counting these incorrectly, # we'd have a new journal_entry l1.restart() # the number of account + income events should be unchanged incomes = l1.rpc.bkpr_listincome()['income_events'] assert len(find_tags(incomes, 'journal_entry')) == 0 assert len(incomes) == 2 events = l1.rpc.bkpr_listaccountevents()['events'] assert len(events) == 3 assert len(find_tags(events, 'journal_entry')) == 0 # the wallet balance should be unchanged btc_balance = only_one(only_one(l1.rpc.bkpr_listbalances()['accounts'])['balances']) assert btc_balance['balance_msat'] == amount_msat * 2 # ok now we mine a block bitcoind.generate_block(1) sync_blockheight(bitcoind, [l1]) # expect the withdrawal to appear in the incomes # and there should be an onchain fee incomes = l1.rpc.bkpr_listincome()['income_events'] # 2 wallet deposits, 1 wallet withdrawal, 1 onchain_fee assert len(incomes) == 4 withdraw_amt = find_tags(incomes, 'withdrawal')[0]['debit_msat'] assert withdraw_amt == Millisatoshi(amount // 2 * 1000) fee_events = find_tags(incomes, 'onchain_fee') assert len(fee_events) == 1 fees = fee_events[0]['debit_msat'] # wallet balance is decremented now btc_balance = only_one(only_one(l1.rpc.bkpr_listbalances()['accounts'])['balances']) assert btc_balance['balance_msat'] == amount_msat * 2 - withdraw_amt - fees @unittest.skipIf(TEST_NETWORK != 'regtest', "External wallet support doesn't work with elements yet.") @unittest.skipIf(os.getenv('TEST_DB_PROVIDER', 'sqlite3') != 'sqlite3', "Depends on sqlite3 database location") def test_bookkeeping_external_withdraw_missing(node_factory, bitcoind): """ Withdrawals to an external address turn up as extremely large onchain_fees when they happen before our accounting plugin is attached""" l1 = node_factory.get_node() basedir = l1.daemon.opts.get("lightning-dir") addr = l1.rpc.newaddr()['bech32'] amount = 1111111 amount_msat = Millisatoshi(amount * 1000) bitcoind.rpc.sendtoaddress(addr, amount / 10**8) bitcoind.rpc.sendtoaddress(addr, amount / 10**8) bitcoind.generate_block(1) wait_for(lambda: len(l1.rpc.listfunds()['outputs']) == 2) waddr = l1.bitcoin.rpc.getnewaddress() # Ok, now we send some funds to an external address l1.rpc.withdraw(waddr, amount // 2) # Only two income events: deposits assert len(l1.rpc.bkpr_listincome()['income_events']) == 2 # 4 account events: 2 wallet deposits, 1 external deposit assert len(l1.rpc.bkpr_listaccountevents()['events']) == 3 # Stop node and remove the accounts data l1.stop() os.remove(os.path.join(basedir, TEST_NETWORK, 'accounts.sqlite3')) l1.start() # Number of income events should be unchanged assert len(l1.rpc.bkpr_listincome()['income_events']) == 2 # we're now missing the external deposit events = l1.rpc.bkpr_listaccountevents()['events'] assert len(events) == 2 assert len([e for e in events if e['account'] == 'external']) == 0 assert len(find_tags(events, 'journal_entry')) == 0 # the wallet balance should be unchanged btc_balance = only_one(only_one(l1.rpc.bkpr_listbalances()['accounts'])['balances']) assert btc_balance['balance_msat'] == amount_msat * 2 # ok now we mine a block bitcoind.generate_block(1) sync_blockheight(bitcoind, [l1]) # expect the withdrawal to appear in the incomes # and there should be an onchain fee incomes = l1.rpc.bkpr_listincome()['income_events'] # 2 wallet deposits, 1 onchain_fee assert len(incomes) == 3 assert len(find_tags(incomes, 'withdrawal')) == 0 fee_events = find_tags(incomes, 'onchain_fee') assert len(fee_events) == 1 fees = fee_events[0]['debit_msat'] assert fees > Millisatoshi(amount // 2 * 1000) # wallet balance is decremented now bal = only_one(only_one(l1.rpc.bkpr_listbalances()['accounts'])['balances']) assert bal['balance_msat'] == amount_msat * 2 - fees @unittest.skipIf(TEST_NETWORK != 'regtest', "External wallet support doesn't work with elements yet.") def test_bookkeeping_rbf_withdraw(node_factory, bitcoind): """ If a withdraw to an external gets RBF'd, it should *not* show up in our income ever. (but it will show up in our account events) """ l1 = node_factory.get_node() addr = l1.rpc.newaddr()['bech32'] amount = 1111111 event_counter = 0 income_counter = 0 bitcoind.rpc.sendtoaddress(addr, amount / 10**8) event_counter += 1 income_counter += 1 bitcoind.generate_block(1) wait_for(lambda: len(l1.rpc.listfunds()['outputs']) == 1) assert len(l1.rpc.bkpr_listaccountevents()['events']) == event_counter assert len(l1.rpc.bkpr_listincome()['income_events']) == income_counter # Ok, now we send some funds to an external address waddr = l1.bitcoin.rpc.getnewaddress() out1 = l1.rpc.withdraw(waddr, amount // 2, feerate='253perkw') event_counter += 1 mempool = bitcoind.rpc.getrawmempool(True) assert len(list(mempool.keys())) == 1 assert out1['txid'] in list(mempool.keys()) # another account event, still one income event assert len(l1.rpc.bkpr_listaccountevents()['events']) == event_counter assert len(l1.rpc.bkpr_listincome()['income_events']) == income_counter # unreserve the existing output l1.rpc.unreserveinputs(out1['psbt'], 200) # resend the tx out2 = l1.rpc.withdraw(waddr, amount // 2, feerate='1000perkw') mempool = bitcoind.rpc.getrawmempool(True) event_counter += 1 assert len(list(mempool.keys())) == 1 assert out2['txid'] in list(mempool.keys()) # another account event, still one income event assert len(l1.rpc.bkpr_listaccountevents()['events']) == event_counter assert len(l1.rpc.bkpr_listincome()['income_events']) == income_counter # ok now we mine a block bitcoind.generate_block(1) sync_blockheight(bitcoind, [l1]) acct_evs = l1.rpc.bkpr_listaccountevents()['events'] externs = [e for e in acct_evs if e['account'] == 'external'] assert len(externs) == 2 assert externs[0]['outpoint'][:-2] == out1['txid'] assert externs[0]['blockheight'] == 0 assert externs[1]['outpoint'][:-2] == out2['txid'] assert externs[1]['blockheight'] > 0 withdraws = find_tags(l1.rpc.bkpr_listincome()['income_events'], 'withdrawal') assert len(withdraws) == 1 assert withdraws[0]['outpoint'][:-2] == out2['txid'] # make sure no onchain fees are counted for the replaced tx fees = find_tags(acct_evs, 'onchain_fee') assert len(fees) > 1 for fee in fees: assert fee['txid'] == out2['txid'] fees = find_tags(l1.rpc.bkpr_listincome(consolidate_fees=False)['income_events'], 'onchain_fee') assert len(fees) == 2 fees = find_tags(l1.rpc.bkpr_listincome(consolidate_fees=True)['income_events'], 'onchain_fee') assert len(fees) == 1 @pytest.mark.openchannel('v2') @unittest.skipIf(os.getenv('TEST_DB_PROVIDER', 'sqlite3') != 'sqlite3', "turns off bookkeeper at start") @unittest.skipIf(TEST_NETWORK != 'regtest', "network fees hardcoded") def test_bookkeeping_missed_chans_leases(node_factory, bitcoind): """ Test that a lease is correctly recorded if bookkeeper was off """ coin_mvt_plugin = Path(__file__).parent / "plugins" / "coin_movements.py" opts = {'funder-policy': 'match', 'funder-policy-mod': 100, 'lease-fee-base-sat': '100sat', 'lease-fee-basis': 100, 'plugin': str(coin_mvt_plugin), 'disable-plugin': 'bookkeeper'} l1, l2 = node_factory.get_nodes(2, opts=opts) open_amt = 500000 feerate = 2000 lease_fee = 6432000 invoice_msat = 11000000 l1.fundwallet(open_amt * 1000) l2.fundwallet(open_amt * 1000) l1.rpc.connect(l2.info['id'], 'localhost', l2.port) # l1 leases a channel from l2 compact_lease = l2.rpc.funderupdate()['compact_lease'] txid = l1.rpc.fundchannel(l2.info['id'], open_amt, request_amt=open_amt, feerate='{}perkw'.format(feerate), compact_lease=compact_lease)['txid'] bitcoind.generate_block(1, wait_for_mempool=[txid]) wait_for(lambda: l1.channel_state(l2) == 'CHANNELD_NORMAL') scid = l1.get_channel_scid(l2) l1.wait_local_channel_active(scid) channel_id = first_channel_id(l1, l2) l1.pay(l2, invoice_msat) l1.daemon.wait_for_log(r'coin movement:.*\'invoice\'') # Now turn the bookkeeper on and restart l1.stop() l2.stop() del l1.daemon.opts['disable-plugin'] del l2.daemon.opts['disable-plugin'] l1.start() l2.start() # Wait for the balance snapshot to fire/finish l1.daemon.wait_for_log('Snapshot balances updated') l2.daemon.wait_for_log('Snapshot balances updated') def _check_events(node, channel_id, exp_events): chan_events = [ev for ev in node.rpc.bkpr_listaccountevents()['events'] if ev['account'] == channel_id] assert len(chan_events) == len(exp_events) for ev, exp in zip(chan_events, exp_events): assert ev['tag'] == exp[0] assert ev['credit_msat'] == Millisatoshi(exp[1]) assert ev['debit_msat'] == Millisatoshi(exp[2]) # l1 events exp_events = [('channel_open', open_amt * 1000 + lease_fee, 0), ('onchain_fee', 1320000, 0), ('lease_fee', 0, lease_fee), ('journal_entry', 0, invoice_msat)] _check_events(l1, channel_id, exp_events) exp_events = [('channel_open', open_amt * 1000, 0), ('onchain_fee', 892000, 0), ('lease_fee', lease_fee, 0), ('journal_entry', invoice_msat, 0)] _check_events(l2, channel_id, exp_events) @unittest.skipIf(os.getenv('TEST_DB_PROVIDER', 'sqlite3') != 'sqlite3', "turns off bookkeeper at start") @unittest.skipIf(TEST_NETWORK != 'regtest', "network fees hardcoded") @pytest.mark.openchannel('v1', 'Uses push-msat') def test_bookkeeping_missed_chans_pushed(node_factory, bitcoind): """ Test for a push_msat value in a missed channel open. """ coin_mvt_plugin = Path(__file__).parent / "plugins" / "coin_movements.py" l1, l2 = node_factory.get_nodes(2, opts={'disable-plugin': 'bookkeeper', 'plugin': str(coin_mvt_plugin)}) # Double check there's no bookkeeper plugin on assert l1.daemon.opts['disable-plugin'] == 'bookkeeper' assert l2.daemon.opts['disable-plugin'] == 'bookkeeper' open_amt = 10**7 push_amt = 10**6 * 1000 invoice_msat = 11000000 l1.fundwallet(200000000) l1.rpc.connect(l2.info['id'], 'localhost', l2.port) txid = l1.rpc.fundchannel(l2.info['id'], open_amt, push_msat=push_amt)['txid'] bitcoind.generate_block(1, wait_for_mempool=[txid]) wait_for(lambda: l1.channel_state(l2) == 'CHANNELD_NORMAL') scid = l1.get_channel_scid(l2) l1.wait_local_channel_active(scid) channel_id = first_channel_id(l1, l2) # Send l2 funds via the channel l1.pay(l2, invoice_msat) l1.daemon.wait_for_log(r'coin movement:.*\'invoice\'') # Now turn the bookkeeper on and restart l1.stop() l2.stop() del l1.daemon.opts['disable-plugin'] del l2.daemon.opts['disable-plugin'] l1.start() l2.start() # Wait for the balance snapshot to fire/finish l1.daemon.wait_for_log('Snapshot balances updated') l2.daemon.wait_for_log('Snapshot balances updated') def _check_events(node, channel_id, exp_events): chan_events = [ev for ev in node.rpc.bkpr_listaccountevents()['events'] if ev['account'] == channel_id] assert len(chan_events) == len(exp_events) for ev, exp in zip(chan_events, exp_events): assert ev['tag'] == exp[0] assert ev['credit_msat'] == Millisatoshi(exp[1]) assert ev['debit_msat'] == Millisatoshi(exp[2]) # l1 events exp_events = [('channel_open', open_amt * 1000, 0), ('onchain_fee', 4927000, 0), ('pushed', 0, push_amt), ('journal_entry', 0, invoice_msat)] _check_events(l1, channel_id, exp_events) # l2 events exp_events = [('channel_open', 0, 0), ('pushed', push_amt, 0), ('journal_entry', invoice_msat, 0)] _check_events(l2, channel_id, exp_events) @unittest.skipIf(TEST_NETWORK != 'regtest', "network fees hardcoded") @pytest.mark.openchannel('v1') def test_bookkeeping_inspect_multifundchannel(node_factory, bitcoind): """ Test that bookkeeper splits multifundchannel fees correctly for single funded channels. For single funded channels, l1 pays the entirety of the fee associated with multifundchannel, and the fee is split into each channel and is viewed from the opener's perspective. """ l1, l2, l3, l4 = node_factory.get_nodes(4) l1.fundwallet(200000000) l1.rpc.connect(l2.info['id'], 'localhost', l2.port) l1.rpc.connect(l3.info['id'], 'localhost', l3.port) l1.rpc.connect(l4.info['id'], 'localhost', l4.port) destinations = [{"id": '{}@localhost:{}'.format(l2.info['id'], l2.port), "amount": 25000}, {"id": '{}@localhost:{}'.format(l3.info['id'], l3.port), "amount": 25000}, {"id": '{}@localhost:{}'.format(l4.info['id'], l4.port), "amount": 25000}] multifundchannel_return = l1.rpc.multifundchannel(destinations) multifundchannel_txid = multifundchannel_return['txid'] channel_ids = multifundchannel_return['channel_ids'] channel_12_channel_id = channel_ids[0]['channel_id'] channel_13_channel_id = channel_ids[1]['channel_id'] channel_14_channel_id = channel_ids[2]['channel_id'] bitcoind.generate_block(1, wait_for_mempool=[multifundchannel_txid]) wait_for(lambda: l1.channel_state(l2) == 'CHANNELD_NORMAL') wait_for(lambda: l1.channel_state(l3) == 'CHANNELD_NORMAL') wait_for(lambda: l1.channel_state(l4) == 'CHANNELD_NORMAL') # now use getblock to get the tx fee from bitcoin-cli's perspective multifundchannel_rawtx = l1.bitcoin.rpc.getrawtransaction(multifundchannel_txid, True) blockhash = multifundchannel_rawtx['blockhash'] getblock_tx = l1.bitcoin.rpc.getblock(blockhash, 2)['tx'] getblock_fee_btc = 0 for tx in getblock_tx: if tx['txid'] == multifundchannel_txid: getblock_fee_btc = tx['fee'] # now sum bookkeeper fees for each channel to get the total fees for this tx channel_12_multifundchannel_fee_msat = l1.rpc.bkpr_inspect(channel_12_channel_id)['txs'][0]['fees_paid_msat'] channel_13_multifundchannel_fee_msat = l1.rpc.bkpr_inspect(channel_13_channel_id)['txs'][0]['fees_paid_msat'] channel_14_multifundchannel_fee_msat = l1.rpc.bkpr_inspect(channel_14_channel_id)['txs'][0]['fees_paid_msat'] bkpr_total_fee_msat = (channel_12_multifundchannel_fee_msat + channel_13_multifundchannel_fee_msat + channel_14_multifundchannel_fee_msat) assert bkpr_total_fee_msat == int(getblock_fee_btc * 100000000000) @unittest.skipIf(TEST_NETWORK != 'regtest', "network fees hardcoded") @pytest.mark.openchannel('v2') def test_bookkeeping_inspect_mfc_dual_funded(node_factory, bitcoind): """ Test that bookkeeper splits multifundchannel fees correctly for dual funded channels. For dual funded channels, the other nodes also pay part of the fees associated with multifundchannel, since they are also funding the channel. To calculate the total fees spent for the multifundchannel tx, the other nodes' fees paid must be included. """ opts = {'experimental-dual-fund': None, 'funder-policy': 'match', 'funder-policy-mod': 100, 'funder-lease-requests-only': False} l1, l2, l3, l4 = node_factory.get_nodes(4, opts=opts) l1.fundwallet(2000000000) l2.fundwallet(2000000000) l3.fundwallet(2000000000) l4.fundwallet(2000000000) l1.rpc.connect(l2.info['id'], 'localhost', l2.port) l1.rpc.connect(l3.info['id'], 'localhost', l3.port) l1.rpc.connect(l4.info['id'], 'localhost', l4.port) destinations = [{"id": '{}@localhost:{}'.format(l2.info['id'], l2.port), "amount": 25000, "announce": True}, {"id": '{}@localhost:{}'.format(l3.info['id'], l3.port), "amount": 25000, "announce": True}, {"id": '{}@localhost:{}'.format(l4.info['id'], l4.port), "amount": 25000, "announce": True}] multifundchannel_return = l1.rpc.multifundchannel(destinations) multifundchannel_txid = multifundchannel_return['txid'] channel_ids = multifundchannel_return['channel_ids'] channel_12_channel_id = channel_ids[0]['channel_id'] channel_13_channel_id = channel_ids[1]['channel_id'] channel_14_channel_id = channel_ids[2]['channel_id'] bitcoind.generate_block(5, wait_for_mempool=[multifundchannel_txid]) wait_for(lambda: l1.channel_state(l2) == 'CHANNELD_NORMAL') wait_for(lambda: l1.channel_state(l3) == 'CHANNELD_NORMAL') wait_for(lambda: l1.channel_state(l4) == 'CHANNELD_NORMAL') # now use getblock to get the tx fee from bitcoin-cli's perspective multifundchannel_rawtx = l1.bitcoin.rpc.getrawtransaction(multifundchannel_txid, True) blockhash = multifundchannel_rawtx['blockhash'] getblock_tx = l1.bitcoin.rpc.getblock(blockhash, 2)['tx'] getblock_fee_btc = 0 for tx in getblock_tx: if tx['txid'] == multifundchannel_txid: getblock_fee_btc = tx['fee'] # now sum bookkeeper fees for each node to get the total fees for this tx channel_12_multifundchannel_fee_msat = l1.rpc.bkpr_inspect(channel_12_channel_id)['txs'][0]['fees_paid_msat'] channel_21_multifundchannel_fee_msat = l2.rpc.bkpr_inspect(channel_12_channel_id)['txs'][0]['fees_paid_msat'] channel_13_multifundchannel_fee_msat = l1.rpc.bkpr_inspect(channel_13_channel_id)['txs'][0]['fees_paid_msat'] channel_31_multifundchannel_fee_msat = l3.rpc.bkpr_inspect(channel_13_channel_id)['txs'][0]['fees_paid_msat'] channel_14_multifundchannel_fee_msat = l1.rpc.bkpr_inspect(channel_14_channel_id)['txs'][0]['fees_paid_msat'] channel_41_multifundchannel_fee_msat = l4.rpc.bkpr_inspect(channel_14_channel_id)['txs'][0]['fees_paid_msat'] bkpr_total_fee_msat = (channel_12_multifundchannel_fee_msat + channel_21_multifundchannel_fee_msat + channel_13_multifundchannel_fee_msat + channel_31_multifundchannel_fee_msat + channel_14_multifundchannel_fee_msat + channel_41_multifundchannel_fee_msat) assert bkpr_total_fee_msat == int(getblock_fee_btc * 100000000000) @unittest.skipIf(os.getenv('TEST_DB_PROVIDER', 'sqlite3') != 'sqlite3', "turns off bookkeeper at start") @unittest.skipIf(TEST_NETWORK != 'regtest', "network fees hardcoded") @pytest.mark.openchannel('v1', 'Uses push-msat') def test_bookkeeping_missed_chans_pay_after(node_factory, bitcoind): """ Route a payment through a channel that we didn't have open when the bookkeeper was around """ coin_mvt_plugin = Path(__file__).parent / "plugins" / "coin_movements.py" l1, l2 = node_factory.get_nodes(2, opts={'disable-plugin': 'bookkeeper', 'may_reconnect': True, 'plugin': str(coin_mvt_plugin)}) # Double check there's no bookkeeper plugin on assert l1.daemon.opts['disable-plugin'] == 'bookkeeper' assert l2.daemon.opts['disable-plugin'] == 'bookkeeper' open_amt = 10**7 invoice_msat = 11000000 l1.fundwallet(200000000) l1.rpc.connect(l2.info['id'], 'localhost', l2.port) txid = l1.rpc.fundchannel(l2.info['id'], open_amt)['txid'] bitcoind.generate_block(1, wait_for_mempool=[txid]) wait_for(lambda: l1.channel_state(l2) == 'CHANNELD_NORMAL') scid = l1.get_channel_scid(l2) l1.wait_local_channel_active(scid) channel_id = first_channel_id(l1, l2) # Now turn the bookkeeper on and restart l1.stop() l2.stop() del l1.daemon.opts['disable-plugin'] del l2.daemon.opts['disable-plugin'] l1.start() l2.start() # Wait for the balance snapshot to fire/finish l1.daemon.wait_for_log('Snapshot balances updated') l2.daemon.wait_for_log('Snapshot balances updated') # Should have channel in both, with balances for n in [l1, l2]: accts = [ba['account'] for ba in n.rpc.bkpr_listbalances()['accounts']] assert channel_id in accts # Send a payment, should be ok. l1.rpc.connect(l2.info['id'], 'localhost', l2.port) l1.wait_local_channel_active(scid) l1.pay(l2, invoice_msat) l1.daemon.wait_for_log(r'coin movement:.*\'invoice\'') def _check_events(node, channel_id, exp_events): chan_events = [ev for ev in node.rpc.bkpr_listaccountevents()['events'] if ev['account'] == channel_id] assert len(chan_events) == len(exp_events) for ev, exp in zip(chan_events, exp_events): assert ev['tag'] == exp[0] assert ev['credit_msat'] == Millisatoshi(exp[1]) assert ev['debit_msat'] == Millisatoshi(exp[2]) # l1 events exp_events = [('channel_open', open_amt * 1000, 0), ('onchain_fee', 4927000, 0), ('invoice', 0, invoice_msat)] _check_events(l1, channel_id, exp_events) # l2 events exp_events = [('channel_open', 0, 0), ('invoice', invoice_msat, 0)] _check_events(l2, channel_id, exp_events) @unittest.skipIf(os.getenv('TEST_DB_PROVIDER', 'sqlite3') != 'sqlite3', "turns off bookkeeper at start") def test_bookkeeping_onchaind_txs(node_factory, bitcoind): """ Test for a channel that's closed, but whose close tx re-appears in the rescan """ coin_mvt_plugin = Path(__file__).parent / "plugins" / "coin_movements.py" l1, l2 = node_factory.line_graph(2, wait_for_announce=True, opts={'disable-plugin': 'bookkeeper', 'plugin': str(coin_mvt_plugin)}) # Double check there's no bookkeeper plugin on assert l1.daemon.opts['disable-plugin'] == 'bookkeeper' # Send l2 funds via the channel l1.pay(l2, 11000000) l1.daemon.wait_for_log(r'coin movement:.*\'invoice\'') bitcoind.generate_block(10) # Amicably close the channel, mine 101 blocks (channel forgotten) l1.rpc.close(l2.info['id']) l1.wait_for_channel_onchain(l2.info['id']) bitcoind.generate_block(101) sync_blockheight(bitcoind, [l1]) l1.daemon.wait_for_log('onchaind complete, forgetting peer') # Now turn the bookkeeper on and restart l1.stop() del l1.daemon.opts['disable-plugin'] # Roll back -- close is picked up for a forgotten channel l1.daemon.opts['rescan'] = 102 l1.start() # Wait for the balance snapshot to fire/finish l1.daemon.wait_for_log('Snapshot balances updated') # We should have the deposit events = l1.rpc.bkpr_listaccountevents()['events'] assert len(events) == 2 assert events[0]['account'] == 'wallet' assert events[0]['tag'] == 'deposit' assert events[1]['account'] == 'wallet' assert events[1]['tag'] == 'journal_entry' wallet_bal = only_one(l1.rpc.bkpr_listbalances()['accounts']) assert wallet_bal['account'] == 'wallet' funds = l1.rpc.listfunds() assert len(funds['channels']) == 0 outs = sum([out['amount_msat'] for out in funds['outputs']]) assert outs == only_one(wallet_bal['balances'])['balance_msat'] def test_bookkeeping_descriptions(node_factory, bitcoind, chainparams): """ When an 'invoice' type event comes through, we look up the description details to include about the item. Particularly useful for CSV outputs etc. """ l1, l2 = node_factory.line_graph(2, opts={'experimental-offers': None}) # Send l2 funds via the channel bolt11_desc = 'test "bolt11" description, 🥰🪢' l1.pay(l2, 11000000, label=bolt11_desc) l1.daemon.wait_for_log('coin_move .* [(]invoice[)] 0msat -11000000msat') l2.daemon.wait_for_log('coin_move .* [(]invoice[)] 11000000msat') # Test paying an bolt11 invoice (rcvr) l1_inc_ev = l1.rpc.bkpr_listincome()['income_events'] inv = only_one([ev for ev in l1_inc_ev if ev['tag'] == 'invoice']) assert inv['description'] == bolt11_desc # Test paying an bolt11 invoice (sender) l2_inc_ev = l2.rpc.bkpr_listincome()['income_events'] inv = only_one([ev for ev in l2_inc_ev if ev['tag'] == 'invoice']) assert inv['description'] == bolt11_desc # Make an offer (l1) bolt12_desc = 'test "bolt12" description, 🥰🪢' offer = l1.rpc.call('offer', [100, bolt12_desc]) invoice = l2.rpc.call('fetchinvoice', {'offer': offer['bolt12']}) paid = l2.rpc.pay(invoice['invoice']) l1.daemon.wait_for_log('coin_move .* [(]invoice[)] 100msat') l2.daemon.wait_for_log('coin_move .* [(]invoice[)] 0msat -100msat') # Test paying an offer (bolt12) (rcvr) l1_inc_ev = l1.rpc.bkpr_listincome()['income_events'] inv = only_one([ev for ev in l1_inc_ev if 'payment_id' in ev and ev['payment_id'] == paid['payment_hash']]) assert inv['description'] == bolt12_desc # Test paying an offer (bolt12) (sender) l2_inc_ev = l2.rpc.bkpr_listincome()['income_events'] inv = only_one([ev for ev in l2_inc_ev if 'payment_id' in ev and ev['payment_id'] == paid['payment_hash'] and ev['tag'] == 'invoice']) assert inv['description'] == bolt12_desc # Check the CSVs look groovy l1.rpc.bkpr_dumpincomecsv('koinly', 'koinly.csv') l2.rpc.bkpr_dumpincomecsv('koinly', 'koinly.csv') koinly_path = os.path.join(l1.daemon.lightning_dir, TEST_NETWORK, 'koinly.csv') l1_koinly_csv = open(koinly_path, 'rb').read() bolt11_exp = bytes('invoice,"test \'bolt11\' description, 🥰🪢",', 'utf-8') bolt12_exp = bytes('invoice,"test \'bolt12\' description, 🥰🪢",', 'utf-8') assert l1_koinly_csv.find(bolt11_exp) >= 0 assert l1_koinly_csv.find(bolt12_exp) >= 0 koinly_path = os.path.join(l2.daemon.lightning_dir, TEST_NETWORK, 'koinly.csv') l2_koinly_csv = open(koinly_path, 'rb').read() assert l2_koinly_csv.find(bolt11_exp) >= 0 assert l2_koinly_csv.find(bolt12_exp) >= 0 def test_empty_node(node_factory, bitcoind): """ Make sure that the bookkeeper commands don't blow up on an empty accounts database. """ l1 = node_factory.get_node() bkpr_cmds = [ ('channelsapy', []), ('listaccountevents', []), ('listbalances', []), ('listincome', [])] for cmd, params in bkpr_cmds: l1.rpc.call('bkpr-' + cmd, params) # inspect fails for non-channel accounts # FIXME: implement for all accounts? with pytest.raises(RpcError, match=r'not supported for non-channel accounts'): l1.rpc.bkpr_inspect('wallet') def test_rebalance_tracking(node_factory, bitcoind): """ We identify rebalances (invoices paid and received by our node), this allows us to filter them out of "incomes" (self-transfers are not income/exp) and instead only display the cost incurred to move the payment (correctly marked as a rebalance) 1 -> 2 -> 3 -> 1 """ rebal_amt = 3210 l1, l2, l3 = node_factory.get_nodes(3) l1.rpc.connect(l2.info['id'], 'localhost', l2.port) l2.rpc.connect(l3.info['id'], 'localhost', l3.port) l3.rpc.connect(l1.info['id'], 'localhost', l1.port) c12, _ = l1.fundchannel(l2, 10**7, wait_for_active=True) c23, _ = l2.fundchannel(l3, 10**7, wait_for_active=True) c31, _ = l3.fundchannel(l1, 10**7, wait_for_active=True) # Build a rebalance payment invoice = l1.rpc.invoice(rebal_amt, 'to_self', 'to_self') pay_hash = invoice['payment_hash'] pay_sec = invoice['payment_secret'] route = [{ 'id': l2.info['id'], 'channel': c12, 'direction': int(not l1.info['id'] < l2.info['id']), 'amount_msat': rebal_amt + 1001, 'style': 'tlv', 'delay': 24, }, { 'id': l3.info['id'], 'channel': c23, 'direction': int(not l2.info['id'] < l3.info['id']), 'amount_msat': rebal_amt + 500, 'style': 'tlv', 'delay': 16, }, { 'id': l1.info['id'], 'channel': c31, 'direction': int(not l3.info['id'] < l1.info['id']), 'amount_msat': rebal_amt, 'style': 'tlv', 'delay': 8, }] l1.rpc.sendpay(route, pay_hash, payment_secret=pay_sec) result = l1.rpc.waitsendpay(pay_hash, TIMEOUT) assert result['status'] == 'complete' wait_for(lambda: 'invoice' not in [ev['tag'] for ev in l1.rpc.bkpr_listincome()['income_events']]) inc_evs = l1.rpc.bkpr_listincome()['income_events'] outbound_chan_id = only_one(l1.rpc.listpeerchannels(l2.info['id'])['channels'])['channel_id'] outbound_ev = only_one([ev for ev in inc_evs if ev['tag'] == 'rebalance_fee']) assert outbound_ev['account'] == outbound_chan_id assert outbound_ev['debit_msat'] == Millisatoshi(1001) assert outbound_ev['credit_msat'] == Millisatoshi(0) assert outbound_ev['payment_id'] == pay_hash @unittest.skipIf(os.getenv('TEST_DB_PROVIDER', 'sqlite3') != 'sqlite3', "This test is based on a sqlite3 snapshot") def test_bookkeeper_lease_fee_dupe_migration(node_factory): """ Check that if there's duplicate lease_fees, we remove them""" l1 = node_factory.get_node(bkpr_dbfile='dupe_lease_fee.sqlite3.xz') wait_for(lambda: l1.daemon.is_in_log('Duplicate \'lease_fee\' found for account')) accts_db_path = os.path.join(l1.lightning_dir, TEST_NETWORK, 'accounts.sqlite3') accts_db = Sqlite3Db(accts_db_path) assert accts_db.query('SELECT tag from channel_events where tag = \'lease_fee\';') == [{'tag': 'lease_fee'}]