From 59bb6b90ac9c786605a620a6ca04b450cbe617e7 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 4 Jun 2020 12:15:16 +0930 Subject: [PATCH] pyln: new module pyln.proto.message.bolts This contains the CSVs for the current bolts (autogenerated). It's a separate module because I expect it to be updated alongside the spec. Signed-off-by: Rusty Russell Changelog-Added: pyln: new module pyln.proto.message.bolts --- .../pyln/proto/message/bolts/Makefile | 14 + .../pyln/proto/message/bolts/__init__.py | 10 + .../pyln/proto/message/bolts/bolts.py | 273 ++++++++++++++++++ contrib/pyln-proto/setup.py | 2 +- contrib/pyln-proto/tests/test_bolts.py | 69 +++++ 5 files changed, 367 insertions(+), 1 deletion(-) create mode 100755 contrib/pyln-proto/pyln/proto/message/bolts/Makefile create mode 100644 contrib/pyln-proto/pyln/proto/message/bolts/__init__.py create mode 100755 contrib/pyln-proto/pyln/proto/message/bolts/bolts.py create mode 100644 contrib/pyln-proto/tests/test_bolts.py diff --git a/contrib/pyln-proto/pyln/proto/message/bolts/Makefile b/contrib/pyln-proto/pyln/proto/message/bolts/Makefile new file mode 100755 index 000000000..12271d104 --- /dev/null +++ b/contrib/pyln-proto/pyln/proto/message/bolts/Makefile @@ -0,0 +1,14 @@ +#! /usr/bin/make + +SPECDIR := ../../../../../../../lightning-rfc + +# Only consider specs which have types in them. +SPECS := $(shell fgrep -l '1. type' $(SPECDIR)/[0-9]*.md) + +bolts.py: $(SPECS) Makefile + for f in $(SPECS); do SPECNUM=`basename $$f | sed 's/-.*//'`; echo bolt_$${SPECNUM}_csv = '['; python3 $(SPECDIR)/tools/extract-formats.py $$f | sed 's/\(.*\)/ "\1",/'; echo ']'; done > $@ + chmod a+x $@ + +CSVFILES = $(SPECS:%.md=bolt%.py) + +refresh: $(CSVFILES) diff --git a/contrib/pyln-proto/pyln/proto/message/bolts/__init__.py b/contrib/pyln-proto/pyln/proto/message/bolts/__init__.py new file mode 100644 index 000000000..3fe7e35af --- /dev/null +++ b/contrib/pyln-proto/pyln/proto/message/bolts/__init__.py @@ -0,0 +1,10 @@ +from .bolts import bolt_01_csv, bolt_02_csv, bolt_04_csv, bolt_07_csv + +__version__ = '0.0.1' + +__all__ = [ + "bolt_01_csv", + "bolt_02_csv", + "bolt_04_csv", + "bolt_07_csv", +] diff --git a/contrib/pyln-proto/pyln/proto/message/bolts/bolts.py b/contrib/pyln-proto/pyln/proto/message/bolts/bolts.py new file mode 100755 index 000000000..b2a0ffb21 --- /dev/null +++ b/contrib/pyln-proto/pyln/proto/message/bolts/bolts.py @@ -0,0 +1,273 @@ +bolt_01_csv = [ + "msgtype,init,16", + "msgdata,init,gflen,u16,", + "msgdata,init,globalfeatures,byte,gflen", + "msgdata,init,flen,u16,", + "msgdata,init,features,byte,flen", + "msgdata,init,tlvs,init_tlvs,", + "tlvtype,init_tlvs,networks,1", + "tlvdata,init_tlvs,networks,chains,chain_hash,...", + "msgtype,error,17", + "msgdata,error,channel_id,channel_id,", + "msgdata,error,len,u16,", + "msgdata,error,data,byte,len", + "msgtype,ping,18", + "msgdata,ping,num_pong_bytes,u16,", + "msgdata,ping,byteslen,u16,", + "msgdata,ping,ignored,byte,byteslen", + "msgtype,pong,19", + "msgdata,pong,byteslen,u16,", + "msgdata,pong,ignored,byte,byteslen", + "tlvtype,n1,tlv1,1", + "tlvdata,n1,tlv1,amount_msat,tu64,", + "tlvtype,n1,tlv2,2", + "tlvdata,n1,tlv2,scid,short_channel_id,", + "tlvtype,n1,tlv3,3", + "tlvdata,n1,tlv3,node_id,point,", + "tlvdata,n1,tlv3,amount_msat_1,u64,", + "tlvdata,n1,tlv3,amount_msat_2,u64,", + "tlvtype,n1,tlv4,254", + "tlvdata,n1,tlv4,cltv_delta,u16,", + "tlvtype,n2,tlv1,0", + "tlvdata,n2,tlv1,amount_msat,tu64,", + "tlvtype,n2,tlv2,11", + "tlvdata,n2,tlv2,cltv_expiry,tu32,", +] +bolt_02_csv = [ + "msgtype,open_channel,32", + "msgdata,open_channel,chain_hash,chain_hash,", + "msgdata,open_channel,temporary_channel_id,byte,32", + "msgdata,open_channel,funding_satoshis,u64,", + "msgdata,open_channel,push_msat,u64,", + "msgdata,open_channel,dust_limit_satoshis,u64,", + "msgdata,open_channel,max_htlc_value_in_flight_msat,u64,", + "msgdata,open_channel,channel_reserve_satoshis,u64,", + "msgdata,open_channel,htlc_minimum_msat,u64,", + "msgdata,open_channel,feerate_per_kw,u32,", + "msgdata,open_channel,to_self_delay,u16,", + "msgdata,open_channel,max_accepted_htlcs,u16,", + "msgdata,open_channel,funding_pubkey,point,", + "msgdata,open_channel,revocation_basepoint,point,", + "msgdata,open_channel,payment_basepoint,point,", + "msgdata,open_channel,delayed_payment_basepoint,point,", + "msgdata,open_channel,htlc_basepoint,point,", + "msgdata,open_channel,first_per_commitment_point,point,", + "msgdata,open_channel,channel_flags,byte,", + "msgdata,open_channel,tlvs,open_channel_tlvs,", + "tlvtype,open_channel_tlvs,upfront_shutdown_script,0", + "tlvdata,open_channel_tlvs,upfront_shutdown_script,shutdown_scriptpubkey,byte,...", + "msgtype,accept_channel,33", + "msgdata,accept_channel,temporary_channel_id,byte,32", + "msgdata,accept_channel,dust_limit_satoshis,u64,", + "msgdata,accept_channel,max_htlc_value_in_flight_msat,u64,", + "msgdata,accept_channel,channel_reserve_satoshis,u64,", + "msgdata,accept_channel,htlc_minimum_msat,u64,", + "msgdata,accept_channel,minimum_depth,u32,", + "msgdata,accept_channel,to_self_delay,u16,", + "msgdata,accept_channel,max_accepted_htlcs,u16,", + "msgdata,accept_channel,funding_pubkey,point,", + "msgdata,accept_channel,revocation_basepoint,point,", + "msgdata,accept_channel,payment_basepoint,point,", + "msgdata,accept_channel,delayed_payment_basepoint,point,", + "msgdata,accept_channel,htlc_basepoint,point,", + "msgdata,accept_channel,first_per_commitment_point,point,", + "msgdata,accept_channel,tlvs,accept_channel_tlvs,", + "tlvtype,accept_channel_tlvs,upfront_shutdown_script,0", + "tlvdata,accept_channel_tlvs,upfront_shutdown_script,shutdown_scriptpubkey,byte,...", + "msgtype,funding_created,34", + "msgdata,funding_created,temporary_channel_id,byte,32", + "msgdata,funding_created,funding_txid,sha256,", + "msgdata,funding_created,funding_output_index,u16,", + "msgdata,funding_created,signature,signature,", + "msgtype,funding_signed,35", + "msgdata,funding_signed,channel_id,channel_id,", + "msgdata,funding_signed,signature,signature,", + "msgtype,funding_locked,36", + "msgdata,funding_locked,channel_id,channel_id,", + "msgdata,funding_locked,next_per_commitment_point,point,", + "msgtype,shutdown,38", + "msgdata,shutdown,channel_id,channel_id,", + "msgdata,shutdown,len,u16,", + "msgdata,shutdown,scriptpubkey,byte,len", + "msgtype,closing_signed,39", + "msgdata,closing_signed,channel_id,channel_id,", + "msgdata,closing_signed,fee_satoshis,u64,", + "msgdata,closing_signed,signature,signature,", + "msgtype,update_add_htlc,128", + "msgdata,update_add_htlc,channel_id,channel_id,", + "msgdata,update_add_htlc,id,u64,", + "msgdata,update_add_htlc,amount_msat,u64,", + "msgdata,update_add_htlc,payment_hash,sha256,", + "msgdata,update_add_htlc,cltv_expiry,u32,", + "msgdata,update_add_htlc,onion_routing_packet,byte,1366", + "msgtype,update_fulfill_htlc,130", + "msgdata,update_fulfill_htlc,channel_id,channel_id,", + "msgdata,update_fulfill_htlc,id,u64,", + "msgdata,update_fulfill_htlc,payment_preimage,byte,32", + "msgtype,update_fail_htlc,131", + "msgdata,update_fail_htlc,channel_id,channel_id,", + "msgdata,update_fail_htlc,id,u64,", + "msgdata,update_fail_htlc,len,u16,", + "msgdata,update_fail_htlc,reason,byte,len", + "msgtype,update_fail_malformed_htlc,135", + "msgdata,update_fail_malformed_htlc,channel_id,channel_id,", + "msgdata,update_fail_malformed_htlc,id,u64,", + "msgdata,update_fail_malformed_htlc,sha256_of_onion,sha256,", + "msgdata,update_fail_malformed_htlc,failure_code,u16,", + "msgtype,commitment_signed,132", + "msgdata,commitment_signed,channel_id,channel_id,", + "msgdata,commitment_signed,signature,signature,", + "msgdata,commitment_signed,num_htlcs,u16,", + "msgdata,commitment_signed,htlc_signature,signature,num_htlcs", + "msgtype,revoke_and_ack,133", + "msgdata,revoke_and_ack,channel_id,channel_id,", + "msgdata,revoke_and_ack,per_commitment_secret,byte,32", + "msgdata,revoke_and_ack,next_per_commitment_point,point,", + "msgtype,update_fee,134", + "msgdata,update_fee,channel_id,channel_id,", + "msgdata,update_fee,feerate_per_kw,u32,", + "msgtype,channel_reestablish,136", + "msgdata,channel_reestablish,channel_id,channel_id,", + "msgdata,channel_reestablish,next_commitment_number,u64,", + "msgdata,channel_reestablish,next_revocation_number,u64,", + "msgdata,channel_reestablish,your_last_per_commitment_secret,byte,32", + "msgdata,channel_reestablish,my_current_per_commitment_point,point,", +] +bolt_04_csv = [ + "tlvtype,tlv_payload,amt_to_forward,2", + "tlvdata,tlv_payload,amt_to_forward,amt_to_forward,tu64,", + "tlvtype,tlv_payload,outgoing_cltv_value,4", + "tlvdata,tlv_payload,outgoing_cltv_value,outgoing_cltv_value,tu32,", + "tlvtype,tlv_payload,short_channel_id,6", + "tlvdata,tlv_payload,short_channel_id,short_channel_id,short_channel_id,", + "tlvtype,tlv_payload,payment_data,8", + "tlvdata,tlv_payload,payment_data,payment_secret,byte,32", + "tlvdata,tlv_payload,payment_data,total_msat,tu64,", + "msgtype,invalid_realm,PERM|1", + "msgtype,temporary_node_failure,NODE|2", + "msgtype,permanent_node_failure,PERM|NODE|2", + "msgtype,required_node_feature_missing,PERM|NODE|3", + "msgtype,invalid_onion_version,BADONION|PERM|4", + "msgdata,invalid_onion_version,sha256_of_onion,sha256,", + "msgtype,invalid_onion_hmac,BADONION|PERM|5", + "msgdata,invalid_onion_hmac,sha256_of_onion,sha256,", + "msgtype,invalid_onion_key,BADONION|PERM|6", + "msgdata,invalid_onion_key,sha256_of_onion,sha256,", + "msgtype,temporary_channel_failure,UPDATE|7", + "msgdata,temporary_channel_failure,len,u16,", + "msgdata,temporary_channel_failure,channel_update,byte,len", + "msgtype,permanent_channel_failure,PERM|8", + "msgtype,required_channel_feature_missing,PERM|9", + "msgtype,unknown_next_peer,PERM|10", + "msgtype,amount_below_minimum,UPDATE|11", + "msgdata,amount_below_minimum,htlc_msat,u64,", + "msgdata,amount_below_minimum,len,u16,", + "msgdata,amount_below_minimum,channel_update,byte,len", + "msgtype,fee_insufficient,UPDATE|12", + "msgdata,fee_insufficient,htlc_msat,u64,", + "msgdata,fee_insufficient,len,u16,", + "msgdata,fee_insufficient,channel_update,byte,len", + "msgtype,incorrect_cltv_expiry,UPDATE|13", + "msgdata,incorrect_cltv_expiry,cltv_expiry,u32,", + "msgdata,incorrect_cltv_expiry,len,u16,", + "msgdata,incorrect_cltv_expiry,channel_update,byte,len", + "msgtype,expiry_too_soon,UPDATE|14", + "msgdata,expiry_too_soon,len,u16,", + "msgdata,expiry_too_soon,channel_update,byte,len", + "msgtype,incorrect_or_unknown_payment_details,PERM|15", + "msgdata,incorrect_or_unknown_payment_details,htlc_msat,u64,", + "msgdata,incorrect_or_unknown_payment_details,height,u32,", + "msgtype,final_incorrect_cltv_expiry,18", + "msgdata,final_incorrect_cltv_expiry,cltv_expiry,u32,", + "msgtype,final_incorrect_htlc_amount,19", + "msgdata,final_incorrect_htlc_amount,incoming_htlc_amt,u64,", + "msgtype,channel_disabled,UPDATE|20", + "msgtype,expiry_too_far,21", + "msgtype,invalid_onion_payload,PERM|22", + "msgdata,invalid_onion_payload,type,varint,", + "msgdata,invalid_onion_payload,offset,u16,", + "msgtype,mpp_timeout,23", +] +bolt_07_csv = [ + "msgtype,announcement_signatures,259", + "msgdata,announcement_signatures,channel_id,channel_id,", + "msgdata,announcement_signatures,short_channel_id,short_channel_id,", + "msgdata,announcement_signatures,node_signature,signature,", + "msgdata,announcement_signatures,bitcoin_signature,signature,", + "msgtype,channel_announcement,256", + "msgdata,channel_announcement,node_signature_1,signature,", + "msgdata,channel_announcement,node_signature_2,signature,", + "msgdata,channel_announcement,bitcoin_signature_1,signature,", + "msgdata,channel_announcement,bitcoin_signature_2,signature,", + "msgdata,channel_announcement,len,u16,", + "msgdata,channel_announcement,features,byte,len", + "msgdata,channel_announcement,chain_hash,chain_hash,", + "msgdata,channel_announcement,short_channel_id,short_channel_id,", + "msgdata,channel_announcement,node_id_1,point,", + "msgdata,channel_announcement,node_id_2,point,", + "msgdata,channel_announcement,bitcoin_key_1,point,", + "msgdata,channel_announcement,bitcoin_key_2,point,", + "msgtype,node_announcement,257", + "msgdata,node_announcement,signature,signature,", + "msgdata,node_announcement,flen,u16,", + "msgdata,node_announcement,features,byte,flen", + "msgdata,node_announcement,timestamp,u32,", + "msgdata,node_announcement,node_id,point,", + "msgdata,node_announcement,rgb_color,byte,3", + "msgdata,node_announcement,alias,byte,32", + "msgdata,node_announcement,addrlen,u16,", + "msgdata,node_announcement,addresses,byte,addrlen", + "msgtype,channel_update,258", + "msgdata,channel_update,signature,signature,", + "msgdata,channel_update,chain_hash,chain_hash,", + "msgdata,channel_update,short_channel_id,short_channel_id,", + "msgdata,channel_update,timestamp,u32,", + "msgdata,channel_update,message_flags,byte,", + "msgdata,channel_update,channel_flags,byte,", + "msgdata,channel_update,cltv_expiry_delta,u16,", + "msgdata,channel_update,htlc_minimum_msat,u64,", + "msgdata,channel_update,fee_base_msat,u32,", + "msgdata,channel_update,fee_proportional_millionths,u32,", + "msgdata,channel_update,htlc_maximum_msat,u64,,option_channel_htlc_max", + "msgtype,query_short_channel_ids,261,gossip_queries", + "msgdata,query_short_channel_ids,chain_hash,chain_hash,", + "msgdata,query_short_channel_ids,len,u16,", + "msgdata,query_short_channel_ids,encoded_short_ids,byte,len", + "msgdata,query_short_channel_ids,tlvs,query_short_channel_ids_tlvs,", + "tlvtype,query_short_channel_ids_tlvs,query_flags,1", + "tlvdata,query_short_channel_ids_tlvs,query_flags,encoding_type,u8,", + "tlvdata,query_short_channel_ids_tlvs,query_flags,encoded_query_flags,byte,...", + "msgtype,reply_short_channel_ids_end,262,gossip_queries", + "msgdata,reply_short_channel_ids_end,chain_hash,chain_hash,", + "msgdata,reply_short_channel_ids_end,full_information,byte,", + "msgtype,query_channel_range,263,gossip_queries", + "msgdata,query_channel_range,chain_hash,chain_hash,", + "msgdata,query_channel_range,first_blocknum,u32,", + "msgdata,query_channel_range,number_of_blocks,u32,", + "msgdata,query_channel_range,tlvs,query_channel_range_tlvs,", + "tlvtype,query_channel_range_tlvs,query_option,1", + "tlvdata,query_channel_range_tlvs,query_option,query_option_flags,varint,", + "msgtype,reply_channel_range,264,gossip_queries", + "msgdata,reply_channel_range,chain_hash,chain_hash,", + "msgdata,reply_channel_range,first_blocknum,u32,", + "msgdata,reply_channel_range,number_of_blocks,u32,", + "msgdata,reply_channel_range,full_information,byte,", + "msgdata,reply_channel_range,len,u16,", + "msgdata,reply_channel_range,encoded_short_ids,byte,len", + "msgdata,reply_channel_range,tlvs,reply_channel_range_tlvs,", + "tlvtype,reply_channel_range_tlvs,timestamps_tlv,1", + "tlvdata,reply_channel_range_tlvs,timestamps_tlv,encoding_type,u8,", + "tlvdata,reply_channel_range_tlvs,timestamps_tlv,encoded_timestamps,byte,...", + "tlvtype,reply_channel_range_tlvs,checksums_tlv,3", + "tlvdata,reply_channel_range_tlvs,checksums_tlv,checksums,channel_update_checksums,...", + "subtype,channel_update_timestamps", + "subtypedata,channel_update_timestamps,timestamp_node_id_1,u32,", + "subtypedata,channel_update_timestamps,timestamp_node_id_2,u32,", + "subtype,channel_update_checksums", + "subtypedata,channel_update_checksums,checksum_node_id_1,u32,", + "subtypedata,channel_update_checksums,checksum_node_id_2,u32,", + "msgtype,gossip_timestamp_filter,265,gossip_queries", + "msgdata,gossip_timestamp_filter,chain_hash,chain_hash,", + "msgdata,gossip_timestamp_filter,first_timestamp,u32,", + "msgdata,gossip_timestamp_filter,timestamp_range,u32,", +] diff --git a/contrib/pyln-proto/setup.py b/contrib/pyln-proto/setup.py index 9a94232cd..51df7af7d 100644 --- a/contrib/pyln-proto/setup.py +++ b/contrib/pyln-proto/setup.py @@ -17,7 +17,7 @@ setup(name='pyln-proto', author='Christian Decker', author_email='decker.christian@gmail.com', license='MIT', - packages=['pyln.proto', 'pyln.proto.message'], + packages=['pyln.proto', 'pyln.proto.message', 'pyln.proto.message.bolts'], scripts=[], zip_safe=True, install_requires=requirements) diff --git a/contrib/pyln-proto/tests/test_bolts.py b/contrib/pyln-proto/tests/test_bolts.py new file mode 100644 index 000000000..62cda659c --- /dev/null +++ b/contrib/pyln-proto/tests/test_bolts.py @@ -0,0 +1,69 @@ +#! /usr/bin/python3 +from pyln.proto.message import Message, MessageNamespace +from pyln.proto.message.bolts import bolt_01_csv, bolt_02_csv, bolt_04_csv, bolt_07_csv + + +def test_bolt_01_csv_tlv(): + ns = MessageNamespace(bolt_01_csv) + + n1 = ns.get_tlvtype('n1') + + # FIXME: Test failure cases too! + for t in [['0x', ''], + ['0x21 00', '33='], + ['0xfd0201 00', '513='], + ['0xfd00fd 00', '253='], + ['0xfd00ff 00', '255='], + ['0xfe02000001 00', '33554433='], + ['0xff0200000000000001 00', '144115188075855873='], + ['0x01 00', 'tlv1={amount_msat=0}'], + ['0x01 01 01', 'tlv1={amount_msat=1}'], + ['0x01 02 0100', 'tlv1={amount_msat=256}'], + ['0x01 03 010000', 'tlv1={amount_msat=65536}'], + ['0x01 04 01000000', 'tlv1={amount_msat=16777216}'], + ['0x01 05 0100000000', 'tlv1={amount_msat=4294967296}'], + ['0x01 06 010000000000', 'tlv1={amount_msat=1099511627776}'], + ['0x01 07 01000000000000', 'tlv1={amount_msat=281474976710656}'], + ['0x01 08 0100000000000000', 'tlv1={amount_msat=72057594037927936}'], + ['0x02 08 0000000000000226', 'tlv2={scid=0x0x550}'], + ['0x03 31 023da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb00000000000000010000000000000002', 'tlv3={node_id=023da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb,amount_msat_1=1,amount_msat_2=2}'], + ['0xfd00fe 02 0226', 'tlv4={cltv_delta=550}']]: + msg = bytes.fromhex(t[0][2:].replace(' ', '')) + + val, size = n1.val_from_bin(msg, None) + assert size == len(msg) + assert n1.val_to_str(val, None) == '{' + t[1] + '}' + + +def test_bolt_01_csv(): + ns = MessageNamespace(bolt_01_csv) + # string [expected string] + for t in [['init globalfeatures= features=80', + 'init globalfeatures= features=80 tlvs={}'], + ['init globalfeatures= features=80 tlvs={}'], + ['init globalfeatures= features=80 tlvs={networks={chains=[6fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d6190000000000]}}'], + ['init globalfeatures= features=80 tlvs={networks={chains=[6fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d6190000000000,1fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d6190000000000]}}'], + ['error channel_id=0000000000000000000000000000000000000000000000000000000000000000 data=00'], + ['ping num_pong_bytes=0 ignored='], + ['ping num_pong_bytes=3 ignored=0000'], + ['pong ignored='], + ['pong ignored=000000']]: + m = Message.from_str(ns, t[0]) + b = m.to_bin() + m2 = Message.from_bin(ns, b) + assert m2.to_str() == t[-1] + + +def test_bolt_02_csv(): + MessageNamespace(bolt_02_csv) + # FIXME: Add tests. + + +def test_bolt_04_csv(): + MessageNamespace(bolt_04_csv) + # FIXME: Add tests. + + +def test_bolt_07_csv(): + MessageNamespace(bolt_07_csv) + # FIXME: Add tests.