From ed6eaf9171f85753774ed96896a7b66ca60bb5d7 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 18 Oct 2021 10:43:33 +1030 Subject: [PATCH] experimental-websocket-port: option to create a WebSocket port. Signed-off-by: Rusty Russell --- doc/lightning-listconfigs.7.md | 3 +- doc/lightningd-config.5.md | 7 ++++ doc/schemas/listconfigs.schema.json | 4 ++ lightningd/connect_control.c | 7 +++- lightningd/lightningd.c | 1 + lightningd/lightningd.h | 3 ++ lightningd/options.c | 25 +++++++++++ requirements.lock | 60 +++++++++++++-------------- requirements.txt | 1 + tests/test_connection.py | 64 +++++++++++++++++++++++++++++ 10 files changed, 142 insertions(+), 33 deletions(-) diff --git a/doc/lightning-listconfigs.7.md b/doc/lightning-listconfigs.7.md index 7c71339d5..352bd1c40 100644 --- a/doc/lightning-listconfigs.7.md +++ b/doc/lightning-listconfigs.7.md @@ -56,6 +56,7 @@ On success, an object is returned, containing: - **experimental-onion-messages** (boolean, optional): `experimental-onion-messages` field from config or cmdline, or default - **experimental-offers** (boolean, optional): `experimental-offers` field from config or cmdline, or default - **experimental-shutdown-wrong-funding** (boolean, optional): `experimental-shutdown-wrong-funding` field from config or cmdline, or default +- **experimental-websocket-port** (u16, optional): `experimental-websocket-port` field from config or cmdline, or default - **rgb** (hex, optional): `rgb` field from config or cmdline, or default (always 6 characters) - **alias** (string, optional): `alias` field from config or cmdline, or default - **pid-file** (string, optional): `pid-file` field from config or cmdline, or default @@ -205,4 +206,4 @@ RESOURCES --------- Main web site: -[comment]: # ( SHA256STAMP:7bb40fc8fac201b32d9701b02596d0fa59eb14a3baf606439cbf96dc11548ed4) +[comment]: # ( SHA256STAMP:47c067588120e0f9a71206313685cebb2a8c515e9b04b688b202d2772c8f8146) diff --git a/doc/lightningd-config.5.md b/doc/lightningd-config.5.md index 488db84c9..05c096944 100644 --- a/doc/lightningd-config.5.md +++ b/doc/lightningd-config.5.md @@ -517,6 +517,13 @@ about whether to add funds or not to a proposed channel is handled automatically by a plugin that implements the appropriate logic for your needs. The default behavior is to not contribute funds. + **experimental-websocket-port** + +Specifying this enables support for accepting incoming WebSocket +connections on that port, on any IPv4 and IPv6 addresses you listen +to. The normal protocol is expected to be sent over WebSocket binary +frames once the connection is upgraded. + BUGS ---- diff --git a/doc/schemas/listconfigs.schema.json b/doc/schemas/listconfigs.schema.json index 877cea210..9f910292f 100644 --- a/doc/schemas/listconfigs.schema.json +++ b/doc/schemas/listconfigs.schema.json @@ -121,6 +121,10 @@ "type": "boolean", "description": "`experimental-shutdown-wrong-funding` field from config or cmdline, or default" }, + "experimental-websocket-port": { + "type": "u16", + "description": "`experimental-websocket-port` field from config or cmdline, or default" + }, "rgb": { "type": "hex", "description": "`rgb` field from config or cmdline, or default", diff --git a/lightningd/connect_control.c b/lightningd/connect_control.c index 314efccb4..d17d3e780 100644 --- a/lightningd/connect_control.c +++ b/lightningd/connect_control.c @@ -350,7 +350,10 @@ int connectd_init(struct lightningd *ld) int hsmfd; struct wireaddr_internal *wireaddrs = ld->proposed_wireaddr; enum addr_listen_announce *listen_announce = ld->proposed_listen_announce; - const char *websocket_helper_path = ""; + const char *websocket_helper_path; + + websocket_helper_path = subdaemon_path(tmpctx, ld, + "lightning_websocketd"); if (socketpair(AF_LOCAL, SOCK_STREAM, 0, fds) != 0) fatal("Could not socketpair for connectd<->gossipd"); @@ -384,7 +387,7 @@ int connectd_init(struct lightningd *ld) ld->config.use_v3_autotor, ld->config.connection_timeout_secs, websocket_helper_path, - 0); + ld->websocket_port); subd_req(ld->connectd, ld->connectd, take(msg), -1, 0, connect_init_done, NULL); diff --git a/lightningd/lightningd.c b/lightningd/lightningd.c index 84f9c9cd5..878856c94 100644 --- a/lightningd/lightningd.c +++ b/lightningd/lightningd.c @@ -216,6 +216,7 @@ static struct lightningd *new_lightningd(const tal_t *ctx) ld->always_use_proxy = false; ld->pure_tor_setup = false; ld->tor_service_password = NULL; + ld->websocket_port = 0; /*~ This is initialized later, but the plugin loop examines this, * so set it to NULL explicitly now. */ diff --git a/lightningd/lightningd.h b/lightningd/lightningd.h index 274995422..532841dca 100644 --- a/lightningd/lightningd.h +++ b/lightningd/lightningd.h @@ -284,6 +284,9 @@ struct lightningd { /* Array of (even) TLV types that we should allow. This is required * since we otherwise would outright reject them. */ u64 *accept_extra_tlv_types; + + /* EXPERIMENTAL: websocket port if non-zero */ + u16 websocket_port; }; /* Turning this on allows a tal allocation to return NULL, rather than aborting. diff --git a/lightningd/options.c b/lightningd/options.c index b88cc8234..ef65f9dd2 100644 --- a/lightningd/options.c +++ b/lightningd/options.c @@ -850,6 +850,21 @@ static char *opt_set_wumbo(struct lightningd *ld) return NULL; } +static char *opt_set_websocket_port(const char *arg, struct lightningd *ld) +{ + u32 port COMPILER_WANTS_INIT("9.3.0 -O2"); + char *err; + + err = opt_set_u32(arg, &port); + if (err) + return err; + + ld->websocket_port = port; + if (ld->websocket_port != port) + return tal_fmt(NULL, "'%s' is out of range", arg); + return NULL; +} + static char *opt_set_dual_fund(struct lightningd *ld) { /* Dual funding implies anchor outputs */ @@ -1051,6 +1066,11 @@ static void register_opts(struct lightningd *ld) "--subdaemon=hsmd:remote_signer " "would use a hypothetical remote signing subdaemon."); + opt_register_arg("--experimental-websocket-port", + opt_set_websocket_port, NULL, + ld, + "experimental: alternate port for peers to connect" + " using WebSockets (RFC6455)"); opt_register_logging(ld); opt_register_version(); @@ -1463,6 +1483,11 @@ static void add_config(struct lightningd *ld, json_add_opt_disable_plugins(response, ld->plugins); } else if (opt->cb_arg == (void *)opt_force_feerates) { answer = fmt_force_feerates(name0, ld->force_feerates); + } else if (opt->cb_arg == (void *)opt_set_websocket_port) { + if (ld->websocket_port) + json_add_u32(response, name0, + ld->websocket_port); + return; } else if (opt->cb_arg == (void *)opt_important_plugin) { /* Do nothing, this is already handled by * opt_add_plugin. */ diff --git a/requirements.lock b/requirements.lock index 6c95dc9e5..cc193eea8 100644 --- a/requirements.lock +++ b/requirements.lock @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with python 3.8 # To update, run: # -# pip-compile --output-file=requirements.lock requirements.in +# pip-compile --output-file=requirements.lock requirements.txt # alabaster==0.7.12 # via sphinx @@ -15,9 +15,9 @@ attrs==21.2.0 babel==2.9.1 # via sphinx base58==2.0.1 - # via -r requirements.in + # via pyln.proto bitstring==3.1.9 - # via -r requirements.in + # via pyln.proto certifi==2021.5.30 # via requests cffi==1.14.6 @@ -27,17 +27,17 @@ cffi==1.14.6 charset-normalizer==2.0.6 # via requests cheroot==8.5.2 - # via -r requirements.in + # via pyln-testing click==7.1.2 # via flask coincurve==13.0.0 - # via -r requirements.in + # via pyln.proto commonmark==0.9.1 # via recommonmark crc32c==2.2.post0 - # via -r requirements.in + # via -r requirements.txt cryptography==3.4.8 - # via -r requirements.in + # via pyln.proto docutils==0.17.1 # via # recommonmark @@ -45,15 +45,15 @@ docutils==0.17.1 entrypoints==0.3 # via flake8 ephemeral-port-reserve==1.1.1 - # via -r requirements.in + # via pyln-testing execnet==1.9.0 # via pytest-xdist flake8==3.7.9 - # via -r requirements.in + # via -r requirements.txt flaky==3.7.0 - # via -r requirements.in + # via pyln-testing flask==1.1.4 - # via -r requirements.in + # via pyln-testing idna==3.2 # via requests imagesize==1.2.0 @@ -70,9 +70,9 @@ jinja2==2.11.3 # mrkd # sphinx jsonschema==3.2.0 - # via -r requirements.in + # via pyln-testing mako==1.1.5 - # via -r requirements.in + # via -r requirements.txt markupsafe==2.0.1 # via # jinja2 @@ -88,9 +88,9 @@ more-itertools==8.10.0 # cheroot # jaraco.functools mrkd==0.1.6 - # via -r requirements.in + # via -r requirements.txt mypy==0.910 - # via -r requirements.in + # via pyln.proto mypy-extensions==0.4.3 # via mypy packaging==21.0 @@ -102,9 +102,9 @@ plac==1.3.3 pluggy==0.13.1 # via pytest psutil==5.7.3 - # via -r requirements.in + # via pyln-testing psycopg2-binary==2.8.6 - # via -r requirements.in + # via pyln-testing py==1.10.0 # via # pytest @@ -112,9 +112,7 @@ py==1.10.0 pycodestyle==2.5.0 # via flake8 pycparser==2.20 - # via - # -r requirements.in - # cffi + # via cffi pyflakes==2.1.1 # via flake8 pygments==2.10.0 @@ -126,10 +124,10 @@ pyparsing==2.4.7 pyrsistent==0.18.0 # via jsonschema pysocks==1.7.1 - # via -r requirements.in + # via pyln.proto pytest==6.1.2 # via - # -r requirements.in + # pyln-testing # pytest-forked # pytest-rerunfailures # pytest-timeout @@ -137,17 +135,17 @@ pytest==6.1.2 pytest-forked==1.3.0 # via pytest-xdist pytest-rerunfailures==9.1.1 - # via -r requirements.in + # via pyln-testing pytest-timeout==1.4.2 - # via -r requirements.in + # via pyln-testing pytest-xdist==2.2.1 - # via -r requirements.in + # via pyln-testing python-bitcoinlib==0.11.0 - # via -r requirements.in + # via pyln-testing pytz==2021.1 # via babel recommonmark==0.7.1 - # via -r requirements.in + # via pyln-client requests==2.26.0 # via sphinx six==1.16.0 @@ -171,13 +169,15 @@ sphinxcontrib-qthelp==1.0.3 sphinxcontrib-serializinghtml==1.1.5 # via sphinx toml==0.10.2 - # via - # mypy - # pytest + # via pytest +typed-ast==1.4.3 + # via mypy typing-extensions==3.10.0.2 # via mypy urllib3==1.26.7 # via requests +websocket-client==1.2.1 + # via -r requirements.txt werkzeug==1.0.1 # via flask diff --git a/requirements.txt b/requirements.txt index ef5e2b9dc..4986279a2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ mrkd ~= 0.1.6 Mako ~= 1.1.3 flake8 ~= 3.7.8 +websocket-client ./contrib/pyln-client ./contrib/pyln-proto diff --git a/tests/test_connection.py b/tests/test_connection.py index 0d9d58ff5..005032cc2 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -2,7 +2,9 @@ from collections import namedtuple from fixtures import * # noqa: F401,F403 from fixtures import TEST_NETWORK from flaky import flaky # noqa: F401 +from ephemeral_port_reserve import reserve # type: ignore from pyln.client import RpcError, Millisatoshi +import pyln.proto.wire as wire from utils import ( only_one, wait_for, sync_blockheight, TIMEOUT, expected_peer_features, expected_node_features, @@ -20,6 +22,7 @@ import re import shutil import time import unittest +import websocket def test_connect(node_factory): @@ -3740,6 +3743,67 @@ def test_old_feerate(node_factory): l1.pay(l2, 1000) +@pytest.mark.developer("needs --dev-allow-localhost") +def test_websocket(node_factory): + ws_port = reserve() + l1, l2 = node_factory.line_graph(2, + opts=[{'experimental-websocket-port': ws_port, + 'dev-allow-localhost': None}, + {'dev-allow-localhost': None}], + wait_for_announce=True) + assert l1.rpc.listconfigs()['experimental-websocket-port'] == ws_port + + # Adapter to turn websocket into a stream "connection" + class BinWebSocket(object): + def __init__(self, hostname, port): + self.ws = websocket.WebSocket() + self.ws.connect("ws://" + hostname + ":" + str(port)) + self.recvbuf = bytes() + + def send(self, data): + self.ws.send(data, websocket.ABNF.OPCODE_BINARY) + + def recv(self, maxlen): + while len(self.recvbuf) < maxlen: + self.recvbuf += self.ws.recv() + + ret = self.recvbuf[:maxlen] + self.recvbuf = self.recvbuf[maxlen:] + return ret + + ws = BinWebSocket('localhost', ws_port) + lconn = wire.LightningConnection(ws, + wire.PublicKey(bytes.fromhex(l1.info['id'])), + wire.PrivateKey(bytes([1] * 32)), + is_initiator=True) + + l1.daemon.wait_for_log('Websocket connection in from') + + # Perform handshake. + lconn.shake() + + # Expect to receive init msg. + msg = lconn.read_message() + assert int.from_bytes(msg[0:2], 'big') == 16 + + # Echo same message back. + lconn.send_message(msg) + + # Now try sending a ping, ask for 50 bytes + msg = bytes((0, 18, 0, 50, 0, 0)) + lconn.send_message(msg) + + # Could actually reply with some gossip msg! + while True: + msg = lconn.read_message() + if int.from_bytes(msg[0:2], 'big') == 19: + break + + # Check node_announcement has websocket + assert (only_one(l2.rpc.listnodes(l1.info['id'])['nodes'])['addresses'] + == [{'type': 'ipv4', 'address': '127.0.0.1', 'port': l1.port}, {'type': 'websocket', 'port': ws_port}]) + + @pytest.mark.developer("dev-disconnect required") def test_ping_timeout(node_factory): # Disconnects after this, but doesn't know it.