msggen: Start making the `msggen` library a standalone tool
There are quite some things we want to generate from the schema definitions, both inside and outside of CLN itself. By bundling the schemas into the library we can make use of the tooling without running in the CLN source tree. This is in part used to generate some of the interfaces in Greenlight. Changelog-None
This commit is contained in:
Christian Decker
parent
b5bd907245
commit
00fbd5977f
5
Makefile
5
Makefile
|
|
@ -356,6 +356,7 @@ include connectd/Makefile
|
|||
include lightningd/Makefile
|
||||
include cli/Makefile
|
||||
include doc/Makefile
|
||||
include contrib/msggen/Makefile
|
||||
include devtools/Makefile
|
||||
include tools/Makefile
|
||||
include plugins/Makefile
|
||||
|
|
@ -368,8 +369,8 @@ ifneq ($(RUST),0)
|
|||
include cln-rpc/Makefile
|
||||
include cln-grpc/Makefile
|
||||
|
||||
$(MSGGEN_GENALL)&: doc/schemas/*.request.json doc/schemas/*.schema.json
|
||||
PYTHONPATH=contrib/msggen $(PYTHON) contrib/msggen/msggen/__main__.py
|
||||
$(MSGGEN_GENALL)&: contrib/msggen/msggen/schema.json
|
||||
PYTHONPATH=contrib/msggen $(PYTHON) contrib/msggen/msggen/__main__.py generate
|
||||
|
||||
# The compiler assumes that the proto files are in the same
|
||||
# directory structure as the generated files will be. Since we
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
// This file was automatically generated using the following command:
|
||||
//
|
||||
// ```bash
|
||||
// contrib/msggen/msggen/__main__.py
|
||||
// contrib/msggen/msggen/__main__.py generate
|
||||
// ```
|
||||
//
|
||||
// Do not edit this file, it'll be overwritten. Rather edit the schema that
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
contrib/msggen/dist/
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
#! /usr/bin/make
|
||||
|
||||
contrib/msggen/msggen/schema.json: ${SCHEMAS}
|
||||
PYTHONPATH=contrib/msggen ${PYTHON} contrib/msggen/msggen/__main__.py bundle doc/schemas
|
||||
|
|
@ -6,7 +6,7 @@ from msggen.gen.grpc import GrpcGenerator, GrpcConverterGenerator, GrpcUnconvert
|
|||
from msggen.gen.grpc2py import Grpc2PyGenerator
|
||||
from msggen.gen.rust import RustGenerator
|
||||
from msggen.gen.generator import GeneratorChain
|
||||
from msggen.utils import load_jsonrpc_service
|
||||
from msggen.utils import load_jsonrpc_service, combine_schemas
|
||||
import logging
|
||||
from msggen.patch import VersionAnnotationPatch, OptionalPatch, OverridePatch
|
||||
from msggen.checks import VersioningCheck
|
||||
|
|
@ -62,11 +62,9 @@ def write_msggen_meta(meta):
|
|||
os.rename(f'.msggen.json.tmp.{pid}', '.msggen.json')
|
||||
|
||||
|
||||
def run(rootdir: Path):
|
||||
schemadir = rootdir / "doc" / "schemas"
|
||||
def run():
|
||||
meta = load_msggen_meta()
|
||||
service = load_jsonrpc_service(
|
||||
schema_dir=schemadir,
|
||||
)
|
||||
|
||||
p = VersionAnnotationPatch(meta=meta)
|
||||
|
|
@ -91,14 +89,25 @@ def run(rootdir: Path):
|
|||
write_msggen_meta(meta)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
'--rootdir',
|
||||
dest='rootdir',
|
||||
default='.'
|
||||
)
|
||||
subcmds = parser.add_subparsers(required=True, dest='command')
|
||||
bundle = subcmds.add_parser("bundle", help="bundle schemas into package")
|
||||
bundle.add_argument('schema_dir', action='store')
|
||||
|
||||
subcmds.add_parser("generate", help="generate all files")
|
||||
args = parser.parse_args()
|
||||
run(
|
||||
rootdir=Path(args.rootdir)
|
||||
)
|
||||
|
||||
if args.command == 'generate':
|
||||
run()
|
||||
elif args.command == 'bundle':
|
||||
dest = Path(__file__).parent / 'schema.json'
|
||||
src = Path(__file__).parent / '..' / '..' / '..' / 'doc' / 'schemas'
|
||||
|
||||
print(f"Combining schemas from {src.resolve()} into {dest.resolve()}")
|
||||
schema = combine_schemas(src, dest)
|
||||
print(f"Created {dest} from {len(schema)} files")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -1 +1 @@
|
|||
from .utils import load_jsonrpc_method, load_jsonrpc_service # noqa
|
||||
from .utils import load_jsonrpc_method, load_jsonrpc_service, combine_schemas # noqa
|
||||
|
|
|
|||
|
|
@ -1,17 +1,48 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from importlib import resources
|
||||
from msggen.model import Method, CompositeField, Service
|
||||
import functools
|
||||
from collections import OrderedDict
|
||||
|
||||
|
||||
def load_jsonrpc_method(name, schema_dir: Path):
|
||||
def combine_schemas(schema_dir: Path, dest: Path):
|
||||
"""Enumerate all schema files, and combine it into a single JSON file."""
|
||||
bundle = OrderedDict()
|
||||
files = sorted(list(schema_dir.iterdir()))
|
||||
|
||||
for f in files:
|
||||
if not f.name.endswith(".json"):
|
||||
continue
|
||||
bundle[f.name] = json.load(f.open())
|
||||
|
||||
with dest.open(mode='w') as f:
|
||||
json.dump(
|
||||
bundle,
|
||||
f,
|
||||
indent=2,
|
||||
)
|
||||
return bundle
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def get_schema_bundle():
|
||||
"""Load the schema bundle from the combined schema file.
|
||||
|
||||
The combined schema is generated by `combine_schemas`.
|
||||
"""
|
||||
p = resources.open_text("msggen", "schema.json")
|
||||
return json.load(p)
|
||||
|
||||
|
||||
def load_jsonrpc_method(name):
|
||||
"""Load a method based on the file naming conventions for the JSON-RPC.
|
||||
"""
|
||||
base_path = schema_dir
|
||||
req_file = base_path / f"{name.lower()}.request.json"
|
||||
resp_file = base_path / f"{name.lower()}.schema.json"
|
||||
request = CompositeField.from_js(json.load(open(req_file)), path=name)
|
||||
response = CompositeField.from_js(json.load(open(resp_file)), path=name)
|
||||
schema = get_schema_bundle()
|
||||
req_file = f"{name.lower()}.request.json"
|
||||
resp_file = f"{name.lower()}.schema.json"
|
||||
request = CompositeField.from_js(schema[req_file], path=name)
|
||||
response = CompositeField.from_js(schema[resp_file], path=name)
|
||||
|
||||
# Normalize the method request and response typename so they no
|
||||
# longer conflict.
|
||||
|
|
@ -25,7 +56,7 @@ def load_jsonrpc_method(name, schema_dir: Path):
|
|||
)
|
||||
|
||||
|
||||
def load_jsonrpc_service(schema_dir: str):
|
||||
def load_jsonrpc_service():
|
||||
method_names = [
|
||||
"Getinfo",
|
||||
"ListPeers",
|
||||
|
|
@ -116,7 +147,7 @@ def load_jsonrpc_service(schema_dir: str):
|
|||
"StaticBackup",
|
||||
"Bkpr-ListIncome",
|
||||
]
|
||||
methods = [load_jsonrpc_method(name, schema_dir=schema_dir) for name in method_names]
|
||||
methods = [load_jsonrpc_method(name) for name in method_names]
|
||||
service = Service(name="Node", methods=methods)
|
||||
service.includes = ['primitives.proto'] # Make sure we have the primitives included.
|
||||
return service
|
||||
|
|
|
|||
|
|
@ -0,0 +1,195 @@
|
|||
# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand.
|
||||
|
||||
[[package]]
|
||||
name = "atomicwrites"
|
||||
version = "1.4.1"
|
||||
description = "Atomic file writes."
|
||||
optional = false
|
||||
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
|
||||
files = [
|
||||
{file = "atomicwrites-1.4.1.tar.gz", hash = "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "attrs"
|
||||
version = "22.2.0"
|
||||
description = "Classes Without Boilerplate"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
files = [
|
||||
{file = "attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"},
|
||||
{file = "attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"},
|
||||
]
|
||||
|
||||
[package.extras]
|
||||
cov = ["attrs[tests]", "coverage-enable-subprocess", "coverage[toml] (>=5.3)"]
|
||||
dev = ["attrs[docs,tests]"]
|
||||
docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier", "zope.interface"]
|
||||
tests = ["attrs[tests-no-zope]", "zope.interface"]
|
||||
tests-no-zope = ["cloudpickle", "cloudpickle", "hypothesis", "hypothesis", "mypy (>=0.971,<0.990)", "mypy (>=0.971,<0.990)", "pympler", "pympler", "pytest (>=4.3.0)", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-mypy-plugins", "pytest-xdist[psutil]", "pytest-xdist[psutil]"]
|
||||
|
||||
[[package]]
|
||||
name = "colorama"
|
||||
version = "0.4.5"
|
||||
description = "Cross-platform colored terminal text."
|
||||
optional = false
|
||||
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
|
||||
files = [
|
||||
{file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"},
|
||||
{file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "importlib-metadata"
|
||||
version = "4.8.3"
|
||||
description = "Read metadata from Python packages"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
files = [
|
||||
{file = "importlib_metadata-4.8.3-py3-none-any.whl", hash = "sha256:65a9576a5b2d58ca44d133c42a241905cc45e34d2c06fd5ba2bafa221e5d7b5e"},
|
||||
{file = "importlib_metadata-4.8.3.tar.gz", hash = "sha256:766abffff765960fcc18003801f7044eb6755ffae4521c8e8ce8e83b9c9b0668"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""}
|
||||
zipp = ">=0.5"
|
||||
|
||||
[package.extras]
|
||||
docs = ["jaraco.packaging (>=8.2)", "rst.linker (>=1.9)", "sphinx"]
|
||||
perf = ["ipython"]
|
||||
testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pep517", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-flake8", "pytest-mypy", "pytest-perf (>=0.9.2)"]
|
||||
|
||||
[[package]]
|
||||
name = "iniconfig"
|
||||
version = "1.1.1"
|
||||
description = "iniconfig: brain-dead simple config-ini parsing"
|
||||
optional = false
|
||||
python-versions = "*"
|
||||
files = [
|
||||
{file = "iniconfig-1.1.1-py2.py3-none-any.whl", hash = "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3"},
|
||||
{file = "iniconfig-1.1.1.tar.gz", hash = "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "packaging"
|
||||
version = "21.3"
|
||||
description = "Core utilities for Python packages"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
files = [
|
||||
{file = "packaging-21.3-py3-none-any.whl", hash = "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"},
|
||||
{file = "packaging-21.3.tar.gz", hash = "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
pyparsing = ">=2.0.2,<3.0.5 || >3.0.5"
|
||||
|
||||
[[package]]
|
||||
name = "pluggy"
|
||||
version = "1.0.0"
|
||||
description = "plugin and hook calling mechanisms for python"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
files = [
|
||||
{file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"},
|
||||
{file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""}
|
||||
|
||||
[package.extras]
|
||||
dev = ["pre-commit", "tox"]
|
||||
testing = ["pytest", "pytest-benchmark"]
|
||||
|
||||
[[package]]
|
||||
name = "py"
|
||||
version = "1.11.0"
|
||||
description = "library with cross-python path, ini-parsing, io, code, log facilities"
|
||||
optional = false
|
||||
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
|
||||
files = [
|
||||
{file = "py-1.11.0-py2.py3-none-any.whl", hash = "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"},
|
||||
{file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pyparsing"
|
||||
version = "3.0.7"
|
||||
description = "Python parsing module"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
files = [
|
||||
{file = "pyparsing-3.0.7-py3-none-any.whl", hash = "sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484"},
|
||||
{file = "pyparsing-3.0.7.tar.gz", hash = "sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea"},
|
||||
]
|
||||
|
||||
[package.extras]
|
||||
diagrams = ["jinja2", "railroad-diagrams"]
|
||||
|
||||
[[package]]
|
||||
name = "pytest"
|
||||
version = "6.2.5"
|
||||
description = "pytest: simple powerful testing with Python"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
files = [
|
||||
{file = "pytest-6.2.5-py3-none-any.whl", hash = "sha256:7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134"},
|
||||
{file = "pytest-6.2.5.tar.gz", hash = "sha256:131b36680866a76e6781d13f101efb86cf674ebb9762eb70d3082b6f29889e89"},
|
||||
]
|
||||
|
||||
[package.dependencies]
|
||||
atomicwrites = {version = ">=1.0", markers = "sys_platform == \"win32\""}
|
||||
attrs = ">=19.2.0"
|
||||
colorama = {version = "*", markers = "sys_platform == \"win32\""}
|
||||
importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""}
|
||||
iniconfig = "*"
|
||||
packaging = "*"
|
||||
pluggy = ">=0.12,<2.0"
|
||||
py = ">=1.8.2"
|
||||
toml = "*"
|
||||
|
||||
[package.extras]
|
||||
testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "requests", "xmlschema"]
|
||||
|
||||
[[package]]
|
||||
name = "toml"
|
||||
version = "0.10.2"
|
||||
description = "Python Library for Tom's Obvious, Minimal Language"
|
||||
optional = false
|
||||
python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
|
||||
files = [
|
||||
{file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"},
|
||||
{file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typing-extensions"
|
||||
version = "4.1.1"
|
||||
description = "Backported and Experimental Type Hints for Python 3.6+"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
files = [
|
||||
{file = "typing_extensions-4.1.1-py3-none-any.whl", hash = "sha256:21c85e0fe4b9a155d0799430b0ad741cdce7e359660ccbd8b530613e8df88ce2"},
|
||||
{file = "typing_extensions-4.1.1.tar.gz", hash = "sha256:1a9462dcc3347a79b1f1c0271fbe79e844580bb598bafa1ed208b94da3cdcd42"},
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zipp"
|
||||
version = "3.6.0"
|
||||
description = "Backport of pathlib-compatible object wrapper for zip files"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
files = [
|
||||
{file = "zipp-3.6.0-py3-none-any.whl", hash = "sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc"},
|
||||
{file = "zipp-3.6.0.tar.gz", hash = "sha256:71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832"},
|
||||
]
|
||||
|
||||
[package.extras]
|
||||
docs = ["jaraco.packaging (>=8.2)", "rst.linker (>=1.9)", "sphinx"]
|
||||
testing = ["func-timeout", "jaraco.itertools", "pytest (>=4.6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-flake8", "pytest-mypy"]
|
||||
|
||||
[metadata]
|
||||
lock-version = "2.0"
|
||||
python-versions = "^3.6"
|
||||
content-hash = "c297503666e67dc734e0ffc95b912c7bb491ebd3a6ff0553a31027a13c780a8a"
|
||||
|
|
@ -5,6 +5,9 @@ description = "A utility to transform wire messages and JSON-RPC messages to arb
|
|||
authors = ["Christian Decker <decker@blockstream.com>"]
|
||||
license = "BSD-MIT"
|
||||
|
||||
include = ["msggen/schema.json"]
|
||||
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = "^3.6"
|
||||
|
||||
|
|
@ -16,4 +19,4 @@ requires = ["poetry-core>=1.0.0"]
|
|||
build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[tool.poetry.scripts]
|
||||
msggen = 'msggen.__main__:run'
|
||||
msggen = 'msggen.__main__:main'
|
||||
|
|
|
|||
Loading…
Reference in New Issue