From a506c0568449cda337c06ae90c15fdd3850c369a Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Mon, 8 Jul 2024 14:43:07 -0500 Subject: [PATCH 1/8] reckless-rpc: initial boilerplate Trying a single command first - reckless-search to test launching a reckless process and processing the result. --- plugins/Makefile | 9 +- plugins/recklessrpc.c | 190 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 198 insertions(+), 1 deletion(-) create mode 100644 plugins/recklessrpc.c diff --git a/plugins/Makefile b/plugins/Makefile index 398226f99ba3..a0b2d5efe772 100644 --- a/plugins/Makefile +++ b/plugins/Makefile @@ -63,6 +63,9 @@ PLUGIN_FUNDER_HEADER := \ plugins/funder_policy.h PLUGIN_FUNDER_OBJS := $(PLUGIN_FUNDER_SRC:.c=.o) +PLUGIN_RECKLESSRPC_SRC := plugins/recklessrpc.c +PLUGIN_RECKLESSRPC_OBJS := $(PLUGIN_RECKLESSRPC_SRC:.c=.o) + PLUGIN_ALL_SRC := \ $(PLUGIN_AUTOCLEAN_SRC) \ $(PLUGIN_chanbackup_SRC) \ @@ -77,7 +80,8 @@ PLUGIN_ALL_SRC := \ $(PLUGIN_PAY_LIB_SRC) \ $(PLUGIN_PAY_SRC) \ $(PLUGIN_SPENDER_SRC) \ - $(PLUGIN_RECOVER_SRC) + $(PLUGIN_RECOVER_SRC) \ + $(PLUGIN_RECKLESSRPC_SRC) PLUGIN_ALL_HEADER := \ $(PLUGIN_PAY_HEADER) \ @@ -97,6 +101,7 @@ C_PLUGINS := \ plugins/keysend \ plugins/offers \ plugins/pay \ + plugins/recklessrpc \ plugins/recover \ plugins/txprepare \ plugins/cln-renepay \ @@ -216,6 +221,8 @@ plugins/funder: bitcoin/psbt.o common/psbt_open.o $(PLUGIN_FUNDER_OBJS) $(PLUGIN plugins/recover: common/gossmap.o common/sciddir_or_pubkey.o common/fp16.o $(PLUGIN_RECOVER_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) +plugins/recklessrpc: $(PLUGIN_RECKLESSRPC_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) + # This covers all the low-level list RPCs which return simple arrays SQL_LISTRPCS := listchannels listforwards listhtlcs listinvoices listnodes listoffers listpeers listpeerchannels listclosedchannels listtransactions listsendpays bkpr-listaccountevents bkpr-listincome SQL_LISTRPCS_SCHEMAS := $(foreach l,$(SQL_LISTRPCS),doc/schemas/lightning-$l.json) diff --git a/plugins/recklessrpc.c b/plugins/recklessrpc.c new file mode 100644 index 000000000000..416c7230c7f0 --- /dev/null +++ b/plugins/recklessrpc.c @@ -0,0 +1,190 @@ +/* This plugin provides RPC access to the reckless standalone utility. + */ + +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static struct plugin *plugin; + +struct reckless { + struct command *cmd; + int stdinfd; + int stdoutfd; + int stderrfd; + char *stdoutbuf; + char *stderrbuf; + size_t stdout_read; /* running total */ + size_t stdout_new; /* new since last read */ + pid_t pid; +}; + +static struct io_plan *read_more(struct io_conn *conn, struct reckless *rkls) +{ + rkls->stdout_read += rkls->stdout_new; + if (rkls->stdout_read == tal_count(rkls->stdoutbuf)) + tal_resize(&rkls->stdoutbuf, rkls->stdout_read * 2); + return io_read_partial(conn, rkls->stdoutbuf + rkls->stdout_read, + tal_count(rkls->stdoutbuf) - rkls->stdout_read, + &rkls->stdout_new, read_more, rkls); +} + +static struct command_result *reckless_result(struct io_conn *conn, + struct reckless *reckless) +{ + struct json_stream *response; + response = jsonrpc_stream_success(reckless->cmd); + json_array_start(response, "result"); + const jsmntok_t *results, *result, *logs, *log; + size_t i; + jsmn_parser parser; + jsmntok_t *toks; + toks = tal_arr(reckless, jsmntok_t, 500); + jsmn_init(&parser); + if (jsmn_parse(&parser, reckless->stdoutbuf, + strlen(reckless->stdoutbuf), toks, tal_count(toks)) <= 0) { + plugin_log(plugin, LOG_DBG, "need more json tokens"); + assert(false); + } + + results = json_get_member(reckless->stdoutbuf, toks, "result"); + json_for_each_arr(i, result, results) { + json_add_string(response, + NULL, + json_strdup(reckless, reckless->stdoutbuf, + result)); + } + json_array_end(response); + json_array_start(response, "log"); + logs = json_get_member(reckless->stdoutbuf, toks, "log"); + json_for_each_arr(i, log, logs) { + json_add_string(response, + NULL, + json_strdup(reckless, reckless->stdoutbuf, + log)); + } + json_array_end(response); + + return command_finished(reckless->cmd, response); +} + +static void reckless_conn_finish(struct io_conn *conn, + struct reckless *reckless) +{ + /* FIXME: avoid EBADFD - leave stdin fd open? */ + if (errno && errno != 9) + plugin_log(plugin, LOG_DBG, "err: %s", strerror(errno)); + reckless_result(conn, reckless); + if (reckless->pid > 0) { + int status; + pid_t p; + p = waitpid(reckless->pid, &status, WNOHANG); + /* Did the reckless process exit? */ + if (p != reckless->pid && reckless->pid) { + plugin_log(plugin, LOG_DBG, "reckless failed to exit " + "(%i), killing now.", status); + kill(reckless->pid, SIGKILL); + } + } + plugin_log(plugin, LOG_DBG, "Reckless subprocess complete."); + plugin_log(plugin, LOG_DBG, "output: %s", reckless->stdoutbuf); + io_close(conn); + tal_free(reckless); +} + +static struct io_plan *conn_init(struct io_conn *conn, struct reckless *rkls) +{ + io_set_finish(conn, reckless_conn_finish, rkls); + return read_more(conn, rkls); +} + +static struct command_result *reckless_call(struct command *cmd, + const char *call) +{ + if (!call) + return command_fail(cmd, PLUGIN_ERROR, "invalid reckless call"); + char **my_call; + my_call = tal_arrz(tmpctx, char *, 0); + tal_arr_expand(&my_call, "reckless"); + tal_arr_expand(&my_call, "-v"); + tal_arr_expand(&my_call, "--json"); + tal_arr_expand(&my_call, "search"); + tal_arr_expand(&my_call, (char *) call); + tal_arr_expand(&my_call, NULL); + struct reckless *reckless; + reckless = tal(NULL, struct reckless); + reckless->cmd = cmd; + reckless->stdoutbuf = tal_arrz(reckless, char, 1024); + reckless->stderrbuf = tal_arrz(reckless, char, 1024); + reckless->stdout_read = 0; + reckless->stdout_new = 0; + char * full_cmd; + full_cmd = tal_fmt(tmpctx, "calling:"); + for (int i=0; ipid = pipecmdarr(&reckless->stdinfd, &reckless->stdoutfd, + &reckless->stderrfd, my_call); + + /* FIXME: fail if invalid pid*/ + io_new_conn(reckless, reckless->stdoutfd, conn_init, reckless); + tal_free(my_call); + return command_still_pending(cmd); +} + +static struct command_result *json_search(struct command *cmd, + const char *buf, + const jsmntok_t *params) +{ + const char *search_target; + /* Allow check command to evaluate. */ + if (!param(cmd, buf, params, + p_req("plugin", param_string, &search_target), + NULL)) + return command_param_failed(); + return reckless_call(cmd, search_target); +} + +static const char *init(struct plugin *p, + const char *buf UNUSED, + const jsmntok_t *config UNUSED) +{ + plugin = p; + plugin_log(p, LOG_DBG, "plugin initialized!"); + /* FIXME: TODO: scan for reckless config info */ + /* FIXME: TODO: assume default reckless config using ld config dets */ + return NULL; +} + +static const struct plugin_command commands[] = { + { + "reckless-search", + json_search, + }, +}; + +int main(int argc, char **argv) +{ + setup_locale(); + + plugin_main(argv, init, NULL, PLUGIN_RESTARTABLE, true, + NULL, + commands, ARRAY_SIZE(commands), + NULL, 0, /* Notifications */ + NULL, 0, /* Hooks */ + NULL, 0, /* Notification topics */ + NULL); /* plugin options */ + + return 0; +} + From ad6178533356851b67777103deb7145933303229 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 25 Jul 2024 15:35:17 -0500 Subject: [PATCH 2/8] reckless-rpc: read lightningdir and network from listconfigs --- plugins/recklessrpc.c | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/plugins/recklessrpc.c b/plugins/recklessrpc.c index 416c7230c7f0..17bdd7a706e4 100644 --- a/plugins/recklessrpc.c +++ b/plugins/recklessrpc.c @@ -5,10 +5,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -27,6 +29,12 @@ struct reckless { pid_t pid; }; +struct lconfig { + char *lightningdir; + char *config; + char *network; +} lconfig; + static struct io_plan *read_more(struct io_conn *conn, struct reckless *rkls) { rkls->stdout_read += rkls->stdout_new; @@ -116,6 +124,14 @@ static struct command_result *reckless_call(struct command *cmd, tal_arr_expand(&my_call, "reckless"); tal_arr_expand(&my_call, "-v"); tal_arr_expand(&my_call, "--json"); + tal_arr_expand(&my_call, "-l"); + tal_arr_expand(&my_call, lconfig.lightningdir); + tal_arr_expand(&my_call, "--network"); + tal_arr_expand(&my_call, lconfig.network); + if (lconfig.config) { + tal_arr_expand(&my_call, "--conf"); + tal_arr_expand(&my_call, lconfig.config); + } tal_arr_expand(&my_call, "search"); tal_arr_expand(&my_call, (char *) call); tal_arr_expand(&my_call, NULL); @@ -160,9 +176,24 @@ static const char *init(struct plugin *p, const jsmntok_t *config UNUSED) { plugin = p; + rpc_scan(p, "listconfigs", + take(json_out_obj(NULL, NULL, NULL)), + "{configs:{" + "conf?:{value_str:%}," + "lightning-dir:{value_str:%}," + "network:{value_str:%}" + "}}", + JSON_SCAN_TAL(p, json_strdup, &lconfig.config), + JSON_SCAN_TAL(p, json_strdup, &lconfig.lightningdir), + JSON_SCAN_TAL(p, json_strdup, &lconfig.network)); + /* These lightning config parameters need to stick around for each + * reckless call. */ + if (lconfig.config) + notleak(lconfig.config); + notleak(lconfig.lightningdir); + notleak(lconfig.network); plugin_log(p, LOG_DBG, "plugin initialized!"); - /* FIXME: TODO: scan for reckless config info */ - /* FIXME: TODO: assume default reckless config using ld config dets */ + plugin_log(p, LOG_DBG, "lightning-dir: %s", lconfig.lightningdir); return NULL; } From 87b44808949f9b12373d8cea99a9fe295cadb1d1 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 25 Jul 2024 15:44:37 -0500 Subject: [PATCH 3/8] reckless-rpc: auto-accept reckless config creation dialog This would interupt most commands during the config reading step until the user accepts the prompt to create a new empty config. --- plugins/recklessrpc.c | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/plugins/recklessrpc.c b/plugins/recklessrpc.c index 17bdd7a706e4..09c193ac9e6b 100644 --- a/plugins/recklessrpc.c +++ b/plugins/recklessrpc.c @@ -26,6 +26,8 @@ struct reckless { char *stderrbuf; size_t stdout_read; /* running total */ size_t stdout_new; /* new since last read */ + size_t stderr_read; + size_t stderr_new; pid_t pid; }; @@ -35,6 +37,17 @@ struct lconfig { char *network; } lconfig; +static struct io_plan *reckless_in_init(struct io_conn *conn, + struct reckless *reckless) +{ + return io_write(conn, "Y", 1, io_close_cb, NULL); +} + +static void reckless_send_yes(struct reckless *reckless) +{ + io_new_conn(reckless, reckless->stdinfd, reckless_in_init, reckless); +} + static struct io_plan *read_more(struct io_conn *conn, struct reckless *rkls) { rkls->stdout_read += rkls->stdout_new; @@ -114,6 +127,33 @@ static struct io_plan *conn_init(struct io_conn *conn, struct reckless *rkls) return read_more(conn, rkls); } +static void stderr_conn_finish(struct io_conn *conn, void *reckless UNUSED) +{ + io_close(conn); +} + +static struct io_plan *stderr_read_more(struct io_conn *conn, + struct reckless *rkls) +{ + rkls->stderr_read += rkls->stderr_new; + if (rkls->stderr_read == tal_count(rkls->stderrbuf)) + tal_resize(&rkls->stderrbuf, rkls->stderr_read * 2); + if (strends(rkls->stderrbuf, "[Y] to create one now.\n")) { + plugin_log(plugin, LOG_DBG, "confirming config creation"); + reckless_send_yes(rkls); + } + return io_read_partial(conn, rkls->stderrbuf + rkls->stderr_read, + tal_count(rkls->stderrbuf) - rkls->stderr_read, + &rkls->stderr_new, stderr_read_more, rkls); +} + +static struct io_plan *stderr_conn_init(struct io_conn *conn, + struct reckless *reckless) +{ + io_set_finish(conn, stderr_conn_finish, NULL); + return stderr_read_more(conn, reckless); +} + static struct command_result *reckless_call(struct command *cmd, const char *call) { @@ -142,6 +182,8 @@ static struct command_result *reckless_call(struct command *cmd, reckless->stderrbuf = tal_arrz(reckless, char, 1024); reckless->stdout_read = 0; reckless->stdout_new = 0; + reckless->stderr_read = 0; + reckless->stderr_new = 0; char * full_cmd; full_cmd = tal_fmt(tmpctx, "calling:"); for (int i=0; istdoutfd, conn_init, reckless); + io_new_conn(reckless, reckless->stderrfd, stderr_conn_init, reckless); tal_free(my_call); return command_still_pending(cmd); } From 1f0973ef5988c955994ae4826b7d04afa15a1221 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Wed, 31 Jul 2024 15:06:36 -0500 Subject: [PATCH 4/8] reckless-rpc: accept and pass generic subcommands This allows generic subcommands to be passed to reckless-rpc along with the target to search/install/etc.. These commands are unvalidated so far and may crash the reckless process. Changelog-Added: reckless-rpc plugin: issue commands to reckless over rpc. --- plugins/recklessrpc.c | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/plugins/recklessrpc.c b/plugins/recklessrpc.c index 09c193ac9e6b..b5b43cf4ec1c 100644 --- a/plugins/recklessrpc.c +++ b/plugins/recklessrpc.c @@ -155,9 +155,11 @@ static struct io_plan *stderr_conn_init(struct io_conn *conn, } static struct command_result *reckless_call(struct command *cmd, - const char *call) + const char *subcommand, + const char *target, + const char *target2) { - if (!call) + if (!subcommand || !target) return command_fail(cmd, PLUGIN_ERROR, "invalid reckless call"); char **my_call; my_call = tal_arrz(tmpctx, char *, 0); @@ -172,8 +174,10 @@ static struct command_result *reckless_call(struct command *cmd, tal_arr_expand(&my_call, "--conf"); tal_arr_expand(&my_call, lconfig.config); } - tal_arr_expand(&my_call, "search"); - tal_arr_expand(&my_call, (char *) call); + tal_arr_expand(&my_call, (char *) subcommand); + tal_arr_expand(&my_call, (char *) target); + if (target2) + tal_arr_expand(&my_call, (char *) target2); tal_arr_expand(&my_call, NULL); struct reckless *reckless; reckless = tal(NULL, struct reckless); @@ -201,17 +205,21 @@ static struct command_result *reckless_call(struct command *cmd, return command_still_pending(cmd); } -static struct command_result *json_search(struct command *cmd, - const char *buf, - const jsmntok_t *params) +static struct command_result *json_reckless(struct command *cmd, + const char *buf, + const jsmntok_t *params) { - const char *search_target; + const char *command; + const char *target; + const char *target2; /* Allow check command to evaluate. */ if (!param(cmd, buf, params, - p_req("plugin", param_string, &search_target), + p_req("command", param_string, &command), + p_req("target/subcommand", param_string, &target), + p_opt("target", param_string, &target2), NULL)) return command_param_failed(); - return reckless_call(cmd, search_target); + return reckless_call(cmd, command, target, target2); } static const char *init(struct plugin *p, @@ -242,8 +250,8 @@ static const char *init(struct plugin *p, static const struct plugin_command commands[] = { { - "reckless-search", - json_search, + "reckless", + json_reckless, }, }; From aec0bfaae9a925d7b3671839e5e4b763c5d2cc88 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Thu, 1 Aug 2024 17:05:36 -0500 Subject: [PATCH 5/8] reckless-rpc: catch old installed version If the rpc plugin is run while an older version of reckless is found on PATH, it produces an error: 2024-08-01T18:32:00.849Z DEBUG plugin-recklessrpc: reckless-stderr:usage: reckless [-h] [-d RECKLESS_DIR] [-l LIGHTNING] [-c CONF] [-r] [--network NETWORK] [-v] {install,uninstall,search,enable,disable,source,help} ... reckless: error: unrecognized arguments: --json Catch this and don't try to parse the output as json. --- plugins/recklessrpc.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/plugins/recklessrpc.c b/plugins/recklessrpc.c index b5b43cf4ec1c..932abfbf3dfd 100644 --- a/plugins/recklessrpc.c +++ b/plugins/recklessrpc.c @@ -29,6 +29,7 @@ struct reckless { size_t stderr_read; size_t stderr_new; pid_t pid; + char *process_failed; }; struct lconfig { @@ -62,6 +63,12 @@ static struct command_result *reckless_result(struct io_conn *conn, struct reckless *reckless) { struct json_stream *response; + if (reckless->process_failed) { + response = jsonrpc_stream_fail(reckless->cmd, + PLUGIN_ERROR, + reckless->process_failed); + return command_finished(reckless->cmd, response); + } response = jsonrpc_stream_success(reckless->cmd); json_array_start(response, "result"); const jsmntok_t *results, *result, *logs, *log; @@ -142,6 +149,15 @@ static struct io_plan *stderr_read_more(struct io_conn *conn, plugin_log(plugin, LOG_DBG, "confirming config creation"); reckless_send_yes(rkls); } + /* Old version of reckless installed? */ + if (strstr(rkls->stderrbuf, "error: unrecognized arguments: --json")) { + plugin_log(plugin, LOG_DBG, "Reckless call failed due to old " + "installed version."); + rkls->process_failed = tal_strdup(plugin, "The installed " + "reckless utility is out of " + "date. Please update to use " + "the RPC plugin."); + } return io_read_partial(conn, rkls->stderrbuf + rkls->stderr_read, tal_count(rkls->stderrbuf) - rkls->stderr_read, &rkls->stderr_new, stderr_read_more, rkls); @@ -188,6 +204,7 @@ static struct command_result *reckless_call(struct command *cmd, reckless->stdout_new = 0; reckless->stderr_read = 0; reckless->stderr_new = 0; + reckless->process_failed = NULL; char * full_cmd; full_cmd = tal_fmt(tmpctx, "calling:"); for (int i=0; i Date: Mon, 5 Aug 2024 12:13:18 -0500 Subject: [PATCH 6/8] reckless-rpc: add documentation --- contrib/msggen/msggen/schema.json | 191 ++++++++++++++++++++++++++++ doc/Makefile | 1 + doc/index.rst | 1 + doc/schemas/lightning-reckless.json | 191 ++++++++++++++++++++++++++++ 4 files changed, 384 insertions(+) create mode 100644 doc/schemas/lightning-reckless.json diff --git a/contrib/msggen/msggen/schema.json b/contrib/msggen/msggen/schema.json index 7dc35385a9fe..ebee1844c89e 100644 --- a/contrib/msggen/msggen/schema.json +++ b/contrib/msggen/msggen/schema.json @@ -25995,6 +25995,197 @@ "Main web site: " ] }, + "lightning-reckless.json": { + "$schema": "../rpc-schema-draft.json", + "type": "object", + "additionalProperties": false, + "rpc": "reckless", + "title": "Issue a command to the reckless plugin manager utility", + "description": [ + "The **reckless** RPC starts a reckless process with the *command* and *target* provided. Node configuration, network, and lightning direrctory are automatically passed to the reckless utility." + ], + "request": { + "required": [ + "command" + ], + "properties": { + "command": { + "type": "string", + "enum": [ + "install", + "uninstall", + "search", + "enable", + "disable", + "source", + "--version" + ], + "description": [ + "Determines which command to pass to reckless", + " - *command* **install** takes a *plugin_name* to search for and install a named plugin.", + " - *command* **uninstall** takes a *plugin_name* and attempts to uninstall a plugin of the same name.", + " - *command* **search** takes a *plugin_name* to search for a named plugin.", + "..." + ] + }, + "target/subcommand": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "array" + } + ], + "description": [ + "Target of a reckless command or a subcommand." + ] + }, + "target": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "array" + } + ], + "description": [ + "*name* of a plugin to install/uninstall/search/enable/disable or source to add/remove." + ] + } + } + }, + "response": { + "required": [ + "log", + "result" + ], + "properties": { + "result": { + "type": "array", + "items": { + "type": "string" + }, + "description": [ + "Output of the requested reckless command." + ] + }, + "log": { + "type": "array", + "items": { + "type": "string" + }, + "description": [ + "Verbose log entries of the requested reckless command." + ] + } + } + }, + "author": [ + "Alex Myers <> is mainly responsible." + ], + "see_also": [ + "reckless(7)" + ], + "resources": [ + "Main web site: " + ], + "examples": [ + { + "request": { + "id": "example:reckless#1", + "method": "reckless", + "params": { + "command": "search", + "target/subcommand": "backup" + } + }, + "response": { + "result": [ + "https://github.com/lightningd/plugins" + ], + "log": [ + "DEBUG: Warning: Reckless requires write access", + "DEBUG: fetching from gh API: https://api.github.com/repos/lightningd/plugins/contents/", + "DEBUG: fetching from gh API: https://api.github.com/repos/lightningd/plugins/git/trees/294f93d7060799439c994daa84f534c4d1458325", + "INFO: found backup in source: https://github.com/lightningd/plugins", + "DEBUG: entry: None", + "DEBUG: sub-directory: backup" + ] + } + }, + { + "request": { + "id": "example:reckless#2", + "method": "reckless", + "params": { + "command": "install", + "target/subcommand": [ + "summars", + "currecyrate" + ] + } + }, + "response": { + "result": [ + "/tmp/l1/reckless/summars", + "/tmp/l1/reckless/currencyrate" + ], + "log": [ + "DEBUG: Searching for summars", + "DEBUG: fetching from gh API: https://api.github.com/repos/lightningd/plugins/contents/", + "DEBUG: fetching from gh API: https://api.github.com/repos/lightningd/plugins/git/trees/294f93d7060799439c994daa84f534c4d1458325", + "INFO: found summars in source: https://github.com/lightningd/plugins", + "DEBUG: entry: None", + "DEBUG: sub-directory: summars", + "DEBUG: Retrieving summars from https://github.com/lightningd/plugins", + "DEBUG: Install requested from InstInfo(summars, https://github.com/lightningd/plugins, None, None, None, summars).", + "INFO: cloning Source.GITHUB_REPO InstInfo(summars, https://github.com/lightningd/plugins, None, None, None, summars)", + "DEBUG: cloned_src: InstInfo(summars, /tmp/reckless-726255950dyifh_fh/clone, None, Cargo.toml, Cargo.toml, summars/summars)", + "DEBUG: using latest commit of default branch", + "DEBUG: checked out HEAD: 5e449468bd57db7d0f33178fe0dc867e0da94133", + "DEBUG: using installer rust", + "DEBUG: creating /tmp/l1/reckless/summars", + "DEBUG: creating /tmp/l1/reckless/summars/source", + "DEBUG: copying /tmp/reckless-726255950dyifh_fh/clone/summars/summars tree to /tmp/l1/reckless/summars/source/summars", + "DEBUG: linking source /tmp/l1/reckless/summars/source/summars/Cargo.toml to /tmp/l1/reckless/summars/Cargo.toml", + "DEBUG: InstInfo(summars, /tmp/l1/reckless/summars, None, Cargo.toml, Cargo.toml, source/summars)", + "DEBUG: cargo installing from /tmp/l1/reckless/summars/source/summars", + "DEBUG: rust project compiled successfully", + "INFO: plugin installed: /tmp/l1/reckless/summars", + "DEBUG: activating summars", + "INFO: summars enabled", + "DEBUG: Searching for currencyrate", + "DEBUG: fetching from gh API: https://api.github.com/repos/lightningd/plugins/contents/", + "DEBUG: fetching from gh API: https://api.github.com/repos/lightningd/plugins/git/trees/294f93d7060799439c994daa84f534c4d1458325", + "INFO: found currencyrate in source: https://github.com/lightningd/plugins", + "DEBUG: entry: None", + "DEBUG: sub-directory: currencyrate", + "DEBUG: Retrieving currencyrate from https://github.com/lightningd/plugins", + "DEBUG: Install requested from InstInfo(currencyrate, https://github.com/lightningd/plugins, None, None, None, currencyrate).", + "INFO: cloning Source.GITHUB_REPO InstInfo(currencyrate, https://github.com/lightningd/plugins, None, None, None, currencyrate)", + "DEBUG: cloned_src: InstInfo(currencyrate, /tmp/reckless-192564272t478naxn/clone, None, currencyrate.py, requirements.txt, currencyrate/currencyrate)", + "DEBUG: using latest commit of default branch", + "DEBUG: checked out HEAD: 5e449468bd57db7d0f33178fe0dc867e0da94133", + "DEBUG: using installer python3venv", + "DEBUG: creating /tmp/l1/reckless/currencyrate", + "DEBUG: creating /tmp/l1/reckless/currencyrate/source", + "DEBUG: copying /tmp/reckless-192564272t478naxn/clone/currencyrate/currencyrate tree to /tmp/l1/reckless/currencyrate/source/currencyrate", + "DEBUG: linking source /tmp/l1/reckless/currencyrate/source/currencyrate/currencyrate.py to /tmp/l1/reckless/currencyrate/currencyrate.py", + "DEBUG: InstInfo(currencyrate, /tmp/l1/reckless/currencyrate, None, currencyrate.py, requirements.txt, source/currencyrate)", + "DEBUG: configuring a python virtual environment (pip) in /tmp/l1/reckless/currencyrate/.venv", + "DEBUG: virtual environment created in /tmp/l1/reckless/currencyrate/.venv.", + "INFO: dependencies installed successfully", + "DEBUG: virtual environment for cloned plugin: .venv", + "INFO: plugin installed: /tmp/l1/reckless/currencyrate", + "DEBUG: activating currencyrate", + "INFO: currencyrate enabled" + ] + } + } + ] + }, "lightning-recover.json": { "$schema": "../rpc-schema-draft.json", "type": "object", diff --git a/doc/Makefile b/doc/Makefile index 674ca926270b..fd59c4182646 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -97,6 +97,7 @@ GENERATE_MARKDOWN := doc/lightning-addgossip.7 \ doc/lightning-plugin.7 \ doc/lightning-preapproveinvoice.7 \ doc/lightning-preapprovekeysend.7 \ + doc/lightning-reckless.7 \ doc/lightning-recoverchannel.7 \ doc/lightning-recover.7 \ doc/lightning-renepay.7 \ diff --git a/doc/index.rst b/doc/index.rst index 74937401325e..5515229db141 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -108,6 +108,7 @@ Core Lightning Documentation lightning-plugin lightning-preapproveinvoice lightning-preapprovekeysend + lightning-reckless lightning-recover lightning-recoverchannel lightning-renepay diff --git a/doc/schemas/lightning-reckless.json b/doc/schemas/lightning-reckless.json new file mode 100644 index 000000000000..51075f5ee824 --- /dev/null +++ b/doc/schemas/lightning-reckless.json @@ -0,0 +1,191 @@ +{ + "$schema": "../rpc-schema-draft.json", + "type": "object", + "additionalProperties": false, + "rpc": "reckless", + "title": "Issue a command to the reckless plugin manager utility", + "description": [ + "The **reckless** RPC starts a reckless process with the *command* and *target* provided. Node configuration, network, and lightning direrctory are automatically passed to the reckless utility." + ], + "request": { + "required": [ + "command" + ], + "properties": { + "command": { + "type": "string", + "enum": [ + "install", + "uninstall", + "search", + "enable", + "disable", + "source", + "--version" + ], + "description": [ + "Determines which command to pass to reckless", + " - *command* **install** takes a *plugin_name* to search for and install a named plugin.", + " - *command* **uninstall** takes a *plugin_name* and attempts to uninstall a plugin of the same name.", + " - *command* **search** takes a *plugin_name* to search for a named plugin.", + "..." + ] + }, + "target/subcommand": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "array" + } + ], + "description": [ + "Target of a reckless command or a subcommand." + ] + }, + "target": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "array" + } + ], + "description": [ + "*name* of a plugin to install/uninstall/search/enable/disable or source to add/remove." + ] + } + } + }, + "response": { + "required": [ + "log", + "result" + ], + "properties": { + "result": { + "type": "array", + "items": { + "type": "string" + }, + "description": [ + "Output of the requested reckless command." + ] + }, + "log": { + "type": "array", + "items": { + "type": "string" + }, + "description": [ + "Verbose log entries of the requested reckless command." + ] + } + } + }, + "author": [ + "Alex Myers <> is mainly responsible." + ], + "see_also": [ + "reckless(7)" + ], + "resources": [ + "Main web site: " + ], + "examples": [ + { + "request": { + "id": "example:reckless#1", + "method": "reckless", + "params": { + "command": "search", + "target/subcommand": "backup" + } + }, + "response": { + "result": [ + "https://github.com/lightningd/plugins" + ], + "log": [ + "DEBUG: Warning: Reckless requires write access", + "DEBUG: fetching from gh API: https://api.github.com/repos/lightningd/plugins/contents/", + "DEBUG: fetching from gh API: https://api.github.com/repos/lightningd/plugins/git/trees/294f93d7060799439c994daa84f534c4d1458325", + "INFO: found backup in source: https://github.com/lightningd/plugins", + "DEBUG: entry: None", + "DEBUG: sub-directory: backup" + ] + } + }, + { + "request": { + "id": "example:reckless#2", + "method": "reckless", + "params": { + "command": "install", + "target/subcommand": [ + "summars", + "currecyrate" + ] + } + }, + "response": { + "result": [ + "/tmp/l1/reckless/summars", + "/tmp/l1/reckless/currencyrate" + ], + "log": [ + "DEBUG: Searching for summars", + "DEBUG: fetching from gh API: https://api.github.com/repos/lightningd/plugins/contents/", + "DEBUG: fetching from gh API: https://api.github.com/repos/lightningd/plugins/git/trees/294f93d7060799439c994daa84f534c4d1458325", + "INFO: found summars in source: https://github.com/lightningd/plugins", + "DEBUG: entry: None", + "DEBUG: sub-directory: summars", + "DEBUG: Retrieving summars from https://github.com/lightningd/plugins", + "DEBUG: Install requested from InstInfo(summars, https://github.com/lightningd/plugins, None, None, None, summars).", + "INFO: cloning Source.GITHUB_REPO InstInfo(summars, https://github.com/lightningd/plugins, None, None, None, summars)", + "DEBUG: cloned_src: InstInfo(summars, /tmp/reckless-726255950dyifh_fh/clone, None, Cargo.toml, Cargo.toml, summars/summars)", + "DEBUG: using latest commit of default branch", + "DEBUG: checked out HEAD: 5e449468bd57db7d0f33178fe0dc867e0da94133", + "DEBUG: using installer rust", + "DEBUG: creating /tmp/l1/reckless/summars", + "DEBUG: creating /tmp/l1/reckless/summars/source", + "DEBUG: copying /tmp/reckless-726255950dyifh_fh/clone/summars/summars tree to /tmp/l1/reckless/summars/source/summars", + "DEBUG: linking source /tmp/l1/reckless/summars/source/summars/Cargo.toml to /tmp/l1/reckless/summars/Cargo.toml", + "DEBUG: InstInfo(summars, /tmp/l1/reckless/summars, None, Cargo.toml, Cargo.toml, source/summars)", + "DEBUG: cargo installing from /tmp/l1/reckless/summars/source/summars", + "DEBUG: rust project compiled successfully", + "INFO: plugin installed: /tmp/l1/reckless/summars", + "DEBUG: activating summars", + "INFO: summars enabled", + "DEBUG: Searching for currencyrate", + "DEBUG: fetching from gh API: https://api.github.com/repos/lightningd/plugins/contents/", + "DEBUG: fetching from gh API: https://api.github.com/repos/lightningd/plugins/git/trees/294f93d7060799439c994daa84f534c4d1458325", + "INFO: found currencyrate in source: https://github.com/lightningd/plugins", + "DEBUG: entry: None", + "DEBUG: sub-directory: currencyrate", + "DEBUG: Retrieving currencyrate from https://github.com/lightningd/plugins", + "DEBUG: Install requested from InstInfo(currencyrate, https://github.com/lightningd/plugins, None, None, None, currencyrate).", + "INFO: cloning Source.GITHUB_REPO InstInfo(currencyrate, https://github.com/lightningd/plugins, None, None, None, currencyrate)", + "DEBUG: cloned_src: InstInfo(currencyrate, /tmp/reckless-192564272t478naxn/clone, None, currencyrate.py, requirements.txt, currencyrate/currencyrate)", + "DEBUG: using latest commit of default branch", + "DEBUG: checked out HEAD: 5e449468bd57db7d0f33178fe0dc867e0da94133", + "DEBUG: using installer python3venv", + "DEBUG: creating /tmp/l1/reckless/currencyrate", + "DEBUG: creating /tmp/l1/reckless/currencyrate/source", + "DEBUG: copying /tmp/reckless-192564272t478naxn/clone/currencyrate/currencyrate tree to /tmp/l1/reckless/currencyrate/source/currencyrate", + "DEBUG: linking source /tmp/l1/reckless/currencyrate/source/currencyrate/currencyrate.py to /tmp/l1/reckless/currencyrate/currencyrate.py", + "DEBUG: InstInfo(currencyrate, /tmp/l1/reckless/currencyrate, None, currencyrate.py, requirements.txt, source/currencyrate)", + "DEBUG: configuring a python virtual environment (pip) in /tmp/l1/reckless/currencyrate/.venv", + "DEBUG: virtual environment created in /tmp/l1/reckless/currencyrate/.venv.", + "INFO: dependencies installed successfully", + "DEBUG: virtual environment for cloned plugin: .venv", + "INFO: plugin installed: /tmp/l1/reckless/currencyrate", + "DEBUG: activating currencyrate", + "INFO: currencyrate enabled" + ] + } + } + ] +} From cc3485e2f07add10b0963b0ef383d3c78dbfe883 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Tue, 6 Aug 2024 15:47:46 -0500 Subject: [PATCH 7/8] reckless-rpc: catch failed json parsing of reckless output --- plugins/recklessrpc.c | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/plugins/recklessrpc.c b/plugins/recklessrpc.c index 932abfbf3dfd..6525306aa83a 100644 --- a/plugins/recklessrpc.c +++ b/plugins/recklessrpc.c @@ -69,20 +69,36 @@ static struct command_result *reckless_result(struct io_conn *conn, reckless->process_failed); return command_finished(reckless->cmd, response); } - response = jsonrpc_stream_success(reckless->cmd); - json_array_start(response, "result"); const jsmntok_t *results, *result, *logs, *log; size_t i; jsmn_parser parser; jsmntok_t *toks; - toks = tal_arr(reckless, jsmntok_t, 500); + toks = tal_arr(reckless, jsmntok_t, 5000); jsmn_init(&parser); - if (jsmn_parse(&parser, reckless->stdoutbuf, - strlen(reckless->stdoutbuf), toks, tal_count(toks)) <= 0) { - plugin_log(plugin, LOG_DBG, "need more json tokens"); - assert(false); + int res; + res = jsmn_parse(&parser, reckless->stdoutbuf, + strlen(reckless->stdoutbuf), toks, tal_count(toks)); + const char *err; + if (res == JSMN_ERROR_INVAL) + err = tal_fmt(tmpctx, "reckless returned invalid character in json " + "output"); + else if (res == JSMN_ERROR_PART) + err = tal_fmt(tmpctx, "reckless returned partial output"); + else if (res == JSMN_ERROR_NOMEM ) + err = tal_fmt(tmpctx, "insufficient tokens to parse " + "reckless output."); + else + err = NULL; + + if (err) { + plugin_log(plugin, LOG_UNUSUAL, "failed to parse json: %s", err); + response = jsonrpc_stream_fail(reckless->cmd, PLUGIN_ERROR, + err); + return command_finished(reckless->cmd, response); } + response = jsonrpc_stream_success(reckless->cmd); + json_array_start(response, "result"); results = json_get_member(reckless->stdoutbuf, toks, "result"); json_for_each_arr(i, result, results) { json_add_string(response, From eda01298347d22e01d00d9aa7a91efc422e05c65 Mon Sep 17 00:00:00 2001 From: Alex Myers Date: Wed, 7 Aug 2024 18:22:36 -0500 Subject: [PATCH 8/8] reckless-rpc: catch failed reckless subprocess ... before processing the output. It's probably a python backtrace, not json anyway. --- plugins/recklessrpc.c | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/plugins/recklessrpc.c b/plugins/recklessrpc.c index 6525306aa83a..c64bfb413209 100644 --- a/plugins/recklessrpc.c +++ b/plugins/recklessrpc.c @@ -120,26 +120,52 @@ static struct command_result *reckless_result(struct io_conn *conn, return command_finished(reckless->cmd, response); } +static struct command_result *reckless_fail(struct reckless *reckless, + char *err) +{ + struct json_stream *resp; + resp = jsonrpc_stream_fail(reckless->cmd, PLUGIN_ERROR, err); + return command_finished(reckless->cmd, resp); +} + static void reckless_conn_finish(struct io_conn *conn, struct reckless *reckless) { /* FIXME: avoid EBADFD - leave stdin fd open? */ if (errno && errno != 9) plugin_log(plugin, LOG_DBG, "err: %s", strerror(errno)); - reckless_result(conn, reckless); if (reckless->pid > 0) { - int status; + int status = 0; pid_t p; p = waitpid(reckless->pid, &status, WNOHANG); /* Did the reckless process exit? */ if (p != reckless->pid && reckless->pid) { - plugin_log(plugin, LOG_DBG, "reckless failed to exit " - "(%i), killing now.", status); + plugin_log(plugin, LOG_DBG, "reckless failed to exit, " + "killing now."); kill(reckless->pid, SIGKILL); + reckless_fail(reckless, "reckless process hung"); + /* Reckless process exited and with normal status? */ + } else if (WIFEXITED(status) && !WEXITSTATUS(status)) { + plugin_log(plugin, LOG_DBG, + "Reckless subprocess complete: %s", + reckless->stdoutbuf); + reckless_result(conn, reckless); + /* Don't try to process json if python raised an error. */ + } else { + plugin_log(plugin, LOG_DBG, + "Reckless process has crashed (%i).", + WEXITSTATUS(status)); + char * err; + if (reckless->process_failed) + err = reckless->process_failed; + else + err = tal_strdup(tmpctx, "the reckless process " + "has crashed"); + reckless_fail(reckless, err); + plugin_log(plugin, LOG_UNUSUAL, + "The reckless subprocess has failed."); } } - plugin_log(plugin, LOG_DBG, "Reckless subprocess complete."); - plugin_log(plugin, LOG_DBG, "output: %s", reckless->stdoutbuf); io_close(conn); tal_free(reckless); }