From 8cdedac04679dedcbe6b0ad02d4a0db946c9f047 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Sun, 6 Aug 2023 21:23:40 +0200 Subject: [PATCH 1/3] bitcoind-rpc-public-whitelist: add `getindexinfo` --- modules/bitcoind-rpc-public-whitelist.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/bitcoind-rpc-public-whitelist.nix b/modules/bitcoind-rpc-public-whitelist.nix index ca2fc8b..8568421 100644 --- a/modules/bitcoind-rpc-public-whitelist.nix +++ b/modules/bitcoind-rpc-public-whitelist.nix @@ -2,6 +2,7 @@ [ "echo" "getinfo" + "getindexinfo" "help" "ping" "uptime" From f0bf94cc5a3ae11acf7136c115f692afb19064e6 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Sun, 6 Aug 2023 21:07:44 +0200 Subject: [PATCH 2/3] mempool: init at 2.5.0 --- helper/update-fixed-output-derivation.sh | 2 + pkgs/default.nix | 4 + pkgs/mempool/default.nix | 143 ++++++++++++++++++ pkgs/mempool/frontend-assets-fetch.sh | 31 ++++ pkgs/mempool/frontend-assets-update.sh | 14 ++ pkgs/mempool/generate.sh | 71 +++++++++ .../nginx-conf/mempool/location-static.conf | 44 ++++++ pkgs/mempool/nginx-conf/mempool/mempool.conf | 44 ++++++ pkgs/mempool/nginx-conf/nginx.conf | 82 ++++++++++ 9 files changed, 435 insertions(+) create mode 100644 pkgs/mempool/default.nix create mode 100755 pkgs/mempool/frontend-assets-fetch.sh create mode 100755 pkgs/mempool/frontend-assets-update.sh create mode 100755 pkgs/mempool/generate.sh create mode 100644 pkgs/mempool/nginx-conf/mempool/location-static.conf create mode 100644 pkgs/mempool/nginx-conf/mempool/mempool.conf create mode 100644 pkgs/mempool/nginx-conf/nginx.conf diff --git a/helper/update-fixed-output-derivation.sh b/helper/update-fixed-output-derivation.sh index dafceb4..cc28b9f 100755 --- a/helper/update-fixed-output-derivation.sh +++ b/helper/update-fixed-output-derivation.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + set -euo pipefail # The file that defines the derivation that should be updated diff --git a/pkgs/default.nix b/pkgs/default.nix index 0cdd059..c0e9fdf 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -17,6 +17,10 @@ let self = { lndinit = pkgs.callPackage ./lndinit { }; liquid-swap = pkgs.python3Packages.callPackage ./liquid-swap { }; rtl = pkgs.callPackage ./rtl { inherit (self) fetchNodeModules; }; + inherit (pkgs.callPackage ./mempool { inherit (self) fetchNodeModules; }) + mempool-backend + mempool-frontend + mempool-nginx-conf; # The secp256k1 version used by joinmarket secp256k1 = pkgs.callPackage ./secp256k1 { }; trustedcoin = pkgs.callPackage ./trustedcoin { }; diff --git a/pkgs/mempool/default.nix b/pkgs/mempool/default.nix new file mode 100644 index 0000000..1a45785 --- /dev/null +++ b/pkgs/mempool/default.nix @@ -0,0 +1,143 @@ +{ lib +, stdenvNoCC +, nodejs-18_x +, nodejs-slim-18_x +, fetchFromGitHub +, fetchNodeModules +, runCommand +, makeWrapper +, curl +, cacert +, rsync +}: +rec { + nodejs = nodejs-18_x; + nodejsRuntime = nodejs-slim-18_x; + + src = fetchFromGitHub { + owner = "mempool"; + repo = "mempool"; + rev = "v2.5.0"; + hash = "sha256-8HmfytxRte3fQ0QKOljUVk9YAuaXhQQWuv3EFNmOgfQ="; + }; + + nodeModules = { + frontend = fetchNodeModules { + inherit src nodejs; + preBuild = "cd frontend"; + hash = "sha256-/Z0xNvob7eMGpzdUWolr47vljpFiIutZpGwd0uYhPWI="; + }; + backend = fetchNodeModules { + inherit src nodejs; + preBuild = "cd backend"; + hash = "sha256-HpzzSTuSRWDWGbctVhTcUA01if/7OTI4xN3DAbAAX+U="; + }; + }; + + frontendAssets = fetchFiles { + name = "mempool-frontend-assets"; + hash = "sha256-3TmulAfzJJMf0UFhnHEqjAnzc1TNC5DM2XcsU7eyinY="; + fetcher = ./frontend-assets-fetch.sh; + }; + + mempool-backend = mkDerivationMempool { + pname = "mempool-backend"; + + buildPhase = '' + cd backend + ${sync} --chmod=+w ${nodeModules.backend}/lib/node_modules . + patchShebangs node_modules + + npm run package + + runHook postBuild + ''; + + installPhase = '' + mkdir -p $out/lib/mempool-backend + ${sync} package/ $out/lib/mempool-backend + + makeWrapper ${nodejsRuntime}/bin/node $out/bin/mempool-backend \ + --add-flags $out/lib/mempool-backend/index.js + + runHook postInstall + ''; + + passthru = { + inherit nodejs nodejsRuntime; + }; + }; + + mempool-frontend = mkDerivationMempool { + pname = "mempool-frontend"; + + buildPhase = '' + cd frontend + + ${sync} --chmod=+w ${nodeModules.frontend}/lib/node_modules . + patchShebangs node_modules + + # sync-assets.js is called during `npm run build` and downloads assets from the + # internet. Disable this script and instead add the assets manually after building. + : > sync-assets.js + + # If this produces incomplete output (when run in a different build setup), + # see https://github.com/mempool/mempool/issues/1256 + npm run build + + # Add assets that would otherwise be downloaded by sync-assets.js + ${sync} ${frontendAssets}/ dist/mempool/browser/resources + + runHook postBuild + ''; + + installPhase = '' + ${sync} dist/mempool/browser/ $out + + runHook postInstall + ''; + + passthru = { assets = frontendAssets; }; + }; + + mempool-nginx-conf = runCommand "mempool-nginx-conf" {} '' + ${sync} --chmod=u+w ${./nginx-conf}/ $out + ${sync} ${src}/production/nginx/http-language.conf $out/mempool + ''; + + sync = "${rsync}/bin/rsync -a --inplace"; + + mkDerivationMempool = args: stdenvNoCC.mkDerivation ({ + version = src.rev; + inherit src meta; + + nativeBuildInputs = [ + makeWrapper + nodejs + rsync + ]; + + phases = "unpackPhase patchPhase buildPhase installPhase"; + } // args); + + fetchFiles = { name, hash, fetcher }: stdenvNoCC.mkDerivation { + inherit name; + outputHashMode = "recursive"; + outputHashAlgo = "sha256"; + outputHash = hash; + nativeBuildInputs = [ curl cacert ]; + buildCommand = '' + mkdir $out + cd $out + ${builtins.readFile fetcher} + ''; + }; + + meta = with lib; { + description = "Bitcoin blockchain and mempool explorer"; + homepage = "https://github.com/mempool/mempool/"; + license = licenses.agpl3Plus; + maintainers = with maintainers; [ erikarvstedt ]; + platforms = platforms.unix; + }; +} diff --git a/pkgs/mempool/frontend-assets-fetch.sh b/pkgs/mempool/frontend-assets-fetch.sh new file mode 100755 index 0000000..f114e8c --- /dev/null +++ b/pkgs/mempool/frontend-assets-fetch.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Fetch hash-locked versions of assets that are dynamically fetched via +# https://github.com/mempool/mempool/blob/master/frontend/sync-assets.js +# when running `npm run build` in the frontend. +# +# This file is updated by ./frontend-assets-update.sh + +declare -A revs=( + ["mempool/mining-pools"]=e889230b0924d7d72eb28186db6f96ef94361fa5 + ["mempool/mining-pool-logos"]=9cb443035878c3f112af97384d624de245afe72d +) + +fetchFile() { + repo=$1 + file=$2 + rev=${revs["$repo"]} + curl -fsS "https://raw.githubusercontent.com/$repo/$rev/$file" +} + +fetchRepo() { + repo=$1 + rev=${revs["$repo"]} + curl -fsSL "https://github.com/$repo/archive/$rev.tar.gz" +} + +# shellcheck disable=SC2094 +fetchFile "mempool/mining-pools" pools.json > pools.json +mkdir mining-pools +fetchRepo "mempool/mining-pool-logos" | tar xz --strip-components=1 -C mining-pools diff --git a/pkgs/mempool/frontend-assets-update.sh b/pkgs/mempool/frontend-assets-update.sh new file mode 100755 index 0000000..33fdf2d --- /dev/null +++ b/pkgs/mempool/frontend-assets-update.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +updateRepoHash() { + repo=$1 + echo -n "Fetching latest rev for $repo: " + hash=$(curl -fsS "https://api.github.com/repos/$repo/commits/master" | jq -r '.sha') + echo "$hash" + sed -i -E "s|( +)\[\"$repo(.*)|\1[\"$repo\"]=$hash|" frontend-assets-fetch.sh +} + + /dev/null + git -C "$src" verify-tag $tag + rev=$tag + fi + rm -rf "$src"/.git + hash=$(nix hash path "$src") + + sed -i " + s|\bowner = .*;|owner = \"$owner\";| + s|\brev = .*;|rev = \"$rev\";| + s|\bhash = .*;|hash = \"$hash\";| + " default.nix +} + +updateNodeModulesHash() { + component=$1 + echo + echo "Fetching node modules for mempool-$component" + ../../helper/update-fixed-output-derivation.sh ./default.nix mempool-"$component" "cd $component" +} + +updateFrontendAssets() { + . ./frontend-assets-update.sh + echo + echo "Fetching frontend assets" + ../../helper/update-fixed-output-derivation.sh ./default.nix mempool-frontend.assets "frontendAssets" +} + +if [[ $# == 0 ]]; then + # Each of these can be run separately + updateSrc + updateFrontendAssets + updateNodeModulesHash backend + updateNodeModulesHash frontend +else + "$@" +fi diff --git a/pkgs/mempool/nginx-conf/mempool/location-static.conf b/pkgs/mempool/nginx-conf/mempool/location-static.conf new file mode 100644 index 0000000..2afed65 --- /dev/null +++ b/pkgs/mempool/nginx-conf/mempool/location-static.conf @@ -0,0 +1,44 @@ +# see order of nginx location rules +# https://stackoverflow.com/questions/5238377/nginx-location-priority + +# for exact / requests, redirect based on $lang +# cache redirect for 5 minutes +location = / { + if ($lang != '') { + return 302 $scheme://$host/$lang/; + } + try_files /en-US/index.html =404; + expires 5m; +} + +# cache //main.f40e91d908a068a2.js forever since they never change +location ~ ^/([a-z][a-z])/(.+\..+\.(js|css)) { + try_files $uri =404; + expires 1y; +} +# cache everything else for 5 minutes +location ~ ^/([a-z][a-z])$ { + try_files $uri /$1/index.html /en-US/index.html =404; + expires 5m; +} +location ~ ^/([a-z][a-z])/ { + try_files $uri /$1/index.html /en-US/index.html =404; + expires 5m; +} + +# cache /resources/** for 1 week since they don't change often +location /resources { + try_files $uri /en-US/index.html; + expires 1w; +} +# cache /main.f40e91d908a068a2.js forever since they never change +location ~* ^/.+\..+\.(js|css) { + try_files /$lang/$uri /en-US/$uri =404; + expires 1y; +} +# catch-all for all URLs i.e. /address/foo /tx/foo /block/000 +# cache 5 minutes since they change frequently +location / { + try_files /$lang/$uri $uri /en-US/$uri /en-US/index.html =404; + expires 5m; +} diff --git a/pkgs/mempool/nginx-conf/mempool/mempool.conf b/pkgs/mempool/nginx-conf/mempool/mempool.conf new file mode 100644 index 0000000..78d2813 --- /dev/null +++ b/pkgs/mempool/nginx-conf/mempool/mempool.conf @@ -0,0 +1,44 @@ +access_log /var/log/nginx/access_mempool.log; +error_log /var/log/nginx/error_mempool.log; + +root /var/www/mempool/browser; + +index index.html; + +# enable browser and proxy caching +add_header Cache-Control "public, no-transform"; + +# vary cache if user changes language preference +add_header Vary Accept-Language; +add_header Vary Cookie; + +include mempool/location-static.conf; + +# static API docs +location = /api { + try_files $uri $uri/ /en-US/index.html =404; +} +location = /api/ { + try_files $uri $uri/ /en-US/index.html =404; +} + +location /api/v1/ws { + proxy_pass http://127.0.0.1:8999/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; +} +location /api/v1 { + proxy_pass http://127.0.0.1:8999/api/v1; +} +location /api/ { + proxy_pass http://127.0.0.1:8999/api/v1/; +} + +# mainnet API +location /ws { + proxy_pass http://127.0.0.1:8999/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; +} diff --git a/pkgs/mempool/nginx-conf/nginx.conf b/pkgs/mempool/nginx-conf/nginx.conf new file mode 100644 index 0000000..cce77d7 --- /dev/null +++ b/pkgs/mempool/nginx-conf/nginx.conf @@ -0,0 +1,82 @@ +user nobody; +pid /var/run/nginx.pid; + +worker_processes auto; +worker_rlimit_nofile 100000; + +events { + worker_connections 9000; + multi_accept on; +} + +http { + sendfile on; + tcp_nopush on; + tcp_nodelay on; + + server_tokens off; + server_name_in_redirect off; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/error.log; + + # reset timed out connections freeing ram + reset_timedout_connection on; + # maximum time between packets the client can pause when sending nginx any data + client_body_timeout 10s; + # maximum time the client has to send the entire header to nginx + client_header_timeout 10s; + # timeout which a single keep-alive client connection will stay open + keepalive_timeout 69s; + # maximum time between packets nginx is allowed to pause when sending the client data + send_timeout 69s; + + # number of requests per connection, does not affect SPDY + keepalive_requests 1337; + + # enable gzip compression + gzip on; + gzip_vary on; + gzip_comp_level 6; + gzip_min_length 1000; + gzip_proxied expired no-cache no-store private auth; + # text/html is always compressed by gzip module + gzip_types application/javascript application/json application/ld+json application/manifest+json application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/bmp image/svg+xml image/x-icon text/cache-manifest text/css text/plain text/vcard; + + # limit request body size + client_max_body_size 10m; + + # proxy cache + proxy_cache off; + proxy_cache_path /var/cache/nginx keys_zone=cache:20m levels=1:2 inactive=600s max_size=500m; + types_hash_max_size 2048; + + # exempt localhost from rate limit + geo $limited_ip { + default 1; + 127.0.0.1 0; + } + map $limited_ip $limited_ip_key { + 1 $binary_remote_addr; + 0 ''; + } + + # rate limit requests + limit_req_zone $limited_ip_key zone=api:5m rate=200r/m; + limit_req_zone $limited_ip_key zone=electrs:5m rate=2000r/m; + limit_req_status 429; + + # rate limit connections + limit_conn_zone $limited_ip_key zone=websocket:10m; + limit_conn_status 429; + + include mempool/http-language.conf; + + server { + listen 127.0.0.1:80; + include mempool/mempool.conf; + } +} From 1de259485b76d787c5876829fc08da9064e03546 Mon Sep 17 00:00:00 2001 From: Erik Arvstedt Date: Sun, 6 Aug 2023 21:07:51 +0200 Subject: [PATCH 3/3] mempool: add module --- README.md | 7 +- dev/dev-scenarios.nix | 11 ++ dev/topics/mempool.sh | 22 +++ examples/README.md | 6 + examples/configuration.nix | 20 ++ modules/mempool.nix | 331 +++++++++++++++++++++++++++++++++ modules/modules.nix | 1 + modules/netns-isolation.nix | 12 +- modules/nodeinfo.nix | 6 + modules/onion-services.nix | 3 + modules/presets/enable-tor.nix | 1 + test/tests.nix | 7 + test/tests.py | 15 ++ 13 files changed, 435 insertions(+), 7 deletions(-) create mode 100644 dev/topics/mempool.sh create mode 100644 modules/mempool.nix diff --git a/README.md b/README.md index bbef7ef..24b6629 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,7 @@ NixOS modules ([src](modules/modules.nix)) clightning [via WireGuard](./docs/services.md#use-zeus-mobile-lightning-wallet-via-wireguard) or [Tor](./docs/services.md#use-zeus-mobile-lightning-wallet-via-tor) * [Ride The Lightning](https://github.com/Ride-The-Lightning/RTL): web interface for `lnd` and `clightning` + * [mempool](https://github.com/mempool/mempool): Bitcoin visualizer, explorer, and API service * [electrs](https://github.com/romanz/electrs): Electrum server * [fulcrum](https://github.com/cculianu/Fulcrum): Electrum server (see [the module](modules/fulcrum.nix) for a comparison with electrs) * [btcpayserver](https://github.com/btcpayserver/btcpayserver) @@ -103,12 +104,6 @@ NixOS modules ([src](modules/modules.nix)) * [backups](modules/backups.nix): duplicity backups of all your node's important files * [operator](modules/operator.nix): configures a non-root user who has access to client tools (e.g. `bitcoin-cli`, `lightning-cli`) -### Extension modules -Extension modules are maintained in separate repositories and have their own review -and release process. - -* [Mempool](https://github.com/fort-nix/nix-bitcoin-mempool): Bitcoin visualizer, explorer and API service - Security --- See [SECURITY.md](SECURITY.md) for the security policy and how to report a vulnerability. diff --git a/dev/dev-scenarios.nix b/dev/dev-scenarios.nix index 21f5991..80e2ea2 100644 --- a/dev/dev-scenarios.nix +++ b/dev/dev-scenarios.nix @@ -94,4 +94,15 @@ with lib; test.container.enableWAN = true; }; + + mempool-regtest = { + imports = [ + scenarios.regtestBase + ]; + services.mempool = { + enable = true; + frontend.address = "0.0.0.0"; + }; + nix-bitcoin.nodeinfo.enable = true; + }; } diff --git a/dev/topics/mempool.sh b/dev/topics/mempool.sh new file mode 100644 index 0000000..fb4f4fb --- /dev/null +++ b/dev/topics/mempool.sh @@ -0,0 +1,22 @@ +# Start mempool container +run-tests.sh -s mempool-regtest container + +c systemctl status mempool +c systemctl status mysql +c nodeinfo + +# Check backend +c curl -fsS localhost:8999/api/v1/blocks/1 | jq +c curl -fsS localhost:8999/api/v1/blocks/tip/height | jq +c curl -fsS localhost:8999/api/v1/address/1CGG9qVq2P6F7fo6sZExvNq99Jv2GDpaLE | jq + +# Check frontend +c curl -fsS localhost:60845 +c curl -fsS localhost:60845/api/mempool | jq +c curl -fsS localhost:60845/api/blocks/1 | jq +c curl -fsS localhost:60845/api/v1/blocks/1 | jq +c curl -fsS localhost:60845/api/blocks/tip/height | jq + +# Open frontend +# shellcheck disable=SC2154 +runuser -u "$(logname)" -- xdg-open "http://$ip:60845/" diff --git a/examples/README.md b/examples/README.md index 6825c94..54dd887 100644 --- a/examples/README.md +++ b/examples/README.md @@ -63,3 +63,9 @@ The commands in `shell.nix` allow you to locally run the node in a VM or contain Flakes make it easy to include `nix-bitcoin` in an existing NixOS config. The [flakes example](./flakes/flake.nix) shows how to use `nix-bitcoin` as an input to a system flake. + +### Extending nix-bitcoin with Flakes + +The [mempool extension flake](https://github.com/fort-nix/nix-bitcoin-mempool) shows how to define new +pkgs and modules in a Flake.\ +Since mempool is now a core nix-bitcoin module, this Flake just serves as an example. diff --git a/examples/configuration.nix b/examples/configuration.nix index 5406e78..b6af8e2 100644 --- a/examples/configuration.nix +++ b/examples/configuration.nix @@ -126,6 +126,26 @@ # Automatically enables lightning-loop. # services.rtl.nodes.lnd.loop = true; + ### MEMPOOL + # Set this to enable mempool, a fully featured Bitcoin visualizer, explorer, + # and API service. + # + # services.mempool.enable = true; + # + # Possible options for the Electrum backend server: + # + # - electrs (enabled by default): + # Small database size, slow when querying new addresses. + # + # - fulcrum: + # Large database size, quickly serves arbitrary address queries. + # Enable with: + # services.mempool.electrumServer = "fulcrum"; + # + # Set this to create an onion service to make the mempool web interface + # available via Tor: + # nix-bitcoin.onionServices.mempool-frontend.enable = true; + ### ELECTRS # Set this to enable electrs, an Electrum server implemented in Rust. # services.electrs.enable = true; diff --git a/modules/mempool.nix b/modules/mempool.nix new file mode 100644 index 0000000..de0d0d4 --- /dev/null +++ b/modules/mempool.nix @@ -0,0 +1,331 @@ +{ config, lib, pkgs, ... }: + +with lib; +let + options.services = { + mempool = { + enable = mkOption { + type = types.bool; + default = false; + description = mdDoc '' + Enable Mempool, a fully featured Bitcoin visualizer, explorer, and API service. + + Note: Mempool enables `txindex` in bitcoind (this is a requirement). + + This module has two components: + - A backend service (systemd service `mempool`) + + - An optional web interface run by nginx, defined by options `services.mempool.frontend.*`. + The frontend is enabled by default when mempool is enabled. + For details, see `services.mempool.frontend.enable`. + ''; + }; + + frontend = { + enable = mkOption { + type = types.bool; + default = cfg.enable; + description = mdDoc '' + Enable the mempool frontend (web interface). + This starts a simple nginx instance, configured for local usage with + settings similar to the `mempool/frontend` Docker image. + + IMPORTANT: + If you want to expose the mempool frontend to the internet, you + should create a custom nginx config that includes TLS, backend caching, rate limiting + and performance tuning. + For this task, reuse the config snippets from option `services.mempool.frontend.nginxConfig`. + See also: https://github.com/fort-nix/nixbitcoin.org/blob/master/website/mempool.nix, + which contains a mempool nginx config for public hosting (running at + https://mempool.nixbitcoin.org). + ''; + }; + address = mkOption { + type = types.str; + default = "127.0.0.1"; + description = mdDoc "HTTP server address."; + }; + port = mkOption { + type = types.port; + default = 60845; # A random private port + description = mdDoc "HTTP server port."; + }; + staticContentRoot = mkOption { + type = types.path; + default = nbPkgs.mempool-frontend; + defaultText = "config.nix-bitcoin.pkgs.mempool-frontend"; + description = mdDoc " + Path of the static frontend content root. + "; + }; + nginxConfig = mkOption { + readOnly = true; + default = frontend.nginxConfig; + defaultText = "(See source)"; + description = mdDoc " + An attrset of nginx config snippets for assembling a custom + mempool nginx config. + For details, see the source comments at the point of definition. + "; + }; + }; + + address = mkOption { + type = types.str; + default = "127.0.0.1"; + description = mdDoc "Mempool backend address."; + }; + port = mkOption { + type = types.port; + default = 8999; + description = mdDoc "Mempool backend port."; + }; + electrumServer = mkOption { + type = types.enum [ "electrs" "fulcrum" ]; + default = "electrs"; + description = mdDoc '' + The Electrum server to use for fetching address information. + + Possible options: + - electrs: + Small database size, slow when querying new addresses. + - fulcrum: + Large database size, quickly serves arbitrary address queries. + ''; + }; + settings = mkOption { + type = with types; attrsOf (attrsOf anything); + example = { + MEMPOOL = { + POLL_RATE_MS = 3000; + STDOUT_LOG_MIN_PRIORITY = "debug"; + }; + PRICE_DATA_SERVER = { + CLEARNET_URL = "https://myserver.org/prices"; + }; + }; + description = mdDoc '' + Mempool backend settings. + See here for possible options: + https://github.com/mempool/mempool/blob/master/backend/src/config.ts + ''; + }; + database = { + name = mkOption { + type = types.str; + default = "mempool"; + description = mdDoc "Database name."; + }; + }; + package = mkOption { + type = types.package; + default = nbPkgs.mempool-backend; + defaultText = "config.nix-bitcoin.pkgs.mempool-backend"; + description = mdDoc "The package providing mempool binaries."; + }; + user = mkOption { + type = types.str; + default = "mempool"; + description = mdDoc "The user as which to run Mempool."; + }; + group = mkOption { + type = types.str; + default = cfg.user; + description = mdDoc "The group as which to run Mempool."; + }; + tor = nbLib.tor; + }; + + # Internal read-only options used by `./nodeinfo.nix` and `./onion-services.nix` + mempool-frontend = let + mkAlias = default: mkOption { + internal = true; + readOnly = true; + inherit default; + }; + in { + enable = mkAlias cfg.frontend.enable; + address = mkAlias cfg.frontend.address; + port = mkAlias cfg.frontend.port; + }; + }; + + cfg = config.services.mempool; + nbLib = config.nix-bitcoin.lib; + nbPkgs = config.nix-bitcoin.pkgs; + secretsDir = config.nix-bitcoin.secretsDir; + + configFile = builtins.toFile "mempool-config" (builtins.toJSON cfg.settings); + cacheDir = "/var/cache/mempool"; + + inherit (config.services) + bitcoind + electrs + fulcrum; + + torSocket = config.services.tor.client.socksListenAddress; + + # See the `services.nginx` definition further below below + # on how to use these snippets. + frontend.nginxConfig = { + # This must be added to `services.nginx.commonHttpConfig` when + # `mempool/location-static.conf` is used + httpConfig = '' + include ${nbPkgs.mempool-nginx-conf}/mempool/http-language.conf; + ''; + + # This should be added to `services.nginx.virtualHosts..extraConfig` + staticContent = '' + index index.html; + + add_header Cache-Control "public, no-transform"; + add_header Vary Accept-Language; + add_header Vary Cookie; + + include ${nbPkgs.mempool-nginx-conf}/mempool/location-static.conf; + + # Redirect /api to /docs/api + location = /api { + return 308 https://$host/docs/api; + } + location = /api/ { + return 308 https://$host/docs/api; + } + ''; + + # This should be added to `services.nginx.virtualHosts..extraConfig` + proxyApi = let + backend = "http://${nbLib.addressWithPort cfg.address cfg.port}"; + in '' + location /api/ { + proxy_pass ${backend}/api/v1/; + } + location /api/v1 { + proxy_pass ${backend}; + } + # Websocket API + location /api/v1/ws { + proxy_pass ${backend}; + + # Websocket header settings + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "Upgrade"; + + # Relevant settings from `recommendedProxyConfig` (nixos/nginx/default.nix) + # (In the above api locations, this are inherited from the parent scope) + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + ''; + }; + +in { + inherit options; + + config = mkIf cfg.enable { + services.bitcoind.txindex = true; + services.electrs.enable = mkIf (cfg.electrumServer == "electrs" ) true; + services.fulcrum.enable = mkIf (cfg.electrumServer == "fulcrum" ) true; + services.mysql = { + enable = true; + package = pkgs.mariadb; + ensureDatabases = [ cfg.database.name ]; + ensureUsers = [ + { + name = cfg.user; + ensurePermissions."${cfg.database.name}.*" = "ALL PRIVILEGES"; + } + ]; + }; + + # Available options: + # https://github.com/mempool/mempool/blob/master/backend/src/config.ts + services.mempool.settings = { + MEMPOOL = { + # mempool doesn't support regtest + NETWORK = "mainnet"; + BACKEND = "electrum"; + HTTP_PORT = cfg.port; + CACHE_DIR = "${cacheDir}/cache"; + STDOUT_LOG_MIN_PRIORITY = mkDefault "info"; + }; + CORE_RPC = { + HOST = bitcoind.rpc.address; + PORT = bitcoind.rpc.port; + USERNAME = bitcoind.rpc.users.public.name; + PASSWORD = "@btcRpcPassword@"; + }; + ELECTRUM = let + server = config.services.${cfg.electrumServer}; + in { + HOST = server.address; + PORT = server.port; + TLS_ENABLED = false; + }; + DATABASE = { + ENABLED = true; + DATABASE = cfg.database.name; + SOCKET = "/run/mysqld/mysqld.sock"; + }; + } // optionalAttrs (cfg.tor.proxy) { + # Use Tor for rate fetching + SOCKS5PROXY = { + ENABLED = true; + USE_ONION = true; + HOST = torSocket.addr; + PORT = torSocket.port; + }; + }; + + systemd.services.mempool = { + wantedBy = [ "multi-user.target" ]; + requires = [ "${cfg.electrumServer}.service" ]; + after = [ "${cfg.electrumServer}.service" "mysql.service" ]; + preStart = '' + mkdir -p '${cacheDir}/cache' + <${configFile} sed \ + -e "s|@btcRpcPassword@|$(cat ${secretsDir}/bitcoin-rpcpassword-public)|" \ + > '${cacheDir}/config.json' + ''; + environment.MEMPOOL_CONFIG_FILE = "${cacheDir}/config.json"; + serviceConfig = nbLib.defaultHardening // { + ExecStart = "${cfg.package}/bin/mempool-backend"; + CacheDirectory = "mempool"; + CacheDirectoryMode = "770"; + # Show "mempool" instead of "node" in the journal + SyslogIdentifier = "mempool"; + User = cfg.user; + Restart = "on-failure"; + RestartSec = "10s"; + } // nbLib.allowedIPAddresses cfg.tor.enforce + // nbLib.nodejs; + }; + + services.nginx = mkIf cfg.frontend.enable { + enable = true; + enableReload = true; + recommendedGzipSettings = true; + recommendedOptimisation = true; + recommendedProxySettings = true; + recommendedTlsSettings = true; + commonHttpConfig = frontend.nginxConfig.httpConfig; + virtualHosts."mempool" = { + serverName = "_"; + listen = [ { addr = cfg.frontend.address; port = cfg.frontend.port; } ]; + root = cfg.frontend.staticContentRoot; + extraConfig = + frontend.nginxConfig.staticContent + + frontend.nginxConfig.proxyApi; + }; + }; + + users.users.${cfg.user} = { + isSystemUser = true; + group = cfg.group; + extraGroups = [ "bitcoinrpc-public" ]; + }; + users.groups.${cfg.group} = {}; + }; +} diff --git a/modules/modules.nix b/modules/modules.nix index 1c4d288..1b3c204 100644 --- a/modules/modules.nix +++ b/modules/modules.nix @@ -20,6 +20,7 @@ ./charge-lnd.nix ./lndconnect.nix # Requires onion-addresses.nix ./rtl.nix + ./mempool.nix ./electrs.nix ./fulcrum.nix ./liquid.nix diff --git a/modules/netns-isolation.nix b/modules/netns-isolation.nix index fcd7c99..832bb94 100644 --- a/modules/netns-isolation.nix +++ b/modules/netns-isolation.nix @@ -295,7 +295,14 @@ in { id = 31; connections = [ "bitcoind" ]; }; - # id = 32 reserved for the upcoming mempool module + mempool = { + id = 32; + connections = [ + "bitcoind" + "nginx" + (if (config.services.mempool.electrumServer == "electrs") then "electrs" else "fulcrum") + ]; + }; }; services.bitcoind = { @@ -349,6 +356,9 @@ in { services.rtl.address = netns.rtl.address; services.clightning-rest.address = netns.clightning-rest.address; + + services.mempool.address = netns.mempool.address; + services.mempool.frontend.address = netns.nginx.address; } ]); } diff --git a/modules/nodeinfo.nix b/modules/nodeinfo.nix index d01eaed..f219664 100644 --- a/modules/nodeinfo.nix +++ b/modules/nodeinfo.nix @@ -149,6 +149,12 @@ in { liquidd = mkInfo ""; joinmarket-ob-watcher = mkInfo ""; rtl = mkInfo ""; + mempool = mkInfo ""; + mempool-frontend = name: cfg: mkInfoLong { + inherit name cfg; + systemdServiceName = "nginx"; + extraCode = ""; + }; # Only add sshd when it has an onion service sshd = name: cfg: mkIfOnionPort "sshd" (onionPort: '' add_service("sshd", """info["onion_address"] = get_onion_address("sshd", ${onionPort})""") diff --git a/modules/onion-services.nix b/modules/onion-services.nix index a98b2c0..042ffb5 100644 --- a/modules/onion-services.nix +++ b/modules/onion-services.nix @@ -113,6 +113,9 @@ in { rtl = { externalPort = 80; }; + mempool-frontend = { + externalPort = 80; + }; }; } ]; diff --git a/modules/presets/enable-tor.nix b/modules/presets/enable-tor.nix index 084494b..47766f2 100644 --- a/modules/presets/enable-tor.nix +++ b/modules/presets/enable-tor.nix @@ -27,6 +27,7 @@ in { # disable Tor enforcement until btcpayserver can fetch rates over Tor # btcpayserver = defaultEnableTorProxy; lightning-pool = defaultEnableTorProxy; + mempool = defaultEnableTorProxy; # These services don't make outgoing connections # (or use Tor by default in case of joinmarket) diff --git a/test/tests.nix b/test/tests.nix index cf2cdfd..578a3f8 100644 --- a/test/tests.nix +++ b/test/tests.nix @@ -78,6 +78,9 @@ let echo a > rtl-password ''); + tests.mempool = cfg.mempool.enable; + services.mempool.electrumServer = "fulcrum"; + tests.lnd = cfg.lnd.enable; services.lnd = { port = 9736; @@ -195,6 +198,7 @@ let services.lightning-loop.enable = true; services.lightning-pool.enable = true; services.charge-lnd.enable = true; + services.mempool.enable = true; services.electrs.enable = true; services.fulcrum.enable = true; services.liquidd.enable = true; @@ -219,6 +223,8 @@ let tests.secure-node = true; tests.restart-bitcoind = true; + nix-bitcoin.onionServices.mempool-frontend.enable = true; + # Stop electrs from spamming the test log with 'WARN - wait until IBD is over' messages tests.stop-electrs = true; }; @@ -237,6 +243,7 @@ let services.clightning-rest.enable = true; services.liquidd.enable = true; services.rtl.enable = true; + services.mempool.enable = true; services.lnd.enable = true; services.lightning-loop.enable = true; services.lightning-pool.enable = true; diff --git a/test/tests.py b/test/tests.py index a106942..784a5e6 100644 --- a/test/tests.py +++ b/test/tests.py @@ -251,6 +251,15 @@ def _(): log_has_string("clightning-rest", "cl-rest api server is ready and listening") ) +@test("mempool") +def _(): + assert_running("mempool") + assert_running("nginx") + machine.wait_until_succeeds( + log_has_string("mempool", "Mempool Server is running on port 8999") + ) + assert_matches(f"curl -L {ip('nginx')}:60845", "mempool - Bitcoin Explorer") + @test("joinmarket") def _(): assert_running("joinmarket") @@ -427,6 +436,12 @@ def _(): if enabled("btcpayserver"): machine.wait_until_succeeds(log_has_string("nbxplorer", f"At height: {num_blocks}")) + if enabled("mempool"): + assert_running("nginx") + assert_full_match( + f"curl -fsS http://{ip('nginx')}:60845/api/v1/blocks/tip/height", str(num_blocks) + ) + @test("trustedcoin") def _(): def expect_clightning_log(str):