Merge fort-nix/nix-bitcoin#505: Add mempool

1de259485b mempool: add module (Erik Arvstedt)
f0bf94cc5a mempool: init at 2.5.0 (Erik Arvstedt)
8cdedac046 bitcoind-rpc-public-whitelist: add `getindexinfo` (Erik Arvstedt)

Pull request description:

ACKs for top commit:
  jonasnick:
    ACK 1de259485b

Tree-SHA512: f3cbe1eaac6994c6b05f2e560bb2dce564236bcd650e54e4d80ca2b15e20b1d9aa5a9a9c6b9e46ab4f0cf64e8864cbafe884190260c43ba6a1201513605c24f0

README.md

@@ -90,6 +90,7 @@ NixOS modules ([src](modules/modules.nix))
     clightning [via WireGuard](./docs/services.md#use-zeus-mobile-lightning-wallet-via-wireguard) or
     [Tor](./docs/services.md#use-zeus-mobile-lightning-wallet-via-tor)
   * [Ride The Lightning](https://github.com/Ride-The-Lightning/RTL): web interface for `lnd` and `clightning`
+  * [mempool](https://github.com/mempool/mempool): Bitcoin visualizer, explorer, and API service
   * [electrs](https://github.com/romanz/electrs): Electrum server
   * [fulcrum](https://github.com/cculianu/Fulcrum): Electrum server (see [the module](modules/fulcrum.nix) for a comparison with electrs)
   * [btcpayserver](https://github.com/btcpayserver/btcpayserver)
@@ -103,12 +104,6 @@ NixOS modules ([src](modules/modules.nix))
   * [backups](modules/backups.nix): duplicity backups of all your node's important files
   * [operator](modules/operator.nix): configures a non-root user who has access to client tools (e.g. `bitcoin-cli`, `lightning-cli`)
 
-### Extension modules
-Extension modules are maintained in separate repositories and have their own review
-and release process.
-
-* [Mempool](https://github.com/fort-nix/nix-bitcoin-mempool): Bitcoin visualizer, explorer and API service
-
 Security
 ---
 See [SECURITY.md](SECURITY.md) for the security policy and how to report a vulnerability.

dev/dev-scenarios.nix

@@ -94,4 +94,15 @@ with lib;
 
     test.container.enableWAN = true;
   };
+
+  mempool-regtest = {
+    imports = [
+      scenarios.regtestBase
+    ];
+    services.mempool = {
+      enable = true;
+      frontend.address = "0.0.0.0";
+    };
+    nix-bitcoin.nodeinfo.enable = true;
+  };
 }

dev/topics/mempool.sh

@@ -0,0 +1,22 @@
+# Start mempool container
+run-tests.sh -s mempool-regtest container
+
+c systemctl status mempool
+c systemctl status mysql
+c nodeinfo
+
+# Check backend
+c curl -fsS localhost:8999/api/v1/blocks/1 | jq
+c curl -fsS localhost:8999/api/v1/blocks/tip/height | jq
+c curl -fsS localhost:8999/api/v1/address/1CGG9qVq2P6F7fo6sZExvNq99Jv2GDpaLE | jq
+
+# Check frontend
+c curl -fsS localhost:60845
+c curl -fsS localhost:60845/api/mempool | jq
+c curl -fsS localhost:60845/api/blocks/1 | jq
+c curl -fsS localhost:60845/api/v1/blocks/1 | jq
+c curl -fsS localhost:60845/api/blocks/tip/height | jq
+
+# Open frontend
+# shellcheck disable=SC2154
+runuser -u "$(logname)" -- xdg-open "http://$ip:60845/"

examples/README.md

@@ -63,3 +63,9 @@ The commands in `shell.nix` allow you to locally run the node in a VM or contain
 
 Flakes make it easy to include `nix-bitcoin` in an existing NixOS config.
 The [flakes example](./flakes/flake.nix) shows how to use `nix-bitcoin` as an input to a system flake.
+
+### Extending nix-bitcoin with Flakes
+
+The [mempool extension flake](https://github.com/fort-nix/nix-bitcoin-mempool) shows how to define new
+pkgs and modules in a Flake.\
+Since mempool is now a core nix-bitcoin module, this Flake just serves as an example.

examples/configuration.nix

@@ -126,6 +126,26 @@
   # Automatically enables lightning-loop.
   # services.rtl.nodes.lnd.loop = true;
 
+  ### MEMPOOL
+  # Set this to enable mempool, a fully featured Bitcoin visualizer, explorer,
+  # and API service.
+  #
+  # services.mempool.enable = true;
+  #
+  # Possible options for the Electrum backend server:
+  #
+  # - electrs (enabled by default):
+  #   Small database size, slow when querying new addresses.
+  #
+  # - fulcrum:
+  #   Large database size, quickly serves arbitrary address queries.
+  #   Enable with:
+  #     services.mempool.electrumServer = "fulcrum";
+  #
+  # Set this to create an onion service to make the mempool web interface
+  # available via Tor:
+  # nix-bitcoin.onionServices.mempool-frontend.enable = true;
+
   ### ELECTRS
   # Set this to enable electrs, an Electrum server implemented in Rust.
   # services.electrs.enable = true;

helper/update-fixed-output-derivation.sh

@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
 set -euo pipefail
 
 # The file that defines the derivation that should be updated

modules/bitcoind-rpc-public-whitelist.nix

@@ -2,6 +2,7 @@
 [
   "echo"
   "getinfo"
+  "getindexinfo"
   "help"
   "ping"
   "uptime"

modules/mempool.nix

@@ -0,0 +1,331 @@
+{ config, lib, pkgs, ... }:
+
+with lib;
+let
+  options.services = {
+    mempool = {
+      enable = mkOption {
+        type = types.bool;
+        default = false;
+        description = mdDoc ''
+          Enable Mempool, a fully featured Bitcoin visualizer, explorer, and API service.
+
+          Note: Mempool enables `txindex` in bitcoind (this is a requirement).
+
+          This module has two components:
+          - A backend service (systemd service `mempool`)
+
+          - An optional web interface run by nginx, defined by options `services.mempool.frontend.*`.
+            The frontend is enabled by default when mempool is enabled.
+            For details, see `services.mempool.frontend.enable`.
+        '';
+      };
+
+      frontend = {
+        enable = mkOption {
+          type = types.bool;
+          default = cfg.enable;
+          description = mdDoc ''
+            Enable the mempool frontend (web interface).
+            This starts a simple nginx instance, configured for local usage with
+            settings similar to the `mempool/frontend` Docker image.
+
+            IMPORTANT:
+            If you want to expose the mempool frontend to the internet, you
+            should create a custom nginx config that includes TLS, backend caching, rate limiting
+            and performance tuning.
+            For this task, reuse the config snippets from option `services.mempool.frontend.nginxConfig`.
+            See also: https://github.com/fort-nix/nixbitcoin.org/blob/master/website/mempool.nix,
+            which contains a mempool nginx config for public hosting (running at
+            https://mempool.nixbitcoin.org).
+          '';
+        };
+        address = mkOption {
+          type = types.str;
+          default = "127.0.0.1";
+          description = mdDoc "HTTP server address.";
+        };
+        port = mkOption {
+          type = types.port;
+          default = 60845; # A random private port
+          description = mdDoc "HTTP server port.";
+        };
+        staticContentRoot = mkOption {
+          type = types.path;
+          default = nbPkgs.mempool-frontend;
+          defaultText = "config.nix-bitcoin.pkgs.mempool-frontend";
+          description = mdDoc "
+            Path of the static frontend content root.
+          ";
+        };
+        nginxConfig = mkOption {
+          readOnly = true;
+          default = frontend.nginxConfig;
+          defaultText = "(See source)";
+          description = mdDoc "
+            An attrset of nginx config snippets for assembling a custom
+            mempool nginx config.
+            For details, see the source comments at the point of definition.
+          ";
+        };
+      };
+
+      address = mkOption {
+        type = types.str;
+        default = "127.0.0.1";
+        description = mdDoc "Mempool backend address.";
+      };
+      port = mkOption {
+        type = types.port;
+        default = 8999;
+        description = mdDoc "Mempool backend port.";
+      };
+      electrumServer = mkOption {
+        type = types.enum [ "electrs" "fulcrum" ];
+        default = "electrs";
+        description = mdDoc ''
+          The Electrum server to use for fetching address information.
+
+          Possible options:
+          - electrs:
+            Small database size, slow when querying new addresses.
+          - fulcrum:
+            Large database size, quickly serves arbitrary address queries.
+        '';
+      };
+      settings = mkOption {
+        type = with types; attrsOf (attrsOf anything);
+        example = {
+          MEMPOOL = {
+            POLL_RATE_MS = 3000;
+            STDOUT_LOG_MIN_PRIORITY = "debug";
+          };
+          PRICE_DATA_SERVER = {
+            CLEARNET_URL = "https://myserver.org/prices";
+          };
+        };
+        description = mdDoc ''
+          Mempool backend settings.
+          See here for possible options:
+          https://github.com/mempool/mempool/blob/master/backend/src/config.ts
+        '';
+      };
+      database = {
+        name = mkOption {
+          type = types.str;
+          default = "mempool";
+          description = mdDoc "Database name.";
+        };
+      };
+      package = mkOption {
+        type = types.package;
+        default = nbPkgs.mempool-backend;
+        defaultText = "config.nix-bitcoin.pkgs.mempool-backend";
+        description = mdDoc "The package providing mempool binaries.";
+      };
+      user = mkOption {
+        type = types.str;
+        default = "mempool";
+        description = mdDoc "The user as which to run Mempool.";
+      };
+      group = mkOption {
+        type = types.str;
+        default = cfg.user;
+        description = mdDoc "The group as which to run Mempool.";
+      };
+      tor = nbLib.tor;
+    };
+
+    # Internal read-only options used by `./nodeinfo.nix` and `./onion-services.nix`
+    mempool-frontend = let
+      mkAlias = default: mkOption {
+        internal = true;
+        readOnly = true;
+        inherit default;
+      };
+    in {
+      enable = mkAlias cfg.frontend.enable;
+      address = mkAlias cfg.frontend.address;
+      port = mkAlias cfg.frontend.port;
+    };
+  };
+
+  cfg = config.services.mempool;
+  nbLib = config.nix-bitcoin.lib;
+  nbPkgs = config.nix-bitcoin.pkgs;
+  secretsDir = config.nix-bitcoin.secretsDir;
+
+  configFile = builtins.toFile "mempool-config" (builtins.toJSON cfg.settings);
+  cacheDir = "/var/cache/mempool";
+
+  inherit (config.services)
+    bitcoind
+    electrs
+    fulcrum;
+
+  torSocket = config.services.tor.client.socksListenAddress;
+
+  # See the `services.nginx` definition further below below
+  # on how to use these snippets.
+  frontend.nginxConfig = {
+    # This must be added to `services.nginx.commonHttpConfig` when
+    # `mempool/location-static.conf` is used
+    httpConfig = ''
+      include ${nbPkgs.mempool-nginx-conf}/mempool/http-language.conf;
+    '';
+
+    # This should be added to `services.nginx.virtualHosts.<mempool server name>.extraConfig`
+    staticContent = ''
+      index index.html;
+
+      add_header Cache-Control "public, no-transform";
+      add_header Vary Accept-Language;
+      add_header Vary Cookie;
+
+      include ${nbPkgs.mempool-nginx-conf}/mempool/location-static.conf;
+
+      # Redirect /api to /docs/api
+      location = /api {
+        return 308 https://$host/docs/api;
+      }
+      location = /api/ {
+        return 308 https://$host/docs/api;
+      }
+    '';
+
+    # This should be added to `services.nginx.virtualHosts.<mempool server name>.extraConfig`
+    proxyApi = let
+      backend = "http://${nbLib.addressWithPort cfg.address cfg.port}";
+    in ''
+      location /api/ {
+          proxy_pass ${backend}/api/v1/;
+      }
+      location /api/v1 {
+          proxy_pass ${backend};
+      }
+      # Websocket API
+      location /api/v1/ws {
+          proxy_pass ${backend};
+
+          # Websocket header settings
+          proxy_set_header Upgrade $http_upgrade;
+          proxy_set_header Connection "Upgrade";
+
+          # Relevant settings from `recommendedProxyConfig` (nixos/nginx/default.nix)
+          # (In the above api locations, this are inherited from the parent scope)
+          proxy_set_header Host $host;
+          proxy_set_header X-Real-IP $remote_addr;
+          proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+          proxy_set_header X-Forwarded-Proto $scheme;
+      }
+    '';
+  };
+
+in {
+  inherit options;
+
+  config = mkIf cfg.enable {
+    services.bitcoind.txindex = true;
+    services.electrs.enable = mkIf (cfg.electrumServer == "electrs" ) true;
+    services.fulcrum.enable = mkIf (cfg.electrumServer == "fulcrum" ) true;
+    services.mysql = {
+      enable = true;
+      package = pkgs.mariadb;
+      ensureDatabases = [ cfg.database.name ];
+      ensureUsers = [
+        {
+          name = cfg.user;
+          ensurePermissions."${cfg.database.name}.*" = "ALL PRIVILEGES";
+        }
+      ];
+    };
+
+    # Available options:
+    # https://github.com/mempool/mempool/blob/master/backend/src/config.ts
+    services.mempool.settings = {
+      MEMPOOL = {
+        # mempool doesn't support regtest
+        NETWORK = "mainnet";
+        BACKEND = "electrum";
+        HTTP_PORT = cfg.port;
+        CACHE_DIR = "${cacheDir}/cache";
+        STDOUT_LOG_MIN_PRIORITY = mkDefault "info";
+      };
+      CORE_RPC = {
+        HOST = bitcoind.rpc.address;
+        PORT = bitcoind.rpc.port;
+        USERNAME = bitcoind.rpc.users.public.name;
+        PASSWORD = "@btcRpcPassword@";
+      };
+      ELECTRUM = let
+        server = config.services.${cfg.electrumServer};
+      in {
+        HOST = server.address;
+        PORT = server.port;
+        TLS_ENABLED = false;
+      };
+      DATABASE = {
+        ENABLED = true;
+        DATABASE = cfg.database.name;
+        SOCKET = "/run/mysqld/mysqld.sock";
+      };
+    } // optionalAttrs (cfg.tor.proxy) {
+      # Use Tor for rate fetching
+      SOCKS5PROXY = {
+        ENABLED = true;
+        USE_ONION = true;
+        HOST = torSocket.addr;
+        PORT = torSocket.port;
+      };
+    };
+
+    systemd.services.mempool = {
+      wantedBy = [ "multi-user.target" ];
+      requires = [ "${cfg.electrumServer}.service" ];
+      after = [ "${cfg.electrumServer}.service" "mysql.service" ];
+      preStart = ''
+        mkdir -p '${cacheDir}/cache'
+        <${configFile} sed \
+          -e "s|@btcRpcPassword@|$(cat ${secretsDir}/bitcoin-rpcpassword-public)|" \
+          > '${cacheDir}/config.json'
+      '';
+      environment.MEMPOOL_CONFIG_FILE = "${cacheDir}/config.json";
+      serviceConfig = nbLib.defaultHardening // {
+        ExecStart = "${cfg.package}/bin/mempool-backend";
+        CacheDirectory = "mempool";
+        CacheDirectoryMode = "770";
+        # Show "mempool" instead of "node" in the journal
+        SyslogIdentifier = "mempool";
+        User = cfg.user;
+        Restart = "on-failure";
+        RestartSec = "10s";
+      } // nbLib.allowedIPAddresses cfg.tor.enforce
+        // nbLib.nodejs;
+    };
+
+    services.nginx = mkIf cfg.frontend.enable {
+      enable = true;
+      enableReload = true;
+      recommendedGzipSettings = true;
+      recommendedOptimisation = true;
+      recommendedProxySettings = true;
+      recommendedTlsSettings = true;
+      commonHttpConfig = frontend.nginxConfig.httpConfig;
+      virtualHosts."mempool" = {
+        serverName = "_";
+        listen = [ { addr = cfg.frontend.address; port = cfg.frontend.port; } ];
+        root = cfg.frontend.staticContentRoot;
+        extraConfig =
+          frontend.nginxConfig.staticContent +
+          frontend.nginxConfig.proxyApi;
+      };
+    };
+
+    users.users.${cfg.user} = {
+      isSystemUser = true;
+      group = cfg.group;
+      extraGroups = [ "bitcoinrpc-public" ];
+    };
+    users.groups.${cfg.group} = {};
+  };
+}

modules/modules.nix

@@ -20,6 +20,7 @@
     ./charge-lnd.nix
     ./lndconnect.nix # Requires onion-addresses.nix
     ./rtl.nix
+    ./mempool.nix
     ./electrs.nix
     ./fulcrum.nix
     ./liquid.nix

modules/netns-isolation.nix

@@ -295,7 +295,14 @@ in {
         id = 31;
         connections = [ "bitcoind" ];
       };
-      # id = 32 reserved for the upcoming mempool module
+      mempool = {
+        id = 32;
+        connections = [
+          "bitcoind"
+          "nginx"
+          (if (config.services.mempool.electrumServer == "electrs") then "electrs" else "fulcrum")
+        ];
+      };
     };
 
     services.bitcoind = {
@@ -349,6 +356,9 @@ in {
     services.rtl.address = netns.rtl.address;
 
     services.clightning-rest.address = netns.clightning-rest.address;
+
+    services.mempool.address = netns.mempool.address;
+    services.mempool.frontend.address = netns.nginx.address;
   }
   ]);
 }

modules/nodeinfo.nix

@@ -149,6 +149,12 @@ in {
       liquidd = mkInfo "";
       joinmarket-ob-watcher = mkInfo "";
       rtl = mkInfo "";
+      mempool = mkInfo "";
+      mempool-frontend = name: cfg: mkInfoLong {
+        inherit name cfg;
+        systemdServiceName = "nginx";
+        extraCode = "";
+      };
       # Only add sshd when it has an onion service
       sshd = name: cfg: mkIfOnionPort "sshd" (onionPort: ''
         add_service("sshd", """info["onion_address"] = get_onion_address("sshd", ${onionPort})""")

modules/onion-services.nix

@@ -113,6 +113,9 @@ in {
         rtl = {
           externalPort = 80;
         };
+        mempool-frontend = {
+          externalPort = 80;
+        };
       };
     }
   ];

modules/presets/enable-tor.nix

@@ -27,6 +27,7 @@ in {
     # disable Tor enforcement until btcpayserver can fetch rates over Tor
     # btcpayserver = defaultEnableTorProxy;
     lightning-pool = defaultEnableTorProxy;
+    mempool = defaultEnableTorProxy;
 
     # These services don't make outgoing connections
     # (or use Tor by default in case of joinmarket)

pkgs/default.nix

@@ -17,6 +17,10 @@ let self = {
   lndinit = pkgs.callPackage ./lndinit { };
   liquid-swap = pkgs.python3Packages.callPackage ./liquid-swap { };
   rtl = pkgs.callPackage ./rtl { inherit (self) fetchNodeModules; };
+  inherit (pkgs.callPackage ./mempool { inherit (self) fetchNodeModules; })
+    mempool-backend
+    mempool-frontend
+    mempool-nginx-conf;
   # The secp256k1 version used by joinmarket
   secp256k1 = pkgs.callPackage ./secp256k1 { };
   trustedcoin = pkgs.callPackage ./trustedcoin { };

pkgs/mempool/default.nix

@@ -0,0 +1,143 @@
+{ lib
+, stdenvNoCC
+, nodejs-18_x
+, nodejs-slim-18_x
+, fetchFromGitHub
+, fetchNodeModules
+, runCommand
+, makeWrapper
+, curl
+, cacert
+, rsync
+}:
+rec {
+  nodejs = nodejs-18_x;
+  nodejsRuntime = nodejs-slim-18_x;
+
+  src = fetchFromGitHub {
+    owner = "mempool";
+    repo = "mempool";
+    rev = "v2.5.0";
+    hash = "sha256-8HmfytxRte3fQ0QKOljUVk9YAuaXhQQWuv3EFNmOgfQ=";
+  };
+
+  nodeModules = {
+    frontend = fetchNodeModules {
+      inherit src nodejs;
+      preBuild = "cd frontend";
+      hash = "sha256-/Z0xNvob7eMGpzdUWolr47vljpFiIutZpGwd0uYhPWI=";
+    };
+    backend = fetchNodeModules {
+      inherit src nodejs;
+      preBuild = "cd backend";
+      hash = "sha256-HpzzSTuSRWDWGbctVhTcUA01if/7OTI4xN3DAbAAX+U=";
+    };
+  };
+
+  frontendAssets = fetchFiles {
+    name = "mempool-frontend-assets";
+    hash = "sha256-3TmulAfzJJMf0UFhnHEqjAnzc1TNC5DM2XcsU7eyinY=";
+    fetcher = ./frontend-assets-fetch.sh;
+  };
+
+  mempool-backend = mkDerivationMempool {
+    pname = "mempool-backend";
+
+    buildPhase = ''
+      cd backend
+      ${sync} --chmod=+w ${nodeModules.backend}/lib/node_modules .
+      patchShebangs node_modules
+
+      npm run package
+
+      runHook postBuild
+    '';
+
+    installPhase = ''
+      mkdir -p $out/lib/mempool-backend
+      ${sync} package/ $out/lib/mempool-backend
+
+      makeWrapper ${nodejsRuntime}/bin/node $out/bin/mempool-backend \
+        --add-flags $out/lib/mempool-backend/index.js
+
+      runHook postInstall
+    '';
+
+    passthru = {
+      inherit nodejs nodejsRuntime;
+    };
+  };
+
+  mempool-frontend = mkDerivationMempool {
+    pname = "mempool-frontend";
+
+    buildPhase = ''
+      cd frontend
+
+      ${sync} --chmod=+w ${nodeModules.frontend}/lib/node_modules .
+      patchShebangs node_modules
+
+      # sync-assets.js is called during `npm run build` and downloads assets from the
+      # internet. Disable this script and instead add the assets manually after building.
+      : > sync-assets.js
+
+      # If this produces incomplete output (when run in a different build setup),
+      # see https://github.com/mempool/mempool/issues/1256
+      npm run build
+
+      # Add assets that would otherwise be downloaded by sync-assets.js
+      ${sync} ${frontendAssets}/ dist/mempool/browser/resources
+
+      runHook postBuild
+    '';
+
+    installPhase = ''
+      ${sync} dist/mempool/browser/ $out
+
+      runHook postInstall
+    '';
+
+    passthru = { assets = frontendAssets; };
+  };
+
+  mempool-nginx-conf = runCommand "mempool-nginx-conf" {} ''
+    ${sync} --chmod=u+w ${./nginx-conf}/ $out
+    ${sync} ${src}/production/nginx/http-language.conf $out/mempool
+  '';
+
+  sync = "${rsync}/bin/rsync -a --inplace";
+
+  mkDerivationMempool = args: stdenvNoCC.mkDerivation ({
+    version = src.rev;
+    inherit src meta;
+
+    nativeBuildInputs = [
+      makeWrapper
+      nodejs
+      rsync
+    ];
+
+    phases = "unpackPhase patchPhase buildPhase installPhase";
+  } // args);
+
+  fetchFiles = { name, hash, fetcher }: stdenvNoCC.mkDerivation {
+    inherit name;
+    outputHashMode = "recursive";
+    outputHashAlgo = "sha256";
+    outputHash = hash;
+    nativeBuildInputs = [ curl cacert ];
+    buildCommand = ''
+      mkdir $out
+      cd $out
+      ${builtins.readFile fetcher}
+    '';
+  };
+
+  meta = with lib; {
+    description = "Bitcoin blockchain and mempool explorer";
+    homepage = "https://github.com/mempool/mempool/";
+    license = licenses.agpl3Plus;
+    maintainers = with maintainers; [ erikarvstedt ];
+    platforms = platforms.unix;
+  };
+}

pkgs/mempool/frontend-assets-fetch.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# Fetch hash-locked versions of assets that are dynamically fetched via
+# https://github.com/mempool/mempool/blob/master/frontend/sync-assets.js
+# when running `npm run build` in the frontend.
+#
+# This file is updated by ./frontend-assets-update.sh
+
+declare -A revs=(
+    ["mempool/mining-pools"]=e889230b0924d7d72eb28186db6f96ef94361fa5
+    ["mempool/mining-pool-logos"]=9cb443035878c3f112af97384d624de245afe72d
+)
+
+fetchFile() {
+    repo=$1
+    file=$2
+    rev=${revs["$repo"]}
+    curl -fsS "https://raw.githubusercontent.com/$repo/$rev/$file"
+}
+
+fetchRepo() {
+    repo=$1
+    rev=${revs["$repo"]}
+    curl -fsSL "https://github.com/$repo/archive/$rev.tar.gz"
+}
+
+# shellcheck disable=SC2094
+fetchFile "mempool/mining-pools" pools.json > pools.json
+mkdir mining-pools
+fetchRepo "mempool/mining-pool-logos" | tar xz --strip-components=1 -C mining-pools

pkgs/mempool/frontend-assets-update.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+updateRepoHash() {
+    repo=$1
+    echo -n "Fetching latest rev for $repo: "
+    hash=$(curl -fsS "https://api.github.com/repos/$repo/commits/master" | jq -r '.sha')
+    echo "$hash"
+    sed -i -E "s|( +)\[\"$repo(.*)|\1[\"$repo\"]=$hash|" frontend-assets-fetch.sh
+}
+
+<frontend-assets-fetch.sh sed -nE 's| +\["([^"]+).*|\1|p' | while read -r repo; do
+    updateRepoHash "$repo"
+done

pkgs/mempool/generate.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env nix-shell
+#! nix-shell -i bash -p gnupg gnused jq
+set -euo pipefail
+
+# Use this to start a debug shell at the location of this statement
+# . "${BASH_SOURCE[0]%/*}/../../helper/start-bash-session.sh"
+
+version=2.5.0
+# You can also specify a rev instead:
+# rev=57eddac7f0b99b4fe84d91c0f4a50a4f7ccfe55f
+owner=mempool
+repo=https://github.com/$owner/mempool
+
+cd "${BASH_SOURCE[0]%/*}"
+
+updateSrc() {
+    TMPDIR="$(mktemp -d /tmp/mempool.XXX)"
+    trap 'rm -rf $TMPDIR' EXIT
+
+    # Fetch and verify source
+    src=$TMPDIR/src
+    mkdir -p "$src"
+    if [[ -v rev ]]; then
+        # Fetch revision
+        git -C "$src" init
+        git -C "$src" fetch --depth 1 "$repo" "$rev:src"
+        git -C "$src" checkout src
+    else
+        tag=v$version
+        # Fetch and GPG-verify version tag
+        git clone --depth 1 --branch "$tag" -c advice.detachedHead=false $repo "$src"
+        git -C "$src" checkout tags/$tag
+        export GNUPGHOME=$TMPDIR
+        # Fetch wiz' key
+        gpg --keyserver hkps://keyserver.ubuntu.com --recv-keys 913C5FF1F579B66CA10378DBA394E332255A6173 2> /dev/null
+        git -C "$src" verify-tag $tag
+        rev=$tag
+    fi
+    rm -rf "$src"/.git
+    hash=$(nix hash path "$src")
+
+    sed -i "
+      s|\bowner = .*;|owner = \"$owner\";|
+      s|\brev = .*;|rev = \"$rev\";|
+      s|\bhash = .*;|hash = \"$hash\";|
+    " default.nix
+}
+
+updateNodeModulesHash() {
+    component=$1
+    echo
+    echo "Fetching node modules for mempool-$component"
+    ../../helper/update-fixed-output-derivation.sh ./default.nix mempool-"$component" "cd $component"
+}
+
+updateFrontendAssets() {
+  . ./frontend-assets-update.sh
+  echo
+  echo "Fetching frontend assets"
+  ../../helper/update-fixed-output-derivation.sh ./default.nix mempool-frontend.assets "frontendAssets"
+}
+
+if [[ $# == 0 ]]; then
+    # Each of these can be run separately
+    updateSrc
+    updateFrontendAssets
+    updateNodeModulesHash backend
+    updateNodeModulesHash frontend
+else
+    "$@"
+fi

pkgs/mempool/nginx-conf/mempool/location-static.conf

@@ -0,0 +1,44 @@
+# see order of nginx location rules
+# https://stackoverflow.com/questions/5238377/nginx-location-priority
+
+# for exact / requests, redirect based on $lang
+# cache redirect for 5 minutes
+location = / {
+	if ($lang != '') {
+		return 302 $scheme://$host/$lang/;
+	}
+	try_files /en-US/index.html =404;
+	expires 5m;
+}
+
+# cache /<lang>/main.f40e91d908a068a2.js forever since they never change
+location ~ ^/([a-z][a-z])/(.+\..+\.(js|css)) {
+	try_files $uri =404;
+	expires 1y;
+}
+# cache everything else for 5 minutes
+location ~ ^/([a-z][a-z])$ {
+	try_files $uri /$1/index.html /en-US/index.html =404;
+	expires 5m;
+}
+location ~ ^/([a-z][a-z])/ {
+	try_files $uri /$1/index.html /en-US/index.html =404;
+	expires 5m;
+}
+
+# cache /resources/** for 1 week since they don't change often
+location /resources {
+	try_files $uri /en-US/index.html;
+	expires 1w;
+}
+# cache /main.f40e91d908a068a2.js forever since they never change
+location ~* ^/.+\..+\.(js|css) {
+	try_files /$lang/$uri /en-US/$uri =404;
+	expires 1y;
+}
+# catch-all for all URLs i.e. /address/foo /tx/foo /block/000
+# cache 5 minutes since they change frequently
+location / {
+	try_files /$lang/$uri $uri /en-US/$uri /en-US/index.html =404;
+	expires 5m;
+}

pkgs/mempool/nginx-conf/mempool/mempool.conf

@@ -0,0 +1,44 @@
+access_log /var/log/nginx/access_mempool.log;
+error_log /var/log/nginx/error_mempool.log;
+
+root /var/www/mempool/browser;
+
+index index.html;
+
+# enable browser and proxy caching
+add_header Cache-Control "public, no-transform";
+
+# vary cache if user changes language preference
+add_header Vary Accept-Language;
+add_header Vary Cookie;
+
+include mempool/location-static.conf;
+
+# static API docs
+location = /api {
+	try_files $uri $uri/ /en-US/index.html =404;
+}
+location = /api/ {
+	try_files $uri $uri/ /en-US/index.html =404;
+}
+
+location /api/v1/ws {
+	proxy_pass http://127.0.0.1:8999/;
+	proxy_http_version 1.1;
+	proxy_set_header Upgrade $http_upgrade;
+	proxy_set_header Connection "Upgrade";
+}
+location /api/v1 {
+	proxy_pass http://127.0.0.1:8999/api/v1;
+}
+location /api/ {
+	proxy_pass http://127.0.0.1:8999/api/v1/;
+}
+
+# mainnet API
+location /ws {
+	proxy_pass http://127.0.0.1:8999/;
+	proxy_http_version 1.1;
+	proxy_set_header Upgrade $http_upgrade;
+	proxy_set_header Connection "Upgrade";
+}

pkgs/mempool/nginx-conf/nginx.conf

@@ -0,0 +1,82 @@
+user nobody;
+pid /var/run/nginx.pid;
+
+worker_processes auto;
+worker_rlimit_nofile 100000;
+
+events {
+    worker_connections 9000;
+    multi_accept on;
+}
+
+http {
+    sendfile on;
+    tcp_nopush on;
+    tcp_nodelay on;
+
+    server_tokens off;
+    server_name_in_redirect off;
+
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    access_log /var/log/nginx/access.log;
+    error_log /var/log/nginx/error.log;
+
+    # reset timed out connections freeing ram
+    reset_timedout_connection on;
+    # maximum time between packets the client can pause when sending nginx any data
+    client_body_timeout 10s;
+    # maximum time the client has to send the entire header to nginx
+    client_header_timeout 10s;
+    # timeout which a single keep-alive client connection will stay open
+    keepalive_timeout 69s;
+    # maximum time between packets nginx is allowed to pause when sending the client data
+    send_timeout 69s;
+
+    # number of requests per connection, does not affect SPDY
+    keepalive_requests 1337;
+
+    # enable gzip compression
+    gzip on;
+    gzip_vary on;
+    gzip_comp_level 6;
+    gzip_min_length 1000;
+    gzip_proxied expired no-cache no-store private auth;
+    # text/html is always compressed by gzip module
+    gzip_types application/javascript application/json application/ld+json application/manifest+json application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/bmp image/svg+xml image/x-icon text/cache-manifest text/css text/plain text/vcard;
+
+    # limit request body size
+    client_max_body_size 10m;
+
+    # proxy cache
+    proxy_cache off;
+    proxy_cache_path /var/cache/nginx keys_zone=cache:20m levels=1:2 inactive=600s max_size=500m;
+    types_hash_max_size 2048;
+
+    # exempt localhost from rate limit
+    geo $limited_ip {
+        default        1;
+        127.0.0.1    0;
+    }
+    map $limited_ip $limited_ip_key {
+        1 $binary_remote_addr;
+        0 '';
+    }
+
+    # rate limit requests
+    limit_req_zone $limited_ip_key zone=api:5m rate=200r/m;
+    limit_req_zone $limited_ip_key zone=electrs:5m rate=2000r/m;
+    limit_req_status 429;
+
+    # rate limit connections
+    limit_conn_zone $limited_ip_key zone=websocket:10m;
+    limit_conn_status 429;
+
+    include mempool/http-language.conf;
+
+    server {
+        listen 127.0.0.1:80;
+        include mempool/mempool.conf;
+    }
+}

test/tests.nix

@@ -78,6 +78,9 @@ let
         echo a > rtl-password
       '');
 
+      tests.mempool = cfg.mempool.enable;
+      services.mempool.electrumServer = "fulcrum";
+
       tests.lnd = cfg.lnd.enable;
       services.lnd = {
         port = 9736;
@@ -195,6 +198,7 @@ let
       services.lightning-loop.enable = true;
       services.lightning-pool.enable = true;
       services.charge-lnd.enable = true;
+      services.mempool.enable = true;
       services.electrs.enable = true;
       services.fulcrum.enable = true;
       services.liquidd.enable = true;
@@ -219,6 +223,8 @@ let
       tests.secure-node = true;
       tests.restart-bitcoind = true;
 
+      nix-bitcoin.onionServices.mempool-frontend.enable = true;
+
       # Stop electrs from spamming the test log with 'WARN - wait until IBD is over' messages
       tests.stop-electrs = true;
     };
@@ -237,6 +243,7 @@ let
       services.clightning-rest.enable = true;
       services.liquidd.enable = true;
       services.rtl.enable = true;
+      services.mempool.enable = true;
       services.lnd.enable = true;
       services.lightning-loop.enable = true;
       services.lightning-pool.enable = true;

test/tests.py

@@ -251,6 +251,15 @@ def _():
         log_has_string("clightning-rest", "cl-rest api server is ready and listening")
     )
 
+@test("mempool")
+def _():
+    assert_running("mempool")
+    assert_running("nginx")
+    machine.wait_until_succeeds(
+        log_has_string("mempool", "Mempool Server is running on port 8999")
+    )
+    assert_matches(f"curl -L {ip('nginx')}:60845", "mempool - Bitcoin Explorer")
+
 @test("joinmarket")
 def _():
     assert_running("joinmarket")
@@ -427,6 +436,12 @@ def _():
     if enabled("btcpayserver"):
         machine.wait_until_succeeds(log_has_string("nbxplorer", f"At height: {num_blocks}"))
 
+    if enabled("mempool"):
+        assert_running("nginx")
+        assert_full_match(
+            f"curl -fsS http://{ip('nginx')}:60845/api/v1/blocks/tip/height", str(num_blocks)
+        )
+
 @test("trustedcoin")
 def _():
     def expect_clightning_log(str):