Alpine: Use ZFS for /boot, updated bootloader instructions

Signed-off-by: Maurice Zhou <ja@apvc.uk>

docs/Getting Started/Alpine Linux/Root on ZFS/1-preparation.rst

@@ -36,8 +36,21 @@ Preparation
 
     ssh root@192.168.1.19
 
+#. Configure NTP client for time synchronization::
+
+     setup-ntp chrony
+
+#. Pick a mirror from `Alpine Mirrors <https://mirrors.alpinelinux.org/>`__
+   and add it to package manager configuration::
+
+     tee -a /etc/apk/repositories <<EOF
+     https://dl-5.alpinelinux.org/alpine/latest-stable/community/
+     https://dl-5.alpinelinux.org/alpine/latest-stable/main/
+     EOF
+
 #. Throughout this guide, we use predictable disk names generated by udev::
 
+     apk update
      apk add eudev
      setup-udev
 
@@ -59,21 +72,8 @@ Preparation
 
     DISK='/dev/disk/by-id/disk1'
 
-#. Configure NTP client for time synchronization::
-
-     setup-ntp chrony
-
-#. Pick a mirror from `Alpine Mirrors <https://mirrors.alpinelinux.org/>`__
-   and add it to package manager configuration::
-
-     tee -a /etc/apk/repositories <<EOF
-     https://dl-5.alpinelinux.org/alpine/latest-stable/community/
-     https://dl-5.alpinelinux.org/alpine/latest-stable/main/
-     EOF
-
 #. Install ZFS support and partition tool::
 
-    apk update
     apk add zfs zfs-lts sgdisk e2fsprogs
     modprobe zfs
 

docs/Getting Started/Alpine Linux/Root on ZFS/2-system-installation.rst

@@ -37,9 +37,41 @@ System Installation
 
 #. Create boot partition::
 
-    for i in ${DISK}; do
+     tee -a /root/grub2 <<EOF
-      mkfs.ext4 -F $i-part2
+     # Features which are supported by GRUB2
-    done
+     async_destroy
+     bookmarks
+     embedded_data
+     empty_bpobj
+     enabled_txg
+     extensible_dataset
+     filesystem_limits
+     hole_birth
+     large_blocks
+     lz4_compress
+     spacemap_histogram
+     EOF
+
+     zpool create \
+      -o compatibility=/root/grub2 \
+      -o ashift=12 \
+      -o autotrim=on \
+      -O acltype=posixacl \
+      -O canmount=off \
+      -O compression=lz4 \
+      -O devices=off \
+      -O normalization=formD \
+      -O relatime=on \
+      -O xattr=sa \
+      -O mountpoint=/boot \
+      -R /mnt \
+      bpool \
+      mirror \
+      $(for i in ${DISK}; do
+         printf "$i-part2 ";
+        done)
+
+   If not using a multi-disk setup, remove ``mirror``.
 
 #. Create root pool::
 
@@ -91,14 +123,17 @@ System Installation
      zfs create -o canmount=off -o mountpoint=/var  rpool/alpine/var
      zfs create -o canmount=on  rpool/alpine/var/lib
      zfs create -o canmount=on  rpool/alpine/var/log
-
+     zfs create -o canmount=off  -o mountpoint=none bpool/alpine
-#. Mount /boot::
+     zfs create -o canmount=on  -o mountpoint=/boot bpool/alpine/root
-
-    mkdir -p /mnt/boot
-    mount -t ext4 $(echo $DISK | cut -f1 -d\ )-part2 /mnt/boot/
 
 #. Format and mount ESP::
 
+    for i in ${DISK}; do
+     mkfs.vfat -n EFI ${i}-part1
+     mkdir -p /mnt/boot/efis/${i##*/}-part1
+     mount -t vfat ${i}-part1 /mnt/boot/efis/${i##*/}-part1
+    done
+
     mkdir -p /mnt/boot/efi
     mount -t vfat $(echo $DISK | cut -f1 -d\ )-part1 /mnt/boot/efi
 
@@ -124,22 +159,6 @@ System Installation
 
     chroot /mnt /usr/bin/env DISK=$DISK sh
 
-#. Apply GRUB workaround::
-
-     echo 'export ZPOOL_VDEV_NAME_PATH=YES' >> /etc/profile.d/zpool_vdev_name_path.sh
-     source /etc/profile.d/zpool_vdev_name_path.sh
-
-     sed -i "s|rpool=.*|rpool=rpool|"  /etc/grub.d/10_linux
-
-     sed -i 's|stat -f -c %T /|echo zfs|' /usr/sbin/grub-mkconfig
-
-   This workaround needs to be applied for every GRUB update, as the
-   update will overwrite the changes.
-
-#. Generate GRUB menu::
-
-     grub-mkconfig -o /boot/grub/grub.cfg
-
 #. Rebuild initrd::
 
     mkdir -p /etc/zfs
@@ -172,13 +191,49 @@ System Installation
 
      sed -i 's|,posixacl|,zfsutil,posixacl|' /etc/fstab
 
+#. Apply GRUB workaround::
+
+     echo 'export ZPOOL_VDEV_NAME_PATH=YES' >> /etc/profile.d/zpool_vdev_name_path.sh
+     source /etc/profile.d/zpool_vdev_name_path.sh
+
+     # GRUB fails to detect rpool name, hard code as "rpool"
+     sed -i "s|rpool=.*|rpool=rpool|"  /etc/grub.d/10_linux
+
+     # BusyBox stat does not recognize zfs, replace fs detection with ZFS
+     sed -i 's|stat -f -c %T /|echo zfs|' /usr/sbin/grub-mkconfig
+
+     # grub-probe fails to identify fs mounted at /boot
+     sed -i "s|GRUB_DEVICE_BOOT=.*|GRUB_DEVICE_BOOT=$(echo $DISK | cut -f1 -d\ )-part2|"  /usr/sbin/grub-mkconfig
+
+   This workaround needs to be applied for every GRUB update, as the
+   update will overwrite the changes.
+
 #. Install GRUB::
 
       export ZPOOL_VDEV_NAME_PATH=YES
+      mkdir -p /boot/efi/alpine/grub-bootdir/i386-pc/
+      mkdir -p /boot/efi/alpine/grub-bootdir/x86_64-efi/
       for i in ${DISK}; do
-       grub-install --target=i386-pc $i
+       grub-install --target=i386-pc --boot-directory \
+           /boot/efi/alpine/grub-bootdir/i386-pc/  $i
       done
-      grub-install --target x86_64-efi --bootloader-id alpine --removable
+      grub-install --target x86_64-efi --boot-directory \
+          /boot/efi/alpine/grub-bootdir/x86_64-efi/ --efi-directory \
+	  /boot/efi --bootloader-id alpine --removable
+
+#. Generate GRUB menu::
+
+     grub-mkconfig -o /boot/efi/alpine/grub-bootdir/x86_64-efi/grub/grub.cfg
+     grub-mkconfig -o /boot/efi/alpine/grub-bootdir/i386-pc/grub/grub.cfg
+
+#. For both legacy and EFI booting: mirror ESP content::
+
+    ESP_MIRROR=$(mktemp -d)
+    cp -r /boot/efi/EFI $ESP_MIRROR
+    for i in /boot/efis/*; do
+     cp -r $ESP_MIRROR/EFI $i
+    done
+    rm -rf $ESP_MIRROR
 
 #. Unmount filesystems::
 
@@ -193,11 +248,10 @@ System Installation
    Disconnect the live media and other non-boot storage devices.
    Due to missing support of predictable device names in initrd,
    Alpine Linux will mount whichever disk appears to be /dev/sda or /dev/nvme0
-   at /boot and /boot/efi at boot.
+   at /boot/efi at boot.
 
-   Root filesystem at / is ZFS and imported via pool name thus not affected by the above restriction.
+   Root filesystem at / and /boot are ZFS and imported via pool name thus not affected by the above restriction.
 
 #. Post-install:
 
-   #. Setup mirroring of /boot partition and /boot/efi via dd.
    #. Setup swap.

docs/Getting Started/Arch Linux/Root on ZFS/1-preparation.rst

@@ -50,11 +50,6 @@ Preparation
 
     INST_PARTSIZE_RPOOL=
 
-#. Check kernel version::
-
-     uname -r
-     #5.18.7-arch1-1
-
 #. Add ZFS repo::
 
      curl -L https://archzfs.com/archzfs.gpg |  pacman-key -a -
@@ -70,6 +65,11 @@ Preparation
      Include = /etc/pacman.d/mirrorlist-archzfs
      EOF
 
+#. Check kernel version::
+
+     uname -r
+     #5.18.7-arch1-1
+
 #. Find a ZFS package compatible with the kernel:
 
    Search kernel version string (e.g. 5.18.7) in both pages:

docs/Getting Started/Arch Linux/Root on ZFS/2-system-installation.rst

@@ -118,7 +118,8 @@ System Installation
 
    Create boot dataset::
 
-     zfs create -o canmount=on -o mountpoint=/boot bpool/archlinux
+     zfs create -o canmount=off -o mountpoint=none bpool/archlinux
+     zfs create -o canmount=on -o mountpoint=/boot bpool/archlinux/root
 
 #. Format and mount ESP::
 

docs/Getting Started/Arch Linux/Root on ZFS/5-bootloader.rst

@@ -6,40 +6,6 @@ Bootloader
 .. contents:: Table of Contents
    :local:
 
-Apply workarounds
-~~~~~~~~~~~~~~~~~~~~
-Currently GRUB has multiple compatibility problems with ZFS,
-especially with regards to newer ZFS features.
-Workarounds have to be applied.
-
-#. grub-probe fails to get canonical path
-
-   When persistent device names ``/dev/disk/by-id/*`` are used
-   with ZFS, GRUB will fail to resolve the path of the boot pool
-   device. Error::
-
-     # /usr/bin/grub-probe: error: failed to get canonical path of `/dev/virtio-pci-0000:06:00.0-part3'.
-
-   Solution::
-
-    echo 'export ZPOOL_VDEV_NAME_PATH=YES' >> /etc/profile.d/zpool_vdev_name_path.sh
-    source /etc/profile.d/zpool_vdev_name_path.sh
-
-#. Pool name missing
-
-   See `this bug report <https://savannah.gnu.org/bugs/?59614>`__.
-   Root pool name is missing from ``root=ZFS=rpool_$INST_UUID/ROOT/default``
-   kernel cmdline in generated ``grub.cfg`` file.
-
-   A workaround is to replace the pool name detection with ``zdb``
-   command::
-
-     sed -i "s|rpool=.*|rpool=\`zdb -l \${GRUB_DEVICE} \| grep -E '[[:blank:]]name' \| cut -d\\\' -f 2\`|"  /etc/grub.d/10_linux
-
-   Caution:  this fix must be applied after every GRUB update and before generating the menu.
-
-Install GRUB
-~~~~~~~~~~~~~~~~~~~~
 #. Create empty cache file and generate initrd::
 
     rm -f /etc/zfs/zpool.cache
@@ -49,33 +15,40 @@ Install GRUB
 
     mkinitcpio -P
 
-#. If using legacy booting, install GRUB to every disk::
+#. Apply GRUB workaround::
 
-    for i in ${DISK}; do
+     echo 'export ZPOOL_VDEV_NAME_PATH=YES' >> /etc/profile.d/zpool_vdev_name_path.sh
-     grub-install --target=i386-pc $i
+     source /etc/profile.d/zpool_vdev_name_path.sh
-    done
 
-#. If using EFI::
+     # GRUB fails to detect rpool name, hard code as "rpool"
-
+     sed -i "s|rpool=.*|rpool=rpool|"  /etc/grub.d/10_linux
-     grub-install --target x86_64-efi
-     grub-install --target x86_64-efi --removable
-     for i in ${DISK}; do
-      efibootmgr -cgp 1 -l "\EFI\arch\grubx64.efi" \
-      -L "arch-${i##*/}" -d ${i}
-     done
-
-#. Generate GRUB Menu:
-
-   Generate menu::
 
      echo GRUB_CMDLINE_LINUX=\"zfs_import_dir=/dev/disk/by-id/\" >> /etc/default/grub
-    grub-mkconfig -o /boot/grub/grub.cfg
+
-    cp /boot/grub/grub.cfg /boot/efi/EFI/arch/
+   This workaround needs to be applied for every GRUB update, as the
+   update will overwrite the changes.
+
+#. Install GRUB::
+
+      export ZPOOL_VDEV_NAME_PATH=YES
+      mkdir -p /boot/efi/arch/grub-bootdir/i386-pc/
+      mkdir -p /boot/efi/arch/grub-bootdir/x86_64-efi/
+      for i in ${DISK}; do
+       grub-install --target=i386-pc --boot-directory \
+           /boot/efi/arch/grub-bootdir/i386-pc/  $i
+      done
+      grub-install --target x86_64-efi --boot-directory \
+          /boot/efi/arch/grub-bootdir/x86_64-efi/ --efi-directory \
+	  /boot/efi --bootloader-id arch --removable
+
+#. Generate GRUB menu::
+
+     grub-mkconfig -o /boot/efi/arch/grub-bootdir/x86_64-efi/grub/grub.cfg
+     grub-mkconfig -o /boot/efi/arch/grub-bootdir/i386-pc/grub/grub.cfg
 
 #. For both legacy and EFI booting: mirror ESP content::
 
     ESP_MIRROR=$(mktemp -d)
-    unalias -a
     cp -r /boot/efi/EFI $ESP_MIRROR
     for i in /boot/efis/*; do
      cp -r $ESP_MIRROR/EFI $i

docs/Getting Started/Fedora/Root on ZFS/2-system-installation.rst

@@ -118,7 +118,8 @@ System Installation
 
    Create boot dataset::
 
-     zfs create -o canmount=on -o mountpoint=/boot bpool/redhat
+     zfs create -o canmount=off -o mountpoint=none bpool/redhat
+     zfs create -o canmount=on -o mountpoint=/boot bpool/redhat/root
 
 #. Format and mount ESP::
 

docs/Getting Started/Fedora/Root on ZFS/3-system-configuration.rst

@@ -61,7 +61,7 @@ System Configuration
 
     fixfiles -F onboot
 
-#. Set root password::
+#. Set root password, the password set earlier does not work due to SELinux::
 
     passwd
 

docs/Getting Started/Fedora/Root on ZFS/5-bootloader.rst

@@ -6,40 +6,6 @@ Bootloader
 .. contents:: Table of Contents
    :local:
 
-Apply workarounds
-~~~~~~~~~~~~~~~~~~~~
-Currently GRUB has multiple compatibility problems with ZFS,
-especially with regards to newer ZFS features.
-Workarounds have to be applied.
-
-#. grub2-probe fails to get canonical path
-
-   When persistent device names ``/dev/disk/by-id/*`` are used
-   with ZFS, GRUB will fail to resolve the path of the boot pool
-   device. Error::
-
-     # /usr/bin/grub2-probe: error: failed to get canonical path of `/dev/virtio-pci-0000:06:00.0-part3'.
-
-   Solution::
-
-    echo 'export ZPOOL_VDEV_NAME_PATH=YES' >> /etc/profile.d/zpool_vdev_name_path.sh
-    source /etc/profile.d/zpool_vdev_name_path.sh
-
-#. Pool name missing
-
-   See `this bug report <https://savannah.gnu.org/bugs/?59614>`__.
-   Root pool name is missing from ``root=ZFS=rpool_$INST_UUID/ROOT/default``
-   kernel cmdline in generated ``grub.cfg`` file.
-
-   A workaround is to replace the pool name detection with ``zdb``
-   command::
-
-     sed -i "s|rpool=.*|rpool=\`zdb -l \${GRUB_DEVICE} \| grep -E '[[:blank:]]name' \| cut -d\\\' -f 2\`|"  /etc/grub.d/10_linux
-
-   Caution:  this fix must be applied after every GRUB update and before generating the menu.
-
-Install GRUB
-~~~~~~~~~~~~~~~~~~~~
 
 #. If using virtio disk, add driver to initrd::
 
@@ -61,26 +27,33 @@ Install GRUB
 
     echo 'GRUB_ENABLE_BLSCFG=false' >> /etc/default/grub
 
-#. If using legacy booting, install GRUB to every disk::
+#. Apply GRUB workaround::
 
+     echo 'export ZPOOL_VDEV_NAME_PATH=YES' >> /etc/profile.d/zpool_vdev_name_path.sh
+     source /etc/profile.d/zpool_vdev_name_path.sh
+
+     # GRUB fails to detect rpool name, hard code as "rpool"
+     sed -i "s|rpool=.*|rpool=rpool|"  /etc/grub.d/10_linux
+
+   This workaround needs to be applied for every GRUB update, as the
+   update will overwrite the changes.
+
+#. Install GRUB::
+
+      export ZPOOL_VDEV_NAME_PATH=YES
+      mkdir -p /boot/efi/fedora/grub-bootdir/i386-pc/
+      mkdir -p /boot/efi/fedora/grub-bootdir/x86_64-efi/
       for i in ${DISK}; do
-     grub2-install --target=i386-pc $i
+       grub2-install --target=i386-pc --boot-directory \
+           /boot/efi/fedora/grub-bootdir/i386-pc/  $i
       done
 
-#. If using EFI::
-
-    for i in ${DISK}; do
-     efibootmgr -cgp 1 -l "\EFI\fedora\shimx64.efi" \
-     -L "fedora-${i##*/}" -d ${i}
-    done
       cp -r /usr/lib/grub/x86_64-efi/ /boot/efi/EFI/fedora/
 
-#. Generate GRUB Menu:
+#. Generate GRUB menu::
 
-   Generate menu::
+     grub2-mkconfig -o /boot/efi/EFI/fedora/grub.cfg
-
+     grub2-mkconfig -o /boot/efi/fedora/grub-bootdir/i386-pc/grub2/grub.cfg
-    grub2-mkconfig -o /boot/grub2/grub.cfg
-    cp /boot/grub2/grub.cfg /boot/efi/EFI/fedora/
 
 #. For both legacy and EFI booting: mirror ESP content::
 

docs/Getting Started/NixOS/Root on ZFS/2-system-installation.rst

@@ -118,7 +118,8 @@ System Installation
 
    Create boot dataset::
 
-     zfs create -o canmount=on -o mountpoint=/boot bpool/nixos
+     zfs create -o canmount=off -o mountpoint=none bpool/nixos
+     zfs create -o canmount=on -o mountpoint=/boot bpool/nixos/root
 
 #. Format and mount ESP::
 

docs/Getting Started/NixOS/Root on ZFS/3-system-configuration.rst

@@ -8,11 +8,11 @@ System Configuration
 
 #. Disable cache, stale cache will prevent system from booting::
 
-     mkdir -p /mnt/state/etc/zfs/
+     mkdir -p /mnt/etc/zfs/
-     rm -f /mnt/state/etc/zfs/zpool.cache
+     rm -f /mnt/etc/zfs/zpool.cache
-     touch /mnt/state/etc/zfs/zpool.cache
+     touch /mnt/etc/zfs/zpool.cache
-     chmod a-w /mnt/state/etc/zfs/zpool.cache
+     chmod a-w /mnt/etc/zfs/zpool.cache
-     chattr +i /mnt/state/etc/zfs/zpool.cache
+     chattr +i /mnt/etc/zfs/zpool.cache
 
 #. Generate initial system configuration::
 

docs/Getting Started/RHEL-based distro/RHEL-based distro Root on ZFS/2-system-installation.rst

@@ -118,7 +118,8 @@ System Installation
 
    Create boot dataset::
 
-     zfs create -o canmount=on -o mountpoint=/boot bpool/redhat
+     zfs create -o canmount=off -o mountpoint=none bpool/redhat
+     zfs create -o canmount=on -o mountpoint=/boot bpool/redhat/root
 
 #. Format and mount ESP::
 

docs/Getting Started/RHEL-based distro/RHEL-based distro Root on ZFS/3-system-configuration.rst

@@ -61,6 +61,6 @@ System Configuration
 
     fixfiles -F onboot
 
-#. Set root password::
+#. Set root password, the password set earlier does not work due to SELinux::
 
     passwd

docs/Getting Started/RHEL-based distro/RHEL-based distro Root on ZFS/5-bootloader.rst

@@ -6,40 +6,6 @@ Bootloader
 .. contents:: Table of Contents
    :local:
 
-Apply workarounds
-~~~~~~~~~~~~~~~~~~~~
-Currently GRUB has multiple compatibility problems with ZFS,
-especially with regards to newer ZFS features.
-Workarounds have to be applied.
-
-#. grub2-probe fails to get canonical path
-
-   When persistent device names ``/dev/disk/by-id/*`` are used
-   with ZFS, GRUB will fail to resolve the path of the boot pool
-   device. Error::
-
-     # /usr/bin/grub2-probe: error: failed to get canonical path of `/dev/virtio-pci-0000:06:00.0-part3'.
-
-   Solution::
-
-    echo 'export ZPOOL_VDEV_NAME_PATH=YES' >> /etc/profile.d/zpool_vdev_name_path.sh
-    source /etc/profile.d/zpool_vdev_name_path.sh
-
-#. Pool name missing
-
-   See `this bug report <https://savannah.gnu.org/bugs/?59614>`__.
-   Root pool name is missing from ``root=ZFS=rpool_$INST_UUID/ROOT/default``
-   kernel cmdline in generated ``grub.cfg`` file.
-
-   A workaround is to replace the pool name detection with ``zdb``
-   command::
-
-     sed -i "s|rpool=.*|rpool=\`zdb -l \${GRUB_DEVICE} \| grep -E '[[:blank:]]name' \| cut -d\\\' -f 2\`|"  /etc/grub.d/10_linux
-
-   Caution:  this fix must be applied after every GRUB update and before generating the menu.
-
-Install GRUB
-~~~~~~~~~~~~~~~~~~~~
 
 #. If using virtio disk, add driver to initrd::
 
@@ -61,26 +27,33 @@ Install GRUB
 
     echo 'GRUB_ENABLE_BLSCFG=false' >> /etc/default/grub
 
-#. If using legacy booting, install GRUB to every disk::
+#. Apply GRUB workaround::
 
+     echo 'export ZPOOL_VDEV_NAME_PATH=YES' >> /etc/profile.d/zpool_vdev_name_path.sh
+     source /etc/profile.d/zpool_vdev_name_path.sh
+
+     # GRUB fails to detect rpool name, hard code as "rpool"
+     sed -i "s|rpool=.*|rpool=rpool|"  /etc/grub.d/10_linux
+
+   This workaround needs to be applied for every GRUB update, as the
+   update will overwrite the changes.
+
+#. Install GRUB::
+
+      export ZPOOL_VDEV_NAME_PATH=YES
+      mkdir -p /boot/efi/almalinux/grub-bootdir/i386-pc/
+      mkdir -p /boot/efi/almalinux/grub-bootdir/x86_64-efi/
       for i in ${DISK}; do
-     grub2-install --target=i386-pc $i
+       grub2-install --target=i386-pc --boot-directory \
+           /boot/efi/almalinux/grub-bootdir/i386-pc/  $i
       done
 
-#. If using EFI::
-
-    for i in ${DISK}; do
-     efibootmgr -cgp 1 -l "\EFI\almalinux\shimx64.efi" \
-     -L "almalinux-${i##*/}" -d ${i}
-    done
       cp -r /usr/lib/grub/x86_64-efi/ /boot/efi/EFI/almalinux/
 
-#. Generate GRUB Menu:
+#. Generate GRUB menu::
 
-   Generate menu::
+     grub2-mkconfig -o /boot/efi/EFI/almalinux/grub.cfg
-
+     grub2-mkconfig -o /boot/efi/almalinux/grub-bootdir/i386-pc/grub2/grub.cfg
-    grub2-mkconfig -o /boot/grub2/grub.cfg
-    cp /boot/grub2/grub.cfg /boot/efi/EFI/almalinux/
 
 #. For both legacy and EFI booting: mirror ESP content::