.. highlight:: sh

Alpine Linux Root on ZFS
========================

.. ifconfig:: zfs_root_test

   ::

    # For the CI/CD test run of this guide,
    # Enable verbose logging of bash shell and fail immediately when
    # a commmand fails.
    set -vxeuf
    distro=${1}

    cp /etc/resolv.conf ./"rootfs-${distro}"/etc/resolv.conf
    arch-chroot ./"rootfs-${distro}" sh <<-'ZFS_ROOT_GUIDE_TEST'

    set -vxeuf

    # install alpine setup scripts
    apk update
    apk add alpine-conf curl


**Customization**

Unless stated otherwise, it is not recommended to customize system
configuration before reboot.

Preparation
---------------------------

#. Disable Secure Boot. ZFS modules can not be loaded if Secure Boot is enabled.
#. Download latest extended variant of `Alpine Linux
   live image
   <https://dl-cdn.alpinelinux.org/alpine/v3.17/releases/x86_64/alpine-extended-3.17.3-x86_64.iso>`__,
   verify `checksum <https://dl-cdn.alpinelinux.org/alpine/v3.17/releases/x86_64/alpine-extended-3.17.3-x86_64.iso.asc>`__
   and boot from it.

   .. code-block:: sh

       gpg --auto-key-retrieve --keyserver hkps://keyserver.ubuntu.com --verify alpine-extended-*.asc

       dd if=input-file of=output-file bs=1M

   .. ifconfig:: zfs_root_test

      # check whether the download page exists
      # alpine version must be in sync with ci/cd test chroot tarball
      curl --head --fail https://dl-cdn.alpinelinux.org/alpine/v3.17/releases/x86_64/alpine-extended-3.17.3-x86_64.iso
      curl --head --fail https://dl-cdn.alpinelinux.org/alpine/v3.17/releases/x86_64/alpine-extended-3.17.3-x86_64.iso.asc

#. Login as root user.  There is no password.
#. Configure Internet

   .. code-block:: sh

     setup-interfaces -r
     # You must use "-r" option to start networking services properly
     # example:
     network interface: wlan0
     WiFi name:         <ssid>
     ip address:        dhcp
     <enter done to finish network config>
     manual netconfig:  n

#. If you are using wireless network and it is not shown, see `Alpine
   Linux wiki
   <https://wiki.alpinelinux.org/wiki/Wi-Fi#wpa_supplicant>`__ for
   further details.  ``wpa_supplicant`` can be installed with ``apk
   add wpa_supplicant`` without internet connection.

#. Configure SSH server

   .. code-block:: sh

     setup-sshd
     # example:
     ssh server:        openssh
     allow root:        "prohibit-password" or "yes"
     ssh key:           "none" or "<public key>"

   Configurations set here will be copied verbatim to the installed system.

#. Set root password or ``/root/.ssh/authorized_keys``.

   Choose a strong root password, as it will be copied to the
   installed system.  However, ``authorized_keys`` is not copied.

#. Connect from another computer

   .. code-block:: sh

    ssh root@192.168.1.91

#. Configure NTP client for time synchronization

   .. code-block:: sh

     setup-ntp busybox

   .. ifconfig:: zfs_root_test

     # this step is unnecessary for chroot and returns 1 when executed

#. Set up apk-repo.  A list of available mirrors is shown.
   Press space bar to continue

   .. code-block:: sh

    setup-apkrepos

#. Throughout this guide, we use predictable disk names generated by
   udev

   .. code-block:: sh

     apk update
     apk add eudev
     setup-devd udev

   It can be removed after reboot with ``setup-devd mdev && apk del eudev``.

   .. ifconfig:: zfs_root_test

      # for some reason, udev is extremely slow in chroot
      # it is not needed for chroot anyway. so, skip this step

#. Target disk

   List available disks with

   .. code-block:: sh

    find /dev/disk/by-id/

   If virtio is used as disk bus, power off the VM and set serial numbers for disk.
   For QEMU, use ``-drive format=raw,file=disk2.img,serial=AaBb``.
   For libvirt, edit domain XML.  See `this page
   <https://bugzilla.redhat.com/show_bug.cgi?id=1245013>`__ for examples.

   Declare disk array

   .. code-block:: sh

    DISK='/dev/disk/by-id/ata-FOO /dev/disk/by-id/nvme-BAR'

   For single disk installation, use

   .. code-block:: sh

    DISK='/dev/disk/by-id/disk1'

   .. ifconfig:: zfs_root_test

    # for github test run, use chroot and loop devices
    DISK="$(losetup -a| grep alpine | cut -f1 -d: | xargs -t -I '{}' printf '{} ')"
    # for maintenance guide test
    DISK="$(losetup -a| grep maintenance | cut -f1 -d: | xargs -t -I '{}' printf '{} ') ${DISK}"

#. Set a mount point
   ::

      MNT=$(mktemp -d)

#. Set partition size:

   Set swap size in GB, set to 1 if you don't want swap to
   take up too much space

   .. code-block:: sh

    SWAPSIZE=4

   .. ifconfig:: zfs_root_test

    # For the test run, use 1GB swap space to avoid hitting CI/CD
    # quota
    SWAPSIZE=1

   Set how much space should be left at the end of the disk, minimum 1GB

   ::

    RESERVE=1

#. Install ZFS support from live media::

    apk add zfs

#. Install bootloader programs and partition tool
   ::

      apk add grub-bios grub-efi parted e2fsprogs cryptsetup util-linux

System Installation
---------------------------

#. Partition the disks.

   Note: you must clear all existing partition tables and data structures from the disks,
   especially those with existing ZFS pools or mdraid and those that have been used as live media.
   Those data structures may interfere with boot process.

   For flash-based storage, this can be done by uncommenting the blkdiscard command below:
   ::

     partition_disk () {
      local disk="${1}"
      #blkdiscard -f "${disk}"

      parted --script --align=optimal  "${disk}" -- \
      mklabel gpt \
      mkpart EFI 2MiB 1GiB \
      mkpart bpool 1GiB 5GiB \
      mkpart rpool 5GiB -$((SWAPSIZE + RESERVE))GiB \
      mkpart swap  -$((SWAPSIZE + RESERVE))GiB -"${RESERVE}"GiB \
      mkpart BIOS 1MiB 2MiB \
      set 1 esp on \
      set 5 bios_grub on \
      set 5 legacy_boot on

      partprobe "${disk}"
     }

     for i in ${DISK}; do
        partition_disk "${i}"
     done

   .. ifconfig:: zfs_root_test

      ::

       # When working with GitHub chroot runners, we are using loop
       # devices as installation target.  However, the alias support for
       # loop device was just introduced in March 2023. See
       # https://github.com/systemd/systemd/pull/26693
       # For now, we will create the aliases maunally as a workaround
       looppart="1 2 3 4 5"
       for i in ${DISK}; do
         for j in ${looppart}; do
           if test -e "${i}p${j}"; then
                    ln -s "${i}p${j}" "${i}-part${j}"
                  fi
         done
       done

#. Setup encrypted swap.  This is useful if the available memory is
   small::

     for i in ${DISK}; do
        cryptsetup open --type plain --key-file /dev/random "${i}"-part4 "${i##*/}"-part4
        mkswap /dev/mapper/"${i##*/}"-part4
        swapon /dev/mapper/"${i##*/}"-part4
     done

#. Load ZFS kernel module

   .. code-block:: sh

       modprobe zfs

#. Create boot pool
   ::

      # shellcheck disable=SC2046
      zpool create -d \
          -o feature@async_destroy=enabled \
          -o feature@bookmarks=enabled \
          -o feature@embedded_data=enabled \
          -o feature@empty_bpobj=enabled \
          -o feature@enabled_txg=enabled \
          -o feature@extensible_dataset=enabled \
          -o feature@filesystem_limits=enabled \
          -o feature@hole_birth=enabled \
          -o feature@large_blocks=enabled \
          -o feature@lz4_compress=enabled \
          -o feature@spacemap_histogram=enabled \
          -o ashift=12 \
          -o autotrim=on \
          -O acltype=posixacl \
          -O canmount=off \
          -O compression=lz4 \
          -O devices=off \
          -O normalization=formD \
          -O relatime=on \
          -O xattr=sa \
          -O mountpoint=/boot \
          -R "${MNT}" \
          bpool \
                 mirror \
          $(for i in ${DISK}; do
             printf '%s ' "${i}-part2";
            done)

   If not using a multi-disk setup, remove ``mirror``.

   You should not need to customize any of the options for the boot pool.

   GRUB does not support all of the zpool features. See ``spa_feature_names``
   in `grub-core/fs/zfs/zfs.c
   <http://git.savannah.gnu.org/cgit/grub.git/tree/grub-core/fs/zfs/zfs.c#n276>`__.
   This step creates a separate boot pool for ``/boot`` with the features
   limited to only those that GRUB supports, allowing the root pool to use
   any/all features.

#. Create root pool
   ::

       # shellcheck disable=SC2046
       zpool create \
           -o ashift=12 \
           -o autotrim=on \
           -R "${MNT}" \
           -O acltype=posixacl \
           -O canmount=off \
           -O compression=zstd \
           -O dnodesize=auto \
           -O normalization=formD \
           -O relatime=on \
           -O xattr=sa \
           -O mountpoint=/ \
           rpool \
           mirror \
          $(for i in ${DISK}; do
             printf '%s ' "${i}-part3";
            done)

   If not using a multi-disk setup, remove ``mirror``.

#. Create root system container:

   - Unencrypted

     ::

      zfs create \
       -o canmount=off \
       -o mountpoint=none \
      rpool/alpinelinux

   - Encrypted:

     Pick a strong password. Once compromised, changing password will not keep your
     data safe. See ``zfs-change-key(8)`` for more info

     .. code-block:: sh

      zfs create \
        -o canmount=off \
               -o mountpoint=none \
               -o encryption=on \
               -o keylocation=prompt \
               -o keyformat=passphrase \
      rpool/alpinelinux

   You can automate this step (insecure) with: ``echo POOLPASS | zfs create ...``.

   Create system datasets,
   manage mountpoints with ``mountpoint=legacy``
   ::

      zfs create -o canmount=noauto -o mountpoint=/  rpool/alpinelinux/root
      zfs mount rpool/alpinelinux/root
      zfs create -o mountpoint=legacy rpool/alpinelinux/home
      mkdir "${MNT}"/home
      mount -t zfs rpool/alpinelinux/home "${MNT}"/home
      zfs create -o mountpoint=legacy  rpool/alpinelinux/var
      zfs create -o mountpoint=legacy rpool/alpinelinux/var/lib
      zfs create -o mountpoint=legacy rpool/alpinelinux/var/log
      zfs create -o mountpoint=none bpool/alpinelinux
      zfs create -o mountpoint=legacy bpool/alpinelinux/root
      mkdir "${MNT}"/boot
      mount -t zfs bpool/alpinelinux/root "${MNT}"/boot
      mkdir -p "${MNT}"/var/log
      mkdir -p "${MNT}"/var/lib
      mount -t zfs rpool/alpinelinux/var/lib "${MNT}"/var/lib
      mount -t zfs rpool/alpinelinux/var/log "${MNT}"/var/log

#. Format and mount ESP
   ::

     for i in ${DISK}; do
      mkfs.vfat -n EFI "${i}"-part1
      mkdir -p "${MNT}"/boot/efis/"${i##*/}"-part1
      mount -t vfat -o iocharset=iso8859-1 "${i}"-part1 "${MNT}"/boot/efis/"${i##*/}"-part1
     done

     mkdir -p "${MNT}"/boot/efi
     mount -t vfat -o iocharset=iso8859-1 "$(echo "${DISK}" | sed "s|^ *||"  | cut -f1 -d' '|| true)"-part1 "${MNT}"/boot/efi


System Configuration 
---------------------------

#. Workaround for GRUB to recognize predictable disk names::

     export ZPOOL_VDEV_NAME_PATH=YES

#. Install system to disk

   .. code-block:: sh

     BOOTLOADER=grub setup-disk -k lts -v "${MNT}"

   GRUB installation will fail and will be reinstalled later.
   The error message about ZFS kernel module can be ignored.

   .. ifconfig:: zfs_root_test

     # lts kernel will pull in tons of firmware
     BOOTLOADER=grub setup-disk -k virt -v "${MNT}"

#. Allow EFI system partition to fail at boot::

    sed -i "s|vfat.*rw|vfat rw,nofail|" "${MNT}"/etc/fstab

#. Chroot

   .. code-block:: sh

    for i in /dev /proc /sys; do mkdir -p "${MNT}"/"${i}"; mount --rbind "${i}" "${MNT}"/"${i}"; done
    chroot "${MNT}" /usr/bin/env DISK="${DISK}" sh

   .. ifconfig:: zfs_root_test

     ::

       for i in /dev /proc /sys; do mkdir -p "${MNT}"/"${i}"; mount --rbind "${i}" "${MNT}"/"${i}"; done
       chroot "${MNT}" /usr/bin/env DISK="${DISK}" sh <<-'ZFS_ROOT_NESTED_CHROOT'

       set -vxeuf

#. Apply GRUB workaround

   ::

     echo 'export ZPOOL_VDEV_NAME_PATH=YES' >> /etc/profile.d/zpool_vdev_name_path.sh
     # shellcheck disable=SC1091
     . /etc/profile.d/zpool_vdev_name_path.sh

     # GRUB fails to detect rpool name, hard code as "rpool"
     sed -i "s|rpool=.*|rpool=rpool|"  /etc/grub.d/10_linux

     # BusyBox stat does not recognize zfs, replace fs detection with ZFS
     sed -i 's|stat -f -c %T /|echo zfs|' /usr/sbin/grub-mkconfig

     # grub-probe fails to identify fs mounted at /boot
     BOOT_DEVICE=$(zpool status -P bpool | grep -- -part2 | head -n1 | sed  "s|.*/dev*|/dev|" | sed "s|part2.*|part2|")
     sed -i "s|GRUB_DEVICE_BOOT=.*|GRUB_DEVICE_BOOT=${BOOT_DEVICE}|"  /usr/sbin/grub-mkconfig

   The ``sed`` workaround for ``grub-mkconfig`` needs to be applied
   for every GRUB update, as the update will overwrite the changes.

#. Install GRUB::

      mkdir -p /boot/efi/alpine/grub-bootdir/i386-pc/
      mkdir -p /boot/efi/alpine/grub-bootdir/x86_64-efi/
      for i in ${DISK}; do
       grub-install --target=i386-pc --boot-directory \
           /boot/efi/alpine/grub-bootdir/i386-pc/  "${i}"
      done
      grub-install --target x86_64-efi --boot-directory \
        /boot/efi/alpine/grub-bootdir/x86_64-efi/ --efi-directory \
        /boot/efi --bootloader-id alpine --removable
      if test -d /sys/firmware/efi/efivars/; then
        apk add efibootmgr
        grub-install --target x86_64-efi --boot-directory \
          /boot/efi/alpine/grub-bootdir/x86_64-efi/ --efi-directory \
          /boot/efi --bootloader-id alpine
      fi

#. Generate GRUB menu::

     mkdir -p /boot/grub
     grub-mkconfig -o /boot/grub/grub.cfg
     cp /boot/grub/grub.cfg \
      /boot/efi/alpine/grub-bootdir/x86_64-efi/grub/grub.cfg
     cp /boot/grub/grub.cfg \
      /boot/efi/alpine/grub-bootdir/i386-pc/grub/grub.cfg

   .. ifconfig:: zfs_root_test

      ::

         find /boot/efis/ -name "grub.cfg" -print0 \
         | xargs -t -0I '{}' grub-script-check -v '{}'

#. For both legacy and EFI booting: mirror ESP content::

    espdir=$(mktemp -d)
    find /boot/efi/ -maxdepth 1 -mindepth 1 -type d -print0 \
    | xargs -t -0I '{}' cp -r '{}' "${espdir}"
    find "${espdir}" -maxdepth 1 -mindepth 1 -type d -print0 \
    | xargs -t -0I '{}' sh -vxc "find /boot/efis/ -maxdepth 1 -mindepth 1 -type d -print0 | xargs -t -0I '[]' cp -r '{}' '[]'"

   .. ifconfig:: zfs_root_test

     ::

      ##################################################
      #
      #
      #         MAINTENANCE SCRIPT ENTRY POINT
      #                 DO NOT TOUCH
      #
      #
      #################################################

#. Exit chroot

   .. code-block:: sh

     exit

   .. ifconfig:: zfs_root_test

      # nested chroot ends here
      ZFS_ROOT_NESTED_CHROOT

   .. ifconfig:: zfs_root_test

    ::

     # list contents of boot dir to confirm
     # that the mirroring succeeded
     find "${MNT}"/boot/efis/ -type d > list_of_efi_dirs
     for i in ${DISK}; do
       if ! grep "${i##*/}-part1/efi\|${i##*/}-part1/EFI" list_of_efi_dirs; then
          echo "disk ${i} not found in efi system partition, installation error";
          cat list_of_efi_dirs
          exit 1
       fi
     done

#. Unmount filesystems and create initial system snapshot
   You can later create a boot environment from this snapshot.
   See `Root on ZFS maintenance page <../zfs_root_maintenance.html>`__.
   ::

    umount -Rl "${MNT}"
    zfs snapshot -r rpool@initial-installation
    zfs snapshot -r bpool@initial-installation
    zpool export -a

#. Reboot

   .. code-block:: sh

     reboot

   .. ifconfig:: zfs_root_test

     # chroot ends here
     ZFS_ROOT_GUIDE_TEST