feat: use zstd compression in place of xz

Initramfs and kernel are compressed with zstd.

Extensions are compressed with zstd for Talos 1.8+.

Signed-off-by: Andrey Smirnov <[email protected]>

.drone.jsonnet

@@ -368,17 +368,10 @@ local ExtensionsStep(with_e2e=true) =
 
   // generates the extension list patch manifest
   local extensions_patch_manifest = Step(
-    'extensions-patch-manifest',
-    with_make=false,
-    environment=creds_env_vars,
+    'installer-with-extensions',
+    environment={ IMAGE_REGISTRY: local_registry },
     depends_on=[
       extensions_artifacts,
-    ],
-    extra_commands=[
-      // create a patch file to pass to the downstream build
-      // ignore nvidia extensions, testing nvidia extensions needs a machine with nvidia graphics card
-      'jq -R < _out/extensions-metadata | jq -s -f hack/test/extensions/extension-patch-filter.jq | yq eval ".[] | split_doc" -P > _out/extensions-patch.yaml',
-      'cat _out/extensions-patch.yaml',
     ]
   );
 
@@ -1056,35 +1049,35 @@ local release = {
     files: [
       '_out/akamai-amd64.raw.gz',
       '_out/akamai-arm64.raw.gz',
-      '_out/aws-amd64.raw.xz',
-      '_out/aws-arm64.raw.xz',
-      '_out/azure-amd64.vhd.xz',
-      '_out/azure-arm64.vhd.xz',
+      '_out/aws-amd64.raw.zst',
+      '_out/aws-arm64.raw.zst',
+      '_out/azure-amd64.vhd.zst',
+      '_out/azure-arm64.vhd.zst',
       '_out/cloud-images.json',
       '_out/digital-ocean-amd64.raw.gz',
       '_out/digital-ocean-arm64.raw.gz',
-      '_out/exoscale-amd64.qcow2.xz',
-      '_out/exoscale-arm64.qcow2.xz',
+      '_out/exoscale-amd64.qcow2.zst',
+      '_out/exoscale-arm64.qcow2.zst',
       '_out/gcp-amd64.raw.tar.gz',
       '_out/gcp-arm64.raw.tar.gz',
-      '_out/hcloud-amd64.raw.xz',
-      '_out/hcloud-arm64.raw.xz',
-      '_out/initramfs-amd64.xz',
-      '_out/initramfs-arm64.xz',
+      '_out/hcloud-amd64.raw.zst',
+      '_out/hcloud-arm64.raw.zst',
+      '_out/initramfs-amd64.zst',
+      '_out/initramfs-arm64.zst',
       '_out/metal-amd64.iso',
       '_out/metal-arm64.iso',
-      '_out/metal-amd64.raw.xz',
-      '_out/metal-arm64.raw.xz',
-      '_out/nocloud-amd64.raw.xz',
-      '_out/nocloud-arm64.raw.xz',
-      '_out/opennebula-amd64.raw.xz',
-      '_out/opennebula-arm64.raw.xz',
-      '_out/openstack-amd64.raw.xz',
-      '_out/openstack-arm64.raw.xz',
-      '_out/oracle-amd64.qcow2.xz',
-      '_out/oracle-arm64.qcow2.xz',
-      '_out/scaleway-amd64.raw.xz',
-      '_out/scaleway-arm64.raw.xz',
+      '_out/metal-amd64.raw.zst',
+      '_out/metal-arm64.raw.zst',
+      '_out/nocloud-amd64.raw.zst',
+      '_out/nocloud-arm64.raw.zst',
+      '_out/opennebula-amd64.raw.zst',
+      '_out/opennebula-arm64.raw.zst',
+      '_out/openstack-amd64.raw.zst',
+      '_out/openstack-arm64.raw.zst',
+      '_out/oracle-amd64.qcow2.zst',
+      '_out/oracle-arm64.qcow2.zst',
+      '_out/scaleway-amd64.raw.zst',
+      '_out/scaleway-arm64.raw.zst',
       '_out/sd-boot-amd64.efi',
       '_out/sd-boot-arm64.efi',
       '_out/sd-stub-amd64.efi',
@@ -1099,14 +1092,14 @@ local release = {
       '_out/talosctl-linux-arm64',
       '_out/talosctl-linux-armv7',
       '_out/talosctl-windows-amd64.exe',
-      '_out/upcloud-amd64.raw.xz',
-      '_out/upcloud-arm64.raw.xz',
+      '_out/upcloud-amd64.raw.zst',
+      '_out/upcloud-arm64.raw.zst',
       '_out/vmware-amd64.ova',
       '_out/vmware-arm64.ova',
       '_out/vmlinuz-amd64',
       '_out/vmlinuz-arm64',
-      '_out/vultr-amd64.raw.xz',
-      '_out/vultr-arm64.raw.xz',
+      '_out/vultr-amd64.raw.zst',
+      '_out/vultr-arm64.raw.zst',
     ],
     checksum: ['sha256', 'sha512'],
   },

Dockerfile

@@ -688,14 +688,16 @@ RUN find /rootfs -print0 \
     | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}"
 
 FROM rootfs-base-arm64 AS rootfs-squashfs-arm64
+ARG ZSTD_COMPRESSION_LEVEL
 RUN find /rootfs -print0 \
     | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}"
-RUN mksquashfs /rootfs /rootfs.sqsh -all-root -noappend -comp xz -Xdict-size 100% -no-progress
+RUN mksquashfs /rootfs /rootfs.sqsh -all-root -noappend -comp zstd -Xcompression-level ${ZSTD_COMPRESSION_LEVEL} -no-progress
 
 FROM rootfs-base-amd64 AS rootfs-squashfs-amd64
+ARG ZSTD_COMPRESSION_LEVEL
 RUN find /rootfs -print0 \
     | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}"
-RUN mksquashfs /rootfs /rootfs.sqsh -all-root -noappend -comp xz -Xdict-size 100% -no-progress
+RUN mksquashfs /rootfs /rootfs.sqsh -all-root -noappend -comp zstd -Xcompression-level ${ZSTD_COMPRESSION_LEVEL} -no-progress
 
 FROM scratch AS squashfs-arm64
 COPY --from=rootfs-squashfs-arm64 /rootfs.sqsh /
@@ -710,6 +712,7 @@ COPY --from=rootfs-base /rootfs /
 
 FROM build AS initramfs-archive-arm64
 WORKDIR /initramfs
+ARG ZSTD_COMPRESSION_LEVEL
 COPY --from=squashfs-arm64 /rootfs.sqsh .
 COPY --from=init-build-arm64 /init .
 RUN find . -print0 \
@@ -718,11 +721,12 @@ RUN set -o pipefail \
     && find . 2>/dev/null \
     | LC_ALL=c sort \
     | cpio --reproducible -H newc -o \
-    | xz -v -C crc32 -0 -e -T 0 -z \
+    | zstd -c -T0 -${ZSTD_COMPRESSION_LEVEL} \
     > /initramfs.xz
 
 FROM build AS initramfs-archive-amd64
 WORKDIR /initramfs
+ARG ZSTD_COMPRESSION_LEVEL
 COPY --from=squashfs-amd64 /rootfs.sqsh .
 COPY --from=init-build-amd64 /init .
 RUN find . -print0 \
@@ -731,7 +735,7 @@ RUN set -o pipefail \
     && find . 2>/dev/null \
     | LC_ALL=c sort \
     | cpio --reproducible -H newc -o \
-    | xz -v -C crc32 -0 -e -T 0 -z \
+    | zstd -c -T0 -${ZSTD_COMPRESSION_LEVEL} \
     > /initramfs.xz
 
 FROM initramfs-archive-${TARGETARCH} AS initramfs-archive
@@ -800,7 +804,8 @@ RUN apk add --no-cache --update --no-scripts \
     util-linux \
     xfsprogs \
     xorriso \
-    xz
+    xz \
+    zstd
 ARG TARGETARCH
 ENV TARGETARCH ${TARGETARCH}
 COPY --from=installer-build /installer /bin/installer

Makefile

@@ -13,12 +13,13 @@ DOCKER_LOGIN_ENABLED ?= true
 NAME = Talos
 
 CLOUD_IMAGES_EXTRA_ARGS ?= ""
+ZSTD_COMPRESSION_LEVEL ?= 18
 
 ARTIFACTS := _out
 TOOLS ?= ghcr.io/siderolabs/tools:v1.8.0-alpha.0
 
 PKGS_PREFIX ?= ghcr.io/siderolabs
-PKGS ?= v1.8.0-alpha.0-7-g718a7da
+PKGS ?= v1.8.0-alpha.0-8-gca6249b
 EXTRAS ?= v1.8.0-alpha.0
 
 PKG_FHS ?= $(PKGS_PREFIX)/fhs:$(PKGS)
@@ -202,6 +203,7 @@ COMMON_ARGS += --build-arg=PKG_RASPBERYPI_FIRMWARE=$(PKG_RASPBERYPI_FIRMWARE)
 COMMON_ARGS += --build-arg=PKG_KERNEL=$(PKG_KERNEL)
 COMMON_ARGS += --build-arg=PKG_TALOSCTL_CNI_BUNDLE_INSTALL=$(PKG_TALOSCTL_CNI_BUNDLE_INSTALL)
 COMMON_ARGS += --build-arg=ABBREV_TAG=$(ABBREV_TAG)
+COMMON_ARGS += --build-arg=ZSTD_COMPRESSION_LEVEL=$(ZSTD_COMPRESSION_LEVEL)
 
 CI_ARGS ?=
 
@@ -524,6 +526,12 @@ provision-tests-track-%:
 		REGISTRY=$(IMAGE_REGISTRY) \
 		ARTIFACTS=$(ARTIFACTS)
 
+installer-with-extensions: $(ARTIFACTS)/extensions-metadata
+	$(MAKE) image-installer \
+		IMAGER_ARGS="--base-installer-image=$(REGISTRY_AND_USERNAME)/installer:$(IMAGE_TAG) $(shell cat $(ARTIFACTS)/extensions-metadata | grep -vE 'tailscale|xen-guest-agent|nvidia' | xargs -n 1 echo --system-extension-image)"
+	crane push $(ARTIFACTS)/installer-amd64.tar $(REGISTRY_AND_USERNAME)/installer:$(IMAGE_TAG)-amd64-extensions
+	echo -n "$(REGISTRY_AND_USERNAME)/installer:$(IMAGE_TAG)-amd64-extensions" | jq -Rs -f hack/test/extensions/extension-patch-filter.jq | yq eval ".[] | split_doc" -P > $(ARTIFACTS)/extensions-patch.yaml
+
 # Assets for releases
 
 .PHONY: $(ARTIFACTS)/$(TALOS_RELEASE)

cmd/installer/pkg/install/extensions.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/siderolabs/talos/pkg/imager/extensions"
 	"github.com/siderolabs/talos/pkg/machinery/constants"
+	"github.com/siderolabs/talos/pkg/machinery/imager/quirks"
 )
 
 func (i *Installer) installExtensions() error {
@@ -19,6 +20,7 @@ func (i *Installer) installExtensions() error {
 		Arch:              i.options.Arch,
 		ExtensionTreePath: constants.SystemExtensionsPath,
 		Printf:            log.Printf,
+		Quirks:            quirks.New(i.options.Version),
 	}
 
 	return builder.Build()

go.mod

@@ -92,6 +92,7 @@ require (
 	github.com/jeromer/syslogparser v1.1.0
 	github.com/jsimonetti/rtnetlink v1.4.1
 	github.com/jxskiss/base62 v1.1.0
+	github.com/klauspost/compress v1.17.7
 	github.com/klauspost/cpuid/v2 v2.2.7
 	github.com/linode/go-metadata v0.2.0
 	github.com/martinlindhe/base36 v1.1.1
@@ -267,7 +268,6 @@ require (
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/josharian/native v1.1.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.17.7 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect