diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 999bea5b77..87db746110 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -162,24 +162,6 @@ jobs:
           -- \
           --skip ui
 
-        if [ "${{ matrix.features }}" != "--no-default-features" -a "${{ matrix.features }}" != "" ]; then
-          # Run UI tests separately, treating warnings as warnings (rather than
-          # as errors, as we do everywhere else in our CI tests). This allows
-          # our UI tests to more accurately reflect what users will see, and
-          # also ensures that we're not spuriously relying on warnings being
-          # errors to ensure compilation failure (if we were, then our code
-          # would be unsound whenever -Dwarnings is not enabled).
-          #
-          # TODO(#560), TODO(#187): Once we migrate to the ui-test crate, we
-          # likely won't have to special-case the UI tests like this.
-          RUSTFLAGS="$RUSTFLAGS -Wwarnings" ./cargo.sh +${{ matrix.toolchain }} test \
-            --package ${{ matrix.crate }} \
-            --target ${{ matrix.target }} \
-            ${{ matrix.features }} \
-            --verbose \
-            ui
-        fi
-
       # Only run tests when targetting x86 (32- or 64-bit) - we're executing on
       # x86_64, so we can't run tests for any non-x86 target.
       #
@@ -187,6 +169,37 @@ jobs:
       # Run compile tests when building for other targets.
       if: contains(matrix.target, 'x86_64') || contains(matrix.target, 'i686')
 
+    - name: Run UI tests
+      run: |
+        # Run UI tests separately, treating warnings as warnings (rather than
+        # as errors, as we do everywhere else in our CI tests). This allows
+        # our UI tests to more accurately reflect what users will see, and
+        # also ensures that we're not spuriously relying on warnings being
+        # errors to ensure compilation failure (if we were, then our code
+        # would be unsound whenever -Dwarnings is not enabled).
+        #
+        # TODO(#560), TODO(#187): Once we migrate to the ui-test crate, we
+        # likely won't have to special-case the UI tests like this.
+        RUSTFLAGS="$RUSTFLAGS -Wwarnings" ./cargo.sh +${{ matrix.toolchain }} test \
+          --package ${{ matrix.crate }} \
+          --target ${{ matrix.target }} \
+          ${{ matrix.features }} \
+          --verbose \
+          ui
+
+      # Only run tests when targetting x86 (32- or 64-bit) - we're executing on
+      # x86_64, so we can't run tests for any non-x86 target.
+      #
+      # TODO(https://github.com/dtolnay/trybuild/issues/184#issuecomment-1269097742):
+      # Run compile tests when building for other targets.
+      #
+      # Only run UI tests for zerocopy-derive, or for zerocopy with the derive
+      # feature.
+      if: |
+        (contains(matrix.target, 'x86_64') || contains(matrix.target, 'i686')) &&
+          (matrix.crate == 'zerocopy-derive' ||
+            (matrix.features != '' && matrix.features != '--no-default-features'))
+
     - name: Run tests under Miri
       run: |
         # Work around https://github.com/rust-lang/miri/issues/3125
diff --git a/zerocopy-derive/tests/ui-msrv/struct.stderr b/zerocopy-derive/tests/ui-msrv/struct.stderr
index 5132d979e0..763342667c 100644
--- a/zerocopy-derive/tests/ui-msrv/struct.stderr
+++ b/zerocopy-derive/tests/ui-msrv/struct.stderr
@@ -1,27 +1,3 @@
-error: cannot derive TryFromBytes with repr(packed)
-  --> tests/ui-msrv/struct.rs:70:8
-   |
-70 | #[repr(packed)]
-   |        ^^^^^^
-
-error: cannot derive TryFromBytes with repr(packed)
-  --> tests/ui-msrv/struct.rs:76:8
-   |
-76 | #[repr(packed(1))]
-   |        ^^^^^^^^^
-
-error: cannot derive TryFromBytes with repr(packed)
-  --> tests/ui-msrv/struct.rs:82:11
-   |
-82 | #[repr(C, packed)]
-   |           ^^^^^^
-
-error: cannot derive TryFromBytes with repr(packed)
-  --> tests/ui-msrv/struct.rs:88:11
-   |
-88 | #[repr(C, packed(1))]
-   |           ^^^^^^^^^
-
 error: unsupported on generic structs that are not repr(transparent) or repr(packed)
   --> tests/ui-msrv/struct.rs:97:10
    |
diff --git a/zerocopy-derive/tests/ui-nightly/struct.stderr b/zerocopy-derive/tests/ui-nightly/struct.stderr
index 76f72b8c6f..d59dc3ac42 100644
--- a/zerocopy-derive/tests/ui-nightly/struct.stderr
+++ b/zerocopy-derive/tests/ui-nightly/struct.stderr
@@ -1,27 +1,3 @@
-error: cannot derive TryFromBytes with repr(packed)
-  --> tests/ui-nightly/struct.rs:70:8
-   |
-70 | #[repr(packed)]
-   |        ^^^^^^
-
-error: cannot derive TryFromBytes with repr(packed)
-  --> tests/ui-nightly/struct.rs:76:8
-   |
-76 | #[repr(packed(1))]
-   |        ^^^^^^^^^
-
-error: cannot derive TryFromBytes with repr(packed)
-  --> tests/ui-nightly/struct.rs:82:11
-   |
-82 | #[repr(C, packed)]
-   |           ^^^^^^
-
-error: cannot derive TryFromBytes with repr(packed)
-  --> tests/ui-nightly/struct.rs:88:11
-   |
-88 | #[repr(C, packed(1))]
-   |           ^^^^^^^^^
-
 error: unsupported on generic structs that are not repr(transparent) or repr(packed)
   --> tests/ui-nightly/struct.rs:97:10
    |
diff --git a/zerocopy-derive/tests/ui-stable/struct.stderr b/zerocopy-derive/tests/ui-stable/struct.stderr
index 6e1fdbf423..19a2ff6206 100644
--- a/zerocopy-derive/tests/ui-stable/struct.stderr
+++ b/zerocopy-derive/tests/ui-stable/struct.stderr
@@ -1,27 +1,3 @@
-error: cannot derive TryFromBytes with repr(packed)
-  --> tests/ui-stable/struct.rs:70:8
-   |
-70 | #[repr(packed)]
-   |        ^^^^^^
-
-error: cannot derive TryFromBytes with repr(packed)
-  --> tests/ui-stable/struct.rs:76:8
-   |
-76 | #[repr(packed(1))]
-   |        ^^^^^^^^^
-
-error: cannot derive TryFromBytes with repr(packed)
-  --> tests/ui-stable/struct.rs:82:11
-   |
-82 | #[repr(C, packed)]
-   |           ^^^^^^
-
-error: cannot derive TryFromBytes with repr(packed)
-  --> tests/ui-stable/struct.rs:88:11
-   |
-88 | #[repr(C, packed(1))]
-   |           ^^^^^^^^^
-
 error: unsupported on generic structs that are not repr(transparent) or repr(packed)
   --> tests/ui-stable/struct.rs:97:10
    |