diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6d36d5c309..d4fe0a0623 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -109,8 +109,8 @@ jobs: # # ...to: # - # toolchain: $ {{ ./cargo.sh --version matrix.toolchain }} # hypothetical syntax - ZC_TOOLCHAIN="$(./cargo.sh --version ${{ matrix.toolchain }})" + # toolchain: $ {{ cargo run -p cargos --version matrix.toolchain }} # hypothetical syntax + ZC_TOOLCHAIN="$(cargo run -p cargos --version ${{ matrix.toolchain }})" echo "Found that the '${{ matrix.toolchain }}' toolchain is $ZC_TOOLCHAIN" | tee -a $GITHUB_STEP_SUMMARY echo "ZC_TOOLCHAIN=$ZC_TOOLCHAIN" >> $GITHUB_ENV @@ -158,10 +158,10 @@ jobs: key: "${{ matrix.target }}" - name: Check tests - run: ./cargo.sh +${{ matrix.toolchain }} check --tests --package ${{ matrix.crate }} --target ${{ matrix.target }} ${{ matrix.features }} --verbose + run: cargo run -p cargos +${{ matrix.toolchain }} check --tests --package ${{ matrix.crate }} --target ${{ matrix.target }} ${{ matrix.features }} --verbose - name: Build - run: ./cargo.sh +${{ matrix.toolchain }} build --package ${{ matrix.crate }} --target ${{ matrix.target }} ${{ matrix.features }} --verbose + run: cargo run -p cargos +${{ matrix.toolchain }} build --package ${{ matrix.crate }} --target ${{ matrix.target }} ${{ matrix.features }} --verbose # When building tests for the i686 target, we need certain libraries which # are not installed by default; `gcc-multilib` includes these libraries. @@ -180,7 +180,7 @@ jobs: - name: Run tests run: | - ./cargo.sh +${{ matrix.toolchain }} test \ + cargo run -p cargos +${{ matrix.toolchain }} test \ --package ${{ matrix.crate }} \ --target ${{ matrix.target }} \ ${{ matrix.features }} \ @@ -206,7 +206,7 @@ jobs: # # TODO(#560), TODO(#187): Once we migrate to the ui-test crate, we # likely won't have to special-case the UI tests like this. - RUSTFLAGS="$RUSTFLAGS -Wwarnings" ./cargo.sh +${{ matrix.toolchain }} test \ + RUSTFLAGS="$RUSTFLAGS -Wwarnings" cargo run -p cargos +${{ matrix.toolchain }} test \ --package ${{ matrix.crate }} \ --target ${{ matrix.target }} \ ${{ matrix.features }} \ @@ -242,7 +242,7 @@ jobs: # Run under both the stacked borrows model (default) and under the tree # borrows model to ensure we're compliant with both. for EXTRA_FLAGS in "" "-Zmiri-tree-borrows"; do - MIRIFLAGS="$MIRIFLAGS $EXTRA_FLAGS" ./cargo.sh +${{ matrix.toolchain }} \ + MIRIFLAGS="$MIRIFLAGS $EXTRA_FLAGS" cargo run -p cargos +${{ matrix.toolchain }} \ miri test \ --package ${{ matrix.crate }} \ --target ${{ matrix.target }} \ @@ -256,7 +256,7 @@ jobs: if: matrix.toolchain == 'nightly' && matrix.target != 'riscv64gc-unknown-linux-gnu' && matrix.target != 'wasm32-wasi' - name: Clippy check - run: ./cargo.sh +${{ matrix.toolchain }} clippy --package ${{ matrix.crate }} --target ${{ matrix.target }} ${{ matrix.features }} --tests --verbose + run: cargo run -p cargos +${{ matrix.toolchain }} clippy --package ${{ matrix.crate }} --target ${{ matrix.target }} ${{ matrix.features }} --tests --verbose # Clippy improves the accuracy of lints over time, and fixes bugs. Only # running Clippy on nightly allows us to avoid having to write code which # is compatible with older versions of Clippy, which sometimes requires @@ -275,7 +275,7 @@ jobs: METADATA_DOCS_RS_RUSTDOC_ARGS="$(cargo metadata --format-version 1 | \ jq -r ".packages[] | select(.name == \"zerocopy\").metadata.docs.rs.\"rustdoc-args\".[]" | tr '\n' ' ')" export RUSTDOCFLAGS="${{ matrix.toolchain == 'nightly' && '-Z unstable-options --document-hidden-items' || '' }} $RUSTDOCFLAGS $METADATA_DOCS_RS_RUSTDOC_ARGS" - ./cargo.sh +${{ matrix.toolchain }} doc --document-private-items --package ${{ matrix.crate }} ${{ matrix.features }} + cargo run -p cargos +${{ matrix.toolchain }} doc --document-private-items --package ${{ matrix.crate }} ${{ matrix.features }} # Check semver compatibility with the most recently-published version on # crates.io. We do this in the matrix rather than in its own job so that it diff --git a/Cargo.toml b/Cargo.toml index 53bb46ef87..f8887940b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,8 +13,13 @@ [workspace] members = [ "zerocopy-derive", + "tools/cargos", "tools/generate-readme", ] +default-members = [ + ".", + "zerocopy-derive", +] [package] edition = "2018" diff --git a/INTERNAL.md b/INTERNAL.md index 4e7f440732..aebf65a99f 100644 --- a/INTERNAL.md +++ b/INTERNAL.md @@ -29,7 +29,7 @@ Updating the versions pinned in CI may cause the UI tests to break. In order to fix UI tests after a version update, run: ``` -$ TRYBUILD=overwrite ./cargo.sh +all test +$ TRYBUILD=overwrite cargo run -p cargos +all test ``` ## Crate versions diff --git a/cargo.sh b/cargo.sh deleted file mode 100755 index 424fd94bbc..0000000000 --- a/cargo.sh +++ /dev/null @@ -1,135 +0,0 @@ -#!/bin/bash -# -# Copyright 2023 The Fuchsia Authors -# -# Licensed under a BSD-style license , Apache License, Version 2.0 -# , or the MIT -# license , at your option. -# This file may not be copied, modified, or distributed except according to -# those terms. - -# This script is a thin wrapper around Cargo that provides human-friendly -# toolchain names which are automatically translated to the toolchain versions -# we have pinned in CI. -# -# cargo.sh --version # looks up the version for the named toolchain -# cargo.sh + [...] # runs cargo commands with the named toolchain -# cargo.sh +all [...] # runs cargo commands with each toolchain -# -# The meta-toolchain "all" instructs this script to run the provided command -# once for each "major" toolchain (msrv, stable, nightly). This does not include -# any toolchain which is listed in the `package.metadata.build-rs` Cargo.toml -# section. -# -# A common task that is especially annoying to perform by hand is to update -# trybuild's stderr files. Using this script: -# -# TRYBUILD=overwrite ./cargo.sh +all test --workspace - -set -eo pipefail - -function print-usage-and-exit { - echo "Usage:" >&2 - echo " $0 --version " >&2 - echo " $0 + [...]" >&2 - echo " $0 +all [...]" >&2 - exit 1 -} - -[[ $# -gt 0 ]] || print-usage-and-exit - -function pkg-meta { - # NOTE(#547): We set `CARGO_TARGET_DIR` here because `cargo metadata` - # sometimes causes the `cargo-metadata` crate to be rebuilt from source using - # the default toolchain. This has the effect of clobbering any existing build - # artifacts from whatever toolchain the user has specified (e.g., `+nightly`), - # causing the subsequent `cargo` invocation to rebuild unnecessarily. By - # specifying a separate build directory here, we ensure that this never - # clobbers the build artifacts used by the later `cargo` invocation. - # - # In CI, make sure to use the default stable toolchain. If we're testing on - # our MSRV, then we also have our MSRV toolchain installed. As of this - # writing, our MSRV is low enough that the correspoding Rust toolchain's Cargo - # doesn't know about the `rust-version` field, and so if we were to use Cargo - # with that toolchain, `pkg-meta` would return `null` when asked to retrieve - # the `rust-version` field. This also requires `RUSTFLAGS=''` to override any - # unstable `RUSTFLAGS` set by the caller. - RUSTFLAGS='' CARGO_TARGET_DIR=target/cargo-sh cargo +stable metadata --format-version 1 | jq -r ".packages[] | select(.name == \"zerocopy\").$1" -} - -function lookup-version { - VERSION="$1" - case "$VERSION" in - msrv) - pkg-meta rust_version - ;; - stable) - pkg-meta 'metadata.ci."pinned-stable"' - ;; - nightly) - pkg-meta 'metadata.ci."pinned-nightly"' - ;; - *) - TOOLCHAIN=$(pkg-meta "metadata.\"build-rs\".\"${VERSION}\"") - if [ "$TOOLCHAIN" != "null" ]; then - echo "$TOOLCHAIN" - else - echo "Unrecognized toolchain name: '$VERSION' (options are 'msrv', 'stable', 'nightly', and any value in Cargo.toml's 'metadata.build-rs' table)" >&2 - return 1 - fi - ;; - esac -} - -function get-rustflags { - [ "$1" == nightly ] && echo "--cfg __INTERNAL_USE_ONLY_NIGHLTY_FEATURES_IN_TESTS" -} - -function prompt { - PROMPT="$1" - YES="$2" - while true; do - read -p "$PROMPT " yn - case "$yn" in - [Yy]) $YES; return $?; ;; - [Nn]) return 1; ;; - *) break; ;; - esac - done -} - -case "$1" in - # cargo.sh --version - --version) - [[ $# -eq 2 ]] || print-usage-and-exit - lookup-version "$2" - ;; - # cargo.sh +all [...] - +all) - echo "[cargo.sh] warning: running the same command for each toolchain (msrv, stable, nightly)" >&2 - for toolchain in msrv stable nightly; do - echo "[cargo.sh] running with toolchain: $toolchain" >&2 - $0 "+$toolchain" ${@:2} - done - exit 0 - ;; - # cargo.sh + [...] - +*) - TOOLCHAIN="$(lookup-version ${1:1})" - - cargo "+$TOOLCHAIN" version &>/dev/null && \ - rustup "+$TOOLCHAIN" component list | grep '^rust-src (installed)$' >/dev/null || { - echo "[cargo.sh] missing either toolchain '$TOOLCHAIN' or component 'rust-src'" >&2 - # If we're running in a GitHub action, then it's better to bail than to - # hang waiting for input we're never going to get. - [ -z ${GITHUB_RUN_ID+x} ] || exit 1 - prompt "[cargo.sh] would you like to install toolchain '$TOOLCHAIN' and component 'rust-src' via 'rustup'?" \ - "rustup toolchain install $TOOLCHAIN -c rust-src" - } || exit 1 - - RUSTFLAGS="$(get-rustflags ${1:1}) $RUSTFLAGS" cargo "+$TOOLCHAIN" ${@:2} - ;; - *) - print-usage-and-exit - ;; -esac diff --git a/tools/cargos/Cargo.toml b/tools/cargos/Cargo.toml new file mode 100644 index 0000000000..8bd4cc648b --- /dev/null +++ b/tools/cargos/Cargo.toml @@ -0,0 +1,17 @@ +# Copyright 2024 The Fuchsia Authors +# +# Licensed under a BSD-style license , Apache License, Version 2.0 +# , or the MIT +# license , at your option. +# This file may not be copied, modified, or distributed except according to +# those terms. + +[package] +edition = "2021" +name = "cargos" +version = "0.0.0" +license = "BSD-2-Clause OR Apache-2.0 OR MIT" +publish = false + +[dependencies] +serde_json = "1" diff --git a/tools/cargos/src/main.rs b/tools/cargos/src/main.rs new file mode 100644 index 0000000000..cf3d95d1d9 --- /dev/null +++ b/tools/cargos/src/main.rs @@ -0,0 +1,253 @@ +// Copyright 2023 The Fuchsia Authors +// +// Licensed under a BSD-style license , Apache License, Version 2.0 +// , or the MIT +// license , at your option. +// This file may not be copied, modified, or distributed except according to +// those terms. + +// This script is a thin wrapper around Cargo that provides human-friendly +// toolchain names which are automatically translated to the toolchain versions +// we have pinned in CI. +// +// cargos --version # looks up the version for the named toolchain +// cargos + [...] # runs cargo commands with the named toolchain +// cargos +all [...] # runs cargo commands with each toolchain +// +// The meta-toolchain "all" instructs this script to run the provided command +// once for each "major" toolchain (msrv, stable, nightly). This does not +// include any toolchain which is listed in the `package.metadata.build-rs` +// Cargo.toml section. +// +// A common task that is especially annoying to perform by hand is to update +// trybuild's stderr files. Using this script: +// +// TRYBUILD=overwrite ./cargos +all test --workspace + +use std::{ + env::{args, vars}, + fmt, + io::{stdin, Read as _}, + process::{exit, Command}, +}; + +use serde_json::{Map, Value}; + +#[derive(Debug)] +enum Error { + NoArguments, + UnrecognizedArgument(String), + MissingToolchainVersion, + UnrecognizedToolchain(String), +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NoArguments => write!(f, "No arguments provided"), + Self::UnrecognizedArgument(arg) => write!(f, "Unrecognized argument: '{arg}'"), + Self::MissingToolchainVersion => write!(f, "No toolchain version specified after '--version'"), + Self::UnrecognizedToolchain(name) => write!(f, "Unrecognized toolchain name: `{name}` (options are 'msrv', 'stable', and 'nightly')") + } + } +} + +impl std::error::Error for Error {} + +struct Versions { + msrv: String, + stable: String, + nightly: String, + build_rs: Map, +} + +impl Versions { + fn get(&self, name: &str) -> Result<&str, Error> { + Ok(match name { + "msrv" => &self.msrv, + "stable" => &self.stable, + "nightly" => &self.nightly, + _ => self + .build_rs + .get(name) + .ok_or(Error::UnrecognizedToolchain(name.to_string())) + .map(|value| value.as_str().unwrap())?, + }) + } +} + +fn get_toolchain_versions() -> Versions { + // NOTE(#547): We set `CARGO_TARGET_DIR` here because `cargo metadata` + // sometimes causes the `cargo-metadata` crate to be rebuilt from source + // using the default toolchain. This has the effect of clobbering any + // existing build artifacts from whatever toolchain the user has specified + // (e.g., `+nightly`), causing the subsequent `cargo` invocation to rebuild + // unnecessarily. By specifying a separate build directory here, we ensure + // that this never clobbers the build artifacts used by the later `cargo` + // invocation. + // + // In CI, make sure to use the default stable toolchain. If we're testing on + // our MSRV, then we also have our MSRV toolchain installed. As of this + // writing, our MSRV is low enough that the correspoding Rust toolchain's + // Cargo doesn't know about the `rust-version` field, and so if we were to + // use Cargo with that toolchain, `pkg-meta` would return `null` when asked + // to retrieve the `rust-version` field. This also requires `RUSTFLAGS=''` + // to override any unstable `RUSTFLAGS` set by the caller. + let output = Command::new("rustup") + .args(["run", "stable", "cargo", "metadata", "--format-version", "1"]) + .env("CARGO_TARGET_DIR", "target/cargos") + .output() + .unwrap(); + + let json = serde_json::from_slice::(&output.stdout).unwrap(); + let packages = json.as_object().unwrap().get("packages").unwrap(); + packages + .as_array() + .unwrap() + .iter() + .filter_map(|p| { + let package = p.as_object().unwrap(); + if package.get("name").unwrap().as_str() == Some("zerocopy") { + let metadata = package.get("metadata").unwrap().as_object().unwrap(); + let ci = metadata.get("ci").unwrap().as_object().unwrap(); + Some(Versions { + msrv: package.get("rust_version").unwrap().as_str().unwrap().to_string(), + stable: ci.get("pinned-stable").unwrap().as_str().unwrap().to_string(), + nightly: ci.get("pinned-nightly").unwrap().as_str().unwrap().to_string(), + build_rs: metadata.get("build-rs").unwrap().as_object().unwrap().clone(), + }) + } else { + None + } + }) + .next() + .unwrap() +} + +fn is_toolchain_installed(versions: &Versions, name: &str) -> Result { + let output = Command::new("rustup") + .args(["run", versions.get(name)?, "cargo", "version"]) + .output() + .unwrap(); + + println!("version: {}", versions.get(name).unwrap()); + println!("stdout: {}", std::str::from_utf8(&output.stdout).unwrap()); + println!("stderr: {}", std::str::from_utf8(&output.stderr).unwrap()); + + if output.status.success() { + let output = Command::new("rustup").args(["component", "list"]).output().unwrap(); + + let stdout = String::from_utf8(output.stdout).unwrap(); + Ok(stdout.contains("rust-src (installed)")) + } else { + Ok(false) + } +} + +fn install_toolchain_or_exit(versions: &Versions, name: &str) -> Result<(), Error> { + eprintln!("[cargos] missing either toolchain '{name}' or component 'rust-src'"); + if vars().any(|v| v.0 == "GITHUB_RUN_ID") { + // If we're running in a GitHub action, then it's better to bail than to + // hang waiting for input we're never going to get. + exit(1); + } + + eprintln!("[cargos] would you like to install toolchain '{name}' and component 'rust-src' via 'rustup'? yn"); + + loop { + let mut input = [0]; + stdin().read(&mut input).unwrap(); + match input[0] as char { + 'y' | 'Y' => break, + 'n' | 'N' => exit(1), + _ => (), + } + } + + let version = versions.get(name)?; + Command::new("rustup") + .args(["toolchain", "install", &version, "-c", "rust-src"]) + .status() + .unwrap(); + + Ok(()) +} + +fn get_rustflags(name: &str) -> &'static str { + if name == "nightly" { + "--cfg __INTERNAL_USE_ONLY_NIGHTLY_FEATURES_IN_TESTS" + } else { + "" + } +} + +fn delegate_cargo() -> Result<(), Error> { + let mut args = args(); + let this = args.next().unwrap(); + let versions = get_toolchain_versions(); + + match args.next().as_deref() { + None => Err(Error::NoArguments), + Some("--version") => { + let name = args.next().ok_or(Error::MissingToolchainVersion)?; + println!("{}", versions.get(&name)?); + Ok(()) + } + Some("+all") => { + eprintln!("[cargos] warning: running the same command for each toolchain (msrv, stable, nightly)"); + let args = args.collect::>(); + + for toolchain in ["msrv", "stable", "nightly"] { + eprintln!("[cargos] running with toolchain: {toolchain}"); + Command::new(this.clone()) + .arg(format!("+{toolchain}")) + .args(args.clone()) + .status() + .unwrap(); + } + Ok(()) + } + Some(arg) => { + if let Some(name) = arg.strip_prefix('+') { + let version = versions.get(&name)?; + + if !is_toolchain_installed(&versions, name)? { + install_toolchain_or_exit(&versions, name)?; + } + + let rustflags = vars() + .filter_map(|(k, v)| if k == "RUSTFLAGS" { Some(v) } else { None }) + .next() + .unwrap_or_default(); + + Command::new("rustup") + .args(["run", &version, "cargo"]) + .args(args) + .env("RUSTFLAGS", format!("{}{}", get_rustflags(name), rustflags)) + .status() + .unwrap(); + + Ok(()) + } else { + Err(Error::UnrecognizedArgument(arg.to_string())) + } + } + } +} + +fn print_usage() { + let name = args().next().unwrap(); + + eprintln!("Usage:"); + eprintln!(" {} --version ", name); + eprintln!(" {} + [...]", name); + eprintln!(" {} +all [...]", name); +} + +fn main() { + if let Err(e) = delegate_cargo() { + eprintln!("Error: {e}"); + print_usage(); + exit(1); + } +}