From 07968a001d9e7dfd2417dfb57d3227e395691418 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Mon, 29 Mar 2021 10:45:09 +0200 Subject: [PATCH 01/52] Merge commit '0969bc6dde001e01e7e1f58c8ccd7750f8a49ae1' into sync_cg_clif-2021-03-29 --- .github/workflows/bootstrap_rustc.yml | 44 ------------- .github/workflows/main.yml | 21 ++++++- .github/workflows/rustc.yml | 82 ++++++++++++++++++++++++ .vscode/settings.json | 2 +- Cargo.lock | 50 +++++++++------ Cargo.toml | 6 +- Readme.md | 76 +++++++---------------- build.sh | 7 +++ build_sysroot/Cargo.lock | 16 ++--- build_sysroot/build_sysroot.sh | 2 +- docs/env_vars.md | 5 +- docs/usage.md | 66 ++++++++++++++++++++ example/mini_core.rs | 1 + example/mini_core_hello_world.rs | 9 +-- prepare.sh | 1 - rust-toolchain | 4 +- scripts/cargo.sh | 2 +- scripts/config.sh | 14 +---- scripts/rustup.sh | 2 +- scripts/setup_rust_fork.sh | 68 ++++++++++++++++++++ scripts/test_bootstrap.sh | 62 +------------------ scripts/test_rustc_tests.sh | 87 ++++++++++++++++++++++++++ scripts/tests.sh | 45 ++++++++++---- src/abi/comments.rs | 27 +++++--- src/abi/mod.rs | 24 +++----- src/abi/pass_mode.rs | 7 +-- src/abi/returning.rs | 6 +- src/allocator.rs | 5 +- src/base.rs | 63 +++++++++---------- src/codegen_i128.rs | 89 +++++++++++++++++++++------ src/common.rs | 3 +- src/compiler_builtins.rs | 41 ++++++++++++ src/constant.rs | 29 +++++---- src/debuginfo/line_info.rs | 4 +- src/driver/aot.rs | 22 +++++-- src/driver/jit.rs | 89 ++++++++++++++++++--------- src/driver/mod.rs | 8 ++- src/inline_asm.rs | 14 +++-- src/intrinsics/simd.rs | 2 +- src/lib.rs | 39 +++++------- src/linkage.rs | 2 + src/main_shim.rs | 9 +-- src/metadata.rs | 70 +++++++++++++-------- src/num.rs | 8 +-- src/optimize/stack2reg.rs | 89 ++++++++++++++------------- src/pointer.rs | 3 +- src/pretty_clif.rs | 24 +++++--- src/trap.rs | 3 +- src/value_and_place.rs | 19 ++++-- 49 files changed, 878 insertions(+), 493 deletions(-) delete mode 100644 .github/workflows/bootstrap_rustc.yml create mode 100644 .github/workflows/rustc.yml create mode 100644 docs/usage.md create mode 100644 scripts/setup_rust_fork.sh create mode 100755 scripts/test_rustc_tests.sh create mode 100644 src/compiler_builtins.rs diff --git a/.github/workflows/bootstrap_rustc.yml b/.github/workflows/bootstrap_rustc.yml deleted file mode 100644 index 8c94a0aa5e6eb..0000000000000 --- a/.github/workflows/bootstrap_rustc.yml +++ /dev/null @@ -1,44 +0,0 @@ -name: Bootstrap rustc using cg_clif - -on: - - push - -jobs: - bootstrap_rustc: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - - name: Cache cargo installed crates - uses: actions/cache@v2 - with: - path: ~/.cargo/bin - key: ${{ runner.os }}-cargo-installed-crates - - - name: Cache cargo registry and index - uses: actions/cache@v2 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - key: ${{ runner.os }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }} - - - name: Cache cargo target dir - uses: actions/cache@v2 - with: - path: target - key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }} - - - name: Prepare dependencies - run: | - git config --global user.email "user@example.com" - git config --global user.name "User" - ./prepare.sh - - - name: Test - run: | - # Enable backtraces for easier debugging - export RUST_BACKTRACE=1 - - ./scripts/test_bootstrap.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e6d3375fb1bab..2ac516381cf7a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,11 +7,18 @@ on: jobs: build: runs-on: ${{ matrix.os }} + timeout-minutes: 60 strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] + include: + - os: ubuntu-latest + - os: macos-latest + # cross-compile from Linux to Windows using mingw + - os: ubuntu-latest + env: + TARGET_TRIPLE: x86_64-pc-windows-gnu steps: - uses: actions/checkout@v2 @@ -36,6 +43,12 @@ jobs: path: target key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }} + - name: Install MinGW toolchain and wine + if: matrix.os == 'ubuntu-latest' && matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu' + run: | + sudo apt-get install -y gcc-mingw-w64-x86-64 wine-stable + rustup target add x86_64-pc-windows-gnu + - name: Prepare dependencies run: | git config --global user.email "user@example.com" @@ -43,6 +56,8 @@ jobs: ./prepare.sh - name: Test + env: + TARGET_TRIPLE: ${{ matrix.env.TARGET_TRIPLE }} run: | # Enable backtraces for easier debugging export RUST_BACKTRACE=1 @@ -51,12 +66,16 @@ jobs: export COMPILE_RUNS=2 export RUN_RUNS=2 + # Enable extra checks + export CG_CLIF_ENABLE_VERIFIER=1 + ./test.sh - name: Package prebuilt cg_clif run: tar cvfJ cg_clif.tar.xz build - name: Upload prebuilt cg_clif + if: matrix.env.TARGET_TRIPLE != 'x86_64-pc-windows-gnu' uses: actions/upload-artifact@v2 with: name: cg_clif-${{ runner.os }} diff --git a/.github/workflows/rustc.yml b/.github/workflows/rustc.yml new file mode 100644 index 0000000000000..e01a92598bab7 --- /dev/null +++ b/.github/workflows/rustc.yml @@ -0,0 +1,82 @@ +name: Various rustc tests + +on: + - push + +jobs: + bootstrap_rustc: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Cache cargo installed crates + uses: actions/cache@v2 + with: + path: ~/.cargo/bin + key: ${{ runner.os }}-cargo-installed-crates + + - name: Cache cargo registry and index + uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: ${{ runner.os }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }} + + - name: Cache cargo target dir + uses: actions/cache@v2 + with: + path: target + key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }} + + - name: Prepare dependencies + run: | + git config --global user.email "user@example.com" + git config --global user.name "User" + ./prepare.sh + + - name: Test + run: | + # Enable backtraces for easier debugging + export RUST_BACKTRACE=1 + + ./scripts/test_bootstrap.sh + rustc_test_suite: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + + - name: Cache cargo installed crates + uses: actions/cache@v2 + with: + path: ~/.cargo/bin + key: ${{ runner.os }}-cargo-installed-crates + + - name: Cache cargo registry and index + uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + key: ${{ runner.os }}-cargo-registry-and-index-${{ hashFiles('**/Cargo.lock') }} + + - name: Cache cargo target dir + uses: actions/cache@v2 + with: + path: target + key: ${{ runner.os }}-cargo-build-target-${{ hashFiles('rust-toolchain', '**/Cargo.lock') }} + + - name: Prepare dependencies + run: | + git config --global user.email "user@example.com" + git config --global user.name "User" + ./prepare.sh + + - name: Test + run: | + # Enable backtraces for easier debugging + export RUST_BACKTRACE=1 + + ./scripts/test_rustc_tests.sh diff --git a/.vscode/settings.json b/.vscode/settings.json index a13d5931ffa89..0cd576e160f86 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,7 +2,7 @@ // source for rustc_* is not included in the rust-src component; disable the errors about this "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate", "macro-error"], "rust-analyzer.assist.importMergeBehavior": "last", - "rust-analyzer.cargo.loadOutDirsFromCheck": true, + "rust-analyzer.cargo.runBuildScripts": true, "rust-analyzer.linkedProjects": [ "./Cargo.toml", //"./build_sysroot/sysroot_src/src/libstd/Cargo.toml", diff --git a/Cargo.lock b/Cargo.lock index 76d9f0d27ce46..3cb67032aaa24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,16 +39,16 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cranelift-bforest" -version = "0.70.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058" +version = "0.72.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-codegen" -version = "0.70.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058" +version = "0.72.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" dependencies = [ "byteorder", "cranelift-bforest", @@ -65,8 +65,8 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.70.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058" +version = "0.72.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" dependencies = [ "cranelift-codegen-shared", "cranelift-entity", @@ -74,18 +74,18 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.70.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058" +version = "0.72.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" [[package]] name = "cranelift-entity" -version = "0.70.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058" +version = "0.72.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" [[package]] name = "cranelift-frontend" -version = "0.70.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058" +version = "0.72.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" dependencies = [ "cranelift-codegen", "log", @@ -95,8 +95,8 @@ dependencies = [ [[package]] name = "cranelift-jit" -version = "0.70.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058" +version = "0.72.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" dependencies = [ "anyhow", "cranelift-codegen", @@ -113,8 +113,8 @@ dependencies = [ [[package]] name = "cranelift-module" -version = "0.70.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058" +version = "0.72.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" dependencies = [ "anyhow", "cranelift-codegen", @@ -125,8 +125,8 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.70.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058" +version = "0.72.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" dependencies = [ "cranelift-codegen", "target-lexicon", @@ -134,8 +134,8 @@ dependencies = [ [[package]] name = "cranelift-object" -version = "0.70.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#cdb60ec5a9df087262ae8960a31067e88cd80058" +version = "0.72.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" dependencies = [ "anyhow", "cranelift-codegen", @@ -240,6 +240,15 @@ dependencies = [ "libc", ] +[[package]] +name = "memmap2" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e3e85b970d650e2ae6d70592474087051c11c54da7f7b4949725c5735fbcc6" +dependencies = [ + "libc", +] + [[package]] name = "object" version = "0.23.0" @@ -310,6 +319,7 @@ dependencies = [ "gimli", "indexmap", "libloading", + "memmap2", "object", "smallvec", "target-lexicon", diff --git a/Cargo.toml b/Cargo.toml index 9861af1f8eae2..59542c414fa85 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,12 +16,13 @@ cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime/", branch cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } target-lexicon = "0.11.0" gimli = { version = "0.23.0", default-features = false, features = ["write"]} -object = { version = "0.23.0", default-features = false, features = ["std", "read_core", "write", "coff", "elf", "macho", "pe"] } +object = { version = "0.23.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] } ar = { git = "https://github.com/bjorn3/rust-ar.git", branch = "do_not_remove_cg_clif_ranlib" } indexmap = "1.0.2" libloading = { version = "0.6.0", optional = true } smallvec = "1.6.1" +memmap2 = "0.2.1" # Uncomment to use local checkout of cranelift #[patch."https://github.com/bytecodealliance/wasmtime/"] @@ -75,3 +76,6 @@ debug = false [profile.release.package.syn] opt-level = 0 debug = false + +[package.metadata.rust-analyzer] +rustc_private = true diff --git a/Readme.md b/Readme.md index 6fa5eebdc2f3d..ffe1d9a1e6580 100644 --- a/Readme.md +++ b/Readme.md @@ -34,70 +34,19 @@ rustc_codegen_cranelift can be used as a near-drop-in replacement for `cargo bui Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`prepare.sh` and `build.sh` or `test.sh`). -### Cargo - In the directory with your project (where you can do the usual `cargo build`), run: ```bash -$ $cg_clif_dir/build/cargo.sh run -``` - -This should build and run your project with rustc_codegen_cranelift instead of the usual LLVM backend. - -### Rustc - -> You should prefer using the Cargo method. - -```bash -$ $cg_clif_dir/build/bin/cg_clif my_crate.rs -``` - -### Jit mode - -In jit mode cg_clif will immediately execute your code without creating an executable file. - -> This requires all dependencies to be available as dynamic library. -> The jit mode will probably need cargo integration to make this possible. - -```bash -$ $cg_clif_dir/build/cargo.sh jit -``` - -or - -```bash -$ $cg_clif_dir/build/bin/cg_clif -Cllvm-args=mode=jit -Cprefer-dynamic my_crate.rs -``` - -There is also an experimental lazy jit mode. In this mode functions are only compiled once they are -first called. It currently does not work with multi-threaded programs. When a not yet compiled -function is called from another thread than the main thread, you will get an ICE. - -```bash -$ $cg_clif_dir/build/cargo.sh lazy-jit +$ $cg_clif_dir/build/cargo.sh build ``` -### Shell - -These are a few functions that allow you to easily run rust code from the shell using cg_clif as jit. - -```bash -function jit_naked() { - echo "$@" | $cg_clif_dir/build/bin/cg_clif - -Cllvm-args=mode=jit -Cprefer-dynamic -} - -function jit() { - jit_naked "fn main() { $@ }" -} +This will build your project with rustc_codegen_cranelift instead of the usual LLVM backend. -function jit_calc() { - jit 'println!("0x{:x}", ' $@ ');'; -} -``` +For additional ways to use rustc_codegen_cranelift like the JIT mode see [usage.md](docs/usage.md). ## Env vars -[see env_vars.md](docs/env_vars.md) +See [env_vars.md](docs/env_vars.md) for all env vars used by rustc_codegen_cranelift. ## Not yet supported @@ -106,3 +55,20 @@ function jit_calc() { `llvm_asm!` will remain unimplemented forever. `asm!` doesn't yet support reg classes. You have to specify specific registers instead. * SIMD ([tracked here](https://github.com/bjorn3/rustc_codegen_cranelift/issues/171), some basic things work) + +## License + +Licensed under either of + + * Apache License, Version 2.0 ([LICENSE-APACHE](LICENSE-APACHE) or + http://www.apache.org/licenses/LICENSE-2.0) + * MIT license ([LICENSE-MIT](LICENSE-MIT) or + http://opensource.org/licenses/MIT) + +at your option. + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in the work by you shall be dual licensed as above, without any +additional terms or conditions. diff --git a/build.sh b/build.sh index 090349e54b148..76bc1884334af 100755 --- a/build.sh +++ b/build.sh @@ -55,6 +55,7 @@ ln target/$CHANNEL/*rustc_codegen_cranelift* "$target_dir"/lib ln rust-toolchain scripts/config.sh scripts/cargo.sh "$target_dir" mkdir -p "$target_dir/lib/rustlib/$TARGET_TRIPLE/lib/" +mkdir -p "$target_dir/lib/rustlib/$HOST_TRIPLE/lib/" if [[ "$TARGET_TRIPLE" == "x86_64-pc-windows-gnu" ]]; then cp $(rustc --print sysroot)/lib/rustlib/$TARGET_TRIPLE/lib/*.o "$target_dir/lib/rustlib/$TARGET_TRIPLE/lib/" fi @@ -64,12 +65,18 @@ case "$build_sysroot" in ;; "llvm") cp -r $(rustc --print sysroot)/lib/rustlib/$TARGET_TRIPLE/lib "$target_dir/lib/rustlib/$TARGET_TRIPLE/" + if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then + cp -r $(rustc --print sysroot)/lib/rustlib/$HOST_TRIPLE/lib "$target_dir/lib/rustlib/$HOST_TRIPLE/" + fi ;; "clif") echo "[BUILD] sysroot" dir=$(pwd) cd "$target_dir" time "$dir/build_sysroot/build_sysroot.sh" + if [[ "$HOST_TRIPLE" != "$TARGET_TRIPLE" ]]; then + time TARGET_TRIPLE="$HOST_TRIPLE" "$dir/build_sysroot/build_sysroot.sh" + fi cp lib/rustlib/*/lib/libstd-* lib/ ;; *) diff --git a/build_sysroot/Cargo.lock b/build_sysroot/Cargo.lock index a7650ab995b0f..09c5d7590ab86 100644 --- a/build_sysroot/Cargo.lock +++ b/build_sysroot/Cargo.lock @@ -16,9 +16,9 @@ dependencies = [ [[package]] name = "adler" -version = "0.2.3" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" dependencies = [ "compiler_builtins", "rustc-std-workspace-core", @@ -110,9 +110,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.9.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" dependencies = [ "compiler_builtins", "rustc-std-workspace-alloc", @@ -132,18 +132,18 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.86" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c" +checksum = "8916b1f6ca17130ec6568feccee27c156ad12037880833a3b842a823236502e7" dependencies = [ "rustc-std-workspace-core", ] [[package]] name = "miniz_oxide" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" dependencies = [ "adler", "autocfg", diff --git a/build_sysroot/build_sysroot.sh b/build_sysroot/build_sysroot.sh index 636aa5f3f3dc2..0354304e55bf7 100755 --- a/build_sysroot/build_sysroot.sh +++ b/build_sysroot/build_sysroot.sh @@ -28,7 +28,7 @@ export __CARGO_DEFAULT_LIB_METADATA="cg_clif" if [[ "$1" != "--debug" ]]; then sysroot_channel='release' # FIXME Enable incremental again once rust-lang/rust#74946 is fixed - CARGO_INCREMENTAL=0 RUSTFLAGS="$RUSTFLAGS -Zmir-opt-level=2" cargo build --target "$TARGET_TRIPLE" --release + CARGO_INCREMENTAL=0 RUSTFLAGS="$RUSTFLAGS -Zmir-opt-level=3" cargo build --target "$TARGET_TRIPLE" --release else sysroot_channel='debug' cargo build --target "$TARGET_TRIPLE" diff --git a/docs/env_vars.md b/docs/env_vars.md index f0a0a6ad42ef5..f7fde1b4f3a87 100644 --- a/docs/env_vars.md +++ b/docs/env_vars.md @@ -8,5 +8,8 @@ to make it possible to use incremental mode for all analyses performed by rustc without caching object files when their content should have been changed by a change to cg_clif.
CG_CLIF_DISPLAY_CG_TIME
-
If "1", display the time it took to perform codegen for a crate
+
If "1", display the time it took to perform codegen for a crate.
+
CG_CLIF_ENABLE_VERIFIER
+
Enable the Cranelift ir verifier for all compilation passes. If not set it will only run once + before passing the clif ir to Cranelift for compilation. diff --git a/docs/usage.md b/docs/usage.md new file mode 100644 index 0000000000000..3eee3b554e3b6 --- /dev/null +++ b/docs/usage.md @@ -0,0 +1,66 @@ +# Usage + +rustc_codegen_cranelift can be used as a near-drop-in replacement for `cargo build` or `cargo run` for existing projects. + +Assuming `$cg_clif_dir` is the directory you cloned this repo into and you followed the instructions (`prepare.sh` and `build.sh` or `test.sh`). + +## Cargo + +In the directory with your project (where you can do the usual `cargo build`), run: + +```bash +$ $cg_clif_dir/build/cargo.sh build +``` + +This will build your project with rustc_codegen_cranelift instead of the usual LLVM backend. + +## Rustc + +> You should prefer using the Cargo method. + +```bash +$ $cg_clif_dir/build/bin/cg_clif my_crate.rs +``` + +## Jit mode + +In jit mode cg_clif will immediately execute your code without creating an executable file. + +> This requires all dependencies to be available as dynamic library. +> The jit mode will probably need cargo integration to make this possible. + +```bash +$ $cg_clif_dir/build/cargo.sh jit +``` + +or + +```bash +$ $cg_clif_dir/build/bin/cg_clif -Cllvm-args=mode=jit -Cprefer-dynamic my_crate.rs +``` + +There is also an experimental lazy jit mode. In this mode functions are only compiled once they are +first called. It currently does not work with multi-threaded programs. When a not yet compiled +function is called from another thread than the main thread, you will get an ICE. + +```bash +$ $cg_clif_dir/build/cargo.sh lazy-jit +``` + +## Shell + +These are a few functions that allow you to easily run rust code from the shell using cg_clif as jit. + +```bash +function jit_naked() { + echo "$@" | $cg_clif_dir/build/bin/cg_clif - -Cllvm-args=mode=jit -Cprefer-dynamic +} + +function jit() { + jit_naked "fn main() { $@ }" +} + +function jit_calc() { + jit 'println!("0x{:x}", ' $@ ');'; +} +``` diff --git a/example/mini_core.rs b/example/mini_core.rs index 7c6d7fc106ded..c4834c8040871 100644 --- a/example/mini_core.rs +++ b/example/mini_core.rs @@ -621,6 +621,7 @@ struct PanicLocation { } #[no_mangle] +#[cfg(not(windows))] pub fn get_tls() -> u8 { #[thread_local] static A: u8 = 42; diff --git a/example/mini_core_hello_world.rs b/example/mini_core_hello_world.rs index 237f4d11d57f1..ea37ca98b59a7 100644 --- a/example/mini_core_hello_world.rs +++ b/example/mini_core_hello_world.rs @@ -1,7 +1,4 @@ -#![feature( - no_core, start, lang_items, box_syntax, never_type, linkage, - extern_types, thread_local -)] +#![feature(no_core, lang_items, box_syntax, never_type, linkage, extern_types, thread_local)] #![no_core] #![allow(dead_code, non_camel_case_types)] @@ -239,7 +236,7 @@ fn main() { assert_eq!(((|()| 42u8) as fn(()) -> u8)(()), 42); - #[cfg(not(jit))] + #[cfg(not(any(jit, windows)))] { extern { #[linkage = "extern_weak"] @@ -292,7 +289,7 @@ fn main() { from_decimal_string(); - #[cfg(not(jit))] + #[cfg(not(any(jit, windows)))] test_tls(); #[cfg(all(not(jit), target_os = "linux"))] diff --git a/prepare.sh b/prepare.sh index ee995ffcfa9f7..64c097261c908 100755 --- a/prepare.sh +++ b/prepare.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash set -e -rustup component add rust-src rustc-dev llvm-tools-preview ./build_sysroot/prepare_sysroot_src.sh cargo install hyperfine || echo "Skipping hyperfine install" diff --git a/rust-toolchain b/rust-toolchain index 908ca52135b66..2917fc7ee396d 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1,3 @@ -nightly-2021-03-05 +[toolchain] +channel = "nightly-2021-03-29" +components = ["rust-src", "rustc-dev", "llvm-tools-preview"] diff --git a/scripts/cargo.sh b/scripts/cargo.sh index 669d2d45b71b5..1daa5a78f7bd2 100755 --- a/scripts/cargo.sh +++ b/scripts/cargo.sh @@ -4,7 +4,7 @@ dir=$(dirname "$0") source "$dir/config.sh" # read nightly compiler from rust-toolchain file -TOOLCHAIN=$(cat "$dir/rust-toolchain") +TOOLCHAIN=$(cat "$dir/rust-toolchain" | grep channel | sed "s/channel = \"\(.*\)\"/\1/") cmd=$1 shift || true diff --git a/scripts/config.sh b/scripts/config.sh index c2ed2bf256d59..99b302ee1d94b 100644 --- a/scripts/config.sh +++ b/scripts/config.sh @@ -2,15 +2,7 @@ set -e -unamestr=$(uname) -if [[ "$unamestr" == 'Linux' || "$unamestr" == 'FreeBSD' ]]; then - dylib_ext='so' -elif [[ "$unamestr" == 'Darwin' ]]; then - dylib_ext='dylib' -else - echo "Unsupported os" - exit 1 -fi +dylib=$(echo "" | rustc --print file-names --crate-type dylib --crate-name rustc_codegen_cranelift -) if echo "$RUSTC_WRAPPER" | grep sccache; then echo @@ -24,10 +16,10 @@ dir=$(cd "$(dirname "${BASH_SOURCE[0]}")"; pwd) export RUSTC=$dir"/bin/cg_clif" export RUSTDOCFLAGS=$linker' -Cpanic=abort -Zpanic-abort-tests '\ -'-Zcodegen-backend='$dir'/lib/librustc_codegen_cranelift.'$dylib_ext' --sysroot '$dir +'-Zcodegen-backend='$dir'/lib/'$dylib' --sysroot '$dir # FIXME fix `#[linkage = "extern_weak"]` without this -if [[ "$unamestr" == 'Darwin' ]]; then +if [[ "$(uname)" == 'Darwin' ]]; then export RUSTFLAGS="$RUSTFLAGS -Clink-arg=-undefined -Clink-arg=dynamic_lookup" fi diff --git a/scripts/rustup.sh b/scripts/rustup.sh index 694945a87c268..fa7557653d879 100755 --- a/scripts/rustup.sh +++ b/scripts/rustup.sh @@ -8,7 +8,7 @@ case $1 in echo "=> Installing new nightly" rustup toolchain install --profile minimal "nightly-${TOOLCHAIN}" # Sanity check to see if the nightly exists - echo "nightly-${TOOLCHAIN}" > rust-toolchain + sed -i "s/\"nightly-.*\"/\"nightly-${TOOLCHAIN}\"/" rust-toolchain rustup component add rustfmt || true echo "=> Uninstalling all old nighlies" diff --git a/scripts/setup_rust_fork.sh b/scripts/setup_rust_fork.sh new file mode 100644 index 0000000000000..e8bedf625f796 --- /dev/null +++ b/scripts/setup_rust_fork.sh @@ -0,0 +1,68 @@ +#!/bin/bash +set -e + +./build.sh +source build/config.sh + +echo "[SETUP] Rust fork" +git clone https://github.com/rust-lang/rust.git || true +pushd rust +git fetch +git checkout -- . +git checkout "$(rustc -V | cut -d' ' -f3 | tr -d '(')" + +git apply - < config.toml < config.toml < res.txt - diff -u res.txt examples/regexdna-output.txt + ../build/cargo.sh build --example shootout-regex-dna --target $TARGET_TRIPLE + if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then + cat examples/regexdna-input.txt \ + | ../build/cargo.sh run --example shootout-regex-dna --target $TARGET_TRIPLE \ + | grep -v "Spawned thread" > res.txt + diff -u res.txt examples/regexdna-output.txt + fi - echo "[TEST] rust-lang/regex tests" - ../build/cargo.sh test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options -q + if [[ "$HOST_TRIPLE" = "$TARGET_TRIPLE" ]]; then + echo "[TEST] rust-lang/regex tests" + ../build/cargo.sh test --tests -- --exclude-should-panic --test-threads 1 -Zunstable-options -q + else + echo "[AOT] rust-lang/regex tests" + ../build/cargo.sh build --tests --target $TARGET_TRIPLE + fi popd } diff --git a/src/abi/comments.rs b/src/abi/comments.rs index c3cf90e1e70be..5fbaed7283a67 100644 --- a/src/abi/comments.rs +++ b/src/abi/comments.rs @@ -11,9 +11,11 @@ use cranelift_codegen::entity::EntityRef; use crate::prelude::*; pub(super) fn add_args_header_comment(fx: &mut FunctionCx<'_, '_, '_>) { - fx.add_global_comment( - "kind loc.idx param pass mode ty".to_string(), - ); + if fx.clif_comments.enabled() { + fx.add_global_comment( + "kind loc.idx param pass mode ty".to_string(), + ); + } } pub(super) fn add_arg_comment<'tcx>( @@ -25,6 +27,10 @@ pub(super) fn add_arg_comment<'tcx>( arg_abi_mode: PassMode, arg_layout: TyAndLayout<'tcx>, ) { + if !fx.clif_comments.enabled() { + return; + } + let local = if let Some(local) = local { Cow::Owned(format!("{:?}", local)) } else { @@ -59,10 +65,12 @@ pub(super) fn add_arg_comment<'tcx>( } pub(super) fn add_locals_header_comment(fx: &mut FunctionCx<'_, '_, '_>) { - fx.add_global_comment(String::new()); - fx.add_global_comment( - "kind local ty size align (abi,pref)".to_string(), - ); + if fx.clif_comments.enabled() { + fx.add_global_comment(String::new()); + fx.add_global_comment( + "kind local ty size align (abi,pref)".to_string(), + ); + } } pub(super) fn add_local_place_comments<'tcx>( @@ -70,6 +78,9 @@ pub(super) fn add_local_place_comments<'tcx>( place: CPlace<'tcx>, local: Local, ) { + if !fx.clif_comments.enabled() { + return; + } let TyAndLayout { ty, layout } = place.layout(); let rustc_target::abi::Layout { size, align, abi: _, variants: _, fields: _, largest_niche: _ } = layout; @@ -90,7 +101,7 @@ pub(super) fn add_local_place_comments<'tcx>( } else { Cow::Borrowed("") }; - match ptr.base_and_offset() { + match ptr.debug_base_and_offset() { (crate::pointer::PointerBase::Addr(addr), offset) => { ("reuse", format!("storage={}{}{}", addr, offset, meta).into()) } diff --git a/src/abi/mod.rs b/src/abi/mod.rs index b158d73f3a1a8..0e7829eaa26ac 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -1,6 +1,5 @@ //! Handling of everything related to the calling convention. Also fills `fx.local_map`. -#[cfg(debug_assertions)] mod comments; mod pass_mode; mod returning; @@ -75,8 +74,9 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { let func_id = import_function(self.tcx, self.cx.module, inst); let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func); - #[cfg(debug_assertions)] - self.add_comment(func_ref, format!("{:?}", inst)); + if self.clif_comments.enabled() { + self.add_comment(func_ref, format!("{:?}", inst)); + } func_ref } @@ -92,8 +92,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { let func_id = self.cx.module.declare_function(&name, Linkage::Import, &sig).unwrap(); let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func); let call_inst = self.bcx.ins().call(func_ref, args); - #[cfg(debug_assertions)] - { + if self.clif_comments.enabled() { self.add_comment(call_inst, format!("easy_call {}", name)); } let results = self.bcx.inst_results(call_inst); @@ -149,7 +148,6 @@ fn make_local_place<'tcx>( CPlace::new_stack_slot(fx, layout) }; - #[cfg(debug_assertions)] self::comments::add_local_place_comments(fx, place, local); place @@ -163,7 +161,6 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ let ssa_analyzed = crate::analyze::analyze(fx); - #[cfg(debug_assertions)] self::comments::add_args_header_comment(fx); let mut block_params_iter = fx.bcx.func.dfg.block_params(start_block).to_vec().into_iter(); @@ -228,7 +225,6 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ fx.fn_abi = Some(fn_abi); assert!(block_params_iter.next().is_none(), "arg_value left behind"); - #[cfg(debug_assertions)] self::comments::add_locals_header_comment(fx); for (local, arg_kind, ty) in func_params { @@ -256,7 +252,6 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ CPlace::for_ptr(addr, val.layout()) }; - #[cfg(debug_assertions)] self::comments::add_local_place_comments(fx, place, local); assert_eq!(fx.local_map.push(place), local); @@ -392,8 +387,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( let (func_ref, first_arg) = match instance { // Trait object call Some(Instance { def: InstanceDef::Virtual(_, idx), .. }) => { - #[cfg(debug_assertions)] - { + if fx.clif_comments.enabled() { let nop_inst = fx.bcx.ins().nop(); fx.add_comment( nop_inst, @@ -414,8 +408,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( // Indirect call None => { - #[cfg(debug_assertions)] - { + if fx.clif_comments.enabled() { let nop_inst = fx.bcx.ins().nop(); fx.add_comment(nop_inst, "indirect call"); } @@ -477,10 +470,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( // FIXME find a cleaner way to support varargs if fn_sig.c_variadic { if !matches!(fn_sig.abi, Abi::C { .. }) { - fx.tcx.sess.span_fatal( - span, - &format!("Variadic call for non-C abi {:?}", fn_sig.abi), - ); + fx.tcx.sess.span_fatal(span, &format!("Variadic call for non-C abi {:?}", fn_sig.abi)); } let sig_ref = fx.bcx.func.dfg.call_signature(call_inst).unwrap(); let abi_params = call_args diff --git a/src/abi/pass_mode.rs b/src/abi/pass_mode.rs index d58f952f53c1a..7c275965199e0 100644 --- a/src/abi/pass_mode.rs +++ b/src/abi/pass_mode.rs @@ -208,7 +208,7 @@ pub(super) fn from_casted_value<'tcx>( }); let ptr = Pointer::new(fx.bcx.ins().stack_addr(pointer_ty(fx.tcx), stack_slot, 0)); let mut offset = 0; - let mut block_params_iter = block_params.into_iter().copied(); + let mut block_params_iter = block_params.iter().copied(); for param in abi_params { let val = ptr.offset_i64(fx, offset).store( fx, @@ -248,8 +248,8 @@ pub(super) fn adjust_arg_for_abi<'tcx>( /// as necessary. pub(super) fn cvalue_for_param<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, - #[cfg_attr(not(debug_assertions), allow(unused_variables))] local: Option, - #[cfg_attr(not(debug_assertions), allow(unused_variables))] local_field: Option, + local: Option, + local_field: Option, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, block_params_iter: &mut impl Iterator, ) -> Option> { @@ -263,7 +263,6 @@ pub(super) fn cvalue_for_param<'tcx>( }) .collect::>(); - #[cfg(debug_assertions)] crate::abi::comments::add_arg_comment( fx, "arg", diff --git a/src/abi/returning.rs b/src/abi/returning.rs index 9fa066df69b3c..e1c53224b4f84 100644 --- a/src/abi/returning.rs +++ b/src/abi/returning.rs @@ -84,10 +84,6 @@ pub(super) fn codegen_return_param<'tcx>( } }; - #[cfg(not(debug_assertions))] - let _ = ret_param; - - #[cfg(debug_assertions)] crate::abi::comments::add_arg_comment( fx, "ret", @@ -146,7 +142,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx, T>( let results = fx .bcx .inst_results(call_inst) - .into_iter() + .iter() .copied() .collect::>(); let result = diff --git a/src/allocator.rs b/src/allocator.rs index efb64233ef2c3..f60645a9f97bc 100644 --- a/src/allocator.rs +++ b/src/allocator.rs @@ -3,6 +3,7 @@ use crate::prelude::*; +use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink}; use rustc_ast::expand::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS}; use rustc_span::symbol::sym; @@ -92,7 +93,7 @@ fn codegen_inner( bcx.finalize(); } module - .define_function(func_id, &mut ctx, &mut cranelift_codegen::binemit::NullTrapSink {}) + .define_function(func_id, &mut ctx, &mut NullTrapSink {}, &mut NullStackMapSink {}) .unwrap(); unwind_context.add_function(func_id, &ctx, module.isa()); } @@ -132,7 +133,7 @@ fn codegen_inner( bcx.finalize(); } module - .define_function(func_id, &mut ctx, &mut cranelift_codegen::binemit::NullTrapSink {}) + .define_function(func_id, &mut ctx, &mut NullTrapSink {}, &mut NullStackMapSink {}) .unwrap(); unwind_context.add_function(func_id, &ctx, module.isa()); } diff --git a/src/base.rs b/src/base.rs index 8b5ae9e0541ad..b34a29c25b92e 100644 --- a/src/base.rs +++ b/src/base.rs @@ -1,5 +1,6 @@ //! Codegen of a single function +use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink}; use rustc_index::vec::IndexVec; use rustc_middle::ty::adjustment::PointerCast; use rustc_middle::ty::layout::FnAbiExt; @@ -7,11 +8,7 @@ use rustc_target::abi::call::FnAbi; use crate::prelude::*; -pub(crate) fn codegen_fn<'tcx>( - cx: &mut crate::CodegenCx<'_, 'tcx>, - instance: Instance<'tcx>, - linkage: Linkage, -) { +pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: Instance<'tcx>) { let tcx = cx.tcx; let _inst_guard = @@ -23,7 +20,7 @@ pub(crate) fn codegen_fn<'tcx>( // Declare function let name = tcx.symbol_name(instance).name.to_string(); let sig = get_function_sig(tcx, cx.module.isa().triple(), instance); - let func_id = cx.module.declare_function(&name, linkage, &sig).unwrap(); + let func_id = cx.module.declare_function(&name, Linkage::Local, &sig).unwrap(); cx.cached_context.clear(); @@ -131,7 +128,7 @@ pub(crate) fn codegen_fn<'tcx>( let module = &mut cx.module; tcx.sess.time("define function", || { module - .define_function(func_id, context, &mut cranelift_codegen::binemit::NullTrapSink {}) + .define_function(func_id, context, &mut NullTrapSink {}, &mut NullStackMapSink {}) .unwrap() }); @@ -219,8 +216,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) { codegen_stmt(fx, block, stmt); } - #[cfg(debug_assertions)] - { + if fx.clif_comments.enabled() { let mut terminator_head = "\n".to_string(); bb_data.terminator().kind.fmt_head(&mut terminator_head).unwrap(); let inst = fx.bcx.func.layout.last_inst(block).unwrap(); @@ -433,12 +429,14 @@ fn codegen_stmt<'tcx>( fx.set_debug_loc(stmt.source_info); - #[cfg(false_debug_assertions)] + #[cfg(disabled)] match &stmt.kind { StatementKind::StorageLive(..) | StatementKind::StorageDead(..) => {} // Those are not very useful _ => { - let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap(); - fx.add_comment(inst, format!("{:?}", stmt)); + if fx.clif_comments.enabled() { + let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap(); + fx.add_comment(inst, format!("{:?}", stmt)); + } } } @@ -464,16 +462,16 @@ fn codegen_stmt<'tcx>( let val = crate::constant::codegen_tls_ref(fx, def_id, lval.layout()); lval.write_cvalue(fx, val); } - Rvalue::BinaryOp(bin_op, box (ref lhs, ref rhs)) => { - let lhs = codegen_operand(fx, lhs); - let rhs = codegen_operand(fx, rhs); + Rvalue::BinaryOp(bin_op, ref lhs_rhs) => { + let lhs = codegen_operand(fx, &lhs_rhs.0); + let rhs = codegen_operand(fx, &lhs_rhs.1); let res = crate::num::codegen_binop(fx, bin_op, lhs, rhs); lval.write_cvalue(fx, res); } - Rvalue::CheckedBinaryOp(bin_op, box (ref lhs, ref rhs)) => { - let lhs = codegen_operand(fx, lhs); - let rhs = codegen_operand(fx, rhs); + Rvalue::CheckedBinaryOp(bin_op, ref lhs_rhs) => { + let lhs = codegen_operand(fx, &lhs_rhs.0); + let rhs = codegen_operand(fx, &lhs_rhs.1); let res = if !fx.tcx.sess.overflow_checks() { let val = @@ -659,7 +657,9 @@ fn codegen_stmt<'tcx>( .val .try_to_bits(fx.tcx.data_layout.pointer_size) .unwrap(); - if fx.clif_type(operand.layout().ty) == Some(types::I8) { + if operand.layout().size.bytes() == 0 { + // Do nothing for ZST's + } else if fx.clif_type(operand.layout().ty) == Some(types::I8) { let times = fx.bcx.ins().iconst(fx.pointer_type, times as i64); // FIXME use emit_small_memset where possible let addr = lval.to_ptr().get_addr(fx); @@ -832,25 +832,18 @@ fn codegen_stmt<'tcx>( } } StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"), - StatementKind::CopyNonOverlapping(box rustc_middle::mir::CopyNonOverlapping { - src, - dst, - count, - }) => { - let dst = codegen_operand(fx, dst); + StatementKind::CopyNonOverlapping(inner) => { + let dst = codegen_operand(fx, &inner.dst); let pointee = dst - .layout() - .pointee_info_at(fx, rustc_target::abi::Size::ZERO) - .expect("Expected pointer"); + .layout() + .pointee_info_at(fx, rustc_target::abi::Size::ZERO) + .expect("Expected pointer"); let dst = dst.load_scalar(fx); - let src = codegen_operand(fx, src).load_scalar(fx); - let count = codegen_operand(fx, count).load_scalar(fx); + let src = codegen_operand(fx, &inner.src).load_scalar(fx); + let count = codegen_operand(fx, &inner.count).load_scalar(fx); let elem_size: u64 = pointee.size.bytes(); - let bytes = if elem_size != 1 { - fx.bcx.ins().imul_imm(count, elem_size as i64) - } else { - count - }; + let bytes = + if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count }; fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, bytes); } } diff --git a/src/codegen_i128.rs b/src/codegen_i128.rs index ae75e6508cb0b..ffe1922ab9056 100644 --- a/src/codegen_i128.rs +++ b/src/codegen_i128.rs @@ -32,18 +32,56 @@ pub(crate) fn maybe_codegen<'tcx>( BinOp::Add | BinOp::Sub if !checked => None, BinOp::Mul if !checked => { let val_ty = if is_signed { fx.tcx.types.i128 } else { fx.tcx.types.u128 }; - Some(fx.easy_call("__multi3", &[lhs, rhs], val_ty)) + if fx.tcx.sess.target.is_like_windows { + let ret_place = CPlace::new_stack_slot(fx, lhs.layout()); + let (lhs_ptr, lhs_extra) = lhs.force_stack(fx); + let (rhs_ptr, rhs_extra) = rhs.force_stack(fx); + assert!(lhs_extra.is_none()); + assert!(rhs_extra.is_none()); + let args = + [ret_place.to_ptr().get_addr(fx), lhs_ptr.get_addr(fx), rhs_ptr.get_addr(fx)]; + fx.lib_call( + "__multi3", + vec![ + AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn), + AbiParam::new(pointer_ty(fx.tcx)), + AbiParam::new(pointer_ty(fx.tcx)), + ], + vec![], + &args, + ); + Some(ret_place.to_cvalue(fx)) + } else { + Some(fx.easy_call("__multi3", &[lhs, rhs], val_ty)) + } } BinOp::Add | BinOp::Sub | BinOp::Mul => { assert!(checked); let out_ty = fx.tcx.mk_tup([lhs.layout().ty, fx.tcx.types.bool].iter()); let out_place = CPlace::new_stack_slot(fx, fx.layout_of(out_ty)); - let param_types = vec![ - AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn), - AbiParam::new(types::I128), - AbiParam::new(types::I128), - ]; - let args = [out_place.to_ptr().get_addr(fx), lhs.load_scalar(fx), rhs.load_scalar(fx)]; + let (param_types, args) = if fx.tcx.sess.target.is_like_windows { + let (lhs_ptr, lhs_extra) = lhs.force_stack(fx); + let (rhs_ptr, rhs_extra) = rhs.force_stack(fx); + assert!(lhs_extra.is_none()); + assert!(rhs_extra.is_none()); + ( + vec![ + AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn), + AbiParam::new(pointer_ty(fx.tcx)), + AbiParam::new(pointer_ty(fx.tcx)), + ], + [out_place.to_ptr().get_addr(fx), lhs_ptr.get_addr(fx), rhs_ptr.get_addr(fx)], + ) + } else { + ( + vec![ + AbiParam::special(pointer_ty(fx.tcx), ArgumentPurpose::StructReturn), + AbiParam::new(types::I128), + AbiParam::new(types::I128), + ], + [out_place.to_ptr().get_addr(fx), lhs.load_scalar(fx), rhs.load_scalar(fx)], + ) + }; let name = match (bin_op, is_signed) { (BinOp::Add, false) => "__rust_u128_addo", (BinOp::Add, true) => "__rust_i128_addo", @@ -57,20 +95,33 @@ pub(crate) fn maybe_codegen<'tcx>( Some(out_place.to_cvalue(fx)) } BinOp::Offset => unreachable!("offset should only be used on pointers, not 128bit ints"), - BinOp::Div => { + BinOp::Div | BinOp::Rem => { assert!(!checked); - if is_signed { - Some(fx.easy_call("__divti3", &[lhs, rhs], fx.tcx.types.i128)) - } else { - Some(fx.easy_call("__udivti3", &[lhs, rhs], fx.tcx.types.u128)) - } - } - BinOp::Rem => { - assert!(!checked); - if is_signed { - Some(fx.easy_call("__modti3", &[lhs, rhs], fx.tcx.types.i128)) + let name = match (bin_op, is_signed) { + (BinOp::Div, false) => "__udivti3", + (BinOp::Div, true) => "__divti3", + (BinOp::Rem, false) => "__umodti3", + (BinOp::Rem, true) => "__modti3", + _ => unreachable!(), + }; + if fx.tcx.sess.target.is_like_windows { + let (lhs_ptr, lhs_extra) = lhs.force_stack(fx); + let (rhs_ptr, rhs_extra) = rhs.force_stack(fx); + assert!(lhs_extra.is_none()); + assert!(rhs_extra.is_none()); + let args = [lhs_ptr.get_addr(fx), rhs_ptr.get_addr(fx)]; + let ret = fx.lib_call( + name, + vec![AbiParam::new(pointer_ty(fx.tcx)), AbiParam::new(pointer_ty(fx.tcx))], + vec![AbiParam::new(types::I64X2)], + &args, + )[0]; + // FIXME use bitcast instead of store to get from i64x2 to i128 + let ret_place = CPlace::new_stack_slot(fx, lhs.layout()); + ret_place.to_ptr().store(fx, ret, MemFlags::trusted()); + Some(ret_place.to_cvalue(fx)) } else { - Some(fx.easy_call("__umodti3", &[lhs, rhs], fx.tcx.types.u128)) + Some(fx.easy_call(name, &[lhs, rhs], lhs.layout().ty)) } } BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne => { diff --git a/src/common.rs b/src/common.rs index 6a4a6744a5cf7..b5874f62535ca 100644 --- a/src/common.rs +++ b/src/common.rs @@ -361,8 +361,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { let _ = self.cx.module.define_data(msg_id, &data_ctx); let local_msg_id = self.cx.module.declare_data_in_func(msg_id, self.bcx.func); - #[cfg(debug_assertions)] - { + if self.clif_comments.enabled() { self.add_comment(local_msg_id, msg); } self.bcx.ins().global_value(self.pointer_type, local_msg_id) diff --git a/src/compiler_builtins.rs b/src/compiler_builtins.rs new file mode 100644 index 0000000000000..177f850afb398 --- /dev/null +++ b/src/compiler_builtins.rs @@ -0,0 +1,41 @@ +macro builtin_functions($register:ident; $(fn $name:ident($($arg_name:ident: $arg_ty:ty),*) -> $ret_ty:ty;)*) { + #[cfg(feature = "jit")] + #[allow(improper_ctypes)] + extern "C" { + $(fn $name($($arg_name: $arg_ty),*) -> $ret_ty;)* + } + + #[cfg(feature = "jit")] + pub(crate) fn $register(builder: &mut cranelift_jit::JITBuilder) { + for &(name, val) in &[$((stringify!($name), $name as *const u8)),*] { + builder.symbol(name, val); + } + } +} + +builtin_functions! { + register_functions_for_jit; + + // integers + fn __multi3(a: i128, b: i128) -> i128; + fn __udivti3(n: u128, d: u128) -> u128; + fn __divti3(n: i128, d: i128) -> i128; + fn __umodti3(n: u128, d: u128) -> u128; + fn __modti3(n: i128, d: i128) -> i128; + fn __rust_u128_addo(a: u128, b: u128) -> (u128, bool); + fn __rust_i128_addo(a: i128, b: i128) -> (i128, bool); + fn __rust_u128_subo(a: u128, b: u128) -> (u128, bool); + fn __rust_i128_subo(a: i128, b: i128) -> (i128, bool); + fn __rust_u128_mulo(a: u128, b: u128) -> (u128, bool); + fn __rust_i128_mulo(a: i128, b: i128) -> (i128, bool); + + // floats + fn __floattisf(i: i128) -> f32; + fn __floattidf(i: i128) -> f64; + fn __floatuntisf(i: u128) -> f32; + fn __floatuntidf(i: u128) -> f64; + fn __fixsfti(f: f32) -> i128; + fn __fixdfti(f: f64) -> i128; + fn __fixunssfti(f: f32) -> u128; + fn __fixunsdfti(f: f64) -> u128; +} diff --git a/src/constant.rs b/src/constant.rs index f4cbfb6967ff3..fcd41c844659d 100644 --- a/src/constant.rs +++ b/src/constant.rs @@ -85,8 +85,9 @@ pub(crate) fn codegen_tls_ref<'tcx>( ) -> CValue<'tcx> { let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false); let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); - #[cfg(debug_assertions)] - fx.add_comment(local_data_id, format!("tls {:?}", def_id)); + if fx.clif_comments.enabled() { + fx.add_comment(local_data_id, format!("tls {:?}", def_id)); + } let tls_ptr = fx.bcx.ins().tls_value(fx.pointer_type, local_data_id); CValue::by_val(tls_ptr, layout) } @@ -98,8 +99,9 @@ fn codegen_static_ref<'tcx>( ) -> CPlace<'tcx> { let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false); let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); - #[cfg(debug_assertions)] - fx.add_comment(local_data_id, format!("{:?}", def_id)); + if fx.clif_comments.enabled() { + fx.add_comment(local_data_id, format!("{:?}", def_id)); + } let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id); assert!(!layout.is_unsized(), "unsized statics aren't supported"); assert!( @@ -122,7 +124,9 @@ pub(crate) fn codegen_constant<'tcx>( }; let const_val = match const_.val { ConstKind::Value(const_val) => const_val, - ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted }) if fx.tcx.is_static(def.did) => { + ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted }) + if fx.tcx.is_static(def.did) => + { assert!(substs.is_empty()); assert!(promoted.is_none()); @@ -183,8 +187,9 @@ pub(crate) fn codegen_const_value<'tcx>( data_id_for_alloc_id(fx.cx.module, ptr.alloc_id, alloc.mutability); let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); - #[cfg(debug_assertions)] - fx.add_comment(local_data_id, format!("{:?}", ptr.alloc_id)); + if fx.clif_comments.enabled() { + fx.add_comment(local_data_id, format!("{:?}", ptr.alloc_id)); + } fx.bcx.ins().global_value(fx.pointer_type, local_data_id) } Some(GlobalAlloc::Function(instance)) => { @@ -199,8 +204,9 @@ pub(crate) fn codegen_const_value<'tcx>( let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false); let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); - #[cfg(debug_assertions)] - fx.add_comment(local_data_id, format!("{:?}", def_id)); + if fx.clif_comments.enabled() { + fx.add_comment(local_data_id, format!("{:?}", def_id)); + } fx.bcx.ins().global_value(fx.pointer_type, local_data_id) } None => bug!("missing allocation {:?}", ptr.alloc_id), @@ -241,8 +247,9 @@ fn pointer_for_allocation<'tcx>( let data_id = data_id_for_alloc_id(fx.cx.module, alloc_id, alloc.mutability); let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); - #[cfg(debug_assertions)] - fx.add_comment(local_data_id, format!("{:?}", alloc_id)); + if fx.clif_comments.enabled() { + fx.add_comment(local_data_id, format!("{:?}", alloc_id)); + } let global_ptr = fx.bcx.ins().global_value(fx.pointer_type, local_data_id); crate::pointer::Pointer::new(global_ptr) } diff --git a/src/debuginfo/line_info.rs b/src/debuginfo/line_info.rs index 30ed356c7627f..8578ab33ced68 100644 --- a/src/debuginfo/line_info.rs +++ b/src/debuginfo/line_info.rs @@ -39,11 +39,11 @@ fn osstr_as_utf8_bytes(path: &OsStr) -> &[u8] { #[cfg(unix)] { use std::os::unix::ffi::OsStrExt; - return path.as_bytes(); + path.as_bytes() } #[cfg(not(unix))] { - return path.to_str().unwrap().as_bytes(); + path.to_str().unwrap().as_bytes() } } diff --git a/src/driver/aot.rs b/src/driver/aot.rs index b87dcc41928b6..ed3bdedddced5 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -119,11 +119,10 @@ fn module_codegen( tcx.sess.opts.debuginfo != DebugInfo::None, ); super::predefine_mono_items(&mut cx, &mono_items); - for (mono_item, (linkage, visibility)) in mono_items { - let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); + for (mono_item, _) in mono_items { match mono_item { MonoItem::Fn(inst) => { - cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst, linkage)); + cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst)); } MonoItem::Static(def_id) => { crate::constant::codegen_static(&mut cx.constants_cx, def_id) @@ -163,6 +162,21 @@ pub(super) fn run_aot( metadata: EncodedMetadata, need_metadata_module: bool, ) -> Box<(CodegenResults, FxHashMap)> { + use rustc_span::symbol::sym; + + let crate_attrs = tcx.hir().attrs(rustc_hir::CRATE_HIR_ID); + let subsystem = tcx.sess.first_attr_value_str_by_name(crate_attrs, sym::windows_subsystem); + let windows_subsystem = subsystem.map(|subsystem| { + if subsystem != sym::windows && subsystem != sym::console { + tcx.sess.fatal(&format!( + "invalid windows subsystem `{}`, only \ + `windows` and `console` are allowed", + subsystem + )); + } + subsystem.to_string() + }); + let mut work_products = FxHashMap::default(); let cgus = if tcx.sess.opts.output_types.should_codegen() { @@ -280,7 +294,7 @@ pub(super) fn run_aot( allocator_module, metadata_module, metadata, - windows_subsystem: None, // Windows is not yet supported + windows_subsystem, linker_info: LinkerInfo::new(tcx), crate_info: CrateInfo::new(tcx), }, diff --git a/src/driver/jit.rs b/src/driver/jit.rs index 245df03ffb84d..dbe1ff083f0db 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -5,8 +5,10 @@ use std::cell::RefCell; use std::ffi::CString; use std::os::raw::{c_char, c_int}; +use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink}; use rustc_codegen_ssa::CrateInfo; use rustc_middle::mir::mono::MonoItem; +use rustc_session::config::EntryFnType; use cranelift_jit::{JITBuilder, JITModule}; @@ -28,20 +30,11 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { let mut jit_builder = JITBuilder::with_isa(crate::build_isa(tcx.sess), cranelift_module::default_libcall_names()); jit_builder.hotswap(matches!(backend_config.codegen_mode, CodegenMode::JitLazy)); + crate::compiler_builtins::register_functions_for_jit(&mut jit_builder); jit_builder.symbols(imported_symbols); let mut jit_module = JITModule::new(jit_builder); assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type()); - let sig = Signature { - params: vec![ - AbiParam::new(jit_module.target_config().pointer_type()), - AbiParam::new(jit_module.target_config().pointer_type()), - ], - returns: vec![AbiParam::new(jit_module.target_config().pointer_type() /*isize*/)], - call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)), - }; - let main_func_id = jit_module.declare_function("main", Linkage::Import, &sig).unwrap(); - let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE); let mono_items = cgus .iter() @@ -55,15 +48,12 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { super::time(tcx, "codegen mono items", || { super::predefine_mono_items(&mut cx, &mono_items); - for (mono_item, (linkage, visibility)) in mono_items { - let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); + for (mono_item, _) in mono_items { match mono_item { MonoItem::Fn(inst) => match backend_config.codegen_mode { CodegenMode::Aot => unreachable!(), CodegenMode::Jit => { - cx.tcx - .sess - .time("codegen fn", || crate::base::codegen_fn(&mut cx, inst, linkage)); + cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst)); } CodegenMode::JitLazy => codegen_shim(&mut cx, inst), }, @@ -86,24 +76,17 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { tcx.sess.fatal("Inline asm is not supported in JIT mode"); } - crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, &mut unwind_context); crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context); tcx.sess.abort_if_errors(); jit_module.finalize_definitions(); - let _unwind_register_guard = unsafe { unwind_context.register_jit(&jit_module) }; - let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id); - println!( "Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed" ); - let f: extern "C" fn(c_int, *const *const c_char) -> c_int = - unsafe { ::std::mem::transmute(finalized_main) }; - let args = ::std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new()); let args = std::iter::once(&*tcx.crate_name(LOCAL_CRATE).as_str().to_string()) .chain(args.split(' ')) @@ -118,12 +101,58 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { BACKEND_CONFIG.with(|tls_backend_config| { assert!(tls_backend_config.borrow_mut().replace(backend_config).is_none()) }); - CURRENT_MODULE - .with(|current_module| assert!(current_module.borrow_mut().replace(jit_module).is_none())); - let ret = f(args.len() as c_int, argv.as_ptr()); + let (main_def_id, entry_ty) = tcx.entry_fn(LOCAL_CRATE).unwrap(); + let instance = Instance::mono(tcx, main_def_id.to_def_id()).polymorphize(tcx); + + match entry_ty { + EntryFnType::Main => { + // FIXME set program arguments somehow - std::process::exit(ret); + let main_sig = Signature { + params: vec![], + returns: vec![], + call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)), + }; + let main_func_id = jit_module + .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &main_sig) + .unwrap(); + let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id); + + CURRENT_MODULE.with(|current_module| { + assert!(current_module.borrow_mut().replace(jit_module).is_none()) + }); + + let f: extern "C" fn() = unsafe { ::std::mem::transmute(finalized_main) }; + f(); + std::process::exit(0); + } + EntryFnType::Start => { + let start_sig = Signature { + params: vec![ + AbiParam::new(jit_module.target_config().pointer_type()), + AbiParam::new(jit_module.target_config().pointer_type()), + ], + returns: vec![AbiParam::new( + jit_module.target_config().pointer_type(), /*isize*/ + )], + call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)), + }; + let start_func_id = jit_module + .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &start_sig) + .unwrap(); + let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id); + + CURRENT_MODULE.with(|current_module| { + assert!(current_module.borrow_mut().replace(jit_module).is_none()) + }); + + let f: extern "C" fn(c_int, *const *const c_char) -> c_int = + unsafe { ::std::mem::transmute(finalized_start) }; + let ret = f(args.len() as c_int, argv.as_ptr()); + std::process::exit(ret); + } + } } #[no_mangle] @@ -144,8 +173,7 @@ extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8 jit_module.prepare_for_function_redefine(func_id).unwrap(); let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module, false); - tcx.sess - .time("codegen fn", || crate::base::codegen_fn(&mut cx, instance, Linkage::Export)); + tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, instance)); let (global_asm, _debug_context, unwind_context) = cx.finalize(); assert!(global_asm.is_empty()); @@ -220,7 +248,7 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> { imported_symbols } -pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) { +fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) { let tcx = cx.tcx; let pointer_type = cx.module.target_config().pointer_type(); @@ -267,7 +295,8 @@ pub(super) fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'t .define_function( func_id, &mut Context::for_function(trampoline), - &mut cranelift_codegen::binemit::NullTrapSink {}, + &mut NullTrapSink {}, + &mut NullStackMapSink {}, ) .unwrap(); } diff --git a/src/driver/mod.rs b/src/driver/mod.rs index b994f28ffef5b..d49182a07b79e 100644 --- a/src/driver/mod.rs +++ b/src/driver/mod.rs @@ -44,13 +44,19 @@ fn predefine_mono_items<'tcx>( mono_items: &[(MonoItem<'tcx>, (RLinkage, Visibility))], ) { cx.tcx.sess.time("predefine functions", || { + let is_compiler_builtins = cx.tcx.is_compiler_builtins(LOCAL_CRATE); for &(mono_item, (linkage, visibility)) in mono_items { match mono_item { MonoItem::Fn(instance) => { let name = cx.tcx.symbol_name(instance).name.to_string(); let _inst_guard = crate::PrintOnPanic(|| format!("{:?} {}", instance, name)); let sig = get_function_sig(cx.tcx, cx.module.isa().triple(), instance); - let linkage = crate::linkage::get_clif_linkage(mono_item, linkage, visibility); + let linkage = crate::linkage::get_clif_linkage( + mono_item, + linkage, + visibility, + is_compiler_builtins, + ); cx.module.declare_function(&name, linkage, &sig).unwrap(); } MonoItem::Static(_) | MonoItem::GlobalAsm(_) => {} diff --git a/src/inline_asm.rs b/src/inline_asm.rs index 5b3df2bd38280..1fb5e86aed7df 100644 --- a/src/inline_asm.rs +++ b/src/inline_asm.rs @@ -20,6 +20,10 @@ pub(crate) fn codegen_inline_asm<'tcx>( if template.is_empty() { // Black box return; + } else if template[0] == InlineAsmTemplatePiece::String("int $$0x29".to_string()) { + let true_ = fx.bcx.ins().iconst(types::I32, 1); + fx.bcx.ins().trapnz(true_, TrapCode::User(1)); + return; } let mut slot_size = Size::from_bytes(0); @@ -193,8 +197,9 @@ fn call_inline_asm<'tcx>( offset: None, size: u32::try_from(slot_size.bytes()).unwrap(), }); - #[cfg(debug_assertions)] - fx.add_comment(stack_slot, "inline asm scratch slot"); + if fx.clif_comments.enabled() { + fx.add_comment(stack_slot, "inline asm scratch slot"); + } let inline_asm_func = fx .cx @@ -210,8 +215,9 @@ fn call_inline_asm<'tcx>( ) .unwrap(); let inline_asm_func = fx.cx.module.declare_func_in_func(inline_asm_func, &mut fx.bcx.func); - #[cfg(debug_assertions)] - fx.add_comment(inline_asm_func, asm_name); + if fx.clif_comments.enabled() { + fx.add_comment(inline_asm_func, asm_name); + } for (_reg, offset, value) in inputs { fx.bcx.ins().stack_store(value, stack_slot, i32::try_from(offset.bytes()).unwrap()); diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs index 86df71a0dfc91..c7ce32b385e94 100644 --- a/src/intrinsics/simd.rs +++ b/src/intrinsics/simd.rs @@ -88,7 +88,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( let idx_bytes = match idx_const { ConstValue::ByRef { alloc, offset } => { let ptr = Pointer::new(AllocId(0 /* dummy */), offset); - let size = Size::from_bytes(4 * u64::from(ret_lane_count) /* size_of([u32; ret_lane_count]) */); + let size = Size::from_bytes(4 * ret_lane_count /* size_of([u32; ret_lane_count]) */); alloc.get_bytes(fx, ptr, size).unwrap() } _ => unreachable!("{:?}", idx_const), diff --git a/src/lib.rs b/src/lib.rs index 8edb883ccb5f9..720d2a1253445 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,13 +1,4 @@ -#![feature( - rustc_private, - decl_macro, - type_alias_impl_trait, - associated_type_bounds, - never_type, - try_blocks, - box_patterns, - hash_drain_filter -)] +#![feature(rustc_private, decl_macro, never_type, hash_drain_filter)] #![warn(rust_2018_idioms)] #![warn(unused_lifetimes)] #![warn(unreachable_pub)] @@ -57,6 +48,7 @@ mod base; mod cast; mod codegen_i128; mod common; +mod compiler_builtins; mod constant; mod debuginfo; mod discriminant; @@ -224,8 +216,10 @@ pub struct CraneliftCodegenBackend { impl CodegenBackend for CraneliftCodegenBackend { fn init(&self, sess: &Session) { - if sess.lto() != rustc_session::config::Lto::No && sess.opts.cg.embed_bitcode { - sess.warn("LTO is not supported. You may get a linker error."); + use rustc_session::config::Lto; + match sess.lto() { + Lto::No | Lto::ThinLocal => {} + Lto::Thin | Lto::Fat => sess.warn("LTO is not supported. You may get a linker error."), } } @@ -240,9 +234,9 @@ impl CodegenBackend for CraneliftCodegenBackend { vec![] } - fn codegen_crate<'tcx>( + fn codegen_crate( &self, - tcx: TyCtxt<'tcx>, + tcx: TyCtxt<'_>, metadata: EncodedMetadata, need_metadata_module: bool, ) -> Box { @@ -252,9 +246,7 @@ impl CodegenBackend for CraneliftCodegenBackend { BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args) .unwrap_or_else(|err| tcx.sess.fatal(&err)) }; - let res = driver::codegen_crate(tcx, metadata, need_metadata_module, config); - - res + driver::codegen_crate(tcx, metadata, need_metadata_module, config) } fn join_codegen( @@ -300,9 +292,9 @@ fn build_isa(sess: &Session) -> Box { let mut flags_builder = settings::builder(); flags_builder.enable("is_pic").unwrap(); flags_builder.set("enable_probestack", "false").unwrap(); // __cranelift_probestack is not provided - flags_builder - .set("enable_verifier", if cfg!(debug_assertions) { "true" } else { "false" }) - .unwrap(); + let enable_verifier = + cfg!(debug_assertions) || std::env::var("CG_CLIF_ENABLE_VERIFIER").is_ok(); + flags_builder.set("enable_verifier", if enable_verifier { "true" } else { "false" }).unwrap(); let tls_model = match target_triple.binary_format { BinaryFormat::Elf => "elf_gd", @@ -314,18 +306,17 @@ fn build_isa(sess: &Session) -> Box { flags_builder.set("enable_simd", "true").unwrap(); + flags_builder.set("enable_llvm_abi_extensions", "true").unwrap(); + use rustc_session::config::OptLevel; match sess.opts.optimize { OptLevel::No => { flags_builder.set("opt_level", "none").unwrap(); } OptLevel::Less | OptLevel::Default => {} - OptLevel::Aggressive => { + OptLevel::Size | OptLevel::SizeMin | OptLevel::Aggressive => { flags_builder.set("opt_level", "speed_and_size").unwrap(); } - OptLevel::Size | OptLevel::SizeMin => { - sess.warn("Optimizing for size is not supported. Just ignoring the request"); - } } let flags = settings::Flags::new(flags_builder); diff --git a/src/linkage.rs b/src/linkage.rs index dc1e2107ce712..a564a59f72510 100644 --- a/src/linkage.rs +++ b/src/linkage.rs @@ -6,8 +6,10 @@ pub(crate) fn get_clif_linkage( mono_item: MonoItem<'_>, linkage: RLinkage, visibility: Visibility, + is_compiler_builtins: bool, ) -> Linkage { match (linkage, visibility) { + (RLinkage::External, Visibility::Default) if is_compiler_builtins => Linkage::Hidden, (RLinkage::External, Visibility::Default) => Linkage::Export, (RLinkage::Internal, Visibility::Default) => Linkage::Local, (RLinkage::External, Visibility::Hidden) => Linkage::Hidden, diff --git a/src/main_shim.rs b/src/main_shim.rs index 62e551b186ff7..a6266f507765f 100644 --- a/src/main_shim.rs +++ b/src/main_shim.rs @@ -1,3 +1,4 @@ +use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink}; use rustc_hir::LangItem; use rustc_session::config::EntryFnType; @@ -100,12 +101,8 @@ pub(crate) fn maybe_create_entry_wrapper( bcx.seal_all_blocks(); bcx.finalize(); } - m.define_function( - cmain_func_id, - &mut ctx, - &mut cranelift_codegen::binemit::NullTrapSink {}, - ) - .unwrap(); + m.define_function(cmain_func_id, &mut ctx, &mut NullTrapSink {}, &mut NullStackMapSink {}) + .unwrap(); unwind_context.add_function(cmain_func_id, &ctx, m.isa()); } } diff --git a/src/metadata.rs b/src/metadata.rs index 190c4f45ccafd..c5189c972cd2e 100644 --- a/src/metadata.rs +++ b/src/metadata.rs @@ -1,11 +1,11 @@ //! Reading and writing of the rustc metadata for rlibs and dylibs -use std::convert::TryFrom; use std::fs::File; +use std::ops::Deref; use std::path::Path; use rustc_codegen_ssa::METADATA_FILENAME; -use rustc_data_structures::owning_ref::OwningRef; +use rustc_data_structures::owning_ref::{OwningRef, StableAddress}; use rustc_data_structures::rustc_erase_owner; use rustc_data_structures::sync::MetadataRef; use rustc_middle::middle::cstore::{EncodedMetadata, MetadataLoader}; @@ -17,38 +17,56 @@ use crate::backend::WriteMetadata; pub(crate) struct CraneliftMetadataLoader; +struct StableMmap(memmap2::Mmap); + +impl Deref for StableMmap { + type Target = [u8]; + + fn deref(&self) -> &[u8] { + &*self.0 + } +} + +unsafe impl StableAddress for StableMmap {} + +fn load_metadata_with( + path: &Path, + f: impl for<'a> FnOnce(&'a [u8]) -> Result<&'a [u8], String>, +) -> Result { + let file = File::open(path).map_err(|e| format!("{:?}", e))?; + let data = unsafe { memmap2::MmapOptions::new().map_copy_read_only(&file) } + .map_err(|e| format!("{:?}", e))?; + let metadata = OwningRef::new(StableMmap(data)).try_map(f)?; + return Ok(rustc_erase_owner!(metadata.map_owner_box())); +} + impl MetadataLoader for CraneliftMetadataLoader { fn get_rlib_metadata(&self, _target: &Target, path: &Path) -> Result { - let mut archive = ar::Archive::new(File::open(path).map_err(|e| format!("{:?}", e))?); - // Iterate over all entries in the archive: - while let Some(entry_result) = archive.next_entry() { - let mut entry = entry_result.map_err(|e| format!("{:?}", e))?; - if entry.header().identifier() == METADATA_FILENAME.as_bytes() { - let mut buf = Vec::with_capacity( - usize::try_from(entry.header().size()) - .expect("Rlib metadata file too big to load into memory."), - ); - ::std::io::copy(&mut entry, &mut buf).map_err(|e| format!("{:?}", e))?; - let buf: OwningRef, [u8]> = OwningRef::new(buf); - return Ok(rustc_erase_owner!(buf.map_owner_box())); + load_metadata_with(path, |data| { + let archive = object::read::archive::ArchiveFile::parse(&*data) + .map_err(|e| format!("{:?}", e))?; + + for entry_result in archive.members() { + let entry = entry_result.map_err(|e| format!("{:?}", e))?; + if entry.name() == METADATA_FILENAME.as_bytes() { + return Ok(entry.data()); + } } - } - Err("couldn't find metadata entry".to_string()) + Err("couldn't find metadata entry".to_string()) + }) } fn get_dylib_metadata(&self, _target: &Target, path: &Path) -> Result { use object::{Object, ObjectSection}; - let file = std::fs::read(path).map_err(|e| format!("read:{:?}", e))?; - let file = object::File::parse(&file).map_err(|e| format!("parse: {:?}", e))?; - let buf = file - .section_by_name(".rustc") - .ok_or("no .rustc section")? - .data() - .map_err(|e| format!("failed to read .rustc section: {:?}", e))? - .to_owned(); - let buf: OwningRef, [u8]> = OwningRef::new(buf); - Ok(rustc_erase_owner!(buf.map_owner_box())) + + load_metadata_with(path, |data| { + let file = object::File::parse(&data).map_err(|e| format!("parse: {:?}", e))?; + file.section_by_name(".rustc") + .ok_or("no .rustc section")? + .data() + .map_err(|e| format!("failed to read .rustc section: {:?}", e)) + }) } } diff --git a/src/num.rs b/src/num.rs index da49e1c6c91db..2ebf30da2d8ba 100644 --- a/src/num.rs +++ b/src/num.rs @@ -166,13 +166,11 @@ pub(crate) fn codegen_int_binop<'tcx>( BinOp::Shl => { let lhs_ty = fx.bcx.func.dfg.value_type(lhs); let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1)); - let actual_shift = clif_intcast(fx, actual_shift, types::I8, false); fx.bcx.ins().ishl(lhs, actual_shift) } BinOp::Shr => { let lhs_ty = fx.bcx.func.dfg.value_type(lhs); let actual_shift = fx.bcx.ins().band_imm(rhs, i64::from(lhs_ty.bits() - 1)); - let actual_shift = clif_intcast(fx, actual_shift, types::I8, false); if signed { fx.bcx.ins().sshr(lhs, actual_shift) } else { @@ -387,7 +385,7 @@ pub(crate) fn codegen_ptr_binop<'tcx>( let lhs = in_lhs.load_scalar(fx); let rhs = in_rhs.load_scalar(fx); - return codegen_compare_bin_op(fx, bin_op, false, lhs, rhs); + codegen_compare_bin_op(fx, bin_op, false, lhs, rhs) } BinOp::Offset => { let pointee_ty = in_lhs.layout().ty.builtin_deref(true).unwrap().ty; @@ -396,10 +394,10 @@ pub(crate) fn codegen_ptr_binop<'tcx>( let ptr_diff = fx.bcx.ins().imul_imm(offset, pointee_size as i64); let base_val = base.load_scalar(fx); let res = fx.bcx.ins().iadd(base_val, ptr_diff); - return CValue::by_val(res, base.layout()); + CValue::by_val(res, base.layout()) } _ => unreachable!("{:?}({:?}, {:?})", bin_op, in_lhs, in_rhs), - }; + } } else { let (lhs_ptr, lhs_extra) = in_lhs.load_scalar_pair(fx); let (rhs_ptr, rhs_extra) = in_rhs.load_scalar_pair(fx); diff --git a/src/optimize/stack2reg.rs b/src/optimize/stack2reg.rs index d111f37f5e455..8bb02a3e55854 100644 --- a/src/optimize/stack2reg.rs +++ b/src/optimize/stack2reg.rs @@ -181,7 +181,6 @@ impl<'a> OptimizeContext<'a> { pub(super) fn optimize_function( ctx: &mut Context, - #[cfg_attr(not(debug_assertions), allow(unused_variables))] clif_comments: &mut crate::pretty_clif::CommentWriter, ) { combine_stack_addr_with_load_store(&mut ctx.func); @@ -192,8 +191,7 @@ pub(super) fn optimize_function( remove_unused_stack_addr_and_stack_load(&mut opt_ctx); - #[cfg(debug_assertions)] - { + if clif_comments.enabled() { for (&OrdStackSlot(stack_slot), usage) in &opt_ctx.stack_slot_usage_map { clif_comments.add_comment(stack_slot, format!("used by: {:?}", usage)); } @@ -209,25 +207,27 @@ pub(super) fn optimize_function( for load in users.stack_load.clone().into_iter() { let potential_stores = users.potential_stores_for_load(&opt_ctx.ctx, load); - #[cfg(debug_assertions)] - for &store in &potential_stores { - clif_comments.add_comment( - load, - format!( - "Potential store -> load forwarding {} -> {} ({:?}, {:?})", - opt_ctx.ctx.func.dfg.display_inst(store, None), - opt_ctx.ctx.func.dfg.display_inst(load, None), - spatial_overlap(&opt_ctx.ctx.func, store, load), - temporal_order(&opt_ctx.ctx, store, load), - ), - ); + if clif_comments.enabled() { + for &store in &potential_stores { + clif_comments.add_comment( + load, + format!( + "Potential store -> load forwarding {} -> {} ({:?}, {:?})", + opt_ctx.ctx.func.dfg.display_inst(store, None), + opt_ctx.ctx.func.dfg.display_inst(load, None), + spatial_overlap(&opt_ctx.ctx.func, store, load), + temporal_order(&opt_ctx.ctx, store, load), + ), + ); + } } match *potential_stores { [] => { - #[cfg(debug_assertions)] - clif_comments - .add_comment(load, "[BUG?] Reading uninitialized memory".to_string()); + if clif_comments.enabled() { + clif_comments + .add_comment(load, "[BUG?] Reading uninitialized memory".to_string()); + } } [store] if spatial_overlap(&opt_ctx.ctx.func, store, load) == SpatialOverlap::Full @@ -237,9 +237,12 @@ pub(super) fn optimize_function( // Only one store could have been the origin of the value. let stored_value = opt_ctx.ctx.func.dfg.inst_args(store)[0]; - #[cfg(debug_assertions)] - clif_comments - .add_comment(load, format!("Store to load forward {} -> {}", store, load)); + if clif_comments.enabled() { + clif_comments.add_comment( + load, + format!("Store to load forward {} -> {}", store, load), + ); + } users.change_load_to_alias(&mut opt_ctx.ctx.func, load, stored_value); } @@ -250,33 +253,35 @@ pub(super) fn optimize_function( for store in users.stack_store.clone().into_iter() { let potential_loads = users.potential_loads_of_store(&opt_ctx.ctx, store); - #[cfg(debug_assertions)] - for &load in &potential_loads { - clif_comments.add_comment( - store, - format!( - "Potential load from store {} <- {} ({:?}, {:?})", - opt_ctx.ctx.func.dfg.display_inst(load, None), - opt_ctx.ctx.func.dfg.display_inst(store, None), - spatial_overlap(&opt_ctx.ctx.func, store, load), - temporal_order(&opt_ctx.ctx, store, load), - ), - ); + if clif_comments.enabled() { + for &load in &potential_loads { + clif_comments.add_comment( + store, + format!( + "Potential load from store {} <- {} ({:?}, {:?})", + opt_ctx.ctx.func.dfg.display_inst(load, None), + opt_ctx.ctx.func.dfg.display_inst(store, None), + spatial_overlap(&opt_ctx.ctx.func, store, load), + temporal_order(&opt_ctx.ctx, store, load), + ), + ); + } } if potential_loads.is_empty() { // Never loaded; can safely remove all stores and the stack slot. // FIXME also remove stores when there is always a next store before a load. - #[cfg(debug_assertions)] - clif_comments.add_comment( - store, - format!( - "Remove dead stack store {} of {}", - opt_ctx.ctx.func.dfg.display_inst(store, None), - stack_slot.0 - ), - ); + if clif_comments.enabled() { + clif_comments.add_comment( + store, + format!( + "Remove dead stack store {} of {}", + opt_ctx.ctx.func.dfg.display_inst(store, None), + stack_slot.0 + ), + ); + } users.remove_dead_store(&mut opt_ctx.ctx.func, store); } diff --git a/src/pointer.rs b/src/pointer.rs index 88a78f3214d87..31d827f83bfab 100644 --- a/src/pointer.rs +++ b/src/pointer.rs @@ -39,8 +39,7 @@ impl Pointer { Pointer { base: PointerBase::Dangling(align), offset: Offset32::new(0) } } - #[cfg(debug_assertions)] - pub(crate) fn base_and_offset(self) -> (PointerBase, Offset32) { + pub(crate) fn debug_base_and_offset(self) -> (PointerBase, Offset32) { (self.base, self.offset) } diff --git a/src/pretty_clif.rs b/src/pretty_clif.rs index 9c91b92e515b1..d22ea3772eee7 100644 --- a/src/pretty_clif.rs +++ b/src/pretty_clif.rs @@ -69,13 +69,15 @@ use crate::prelude::*; #[derive(Debug)] pub(crate) struct CommentWriter { + enabled: bool, global_comments: Vec, entity_comments: FxHashMap, } impl CommentWriter { pub(crate) fn new<'tcx>(tcx: TyCtxt<'tcx>, instance: Instance<'tcx>) -> Self { - let global_comments = if cfg!(debug_assertions) { + let enabled = should_write_ir(tcx); + let global_comments = if enabled { vec![ format!("symbol {}", tcx.symbol_name(instance).name), format!("instance {:?}", instance), @@ -86,13 +88,17 @@ impl CommentWriter { vec![] }; - CommentWriter { global_comments, entity_comments: FxHashMap::default() } + CommentWriter { enabled, global_comments, entity_comments: FxHashMap::default() } } } -#[cfg(debug_assertions)] impl CommentWriter { + pub(crate) fn enabled(&self) -> bool { + self.enabled + } + pub(crate) fn add_global_comment>(&mut self, comment: S) { + debug_assert!(self.enabled); self.global_comments.push(comment.into()); } @@ -101,6 +107,8 @@ impl CommentWriter { entity: E, comment: S, ) { + debug_assert!(self.enabled); + use std::collections::hash_map::Entry; match self.entity_comments.entry(entity.into()) { Entry::Occupied(mut occ) => { @@ -179,7 +187,6 @@ impl FuncWriter for &'_ CommentWriter { } } -#[cfg(debug_assertions)] impl FunctionCx<'_, '_, '_> { pub(crate) fn add_global_comment>(&mut self, comment: S) { self.clif_comments.add_global_comment(comment); @@ -198,8 +205,8 @@ pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool { tcx.sess.opts.output_types.contains_key(&OutputType::LlvmAssembly) } -pub(crate) fn write_ir_file<'tcx>( - tcx: TyCtxt<'tcx>, +pub(crate) fn write_ir_file( + tcx: TyCtxt<'_>, name: &str, write: impl FnOnce(&mut dyn Write) -> std::io::Result<()>, ) { @@ -217,10 +224,7 @@ pub(crate) fn write_ir_file<'tcx>( let clif_file_name = clif_output_dir.join(name); - let res: std::io::Result<()> = try { - let mut file = std::fs::File::create(clif_file_name)?; - write(&mut file)?; - }; + let res = std::fs::File::create(clif_file_name).and_then(|mut file| write(&mut file)); if let Err(err) = res { tcx.sess.warn(&format!("error writing ir file: {}", err)); } diff --git a/src/trap.rs b/src/trap.rs index bb63d72addf98..1ab0703e981e7 100644 --- a/src/trap.rs +++ b/src/trap.rs @@ -17,8 +17,7 @@ fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) { ) .unwrap(); let puts = fx.cx.module.declare_func_in_func(puts, &mut fx.bcx.func); - #[cfg(debug_assertions)] - { + if fx.clif_comments.enabled() { fx.add_comment(puts, "puts"); } diff --git a/src/value_and_place.rs b/src/value_and_place.rs index cffaf79ded10b..b97d39009847a 100644 --- a/src/value_and_place.rs +++ b/src/value_and_place.rs @@ -2,7 +2,6 @@ use crate::prelude::*; -use cranelift_codegen::entity::EntityRef; use cranelift_codegen::ir::immediates::Offset32; fn codegen_field<'tcx>( @@ -414,7 +413,7 @@ impl<'tcx> CPlace<'tcx> { self, fx: &mut FunctionCx<'_, '_, 'tcx>, from: CValue<'tcx>, - #[cfg_attr(not(debug_assertions), allow(unused_variables))] method: &'static str, + method: &'static str, ) { fn transmute_value<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, @@ -462,8 +461,7 @@ impl<'tcx> CPlace<'tcx> { assert_eq!(self.layout().size, from.layout().size); - #[cfg(debug_assertions)] - { + if fx.clif_comments.enabled() { use cranelift_codegen::cursor::{Cursor, CursorPosition}; let cur_block = match fx.bcx.cursor().position() { CursorPosition::After(block) => block, @@ -707,6 +705,19 @@ pub(crate) fn assert_assignable<'tcx>( } // dyn for<'r> Trait<'r> -> dyn Trait<'_> is allowed } + (&ty::Adt(adt_def_a, substs_a), &ty::Adt(adt_def_b, substs_b)) + if adt_def_a.did == adt_def_b.did => + { + let mut types_a = substs_a.types(); + let mut types_b = substs_b.types(); + loop { + match (types_a.next(), types_b.next()) { + (Some(a), Some(b)) => assert_assignable(fx, a, b), + (None, None) => return, + (Some(_), None) | (None, Some(_)) => panic!("{:#?}/{:#?}", from_ty, to_ty), + } + } + } _ => { assert_eq!( from_ty, to_ty, From 6e799438b72a40e3212805e357201ca8b3f03553 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Mon, 29 Mar 2021 11:18:52 +0200 Subject: [PATCH 02/52] Add an Mmap wrapper to rustc_data_structures This wrapper implements StableAddress and falls back to directly reading the file on wasm32 --- Cargo.lock | 10 ---------- Cargo.toml | 1 - src/metadata.rs | 21 ++++----------------- 3 files changed, 4 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3cb67032aaa24..dc1cd336e1599 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -240,15 +240,6 @@ dependencies = [ "libc", ] -[[package]] -name = "memmap2" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04e3e85b970d650e2ae6d70592474087051c11c54da7f7b4949725c5735fbcc6" -dependencies = [ - "libc", -] - [[package]] name = "object" version = "0.23.0" @@ -319,7 +310,6 @@ dependencies = [ "gimli", "indexmap", "libloading", - "memmap2", "object", "smallvec", "target-lexicon", diff --git a/Cargo.toml b/Cargo.toml index 59542c414fa85..60946ab280858 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,6 @@ ar = { git = "https://github.com/bjorn3/rust-ar.git", branch = "do_not_remove_cg indexmap = "1.0.2" libloading = { version = "0.6.0", optional = true } smallvec = "1.6.1" -memmap2 = "0.2.1" # Uncomment to use local checkout of cranelift #[patch."https://github.com/bytecodealliance/wasmtime/"] diff --git a/src/metadata.rs b/src/metadata.rs index c5189c972cd2e..dbdc8cbad44c4 100644 --- a/src/metadata.rs +++ b/src/metadata.rs @@ -1,11 +1,11 @@ //! Reading and writing of the rustc metadata for rlibs and dylibs use std::fs::File; -use std::ops::Deref; use std::path::Path; use rustc_codegen_ssa::METADATA_FILENAME; -use rustc_data_structures::owning_ref::{OwningRef, StableAddress}; +use rustc_data_structures::memmap::Mmap; +use rustc_data_structures::owning_ref::OwningRef; use rustc_data_structures::rustc_erase_owner; use rustc_data_structures::sync::MetadataRef; use rustc_middle::middle::cstore::{EncodedMetadata, MetadataLoader}; @@ -17,26 +17,13 @@ use crate::backend::WriteMetadata; pub(crate) struct CraneliftMetadataLoader; -struct StableMmap(memmap2::Mmap); - -impl Deref for StableMmap { - type Target = [u8]; - - fn deref(&self) -> &[u8] { - &*self.0 - } -} - -unsafe impl StableAddress for StableMmap {} - fn load_metadata_with( path: &Path, f: impl for<'a> FnOnce(&'a [u8]) -> Result<&'a [u8], String>, ) -> Result { let file = File::open(path).map_err(|e| format!("{:?}", e))?; - let data = unsafe { memmap2::MmapOptions::new().map_copy_read_only(&file) } - .map_err(|e| format!("{:?}", e))?; - let metadata = OwningRef::new(StableMmap(data)).try_map(f)?; + let data = unsafe { Mmap::map(file) }.map_err(|e| format!("{:?}", e))?; + let metadata = OwningRef::new(data).try_map(f)?; return Ok(rustc_erase_owner!(metadata.map_owner_box())); } From afe74d71e4c2f08696ade0de321a45f7442440d8 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Wed, 31 Mar 2021 11:39:03 +0200 Subject: [PATCH 03/52] Build with `-Cpanic=unwind` by default This doesn't enable unwinding as cg_clif doesn't support it yet. It does allow for linking to a cg_llvm compiled libstd.so, which uses `-Cpanic=unwind`. --- build_sysroot/build_sysroot.sh | 2 +- example/alloc_example.rs | 7 ++++++- scripts/tests.sh | 4 ++-- src/bin/cg_clif.rs | 11 ++++++++++- src/bin/cg_clif_build_sysroot.rs | 6 +++++- 5 files changed, 24 insertions(+), 6 deletions(-) diff --git a/build_sysroot/build_sysroot.sh b/build_sysroot/build_sysroot.sh index 0354304e55bf7..cc1cbf1abc250 100755 --- a/build_sysroot/build_sysroot.sh +++ b/build_sysroot/build_sysroot.sh @@ -23,7 +23,7 @@ rm -r target/*/{debug,release}/{build,deps,examples,libsysroot*,native} 2>/dev/n export CARGO_TARGET_DIR=target # Build libs -export RUSTFLAGS="$RUSTFLAGS -Zforce-unstable-if-unmarked -Cpanic=abort" +export RUSTFLAGS="$RUSTFLAGS -Zforce-unstable-if-unmarked" export __CARGO_DEFAULT_LIB_METADATA="cg_clif" if [[ "$1" != "--debug" ]]; then sysroot_channel='release' diff --git a/example/alloc_example.rs b/example/alloc_example.rs index 71e93e87b6c41..b58d90cb5ae4d 100644 --- a/example/alloc_example.rs +++ b/example/alloc_example.rs @@ -1,4 +1,4 @@ -#![feature(start, box_syntax, core_intrinsics, alloc_prelude, alloc_error_handler)] +#![feature(start, box_syntax, core_intrinsics, alloc_prelude, alloc_error_handler, lang_items)] #![no_std] extern crate alloc; @@ -27,6 +27,11 @@ fn alloc_error_handler(_: alloc::alloc::Layout) -> ! { core::intrinsics::abort(); } +#[lang = "eh_personality"] +fn eh_personality() -> ! { + loop {} +} + #[start] fn main(_argc: isize, _argv: *const *const u8) -> isize { let world: Box<&str> = box "Hello World!\0"; diff --git a/scripts/tests.sh b/scripts/tests.sh index 3afcea8f06bd6..05139c185f2cc 100755 --- a/scripts/tests.sh +++ b/scripts/tests.sh @@ -61,11 +61,11 @@ function base_sysroot_tests() { $RUN_WRAPPER ./target/out/std_example arg echo "[AOT] subslice-patterns-const-eval" - $MY_RUSTC example/subslice-patterns-const-eval.rs --crate-type bin -Cpanic=abort --target "$TARGET_TRIPLE" + $MY_RUSTC example/subslice-patterns-const-eval.rs --crate-type bin --target "$TARGET_TRIPLE" $RUN_WRAPPER ./target/out/subslice-patterns-const-eval echo "[AOT] track-caller-attribute" - $MY_RUSTC example/track-caller-attribute.rs --crate-type bin -Cpanic=abort --target "$TARGET_TRIPLE" + $MY_RUSTC example/track-caller-attribute.rs --crate-type bin --target "$TARGET_TRIPLE" $RUN_WRAPPER ./target/out/track-caller-attribute echo "[AOT] mod_bench" diff --git a/src/bin/cg_clif.rs b/src/bin/cg_clif.rs index 983839d48d2d7..0d73834f165d9 100644 --- a/src/bin/cg_clif.rs +++ b/src/bin/cg_clif.rs @@ -24,7 +24,16 @@ impl rustc_driver::Callbacks for CraneliftPassesCallbacks { self.time_passes = config.opts.prints.is_empty() && (config.opts.debugging_opts.time_passes || config.opts.debugging_opts.time); - config.opts.cg.panic = Some(PanicStrategy::Abort); + if config.opts.test { + // Unwinding is not yet supported by cg_clif. `-Cpanic=abort` in combination with + // `-Zpanic-abort-tests` ensures that tests are run in a subprocess. This avoids + // crashing the test driver on panics, thereby allowing it to report the error and + // continue with other tests. + config.opts.cg.panic = Some(PanicStrategy::Abort); + // Avoid `-Cprefer-dynamic` in case of `-Cpanic=abort` as that will cause a dynamically + // linked libstd with `-Cpanic=unwind` to be linked in, which isn't allowed. + config.opts.cg.prefer_dynamic = false; + } config.opts.debugging_opts.panic_abort_tests = true; config.opts.maybe_sysroot = Some(config.opts.maybe_sysroot.clone().unwrap_or_else(|| { std::env::current_exe().unwrap().parent().unwrap().parent().unwrap().to_owned() diff --git a/src/bin/cg_clif_build_sysroot.rs b/src/bin/cg_clif_build_sysroot.rs index e7cd5edbbf654..70e41f9997e35 100644 --- a/src/bin/cg_clif_build_sysroot.rs +++ b/src/bin/cg_clif_build_sysroot.rs @@ -44,7 +44,11 @@ impl rustc_driver::Callbacks for CraneliftPassesCallbacks { return; } - config.opts.cg.panic = Some(PanicStrategy::Abort); + if config.opts.crate_name.as_deref() == Some("panic_abort") { + // panic_abort must always be built with `-Cpanic=abort` + config.opts.cg.panic = Some(PanicStrategy::Abort); + } + config.opts.debugging_opts.panic_abort_tests = true; config.opts.maybe_sysroot = Some(std::env::current_exe().unwrap().parent().unwrap().parent().unwrap().to_owned()); From a793be8ee8895538e99acc2a855d9c4ae145fc78 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Wed, 31 Mar 2021 12:34:01 +0200 Subject: [PATCH 04/52] Remove the stack2reg optimization completely It is broken and needs to be rewritten from scratch --- src/intrinsics/mod.rs | 2 - src/optimize/mod.rs | 14 +- src/optimize/stack2reg.rs | 486 -------------------------------------- 3 files changed, 3 insertions(+), 499 deletions(-) delete mode 100644 src/optimize/stack2reg.rs diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index 39e047a98f9eb..58a24c4193032 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -827,7 +827,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( volatile_load | unaligned_volatile_load, (c ptr) { // Cranelift treats loads as volatile by default - // FIXME ignore during stack2reg optimization // FIXME correctly handle unaligned_volatile_load let inner_layout = fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty); @@ -836,7 +835,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( }; volatile_store | unaligned_volatile_store, (v ptr, c val) { // Cranelift treats stores as volatile by default - // FIXME ignore during stack2reg optimization // FIXME correctly handle unaligned_volatile_store let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout()); dest.write_cvalue(fx, val); diff --git a/src/optimize/mod.rs b/src/optimize/mod.rs index 389f50e797e89..95e8a449fa210 100644 --- a/src/optimize/mod.rs +++ b/src/optimize/mod.rs @@ -4,7 +4,6 @@ use crate::prelude::*; mod code_layout; pub(crate) mod peephole; -mod stack2reg; pub(crate) fn optimize_function<'tcx>( tcx: TyCtxt<'tcx>, @@ -13,18 +12,11 @@ pub(crate) fn optimize_function<'tcx>( cold_blocks: &EntitySet, clif_comments: &mut crate::pretty_clif::CommentWriter, ) { + // FIXME classify optimizations over opt levels once we have more + // The code_layout optimization is very cheap. self::code_layout::optimize_function(ctx, cold_blocks); - if tcx.sess.opts.optimize == rustc_session::config::OptLevel::No { - return; // FIXME classify optimizations over opt levels - } - - // FIXME(#1142) stack2reg miscompiles lewton - if false { - self::stack2reg::optimize_function(ctx, clif_comments); - } - - crate::pretty_clif::write_clif_file(tcx, "stack2reg", None, instance, &ctx, &*clif_comments); + crate::pretty_clif::write_clif_file(tcx, "preopt", None, instance, &ctx, &*clif_comments); crate::base::verify_func(tcx, &*clif_comments, &ctx.func); } diff --git a/src/optimize/stack2reg.rs b/src/optimize/stack2reg.rs deleted file mode 100644 index 8bb02a3e55854..0000000000000 --- a/src/optimize/stack2reg.rs +++ /dev/null @@ -1,486 +0,0 @@ -//! This optimization replaces stack accesses with SSA variables and removes dead stores when possible. -//! -//! # Undefined behaviour -//! -//! This optimization is based on the assumption that stack slots which don't have their address -//! leaked through `stack_addr` are only accessed using `stack_load` and `stack_store` in the -//! function which has the stack slots. This optimization also assumes that stack slot accesses -//! are never out of bounds. If these assumptions are not correct, then this optimization may remove -//! `stack_store` instruction incorrectly, or incorrectly use a previously stored value as the value -//! being loaded by a `stack_load`. - -use std::collections::BTreeMap; -use std::fmt; -use std::ops::Not; - -use rustc_data_structures::fx::FxHashSet; - -use cranelift_codegen::cursor::{Cursor, FuncCursor}; -use cranelift_codegen::ir::immediates::Offset32; -use cranelift_codegen::ir::{InstructionData, Opcode, ValueDef}; - -use crate::prelude::*; - -/// Workaround for `StackSlot` not implementing `Ord`. -#[derive(Copy, Clone, PartialEq, Eq)] -struct OrdStackSlot(StackSlot); - -impl fmt::Debug for OrdStackSlot { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?}", self.0) - } -} - -impl PartialOrd for OrdStackSlot { - fn partial_cmp(&self, rhs: &Self) -> Option { - self.0.as_u32().partial_cmp(&rhs.0.as_u32()) - } -} - -impl Ord for OrdStackSlot { - fn cmp(&self, rhs: &Self) -> std::cmp::Ordering { - self.0.as_u32().cmp(&rhs.0.as_u32()) - } -} - -#[derive(Debug, Default)] -struct StackSlotUsage { - stack_addr: FxHashSet, - stack_load: FxHashSet, - stack_store: FxHashSet, -} - -impl StackSlotUsage { - fn potential_stores_for_load(&self, ctx: &Context, load: Inst) -> Vec { - self.stack_store - .iter() - .cloned() - .filter(|&store| { - match spatial_overlap(&ctx.func, store, load) { - SpatialOverlap::No => false, // Can never be the source of the loaded value. - SpatialOverlap::Partial | SpatialOverlap::Full => true, - } - }) - .filter(|&store| { - match temporal_order(ctx, store, load) { - TemporalOrder::NeverBefore => false, // Can never be the source of the loaded value. - TemporalOrder::MaybeBefore | TemporalOrder::DefinitivelyBefore => true, - } - }) - .collect::>() - } - - fn potential_loads_of_store(&self, ctx: &Context, store: Inst) -> Vec { - self.stack_load - .iter() - .cloned() - .filter(|&load| { - match spatial_overlap(&ctx.func, store, load) { - SpatialOverlap::No => false, // Can never be the source of the loaded value. - SpatialOverlap::Partial | SpatialOverlap::Full => true, - } - }) - .filter(|&load| { - match temporal_order(ctx, store, load) { - TemporalOrder::NeverBefore => false, // Can never be the source of the loaded value. - TemporalOrder::MaybeBefore | TemporalOrder::DefinitivelyBefore => true, - } - }) - .collect::>() - } - - fn remove_unused_stack_addr(func: &mut Function, inst: Inst) { - func.dfg.detach_results(inst); - func.dfg.replace(inst).nop(); - } - - fn remove_unused_load(func: &mut Function, load: Inst) { - func.dfg.detach_results(load); - func.dfg.replace(load).nop(); - } - - fn remove_dead_store(&mut self, func: &mut Function, store: Inst) { - func.dfg.replace(store).nop(); - self.stack_store.remove(&store); - } - - fn change_load_to_alias(&mut self, func: &mut Function, load: Inst, value: Value) { - let loaded_value = func.dfg.inst_results(load)[0]; - let loaded_type = func.dfg.value_type(loaded_value); - - if func.dfg.value_type(value) == loaded_type { - func.dfg.detach_results(load); - func.dfg.replace(load).nop(); - func.dfg.change_to_alias(loaded_value, value); - } else { - func.dfg.replace(load).bitcast(loaded_type, value); - } - - self.stack_load.remove(&load); - } -} - -struct OptimizeContext<'a> { - ctx: &'a mut Context, - stack_slot_usage_map: BTreeMap, -} - -impl<'a> OptimizeContext<'a> { - fn for_context(ctx: &'a mut Context) -> Self { - ctx.flowgraph(); // Compute cfg and domtree. - - // Record all stack_addr, stack_load and stack_store instructions. - let mut stack_slot_usage_map = BTreeMap::::new(); - - let mut cursor = FuncCursor::new(&mut ctx.func); - while let Some(_block) = cursor.next_block() { - while let Some(inst) = cursor.next_inst() { - match cursor.func.dfg[inst] { - InstructionData::StackLoad { - opcode: Opcode::StackAddr, - stack_slot, - offset: _, - } => { - stack_slot_usage_map - .entry(OrdStackSlot(stack_slot)) - .or_insert_with(StackSlotUsage::default) - .stack_addr - .insert(inst); - } - InstructionData::StackLoad { - opcode: Opcode::StackLoad, - stack_slot, - offset: _, - } => { - stack_slot_usage_map - .entry(OrdStackSlot(stack_slot)) - .or_insert_with(StackSlotUsage::default) - .stack_load - .insert(inst); - } - InstructionData::StackStore { - opcode: Opcode::StackStore, - arg: _, - stack_slot, - offset: _, - } => { - stack_slot_usage_map - .entry(OrdStackSlot(stack_slot)) - .or_insert_with(StackSlotUsage::default) - .stack_store - .insert(inst); - } - _ => {} - } - } - } - - OptimizeContext { ctx, stack_slot_usage_map } - } -} - -pub(super) fn optimize_function( - ctx: &mut Context, - clif_comments: &mut crate::pretty_clif::CommentWriter, -) { - combine_stack_addr_with_load_store(&mut ctx.func); - - let mut opt_ctx = OptimizeContext::for_context(ctx); - - // FIXME Repeat following instructions until fixpoint. - - remove_unused_stack_addr_and_stack_load(&mut opt_ctx); - - if clif_comments.enabled() { - for (&OrdStackSlot(stack_slot), usage) in &opt_ctx.stack_slot_usage_map { - clif_comments.add_comment(stack_slot, format!("used by: {:?}", usage)); - } - } - - for (stack_slot, users) in opt_ctx.stack_slot_usage_map.iter_mut() { - if users.stack_addr.is_empty().not() { - // Stack addr leaked; there may be unknown loads and stores. - // FIXME use stacked borrows to optimize - continue; - } - - for load in users.stack_load.clone().into_iter() { - let potential_stores = users.potential_stores_for_load(&opt_ctx.ctx, load); - - if clif_comments.enabled() { - for &store in &potential_stores { - clif_comments.add_comment( - load, - format!( - "Potential store -> load forwarding {} -> {} ({:?}, {:?})", - opt_ctx.ctx.func.dfg.display_inst(store, None), - opt_ctx.ctx.func.dfg.display_inst(load, None), - spatial_overlap(&opt_ctx.ctx.func, store, load), - temporal_order(&opt_ctx.ctx, store, load), - ), - ); - } - } - - match *potential_stores { - [] => { - if clif_comments.enabled() { - clif_comments - .add_comment(load, "[BUG?] Reading uninitialized memory".to_string()); - } - } - [store] - if spatial_overlap(&opt_ctx.ctx.func, store, load) == SpatialOverlap::Full - && temporal_order(&opt_ctx.ctx, store, load) - == TemporalOrder::DefinitivelyBefore => - { - // Only one store could have been the origin of the value. - let stored_value = opt_ctx.ctx.func.dfg.inst_args(store)[0]; - - if clif_comments.enabled() { - clif_comments.add_comment( - load, - format!("Store to load forward {} -> {}", store, load), - ); - } - - users.change_load_to_alias(&mut opt_ctx.ctx.func, load, stored_value); - } - _ => {} // FIXME implement this - } - } - - for store in users.stack_store.clone().into_iter() { - let potential_loads = users.potential_loads_of_store(&opt_ctx.ctx, store); - - if clif_comments.enabled() { - for &load in &potential_loads { - clif_comments.add_comment( - store, - format!( - "Potential load from store {} <- {} ({:?}, {:?})", - opt_ctx.ctx.func.dfg.display_inst(load, None), - opt_ctx.ctx.func.dfg.display_inst(store, None), - spatial_overlap(&opt_ctx.ctx.func, store, load), - temporal_order(&opt_ctx.ctx, store, load), - ), - ); - } - } - - if potential_loads.is_empty() { - // Never loaded; can safely remove all stores and the stack slot. - // FIXME also remove stores when there is always a next store before a load. - - if clif_comments.enabled() { - clif_comments.add_comment( - store, - format!( - "Remove dead stack store {} of {}", - opt_ctx.ctx.func.dfg.display_inst(store, None), - stack_slot.0 - ), - ); - } - - users.remove_dead_store(&mut opt_ctx.ctx.func, store); - } - } - - if users.stack_store.is_empty() && users.stack_load.is_empty() { - opt_ctx.ctx.func.stack_slots[stack_slot.0].size = 0; - } - } -} - -fn combine_stack_addr_with_load_store(func: &mut Function) { - // Turn load and store into stack_load and stack_store when possible. - let mut cursor = FuncCursor::new(func); - while let Some(_block) = cursor.next_block() { - while let Some(inst) = cursor.next_inst() { - match cursor.func.dfg[inst] { - InstructionData::Load { opcode: Opcode::Load, arg: addr, flags: _, offset } => { - if cursor.func.dfg.ctrl_typevar(inst) == types::I128 - || cursor.func.dfg.ctrl_typevar(inst).is_vector() - { - continue; // WORKAROUD: stack_load.i128 not yet implemented - } - if let Some((stack_slot, stack_addr_offset)) = - try_get_stack_slot_and_offset_for_addr(cursor.func, addr) - { - if let Some(combined_offset) = offset.try_add_i64(stack_addr_offset.into()) - { - let ty = cursor.func.dfg.ctrl_typevar(inst); - cursor.func.dfg.replace(inst).stack_load( - ty, - stack_slot, - combined_offset, - ); - } - } - } - InstructionData::Store { - opcode: Opcode::Store, - args: [value, addr], - flags: _, - offset, - } => { - if cursor.func.dfg.ctrl_typevar(inst) == types::I128 - || cursor.func.dfg.ctrl_typevar(inst).is_vector() - { - continue; // WORKAROUND: stack_store.i128 not yet implemented - } - if let Some((stack_slot, stack_addr_offset)) = - try_get_stack_slot_and_offset_for_addr(cursor.func, addr) - { - if let Some(combined_offset) = offset.try_add_i64(stack_addr_offset.into()) - { - cursor.func.dfg.replace(inst).stack_store( - value, - stack_slot, - combined_offset, - ); - } - } - } - _ => {} - } - } - } -} - -fn remove_unused_stack_addr_and_stack_load(opt_ctx: &mut OptimizeContext<'_>) { - // FIXME incrementally rebuild on each call? - let mut stack_addr_load_insts_users = FxHashMap::>::default(); - - let mut cursor = FuncCursor::new(&mut opt_ctx.ctx.func); - while let Some(_block) = cursor.next_block() { - while let Some(inst) = cursor.next_inst() { - for &arg in cursor.func.dfg.inst_args(inst) { - if let ValueDef::Result(arg_origin, 0) = cursor.func.dfg.value_def(arg) { - match cursor.func.dfg[arg_origin].opcode() { - Opcode::StackAddr | Opcode::StackLoad => { - stack_addr_load_insts_users - .entry(arg_origin) - .or_insert_with(FxHashSet::default) - .insert(inst); - } - _ => {} - } - } - } - } - } - - #[cfg(debug_assertions)] - for inst in stack_addr_load_insts_users.keys() { - let mut is_recorded_stack_addr_or_stack_load = false; - for stack_slot_users in opt_ctx.stack_slot_usage_map.values() { - is_recorded_stack_addr_or_stack_load |= stack_slot_users.stack_addr.contains(inst) - || stack_slot_users.stack_load.contains(inst); - } - assert!(is_recorded_stack_addr_or_stack_load); - } - - // Replace all unused stack_addr and stack_load instructions with nop. - let mut func = &mut opt_ctx.ctx.func; - - for stack_slot_users in opt_ctx.stack_slot_usage_map.values_mut() { - stack_slot_users - .stack_addr - .drain_filter(|inst| { - stack_addr_load_insts_users.get(inst).map(|users| users.is_empty()).unwrap_or(true) - }) - .for_each(|inst| StackSlotUsage::remove_unused_stack_addr(&mut func, inst)); - - stack_slot_users - .stack_load - .drain_filter(|inst| { - stack_addr_load_insts_users.get(inst).map(|users| users.is_empty()).unwrap_or(true) - }) - .for_each(|inst| StackSlotUsage::remove_unused_load(&mut func, inst)); - } -} - -fn try_get_stack_slot_and_offset_for_addr( - func: &Function, - addr: Value, -) -> Option<(StackSlot, Offset32)> { - if let ValueDef::Result(addr_inst, 0) = func.dfg.value_def(addr) { - if let InstructionData::StackLoad { opcode: Opcode::StackAddr, stack_slot, offset } = - func.dfg[addr_inst] - { - return Some((stack_slot, offset)); - } - } - None -} - -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -enum SpatialOverlap { - No, - Partial, - Full, -} - -fn spatial_overlap(func: &Function, src: Inst, dest: Inst) -> SpatialOverlap { - fn inst_info(func: &Function, inst: Inst) -> (StackSlot, Offset32, u32) { - match func.dfg[inst] { - InstructionData::StackLoad { opcode: Opcode::StackAddr, stack_slot, offset } - | InstructionData::StackLoad { opcode: Opcode::StackLoad, stack_slot, offset } - | InstructionData::StackStore { - opcode: Opcode::StackStore, - stack_slot, - offset, - arg: _, - } => (stack_slot, offset, func.dfg.ctrl_typevar(inst).bytes()), - _ => unreachable!("{:?}", func.dfg[inst]), - } - } - - debug_assert_ne!(src, dest); - - let (src_ss, src_offset, src_size) = inst_info(func, src); - let (dest_ss, dest_offset, dest_size) = inst_info(func, dest); - - if src_ss != dest_ss { - return SpatialOverlap::No; - } - - if src_offset == dest_offset && src_size == dest_size { - return SpatialOverlap::Full; - } - - let src_end: i64 = src_offset.try_add_i64(i64::from(src_size)).unwrap().into(); - let dest_end: i64 = dest_offset.try_add_i64(i64::from(dest_size)).unwrap().into(); - if src_end <= dest_offset.into() || dest_end <= src_offset.into() { - return SpatialOverlap::No; - } - - SpatialOverlap::Partial -} - -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -enum TemporalOrder { - /// `src` will never be executed before `dest`. - NeverBefore, - - /// `src` may be executed before `dest`. - MaybeBefore, - - /// `src` will always be executed before `dest`. - /// There may still be other instructions in between. - DefinitivelyBefore, -} - -fn temporal_order(ctx: &Context, src: Inst, dest: Inst) -> TemporalOrder { - debug_assert_ne!(src, dest); - - if ctx.domtree.dominates(src, dest, &ctx.func.layout) { - TemporalOrder::DefinitivelyBefore - } else if ctx.domtree.dominates(src, dest, &ctx.func.layout) { - TemporalOrder::NeverBefore - } else { - TemporalOrder::MaybeBefore - } -} From 2ceb5277284e7a485504d1f7d165a385cbfd797b Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Wed, 31 Mar 2021 12:42:30 +0200 Subject: [PATCH 05/52] Remove the cold block optimization It isn't effective with the new backend framework --- src/abi/mod.rs | 3 +-- src/base.rs | 11 +++-------- src/common.rs | 3 --- src/lib.rs | 1 - src/optimize/code_layout.rs | 34 ---------------------------------- src/optimize/mod.rs | 5 ----- 6 files changed, 4 insertions(+), 53 deletions(-) delete mode 100644 src/optimize/code_layout.rs diff --git a/src/abi/mod.rs b/src/abi/mod.rs index 0e7829eaa26ac..f898348f97bc0 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -295,7 +295,6 @@ pub(crate) fn codegen_fn_prelude<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, start_ pub(crate) fn codegen_terminator_call<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, span: Span, - current_block: Block, func: &Operand<'tcx>, args: &[Operand<'tcx>], destination: Option<(Place<'tcx>, BasicBlock)>, @@ -357,7 +356,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( .map(|inst| fx.tcx.codegen_fn_attrs(inst.def_id()).flags.contains(CodegenFnAttrFlags::COLD)) .unwrap_or(false); if is_cold { - fx.cold_blocks.insert(current_block); + // FIXME Mark current_block block as cold once Cranelift supports it } // Unpack arguments tuple for closures diff --git a/src/base.rs b/src/base.rs index b34a29c25b92e..c4e0c9676f693 100644 --- a/src/base.rs +++ b/src/base.rs @@ -55,7 +55,6 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In block_map, local_map: IndexVec::with_capacity(mir.local_decls.len()), caller_location: None, // set by `codegen_fn_prelude` - cold_blocks: EntitySet::new(), clif_comments, source_info_set: indexmap::IndexSet::new(), @@ -90,7 +89,6 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In let mut clif_comments = fx.clif_comments; let source_info_set = fx.source_info_set; let local_map = fx.local_map; - let cold_blocks = fx.cold_blocks; // Store function in context let context = &mut cx.cached_context; @@ -107,7 +105,6 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In tcx, instance, context, - &cold_blocks, &mut clif_comments, ); }); @@ -205,9 +202,8 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) { // Unwinding after panicking is not supported continue; - // FIXME once unwinding is supported uncomment next lines - // // Unwinding is unlikely to happen, so mark cleanup block's as cold. - // fx.cold_blocks.insert(block); + // FIXME Once unwinding is supported and Cranelift supports marking blocks as cold, do + // so for cleanup blocks. } fx.bcx.ins().nop(); @@ -262,7 +258,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) { let target = fx.get_block(*target); let failure = fx.bcx.create_block(); - fx.cold_blocks.insert(failure); + // FIXME Mark failure block as cold once Cranelift supports it if *expected { fx.bcx.ins().brz(cond, failure, &[]); @@ -358,7 +354,6 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) { crate::abi::codegen_terminator_call( fx, *fn_span, - block, func, args, *destination, diff --git a/src/common.rs b/src/common.rs index b5874f62535ca..fe6cf9a3b01e5 100644 --- a/src/common.rs +++ b/src/common.rs @@ -242,9 +242,6 @@ pub(crate) struct FunctionCx<'m, 'clif, 'tcx> { /// When `#[track_caller]` is used, the implicit caller location is stored in this variable. pub(crate) caller_location: Option>, - /// See [`crate::optimize::code_layout`] for more information. - pub(crate) cold_blocks: EntitySet, - pub(crate) clif_comments: crate::pretty_clif::CommentWriter, pub(crate) source_info_set: indexmap::IndexSet, diff --git a/src/lib.rs b/src/lib.rs index 720d2a1253445..8f4dede33ef44 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -87,7 +87,6 @@ mod prelude { pub(crate) use rustc_index::vec::Idx; - pub(crate) use cranelift_codegen::entity::EntitySet; pub(crate) use cranelift_codegen::ir::condcodes::{FloatCC, IntCC}; pub(crate) use cranelift_codegen::ir::function::Function; pub(crate) use cranelift_codegen::ir::types; diff --git a/src/optimize/code_layout.rs b/src/optimize/code_layout.rs deleted file mode 100644 index ca9ff15ec10ff..0000000000000 --- a/src/optimize/code_layout.rs +++ /dev/null @@ -1,34 +0,0 @@ -//! This optimization moves cold code to the end of the function. -//! -//! Some code is executed much less often than other code. For example panicking or the -//! landingpads for unwinding. By moving this cold code to the end of the function the average -//! amount of jumps is reduced and the code locality is improved. -//! -//! # Undefined behaviour -//! -//! This optimization doesn't assume anything that isn't already assumed by Cranelift itself. - -use crate::prelude::*; - -pub(super) fn optimize_function(ctx: &mut Context, cold_blocks: &EntitySet) { - // FIXME Move the block in place instead of remove and append once - // bytecodealliance/cranelift#1339 is implemented. - - let mut block_insts = FxHashMap::default(); - for block in cold_blocks.keys().filter(|&block| cold_blocks.contains(block)) { - let insts = ctx.func.layout.block_insts(block).collect::>(); - for &inst in &insts { - ctx.func.layout.remove_inst(inst); - } - block_insts.insert(block, insts); - ctx.func.layout.remove_block(block); - } - - // And then append them at the back again. - for block in cold_blocks.keys().filter(|&block| cold_blocks.contains(block)) { - ctx.func.layout.append_block(block); - for inst in block_insts.remove(&block).unwrap() { - ctx.func.layout.append_inst(inst, block); - } - } -} diff --git a/src/optimize/mod.rs b/src/optimize/mod.rs index 95e8a449fa210..137fb5f77313c 100644 --- a/src/optimize/mod.rs +++ b/src/optimize/mod.rs @@ -2,21 +2,16 @@ use crate::prelude::*; -mod code_layout; pub(crate) mod peephole; pub(crate) fn optimize_function<'tcx>( tcx: TyCtxt<'tcx>, instance: Instance<'tcx>, ctx: &mut Context, - cold_blocks: &EntitySet, clif_comments: &mut crate::pretty_clif::CommentWriter, ) { // FIXME classify optimizations over opt levels once we have more - // The code_layout optimization is very cheap. - self::code_layout::optimize_function(ctx, cold_blocks); - crate::pretty_clif::write_clif_file(tcx, "preopt", None, instance, &ctx, &*clif_comments); crate::base::verify_func(tcx, &*clif_comments, &ctx.func); } From ab425a4bca1d6df234596a1621c4cabcc73bb272 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Wed, 31 Mar 2021 13:47:10 +0200 Subject: [PATCH 06/52] Revert "Build with `-Cpanic=unwind` by default" This reverts commit afe74d71e4c2f08696ade0de321a45f7442440d8. It shouldn't have been pushed to master as it isn't ready yet. --- build_sysroot/build_sysroot.sh | 2 +- example/alloc_example.rs | 7 +------ scripts/tests.sh | 4 ++-- src/bin/cg_clif.rs | 11 +---------- src/bin/cg_clif_build_sysroot.rs | 6 +----- 5 files changed, 6 insertions(+), 24 deletions(-) diff --git a/build_sysroot/build_sysroot.sh b/build_sysroot/build_sysroot.sh index cc1cbf1abc250..0354304e55bf7 100755 --- a/build_sysroot/build_sysroot.sh +++ b/build_sysroot/build_sysroot.sh @@ -23,7 +23,7 @@ rm -r target/*/{debug,release}/{build,deps,examples,libsysroot*,native} 2>/dev/n export CARGO_TARGET_DIR=target # Build libs -export RUSTFLAGS="$RUSTFLAGS -Zforce-unstable-if-unmarked" +export RUSTFLAGS="$RUSTFLAGS -Zforce-unstable-if-unmarked -Cpanic=abort" export __CARGO_DEFAULT_LIB_METADATA="cg_clif" if [[ "$1" != "--debug" ]]; then sysroot_channel='release' diff --git a/example/alloc_example.rs b/example/alloc_example.rs index b58d90cb5ae4d..71e93e87b6c41 100644 --- a/example/alloc_example.rs +++ b/example/alloc_example.rs @@ -1,4 +1,4 @@ -#![feature(start, box_syntax, core_intrinsics, alloc_prelude, alloc_error_handler, lang_items)] +#![feature(start, box_syntax, core_intrinsics, alloc_prelude, alloc_error_handler)] #![no_std] extern crate alloc; @@ -27,11 +27,6 @@ fn alloc_error_handler(_: alloc::alloc::Layout) -> ! { core::intrinsics::abort(); } -#[lang = "eh_personality"] -fn eh_personality() -> ! { - loop {} -} - #[start] fn main(_argc: isize, _argv: *const *const u8) -> isize { let world: Box<&str> = box "Hello World!\0"; diff --git a/scripts/tests.sh b/scripts/tests.sh index 05139c185f2cc..3afcea8f06bd6 100755 --- a/scripts/tests.sh +++ b/scripts/tests.sh @@ -61,11 +61,11 @@ function base_sysroot_tests() { $RUN_WRAPPER ./target/out/std_example arg echo "[AOT] subslice-patterns-const-eval" - $MY_RUSTC example/subslice-patterns-const-eval.rs --crate-type bin --target "$TARGET_TRIPLE" + $MY_RUSTC example/subslice-patterns-const-eval.rs --crate-type bin -Cpanic=abort --target "$TARGET_TRIPLE" $RUN_WRAPPER ./target/out/subslice-patterns-const-eval echo "[AOT] track-caller-attribute" - $MY_RUSTC example/track-caller-attribute.rs --crate-type bin --target "$TARGET_TRIPLE" + $MY_RUSTC example/track-caller-attribute.rs --crate-type bin -Cpanic=abort --target "$TARGET_TRIPLE" $RUN_WRAPPER ./target/out/track-caller-attribute echo "[AOT] mod_bench" diff --git a/src/bin/cg_clif.rs b/src/bin/cg_clif.rs index 0d73834f165d9..983839d48d2d7 100644 --- a/src/bin/cg_clif.rs +++ b/src/bin/cg_clif.rs @@ -24,16 +24,7 @@ impl rustc_driver::Callbacks for CraneliftPassesCallbacks { self.time_passes = config.opts.prints.is_empty() && (config.opts.debugging_opts.time_passes || config.opts.debugging_opts.time); - if config.opts.test { - // Unwinding is not yet supported by cg_clif. `-Cpanic=abort` in combination with - // `-Zpanic-abort-tests` ensures that tests are run in a subprocess. This avoids - // crashing the test driver on panics, thereby allowing it to report the error and - // continue with other tests. - config.opts.cg.panic = Some(PanicStrategy::Abort); - // Avoid `-Cprefer-dynamic` in case of `-Cpanic=abort` as that will cause a dynamically - // linked libstd with `-Cpanic=unwind` to be linked in, which isn't allowed. - config.opts.cg.prefer_dynamic = false; - } + config.opts.cg.panic = Some(PanicStrategy::Abort); config.opts.debugging_opts.panic_abort_tests = true; config.opts.maybe_sysroot = Some(config.opts.maybe_sysroot.clone().unwrap_or_else(|| { std::env::current_exe().unwrap().parent().unwrap().parent().unwrap().to_owned() diff --git a/src/bin/cg_clif_build_sysroot.rs b/src/bin/cg_clif_build_sysroot.rs index 70e41f9997e35..e7cd5edbbf654 100644 --- a/src/bin/cg_clif_build_sysroot.rs +++ b/src/bin/cg_clif_build_sysroot.rs @@ -44,11 +44,7 @@ impl rustc_driver::Callbacks for CraneliftPassesCallbacks { return; } - if config.opts.crate_name.as_deref() == Some("panic_abort") { - // panic_abort must always be built with `-Cpanic=abort` - config.opts.cg.panic = Some(PanicStrategy::Abort); - } - + config.opts.cg.panic = Some(PanicStrategy::Abort); config.opts.debugging_opts.panic_abort_tests = true; config.opts.maybe_sysroot = Some(std::env::current_exe().unwrap().parent().unwrap().parent().unwrap().to_owned()); From 8114d933bde512f02db2d755f837cf97a6b1fff2 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Tue, 6 Apr 2021 12:27:45 +0200 Subject: [PATCH 07/52] Rustup to rustc 1.53.0-nightly (d32238532 2021-04-05) --- build_sysroot/Cargo.lock | 4 ++-- rust-toolchain | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build_sysroot/Cargo.lock b/build_sysroot/Cargo.lock index 09c5d7590ab86..8ffed13c843cf 100644 --- a/build_sysroot/Cargo.lock +++ b/build_sysroot/Cargo.lock @@ -132,9 +132,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8916b1f6ca17130ec6568feccee27c156ad12037880833a3b842a823236502e7" +checksum = "56d855069fafbb9b344c0f962150cd2c1187975cb1c22c1522c240d8c4986714" dependencies = [ "rustc-std-workspace-core", ] diff --git a/rust-toolchain b/rust-toolchain index 2917fc7ee396d..19c040f31cd5c 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2021-03-29" +channel = "nightly-2021-04-06" components = ["rust-src", "rustc-dev", "llvm-tools-preview"] From 7a1cf889c72c380e3fa1c77c125d685e94b3f551 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Tue, 6 Apr 2021 13:14:45 +0200 Subject: [PATCH 08/52] Fix rustc test suite by ignoring rustdoc test --- scripts/test_rustc_tests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/test_rustc_tests.sh b/scripts/test_rustc_tests.sh index fbc3feceec7ac..32ecf4d2e98dd 100755 --- a/scripts/test_rustc_tests.sh +++ b/scripts/test_rustc_tests.sh @@ -61,6 +61,8 @@ rm src/test/incremental/issue-49482.rs # same rm src/test/incremental/issue-54059.rs # same rm src/test/incremental/lto.rs # requires lto +rm src/test/run-make/emit-shared-files # requires the rustdoc executable in build/bin/ + rm src/test/pretty/asm.rs # inline asm rm src/test/pretty/raw-str-nonexpr.rs # same From 0c1725f8bb0ab646b432e91c70d65501db76e5f5 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Tue, 6 Apr 2021 13:59:54 +0200 Subject: [PATCH 09/52] Fixup previous commit --- scripts/test_rustc_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test_rustc_tests.sh b/scripts/test_rustc_tests.sh index 32ecf4d2e98dd..e4b73755248c2 100755 --- a/scripts/test_rustc_tests.sh +++ b/scripts/test_rustc_tests.sh @@ -61,7 +61,7 @@ rm src/test/incremental/issue-49482.rs # same rm src/test/incremental/issue-54059.rs # same rm src/test/incremental/lto.rs # requires lto -rm src/test/run-make/emit-shared-files # requires the rustdoc executable in build/bin/ +rm -r src/test/run-make/emit-shared-files # requires the rustdoc executable in build/bin/ rm src/test/pretty/asm.rs # inline asm rm src/test/pretty/raw-str-nonexpr.rs # same From e1a2f0f351695963d0df4453a7baae23ec3b5991 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Wed, 7 Apr 2021 10:16:01 +0200 Subject: [PATCH 10/52] Rustup to rustc 1.53.0-nightly (c051c5ddd 2021-04-06) --- build_sysroot/Cargo.lock | 4 ++-- rust-toolchain | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build_sysroot/Cargo.lock b/build_sysroot/Cargo.lock index 8ffed13c843cf..d96b3e3f5ac4f 100644 --- a/build_sysroot/Cargo.lock +++ b/build_sysroot/Cargo.lock @@ -132,9 +132,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.92" +version = "0.2.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56d855069fafbb9b344c0f962150cd2c1187975cb1c22c1522c240d8c4986714" +checksum = "9385f66bf6105b241aa65a61cb923ef20efc665cb9f9bb50ac2f0c4b7f378d41" dependencies = [ "rustc-std-workspace-core", ] diff --git a/rust-toolchain b/rust-toolchain index 19c040f31cd5c..fe2631bbf5a62 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2021-04-06" +channel = "nightly-2021-04-07" components = ["rust-src", "rustc-dev", "llvm-tools-preview"] From 3d7d08f338354f25cf8f3384376bd5db0c3104f9 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Wed, 7 Apr 2021 10:54:43 +0200 Subject: [PATCH 11/52] Simplify write_metadata --- src/metadata.rs | 44 +++----------------------------------------- 1 file changed, 3 insertions(+), 41 deletions(-) diff --git a/src/metadata.rs b/src/metadata.rs index dbdc8cbad44c4..7bd9ebbb611c3 100644 --- a/src/metadata.rs +++ b/src/metadata.rs @@ -8,9 +8,8 @@ use rustc_data_structures::memmap::Mmap; use rustc_data_structures::owning_ref::OwningRef; use rustc_data_structures::rustc_erase_owner; use rustc_data_structures::sync::MetadataRef; -use rustc_middle::middle::cstore::{EncodedMetadata, MetadataLoader}; +use rustc_middle::middle::cstore::MetadataLoader; use rustc_middle::ty::TyCtxt; -use rustc_session::config; use rustc_target::spec::Target; use crate::backend::WriteMetadata; @@ -58,54 +57,17 @@ impl MetadataLoader for CraneliftMetadataLoader { } // Adapted from https://github.com/rust-lang/rust/blob/da573206f87b5510de4b0ee1a9c044127e409bd3/src/librustc_codegen_llvm/base.rs#L47-L112 -pub(crate) fn write_metadata( - tcx: TyCtxt<'_>, - product: &mut P, -) -> EncodedMetadata { +pub(crate) fn write_metadata(tcx: TyCtxt<'_>, object: &mut O) { use snap::write::FrameEncoder; use std::io::Write; - #[derive(PartialEq, Eq, PartialOrd, Ord)] - enum MetadataKind { - None, - Uncompressed, - Compressed, - } - - let kind = tcx - .sess - .crate_types() - .iter() - .map(|ty| match *ty { - config::CrateType::Executable - | config::CrateType::Staticlib - | config::CrateType::Cdylib => MetadataKind::None, - - config::CrateType::Rlib => MetadataKind::Uncompressed, - - config::CrateType::Dylib | config::CrateType::ProcMacro => MetadataKind::Compressed, - }) - .max() - .unwrap_or(MetadataKind::None); - - if kind == MetadataKind::None { - return EncodedMetadata::new(); - } - let metadata = tcx.encode_metadata(); - if kind == MetadataKind::Uncompressed { - return metadata; - } - - assert!(kind == MetadataKind::Compressed); let mut compressed = tcx.metadata_encoding_version(); FrameEncoder::new(&mut compressed).write_all(&metadata.raw_data).unwrap(); - product.add_rustc_section( + object.add_rustc_section( rustc_middle::middle::exported_symbols::metadata_symbol_name(tcx), compressed, tcx.sess.target.is_like_osx, ); - - metadata } From 1ee0aa9416b9da220d7370007c5e3c402d6714b0 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Wed, 7 Apr 2021 11:02:54 +0200 Subject: [PATCH 12/52] Move BackendConfig to config.rs --- src/config.rs | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 50 ++------------------------------------------------ 2 files changed, 51 insertions(+), 48 deletions(-) create mode 100644 src/config.rs diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 0000000000000..9b3259b622ea2 --- /dev/null +++ b/src/config.rs @@ -0,0 +1,49 @@ +use std::str::FromStr; + +#[derive(Copy, Clone, Debug)] +pub enum CodegenMode { + Aot, + Jit, + JitLazy, +} + +impl Default for CodegenMode { + fn default() -> Self { + CodegenMode::Aot + } +} + +impl FromStr for CodegenMode { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "aot" => Ok(CodegenMode::Aot), + "jit" => Ok(CodegenMode::Jit), + "jit-lazy" => Ok(CodegenMode::JitLazy), + _ => Err(format!("Unknown codegen mode `{}`", s)), + } + } +} + +#[derive(Copy, Clone, Debug, Default)] +pub struct BackendConfig { + pub codegen_mode: CodegenMode, +} + +impl BackendConfig { + pub fn from_opts(opts: &[String]) -> Result { + let mut config = BackendConfig::default(); + for opt in opts { + if let Some((name, value)) = opt.split_once('=') { + match name { + "mode" => config.codegen_mode = value.parse()?, + _ => return Err(format!("Unknown option `{}`", name)), + } + } else { + return Err(format!("Invalid option `{}`", opt)); + } + } + Ok(config) + } +} diff --git a/src/lib.rs b/src/lib.rs index 8f4dede33ef44..77c6247e15766 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,7 +23,6 @@ extern crate rustc_target; extern crate rustc_driver; use std::any::Any; -use std::str::FromStr; use rustc_codegen_ssa::traits::CodegenBackend; use rustc_codegen_ssa::CodegenResults; @@ -36,6 +35,7 @@ use rustc_session::Session; use cranelift_codegen::settings::{self, Configurable}; +pub use crate::config::*; use crate::constant::ConstantCx; use crate::prelude::*; @@ -49,6 +49,7 @@ mod cast; mod codegen_i128; mod common; mod compiler_builtins; +mod config; mod constant; mod debuginfo; mod discriminant; @@ -161,53 +162,6 @@ impl<'m, 'tcx> CodegenCx<'m, 'tcx> { } } -#[derive(Copy, Clone, Debug)] -pub enum CodegenMode { - Aot, - Jit, - JitLazy, -} - -impl Default for CodegenMode { - fn default() -> Self { - CodegenMode::Aot - } -} - -impl FromStr for CodegenMode { - type Err = String; - - fn from_str(s: &str) -> Result { - match s { - "aot" => Ok(CodegenMode::Aot), - "jit" => Ok(CodegenMode::Jit), - "jit-lazy" => Ok(CodegenMode::JitLazy), - _ => Err(format!("Unknown codegen mode `{}`", s)), - } - } -} - -#[derive(Copy, Clone, Debug, Default)] -pub struct BackendConfig { - pub codegen_mode: CodegenMode, -} - -impl BackendConfig { - fn from_opts(opts: &[String]) -> Result { - let mut config = BackendConfig::default(); - for opt in opts { - if let Some((name, value)) = opt.split_once('=') { - match name { - "mode" => config.codegen_mode = value.parse()?, - _ => return Err(format!("Unknown option `{}`", name)), - } - } else { - return Err(format!("Invalid option `{}`", opt)); - } - } - Ok(config) - } -} pub struct CraneliftCodegenBackend { pub config: Option, From 53bfc6729ae4a53c8092473452de7a272224df5d Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Wed, 7 Apr 2021 11:52:11 +0200 Subject: [PATCH 13/52] Centralize all configuration into config.rs --- Readme.md | 5 ++-- docs/env_vars.md | 15 ---------- scripts/ext_config.sh | 2 +- src/backend.rs | 10 +++---- src/config.rs | 66 ++++++++++++++++++++++++++++++++++++++----- src/driver/aot.rs | 27 +++++++++--------- src/driver/jit.rs | 11 ++++---- src/driver/mod.rs | 4 +-- src/lib.rs | 9 +++--- src/pretty_clif.rs | 2 +- 10 files changed, 94 insertions(+), 57 deletions(-) delete mode 100644 docs/env_vars.md diff --git a/Readme.md b/Readme.md index ffe1d9a1e6580..08f9373be6262 100644 --- a/Readme.md +++ b/Readme.md @@ -44,9 +44,10 @@ This will build your project with rustc_codegen_cranelift instead of the usual L For additional ways to use rustc_codegen_cranelift like the JIT mode see [usage.md](docs/usage.md). -## Env vars +## Configuration -See [env_vars.md](docs/env_vars.md) for all env vars used by rustc_codegen_cranelift. +See the documentation on the `BackendConfig` struct in [config.rs](src/config.rs) for all +configuration options. ## Not yet supported diff --git a/docs/env_vars.md b/docs/env_vars.md deleted file mode 100644 index f7fde1b4f3a87..0000000000000 --- a/docs/env_vars.md +++ /dev/null @@ -1,15 +0,0 @@ -# List of env vars recognized by cg_clif - -
-
CG_CLIF_JIT_ARGS
-
When JIT mode is enable pass these arguments to the program.
-
CG_CLIF_INCR_CACHE_DISABLED
-
Don't cache object files in the incremental cache. Useful during development of cg_clif - to make it possible to use incremental mode for all analyses performed by rustc without caching - object files when their content should have been changed by a change to cg_clif.
-
CG_CLIF_DISPLAY_CG_TIME
-
If "1", display the time it took to perform codegen for a crate.
-
CG_CLIF_ENABLE_VERIFIER
-
Enable the Cranelift ir verifier for all compilation passes. If not set it will only run once - before passing the clif ir to Cranelift for compilation. -
diff --git a/scripts/ext_config.sh b/scripts/ext_config.sh index 7971f620df14b..3f98d77d76cad 100644 --- a/scripts/ext_config.sh +++ b/scripts/ext_config.sh @@ -5,7 +5,7 @@ set -e export CG_CLIF_DISPLAY_CG_TIME=1 -export CG_CLIF_INCR_CACHE_DISABLED=1 +export CG_CLIF_DISABLE_INCR_CACHE=1 export HOST_TRIPLE=$(rustc -vV | grep host | cut -d: -f2 | tr -d " ") export TARGET_TRIPLE=${TARGET_TRIPLE:-$HOST_TRIPLE} diff --git a/src/backend.rs b/src/backend.rs index eb7927fc4adeb..eca88dd31727c 100644 --- a/src/backend.rs +++ b/src/backend.rs @@ -5,13 +5,13 @@ use std::convert::{TryFrom, TryInto}; use rustc_data_structures::fx::FxHashMap; use rustc_session::Session; +use cranelift_codegen::isa::TargetIsa; use cranelift_module::FuncId; +use cranelift_object::{ObjectBuilder, ObjectModule, ObjectProduct}; use object::write::*; use object::{RelocationEncoding, SectionKind, SymbolFlags}; -use cranelift_object::{ObjectBuilder, ObjectModule, ObjectProduct}; - use gimli::SectionId; use crate::debuginfo::{DebugReloc, DebugRelocName}; @@ -113,7 +113,7 @@ impl WriteDebugInfo for ObjectProduct { } pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object)) -> Vec { - let triple = crate::build_isa(sess).triple().clone(); + let triple = crate::target_triple(sess); let binary_format = match triple.binary_format { target_lexicon::BinaryFormat::Elf => object::BinaryFormat::Elf, @@ -141,9 +141,9 @@ pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object metadata_object.write().unwrap() } -pub(crate) fn make_module(sess: &Session, name: String) -> ObjectModule { +pub(crate) fn make_module(sess: &Session, isa: Box, name: String) -> ObjectModule { let mut builder = ObjectBuilder::new( - crate::build_isa(sess), + isa, name + ".o", cranelift_module::default_libcall_names(), ) diff --git a/src/config.rs b/src/config.rs index 9b3259b622ea2..0cf02e2bbb565 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,5 +1,10 @@ +use std::env; use std::str::FromStr; +fn bool_env_var(key: &str) -> bool { + env::var(key).as_ref().map(|val| &**val) == Ok("1") +} + #[derive(Copy, Clone, Debug)] pub enum CodegenMode { Aot, @@ -7,12 +12,6 @@ pub enum CodegenMode { JitLazy, } -impl Default for CodegenMode { - fn default() -> Self { - CodegenMode::Aot - } -} - impl FromStr for CodegenMode { type Err = String; @@ -26,24 +25,77 @@ impl FromStr for CodegenMode { } } -#[derive(Copy, Clone, Debug, Default)] +#[derive(Clone, Debug)] pub struct BackendConfig { + /// Should the crate be AOT compiled or JIT executed. + /// + /// Defaults to AOT compilation. Can be set using `-Cllvm-args=mode=...`. pub codegen_mode: CodegenMode, + + /// When JIT mode is enable pass these arguments to the program. + /// + /// Defaults to the value of `CG_CLIF_JIT_ARGS`. + pub jit_args: Vec, + + /// Display the time it took to perform codegen for a crate. + /// + /// Defaults to true when the `CG_CLIF_DISPLAY_CG_TIME` env var is set to 1 or false otherwise. + /// Can be set using `-Cllvm-args=display_cg_time=...`. + pub display_cg_time: bool, + + /// Enable the Cranelift ir verifier for all compilation passes. If not set it will only run + /// once before passing the clif ir to Cranelift for compilation. + /// + /// Defaults to true when the `CG_CLIF_ENABLE_VERIFIER` env var is set to 1 or when cg_clif is + /// compiled with debug assertions enabled or false otherwise. Can be set using + /// `-Cllvm-args=enable_verifier=...`. + pub enable_verifier: bool, + + /// Don't cache object files in the incremental cache. Useful during development of cg_clif + /// to make it possible to use incremental mode for all analyses performed by rustc without + /// caching object files when their content should have been changed by a change to cg_clif. + /// + /// Defaults to true when the `CG_CLIF_DISABLE_INCR_CACHE` env var is set to 1 or false + /// otherwise. Can be set using `-Cllvm-args=disable_incr_cache=...`. + pub disable_incr_cache: bool, +} + +impl Default for BackendConfig { + fn default() -> Self { + BackendConfig { + codegen_mode: CodegenMode::Aot, + jit_args: { + let args = std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new()); + args.split(' ').map(|arg| arg.to_string()).collect() + }, + display_cg_time: bool_env_var("CG_CLIF_DISPLAY_CG_TIME"), + enable_verifier: cfg!(debug_assertions) || bool_env_var("CG_CLIF_ENABLE_VERIFIER"), + disable_incr_cache: bool_env_var("CG_CLIF_DISABLE_INCR_CACHE"), + } + } } impl BackendConfig { pub fn from_opts(opts: &[String]) -> Result { + fn parse_bool(name: &str, value: &str) -> Result { + value.parse().map_err(|_| format!("failed to parse value `{}` for {}", value, name)) + } + let mut config = BackendConfig::default(); for opt in opts { if let Some((name, value)) = opt.split_once('=') { match name { "mode" => config.codegen_mode = value.parse()?, + "display_cg_time" => config.display_cg_time = parse_bool(name, value)?, + "enable_verifier" => config.enable_verifier = parse_bool(name, value)?, + "disable_incr_cache" => config.disable_incr_cache = parse_bool(name, value)?, _ => return Err(format!("Unknown option `{}`", name)), } } else { return Err(format!("Invalid option `{}`", opt)); } } + Ok(config) } } diff --git a/src/driver/aot.rs b/src/driver/aot.rs index ed3bdedddced5..6ce174eec81c2 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -16,12 +16,6 @@ use cranelift_object::ObjectModule; use crate::{prelude::*, BackendConfig}; -fn new_module(tcx: TyCtxt<'_>, name: String) -> ObjectModule { - let module = crate::backend::make_module(tcx.sess, name); - assert_eq!(pointer_ty(tcx), module.target_config().pointer_type()); - module -} - struct ModuleCodegenResult(CompiledModule, Option<(WorkProductId, WorkProduct)>); impl HashStable for ModuleCodegenResult { @@ -32,6 +26,7 @@ impl HashStable for ModuleCodegenResult { fn emit_module( tcx: TyCtxt<'_>, + backend_config: &BackendConfig, name: String, kind: ModuleKind, module: ObjectModule, @@ -52,7 +47,7 @@ fn emit_module( tcx.sess.fatal(&format!("error writing object file: {}", err)); } - let work_product = if std::env::var("CG_CLIF_INCR_CACHE_DISABLED").is_ok() { + let work_product = if backend_config.disable_incr_cache { None } else { rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir( @@ -110,11 +105,13 @@ fn module_codegen( let cgu = tcx.codegen_unit(cgu_name); let mono_items = cgu.items_in_deterministic_order(tcx); - let mut module = new_module(tcx, cgu_name.as_str().to_string()); + let isa = crate::build_isa(tcx.sess, &backend_config); + let mut module = crate::backend::make_module(tcx.sess, isa, cgu_name.as_str().to_string()); + assert_eq!(pointer_ty(tcx), module.target_config().pointer_type()); let mut cx = crate::CodegenCx::new( tcx, - backend_config, + backend_config.clone(), &mut module, tcx.sess.opts.debuginfo != DebugInfo::None, ); @@ -144,6 +141,7 @@ fn module_codegen( let codegen_result = emit_module( tcx, + &backend_config, cgu.name().as_str().to_string(), ModuleKind::Regular, module, @@ -193,14 +191,14 @@ pub(super) fn run_aot( } } - let modules = super::time(tcx, "codegen mono items", || { + let modules = super::time(tcx, backend_config.display_cg_time, "codegen mono items", || { cgus.iter() .map(|cgu| { let cgu_reuse = determine_cgu_reuse(tcx, cgu); tcx.sess.cgu_reuse_tracker.set_actual_reuse(&cgu.name().as_str(), cgu_reuse); match cgu_reuse { - _ if std::env::var("CG_CLIF_INCR_CACHE_DISABLED").is_ok() => {} + _ if backend_config.disable_incr_cache => {} CguReuse::No => {} CguReuse::PreLto => { return reuse_workproduct_for_cgu(tcx, &*cgu, &mut work_products); @@ -212,7 +210,7 @@ pub(super) fn run_aot( let (ModuleCodegenResult(module, work_product), _) = tcx.dep_graph.with_task( dep_node, tcx, - (backend_config, cgu.name()), + (backend_config.clone(), cgu.name()), module_codegen, rustc_middle::dep_graph::hash_result, ); @@ -228,7 +226,9 @@ pub(super) fn run_aot( tcx.sess.abort_if_errors(); - let mut allocator_module = new_module(tcx, "allocator_shim".to_string()); + let isa = crate::build_isa(tcx.sess, &backend_config); + let mut allocator_module = crate::backend::make_module(tcx.sess, isa, "allocator_shim".to_string()); + assert_eq!(pointer_ty(tcx), allocator_module.target_config().pointer_type()); let mut allocator_unwind_context = UnwindContext::new(tcx, allocator_module.isa(), true); let created_alloc_shim = crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context); @@ -236,6 +236,7 @@ pub(super) fn run_aot( let allocator_module = if created_alloc_shim { let ModuleCodegenResult(module, work_product) = emit_module( tcx, + &backend_config, "allocator_shim".to_string(), ModuleKind::Allocator, allocator_module, diff --git a/src/driver/jit.rs b/src/driver/jit.rs index dbe1ff083f0db..e19283af10da7 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -27,8 +27,8 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { let imported_symbols = load_imported_symbols_for_jit(tcx); - let mut jit_builder = - JITBuilder::with_isa(crate::build_isa(tcx.sess), cranelift_module::default_libcall_names()); + let isa = crate::build_isa(tcx.sess, &backend_config); + let mut jit_builder = JITBuilder::with_isa(isa, cranelift_module::default_libcall_names()); jit_builder.hotswap(matches!(backend_config.codegen_mode, CodegenMode::JitLazy)); crate::compiler_builtins::register_functions_for_jit(&mut jit_builder); jit_builder.symbols(imported_symbols); @@ -44,9 +44,9 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { .into_iter() .collect::>(); - let mut cx = crate::CodegenCx::new(tcx, backend_config, &mut jit_module, false); + let mut cx = crate::CodegenCx::new(tcx, backend_config.clone(), &mut jit_module, false); - super::time(tcx, "codegen mono items", || { + super::time(tcx, backend_config.display_cg_time, "codegen mono items", || { super::predefine_mono_items(&mut cx, &mono_items); for (mono_item, _) in mono_items { match mono_item { @@ -87,9 +87,8 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { "Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed" ); - let args = ::std::env::var("CG_CLIF_JIT_ARGS").unwrap_or_else(|_| String::new()); let args = std::iter::once(&*tcx.crate_name(LOCAL_CRATE).as_str().to_string()) - .chain(args.split(' ')) + .chain(backend_config.jit_args.iter().map(|arg| &**arg)) .map(|arg| CString::new(arg).unwrap()) .collect::>(); let mut argv = args.iter().map(|arg| arg.as_ptr()).collect::>(); diff --git a/src/driver/mod.rs b/src/driver/mod.rs index d49182a07b79e..960e10182d812 100644 --- a/src/driver/mod.rs +++ b/src/driver/mod.rs @@ -65,8 +65,8 @@ fn predefine_mono_items<'tcx>( }); } -fn time(tcx: TyCtxt<'_>, name: &'static str, f: impl FnOnce() -> R) -> R { - if std::env::var("CG_CLIF_DISPLAY_CG_TIME").as_ref().map(|val| &**val) == Ok("1") { +fn time(tcx: TyCtxt<'_>, display: bool, name: &'static str, f: impl FnOnce() -> R) -> R { + if display { println!("[{:<30}: {}] start", tcx.crate_name(LOCAL_CRATE), name); let before = std::time::Instant::now(); let res = tcx.sess.time(name, f); diff --git a/src/lib.rs b/src/lib.rs index 77c6247e15766..9a417f35ae200 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -193,7 +193,7 @@ impl CodegenBackend for CraneliftCodegenBackend { metadata: EncodedMetadata, need_metadata_module: bool, ) -> Box { - let config = if let Some(config) = self.config { + let config = if let Some(config) = self.config.clone() { config } else { BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args) @@ -237,7 +237,7 @@ fn target_triple(sess: &Session) -> target_lexicon::Triple { sess.target.llvm_target.parse().unwrap() } -fn build_isa(sess: &Session) -> Box { +fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box { use target_lexicon::BinaryFormat; let target_triple = crate::target_triple(sess); @@ -245,9 +245,8 @@ fn build_isa(sess: &Session) -> Box { let mut flags_builder = settings::builder(); flags_builder.enable("is_pic").unwrap(); flags_builder.set("enable_probestack", "false").unwrap(); // __cranelift_probestack is not provided - let enable_verifier = - cfg!(debug_assertions) || std::env::var("CG_CLIF_ENABLE_VERIFIER").is_ok(); - flags_builder.set("enable_verifier", if enable_verifier { "true" } else { "false" }).unwrap(); + let enable_verifier = if backend_config.enable_verifier { "true" } else { "false" }; + flags_builder.set("enable_verifier", enable_verifier).unwrap(); let tls_model = match target_triple.binary_format { BinaryFormat::Elf => "elf_gd", diff --git a/src/pretty_clif.rs b/src/pretty_clif.rs index d22ea3772eee7..166151ecc70cc 100644 --- a/src/pretty_clif.rs +++ b/src/pretty_clif.rs @@ -248,7 +248,7 @@ pub(crate) fn write_clif_file<'tcx>( &mut clif, &context.func, &DisplayFunctionAnnotations { - isa: Some(&*crate::build_isa(tcx.sess)), + isa, value_ranges: value_ranges.as_ref(), }, ) From a7c4c3ee713aa7e4e5c8638146efa8ad5e7d0de6 Mon Sep 17 00:00:00 2001 From: pierwill Date: Wed, 7 Apr 2021 14:47:01 -0500 Subject: [PATCH 14/52] Fix outdated crate names in compiler docs Changes `librustc_X` to `rustc_X`, only in documentation comments. Plain code comments are left unchanged. Also fix incorrect file paths. --- src/vtable.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/vtable.rs b/src/vtable.rs index 4d2551a061b99..9053d1aa1b05a 100644 --- a/src/vtable.rs +++ b/src/vtable.rs @@ -1,6 +1,6 @@ //! Codegen vtables and vtable accesses. //! -//! See librustc_codegen_llvm/meth.rs for reference +//! See `rustc_codegen_ssa/src/meth.rs` for reference. // FIXME dedup this logic between miri, cg_llvm and cg_clif use crate::prelude::*; From 29a4a551eb23969cde9a895d081bee682254974c Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Mon, 12 Apr 2021 12:09:05 +0200 Subject: [PATCH 15/52] Rustfmt --- src/backend.rs | 8 ++------ src/base.rs | 15 ++------------- src/driver/aot.rs | 3 ++- src/lib.rs | 1 - src/pretty_clif.rs | 5 +---- 5 files changed, 7 insertions(+), 25 deletions(-) diff --git a/src/backend.rs b/src/backend.rs index eca88dd31727c..9f58db62a1fa6 100644 --- a/src/backend.rs +++ b/src/backend.rs @@ -142,12 +142,8 @@ pub(crate) fn with_object(sess: &Session, name: &str, f: impl FnOnce(&mut Object } pub(crate) fn make_module(sess: &Session, isa: Box, name: String) -> ObjectModule { - let mut builder = ObjectBuilder::new( - isa, - name + ".o", - cranelift_module::default_libcall_names(), - ) - .unwrap(); + let mut builder = + ObjectBuilder::new(isa, name + ".o", cranelift_module::default_libcall_names()).unwrap(); // Unlike cg_llvm, cg_clif defaults to disabling -Zfunction-sections. For cg_llvm binary size // is important, while cg_clif cares more about compilation times. Enabling -Zfunction-sections // can easily double the amount of time necessary to perform linking. diff --git a/src/base.rs b/src/base.rs index c4e0c9676f693..8b7a7caade54d 100644 --- a/src/base.rs +++ b/src/base.rs @@ -101,12 +101,7 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In // Perform rust specific optimizations tcx.sess.time("optimize clif ir", || { - crate::optimize::optimize_function( - tcx, - instance, - context, - &mut clif_comments, - ); + crate::optimize::optimize_function(tcx, instance, context, &mut clif_comments); }); // If the return block is not reachable, then the SSA builder may have inserted an `iconst.i128` @@ -351,13 +346,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) { from_hir_call: _, } => { fx.tcx.sess.time("codegen call", || { - crate::abi::codegen_terminator_call( - fx, - *fn_span, - func, - args, - *destination, - ) + crate::abi::codegen_terminator_call(fx, *fn_span, func, args, *destination) }); } TerminatorKind::InlineAsm { diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 6ce174eec81c2..5bfe0b3ad494f 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -227,7 +227,8 @@ pub(super) fn run_aot( tcx.sess.abort_if_errors(); let isa = crate::build_isa(tcx.sess, &backend_config); - let mut allocator_module = crate::backend::make_module(tcx.sess, isa, "allocator_shim".to_string()); + let mut allocator_module = + crate::backend::make_module(tcx.sess, isa, "allocator_shim".to_string()); assert_eq!(pointer_ty(tcx), allocator_module.target_config().pointer_type()); let mut allocator_unwind_context = UnwindContext::new(tcx, allocator_module.isa(), true); let created_alloc_shim = diff --git a/src/lib.rs b/src/lib.rs index 9a417f35ae200..d31e05bdbd538 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -162,7 +162,6 @@ impl<'m, 'tcx> CodegenCx<'m, 'tcx> { } } - pub struct CraneliftCodegenBackend { pub config: Option, } diff --git a/src/pretty_clif.rs b/src/pretty_clif.rs index 166151ecc70cc..30e487a99f431 100644 --- a/src/pretty_clif.rs +++ b/src/pretty_clif.rs @@ -247,10 +247,7 @@ pub(crate) fn write_clif_file<'tcx>( &mut clif_comments, &mut clif, &context.func, - &DisplayFunctionAnnotations { - isa, - value_ranges: value_ranges.as_ref(), - }, + &DisplayFunctionAnnotations { isa, value_ranges: value_ranges.as_ref() }, ) .unwrap(); From b6f7e71c1dd201c11ac85151d5918cb519a7a4f8 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Wed, 14 Apr 2021 10:34:04 +0200 Subject: [PATCH 16/52] Update Cranelift --- Cargo.lock | 44 ++++++++++++++++++++++---------------------- Cargo.toml | 4 ++-- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dc1cd336e1599..16c2732eac9e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,16 +39,16 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cranelift-bforest" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.73.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-codegen" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.73.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" dependencies = [ "byteorder", "cranelift-bforest", @@ -65,8 +65,8 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.73.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" dependencies = [ "cranelift-codegen-shared", "cranelift-entity", @@ -74,18 +74,18 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.73.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" [[package]] name = "cranelift-entity" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.73.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" [[package]] name = "cranelift-frontend" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.73.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" dependencies = [ "cranelift-codegen", "log", @@ -95,8 +95,8 @@ dependencies = [ [[package]] name = "cranelift-jit" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.73.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" dependencies = [ "anyhow", "cranelift-codegen", @@ -113,8 +113,8 @@ dependencies = [ [[package]] name = "cranelift-module" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.73.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" dependencies = [ "anyhow", "cranelift-codegen", @@ -125,8 +125,8 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.73.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" dependencies = [ "cranelift-codegen", "target-lexicon", @@ -134,8 +134,8 @@ dependencies = [ [[package]] name = "cranelift-object" -version = "0.72.0" -source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#8e43e96410a14143d368273cf1e708f8094bb8e0" +version = "0.73.0" +source = "git+https://github.com/bytecodealliance/wasmtime/?branch=main#45bee40f338c631bff4a799288101ba328c7ad36" dependencies = [ "anyhow", "cranelift-codegen", @@ -334,9 +334,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.11.2" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "422045212ea98508ae3d28025bc5aaa2bd4a9cdaecd442a08da2ee620ee9ea95" +checksum = "64ae3b39281e4b14b8123bdbaddd472b7dfe215e444181f2f9d2443c2444f834" [[package]] name = "thiserror" diff --git a/Cargo.toml b/Cargo.toml index 60946ab280858..248540cf1a3a5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,12 +9,12 @@ crate-type = ["dylib"] [dependencies] # These have to be in sync with each other -cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", features = ["unwind", "x64"] } +cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", features = ["unwind"] } cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", optional = true } cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } -target-lexicon = "0.11.0" +target-lexicon = "0.12.0" gimli = { version = "0.23.0", default-features = false, features = ["write"]} object = { version = "0.23.0", default-features = false, features = ["std", "read_core", "write", "archive", "coff", "elf", "macho", "pe"] } From f3b0f425c5a26f611c1743170cb9caf3ac8b6063 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Wed, 7 Apr 2021 14:54:59 +0200 Subject: [PATCH 17/52] Don't deduplicate vtables between functions --- src/base.rs | 8 +++++--- src/common.rs | 3 +++ src/inline_asm.rs | 3 +-- src/lib.rs | 2 -- src/trap.rs | 3 +-- src/vtable.rs | 28 +++++----------------------- 6 files changed, 15 insertions(+), 32 deletions(-) diff --git a/src/base.rs b/src/base.rs index 8b7a7caade54d..1bd303b2db7ee 100644 --- a/src/base.rs +++ b/src/base.rs @@ -18,9 +18,9 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In let mir = tcx.instance_mir(instance.def); // Declare function - let name = tcx.symbol_name(instance).name.to_string(); + let symbol_name = tcx.symbol_name(instance); let sig = get_function_sig(tcx, cx.module.isa().triple(), instance); - let func_id = cx.module.declare_function(&name, Linkage::Local, &sig).unwrap(); + let func_id = cx.module.declare_function(symbol_name.name, Linkage::Local, &sig).unwrap(); cx.cached_context.clear(); @@ -46,8 +46,10 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In cx, tcx, pointer_type, + vtables: FxHashMap::default(), instance, + symbol_name, mir, fn_abi: Some(FnAbi::of_instance(&RevealAllLayoutCx(tcx), instance, &[])), @@ -151,7 +153,7 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In debug_context.define_function( instance, func_id, - &name, + symbol_name.name, isa, context, &source_info_set, diff --git a/src/common.rs b/src/common.rs index fe6cf9a3b01e5..64618dcf3309c 100644 --- a/src/common.rs +++ b/src/common.rs @@ -1,4 +1,5 @@ use rustc_index::vec::IndexVec; +use rustc_middle::ty::SymbolName; use rustc_target::abi::call::FnAbi; use rustc_target::abi::{Integer, Primitive}; use rustc_target::spec::{HasTargetSpec, Target}; @@ -230,8 +231,10 @@ pub(crate) struct FunctionCx<'m, 'clif, 'tcx> { pub(crate) cx: &'clif mut crate::CodegenCx<'m, 'tcx>, pub(crate) tcx: TyCtxt<'tcx>, pub(crate) pointer_type: Type, // Cached from module + pub(crate) vtables: FxHashMap<(Ty<'tcx>, Option>), DataId>, pub(crate) instance: Instance<'tcx>, + pub(crate) symbol_name: SymbolName<'tcx>, pub(crate) mir: &'tcx Body<'tcx>, pub(crate) fn_abi: Option>>, diff --git a/src/inline_asm.rs b/src/inline_asm.rs index 1fb5e86aed7df..7187bc785e46d 100644 --- a/src/inline_asm.rs +++ b/src/inline_asm.rs @@ -92,8 +92,7 @@ pub(crate) fn codegen_inline_asm<'tcx>( let inline_asm_index = fx.inline_asm_index; fx.inline_asm_index += 1; - let asm_name = - format!("{}__inline_asm_{}", fx.tcx.symbol_name(fx.instance).name, inline_asm_index); + let asm_name = format!("{}__inline_asm_{}", fx.symbol_name, inline_asm_index); let generated_asm = generate_asm_wrapper( &asm_name, diff --git a/src/lib.rs b/src/lib.rs index d31e05bdbd538..b79c8c7806bea 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -125,7 +125,6 @@ struct CodegenCx<'m, 'tcx: 'm> { global_asm: String, constants_cx: ConstantCx, cached_context: Context, - vtables: FxHashMap<(Ty<'tcx>, Option>), DataId>, debug_context: Option>, unwind_context: UnwindContext<'tcx>, } @@ -150,7 +149,6 @@ impl<'m, 'tcx> CodegenCx<'m, 'tcx> { global_asm: String::new(), constants_cx: ConstantCx::default(), cached_context: Context::new(), - vtables: FxHashMap::default(), debug_context, unwind_context, } diff --git a/src/trap.rs b/src/trap.rs index 1ab0703e981e7..4e52fb161211c 100644 --- a/src/trap.rs +++ b/src/trap.rs @@ -21,8 +21,7 @@ fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) { fx.add_comment(puts, "puts"); } - let symbol_name = fx.tcx.symbol_name(fx.instance); - let real_msg = format!("trap at {:?} ({}): {}\0", fx.instance, symbol_name, msg); + let real_msg = format!("trap at {:?} ({}): {}\0", fx.instance, fx.symbol_name, msg); let msg_ptr = fx.anonymous_str("trap", &real_msg); fx.bcx.ins().call(puts, &[msg_ptr]); } diff --git a/src/vtable.rs b/src/vtable.rs index 4d2551a061b99..60bc53024a7e3 100644 --- a/src/vtable.rs +++ b/src/vtable.rs @@ -72,11 +72,11 @@ pub(crate) fn get_vtable<'tcx>( layout: TyAndLayout<'tcx>, trait_ref: Option>, ) -> Value { - let data_id = if let Some(data_id) = fx.cx.vtables.get(&(layout.ty, trait_ref)) { + let data_id = if let Some(data_id) = fx.vtables.get(&(layout.ty, trait_ref)) { *data_id } else { let data_id = build_vtable(fx, layout, trait_ref); - fx.cx.vtables.insert((layout.ty, trait_ref), data_id); + fx.vtables.insert((layout.ty, trait_ref), data_id); data_id }; @@ -139,27 +139,9 @@ fn build_vtable<'tcx>( data_ctx.set_align(fx.tcx.data_layout.pointer_align.pref.bytes()); - let data_id = fx - .cx - .module - .declare_data( - &format!( - "__vtable.{}.for.{:?}.{}", - trait_ref - .as_ref() - .map(|trait_ref| format!("{:?}", trait_ref.skip_binder()).into()) - .unwrap_or(std::borrow::Cow::Borrowed("???")), - layout.ty, - fx.cx.vtables.len(), - ), - Linkage::Local, - false, - false, - ) - .unwrap(); - - // FIXME don't duplicate definitions in lazy jit mode - let _ = fx.cx.module.define_data(data_id, &data_ctx); + let data_id = fx.cx.module.declare_anonymous_data(false, false).unwrap(); + + fx.cx.module.define_data(data_id, &data_ctx).unwrap(); data_id } From 65420b50f86296442d26884d2eb0de5b43090b0a Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 8 Apr 2021 12:08:13 +0200 Subject: [PATCH 18/52] Don't deduplicate anonymous allocations --- example/mini_core_hello_world.rs | 5 ++++ scripts/test_rustc_tests.sh | 1 + src/base.rs | 4 +++ src/common.rs | 2 ++ src/constant.rs | 48 ++++++++++++++++++-------------- src/driver/aot.rs | 4 +-- src/driver/jit.rs | 2 +- src/lib.rs | 4 --- 8 files changed, 41 insertions(+), 29 deletions(-) diff --git a/example/mini_core_hello_world.rs b/example/mini_core_hello_world.rs index ea37ca98b59a7..47abe2d1de800 100644 --- a/example/mini_core_hello_world.rs +++ b/example/mini_core_hello_world.rs @@ -296,6 +296,11 @@ fn main() { unsafe { global_asm_test(); } + + // Both statics have a reference that points to the same anonymous allocation. + static REF1: &u8 = &42; + static REF2: &u8 = REF1; + assert_eq!(*REF1, *REF2); } #[cfg(all(not(jit), target_os = "linux"))] diff --git a/scripts/test_rustc_tests.sh b/scripts/test_rustc_tests.sh index e4b73755248c2..6a76f3a721445 100755 --- a/scripts/test_rustc_tests.sh +++ b/scripts/test_rustc_tests.sh @@ -47,6 +47,7 @@ rm src/test/ui/issues/issue-51947.rs # same rm src/test/ui/numbers-arithmetic/saturating-float-casts.rs # intrinsic gives different but valid result rm src/test/ui/mir/mir_misc_casts.rs # depends on deduplication of constants rm src/test/ui/mir/mir_raw_fat_ptr.rs # same +rm src/test/ui/consts/issue-33537.rs # same rm src/test/ui/async-await/async-fn-size-moved-locals.rs # -Cpanic=abort shrinks some generator by one byte rm src/test/ui/async-await/async-fn-size-uninit-locals.rs # same rm src/test/ui/generator/size-moved-locals.rs # same diff --git a/src/base.rs b/src/base.rs index 1bd303b2db7ee..3a5b2be462a93 100644 --- a/src/base.rs +++ b/src/base.rs @@ -6,6 +6,7 @@ use rustc_middle::ty::adjustment::PointerCast; use rustc_middle::ty::layout::FnAbiExt; use rustc_target::abi::call::FnAbi; +use crate::constant::ConstantCx; use crate::prelude::*; pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: Instance<'tcx>) { @@ -47,6 +48,7 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In tcx, pointer_type, vtables: FxHashMap::default(), + constants_cx: ConstantCx::new(), instance, symbol_name, @@ -92,6 +94,8 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In let source_info_set = fx.source_info_set; let local_map = fx.local_map; + fx.constants_cx.finalize(fx.tcx, &mut *fx.cx.module); + // Store function in context let context = &mut cx.cached_context; context.func = func; diff --git a/src/common.rs b/src/common.rs index 64618dcf3309c..6752bb42dc870 100644 --- a/src/common.rs +++ b/src/common.rs @@ -4,6 +4,7 @@ use rustc_target::abi::call::FnAbi; use rustc_target::abi::{Integer, Primitive}; use rustc_target::spec::{HasTargetSpec, Target}; +use crate::constant::ConstantCx; use crate::prelude::*; pub(crate) fn pointer_ty(tcx: TyCtxt<'_>) -> types::Type { @@ -232,6 +233,7 @@ pub(crate) struct FunctionCx<'m, 'clif, 'tcx> { pub(crate) tcx: TyCtxt<'tcx>, pub(crate) pointer_type: Type, // Cached from module pub(crate) vtables: FxHashMap<(Ty<'tcx>, Option>), DataId>, + pub(crate) constants_cx: ConstantCx, pub(crate) instance: Instance<'tcx>, pub(crate) symbol_name: SymbolName<'tcx>, diff --git a/src/constant.rs b/src/constant.rs index fcd41c844659d..339580eb8f4e4 100644 --- a/src/constant.rs +++ b/src/constant.rs @@ -2,7 +2,7 @@ use rustc_span::DUMMY_SP; -use rustc_data_structures::fx::FxHashSet; +use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_errors::ErrorReported; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::mir::interpret::{ @@ -13,12 +13,12 @@ use rustc_middle::ty::ConstKind; use cranelift_codegen::ir::GlobalValueData; use cranelift_module::*; -use crate::prelude::*; +use crate::{prelude::*, CodegenCx}; -#[derive(Default)] pub(crate) struct ConstantCx { todo: Vec, done: FxHashSet, + anon_allocs: FxHashMap, } #[derive(Copy, Clone, Debug)] @@ -28,6 +28,10 @@ enum TodoItem { } impl ConstantCx { + pub(crate) fn new() -> Self { + ConstantCx { todo: vec![], done: FxHashSet::default(), anon_allocs: FxHashMap::default() } + } + pub(crate) fn finalize(mut self, tcx: TyCtxt<'_>, module: &mut dyn Module) { //println!("todo {:?}", self.todo); define_all_allocs(tcx, module, &mut self); @@ -74,8 +78,10 @@ pub(crate) fn check_constants(fx: &mut FunctionCx<'_, '_, '_>) -> bool { all_constants_ok } -pub(crate) fn codegen_static(constants_cx: &mut ConstantCx, def_id: DefId) { +pub(crate) fn codegen_static(cx: &mut CodegenCx<'_, '_>, def_id: DefId) { + let mut constants_cx = ConstantCx::new(); constants_cx.todo.push(TodoItem::Static(def_id)); + constants_cx.finalize(cx.tcx, &mut *cx.module); } pub(crate) fn codegen_tls_ref<'tcx>( @@ -182,9 +188,13 @@ pub(crate) fn codegen_const_value<'tcx>( let alloc_kind = fx.tcx.get_global_alloc(ptr.alloc_id); let base_addr = match alloc_kind { Some(GlobalAlloc::Memory(alloc)) => { - fx.cx.constants_cx.todo.push(TodoItem::Alloc(ptr.alloc_id)); - let data_id = - data_id_for_alloc_id(fx.cx.module, ptr.alloc_id, alloc.mutability); + fx.constants_cx.todo.push(TodoItem::Alloc(ptr.alloc_id)); + let data_id = data_id_for_alloc_id( + &mut fx.constants_cx, + fx.cx.module, + ptr.alloc_id, + alloc.mutability, + ); let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); if fx.clif_comments.enabled() { @@ -243,8 +253,9 @@ fn pointer_for_allocation<'tcx>( alloc: &'tcx Allocation, ) -> crate::pointer::Pointer { let alloc_id = fx.tcx.create_memory_alloc(alloc); - fx.cx.constants_cx.todo.push(TodoItem::Alloc(alloc_id)); - let data_id = data_id_for_alloc_id(fx.cx.module, alloc_id, alloc.mutability); + fx.constants_cx.todo.push(TodoItem::Alloc(alloc_id)); + let data_id = + data_id_for_alloc_id(&mut fx.constants_cx, &mut *fx.cx.module, alloc_id, alloc.mutability); let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); if fx.clif_comments.enabled() { @@ -255,18 +266,14 @@ fn pointer_for_allocation<'tcx>( } fn data_id_for_alloc_id( + cx: &mut ConstantCx, module: &mut dyn Module, alloc_id: AllocId, mutability: rustc_hir::Mutability, ) -> DataId { - module - .declare_data( - &format!(".L__alloc_{:x}", alloc_id.0), - Linkage::Local, - mutability == rustc_hir::Mutability::Mut, - false, - ) - .unwrap() + *cx.anon_allocs.entry(alloc_id).or_insert_with(|| { + module.declare_anonymous_data(mutability == rustc_hir::Mutability::Mut, false).unwrap() + }) } fn data_id_for_static( @@ -344,7 +351,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant GlobalAlloc::Memory(alloc) => alloc, GlobalAlloc::Function(_) | GlobalAlloc::Static(_) => unreachable!(), }; - let data_id = data_id_for_alloc_id(module, alloc_id, alloc.mutability); + let data_id = data_id_for_alloc_id(cx, module, alloc_id, alloc.mutability); (data_id, alloc, None) } TodoItem::Static(def_id) => { @@ -397,7 +404,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant } GlobalAlloc::Memory(target_alloc) => { cx.todo.push(TodoItem::Alloc(reloc)); - data_id_for_alloc_id(module, reloc, target_alloc.mutability) + data_id_for_alloc_id(cx, module, reloc, target_alloc.mutability) } GlobalAlloc::Static(def_id) => { if tcx.codegen_fn_attrs(def_id).flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) @@ -419,8 +426,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant data_ctx.write_data_addr(offset.bytes() as u32, global_value, addend as i64); } - // FIXME don't duplicate definitions in lazy jit mode - let _ = module.define_data(data_id, &data_ctx); + module.define_data(data_id, &data_ctx).unwrap(); cx.done.insert(data_id); } diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 5bfe0b3ad494f..1f731a756ed15 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -121,9 +121,7 @@ fn module_codegen( MonoItem::Fn(inst) => { cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst)); } - MonoItem::Static(def_id) => { - crate::constant::codegen_static(&mut cx.constants_cx, def_id) - } + MonoItem::Static(def_id) => crate::constant::codegen_static(&mut cx, def_id), MonoItem::GlobalAsm(item_id) => { let item = cx.tcx.hir().item(item_id); if let rustc_hir::ItemKind::GlobalAsm(rustc_hir::GlobalAsm { asm }) = item.kind { diff --git a/src/driver/jit.rs b/src/driver/jit.rs index e19283af10da7..ffe449115b967 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -58,7 +58,7 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { CodegenMode::JitLazy => codegen_shim(&mut cx, inst), }, MonoItem::Static(def_id) => { - crate::constant::codegen_static(&mut cx.constants_cx, def_id); + crate::constant::codegen_static(&mut cx, def_id); } MonoItem::GlobalAsm(item_id) => { let item = cx.tcx.hir().item(item_id); diff --git a/src/lib.rs b/src/lib.rs index b79c8c7806bea..f94c9c1dab540 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -36,7 +36,6 @@ use rustc_session::Session; use cranelift_codegen::settings::{self, Configurable}; pub use crate::config::*; -use crate::constant::ConstantCx; use crate::prelude::*; mod abi; @@ -123,7 +122,6 @@ struct CodegenCx<'m, 'tcx: 'm> { tcx: TyCtxt<'tcx>, module: &'m mut dyn Module, global_asm: String, - constants_cx: ConstantCx, cached_context: Context, debug_context: Option>, unwind_context: UnwindContext<'tcx>, @@ -147,7 +145,6 @@ impl<'m, 'tcx> CodegenCx<'m, 'tcx> { tcx, module, global_asm: String::new(), - constants_cx: ConstantCx::default(), cached_context: Context::new(), debug_context, unwind_context, @@ -155,7 +152,6 @@ impl<'m, 'tcx> CodegenCx<'m, 'tcx> { } fn finalize(self) -> (String, Option>, UnwindContext<'tcx>) { - self.constants_cx.finalize(self.tcx, self.module); (self.global_asm, self.debug_context, self.unwind_context) } } From b4bf4b5c0988c0a1d0c4a9c17217a47702ee8ca3 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 8 Apr 2021 17:57:15 +0200 Subject: [PATCH 19/52] Enable and disable some rust-analyzer diagnostics --- .vscode/settings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 0cd576e160f86..9009a532c54dc 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,6 +1,6 @@ { // source for rustc_* is not included in the rust-src component; disable the errors about this - "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate", "macro-error"], + "rust-analyzer.diagnostics.disabled": ["unresolved-extern-crate", "unresolved-macro-call"], "rust-analyzer.assist.importMergeBehavior": "last", "rust-analyzer.cargo.runBuildScripts": true, "rust-analyzer.linkedProjects": [ From b09b8b1bd475f3782818e825b8f5f32d4df77d6c Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 8 Apr 2021 18:08:20 +0200 Subject: [PATCH 20/52] Re-use Context in codegen_shim --- src/driver/jit.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/driver/jit.rs b/src/driver/jit.rs index ffe449115b967..869b7760e8000 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -271,9 +271,12 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) { ) .unwrap(); - let mut trampoline = Function::with_name_signature(ExternalName::default(), sig.clone()); + cx.cached_context.clear(); + let trampoline = &mut cx.cached_context.func; + trampoline.signature = sig.clone(); + let mut builder_ctx = FunctionBuilderContext::new(); - let mut trampoline_builder = FunctionBuilder::new(&mut trampoline, &mut builder_ctx); + let mut trampoline_builder = FunctionBuilder::new(trampoline, &mut builder_ctx); let jit_fn = cx.module.declare_func_in_func(jit_fn, trampoline_builder.func); let sig_ref = trampoline_builder.func.import_signature(sig); @@ -293,7 +296,7 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) { cx.module .define_function( func_id, - &mut Context::for_function(trampoline), + &mut cx.cached_context, &mut NullTrapSink {}, &mut NullStackMapSink {}, ) From 3e2bdb94ec0a87ed3a2c711c6aaffb1447b06dbf Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 8 Apr 2021 22:04:58 +0200 Subject: [PATCH 21/52] Don't store TyCtxt in UnwindContext --- src/allocator.rs | 4 ++-- src/debuginfo/unwind.rs | 18 +++++++++--------- src/driver/aot.rs | 2 +- src/lib.rs | 4 ++-- src/main_shim.rs | 4 ++-- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/allocator.rs b/src/allocator.rs index f60645a9f97bc..a09e32577869e 100644 --- a/src/allocator.rs +++ b/src/allocator.rs @@ -11,7 +11,7 @@ use rustc_span::symbol::sym; pub(crate) fn codegen( tcx: TyCtxt<'_>, module: &mut impl Module, - unwind_context: &mut UnwindContext<'_>, + unwind_context: &mut UnwindContext, ) -> bool { let any_dynamic_crate = tcx.dependency_formats(LOCAL_CRATE).iter().any(|(_, list)| { use rustc_middle::middle::dependency_format::Linkage; @@ -29,7 +29,7 @@ pub(crate) fn codegen( fn codegen_inner( module: &mut impl Module, - unwind_context: &mut UnwindContext<'_>, + unwind_context: &mut UnwindContext, kind: AllocatorKind, ) { let usize_ty = module.target_config().pointer_type(); diff --git a/src/debuginfo/unwind.rs b/src/debuginfo/unwind.rs index 357c9fe6ed83a..aeafc901fa533 100644 --- a/src/debuginfo/unwind.rs +++ b/src/debuginfo/unwind.rs @@ -5,17 +5,19 @@ use crate::prelude::*; use cranelift_codegen::isa::{unwind::UnwindInfo, TargetIsa}; use gimli::write::{Address, CieId, EhFrame, FrameTable, Section}; +use gimli::RunTimeEndian; use crate::backend::WriteDebugInfo; -pub(crate) struct UnwindContext<'tcx> { - tcx: TyCtxt<'tcx>, +pub(crate) struct UnwindContext { + endian: RunTimeEndian, frame_table: FrameTable, cie_id: Option, } -impl<'tcx> UnwindContext<'tcx> { - pub(crate) fn new(tcx: TyCtxt<'tcx>, isa: &dyn TargetIsa, pic_eh_frame: bool) -> Self { +impl UnwindContext { + pub(crate) fn new(tcx: TyCtxt<'_>, isa: &dyn TargetIsa, pic_eh_frame: bool) -> Self { + let endian = super::target_endian(tcx); let mut frame_table = FrameTable::default(); let cie_id = if let Some(mut cie) = isa.create_systemv_cie() { @@ -28,7 +30,7 @@ impl<'tcx> UnwindContext<'tcx> { None }; - UnwindContext { tcx, frame_table, cie_id } + UnwindContext { endian, frame_table, cie_id } } pub(crate) fn add_function(&mut self, func_id: FuncId, context: &Context, isa: &dyn TargetIsa) { @@ -54,8 +56,7 @@ impl<'tcx> UnwindContext<'tcx> { } pub(crate) fn emit(self, product: &mut P) { - let mut eh_frame = - EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(self.tcx))); + let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(self.endian)); self.frame_table.write_eh_frame(&mut eh_frame).unwrap(); if !eh_frame.0.writer.slice().is_empty() { @@ -75,8 +76,7 @@ impl<'tcx> UnwindContext<'tcx> { self, jit_module: &cranelift_jit::JITModule, ) -> Option { - let mut eh_frame = - EhFrame::from(super::emit::WriterRelocate::new(super::target_endian(self.tcx))); + let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(self.endian)); self.frame_table.write_eh_frame(&mut eh_frame).unwrap(); if eh_frame.0.writer.slice().is_empty() { diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 1f731a756ed15..db7d39fea4c59 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -31,7 +31,7 @@ fn emit_module( kind: ModuleKind, module: ObjectModule, debug: Option>, - unwind_context: UnwindContext<'_>, + unwind_context: UnwindContext, ) -> ModuleCodegenResult { let mut product = module.finish(); diff --git a/src/lib.rs b/src/lib.rs index f94c9c1dab540..ee556bd2281a5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -124,7 +124,7 @@ struct CodegenCx<'m, 'tcx: 'm> { global_asm: String, cached_context: Context, debug_context: Option>, - unwind_context: UnwindContext<'tcx>, + unwind_context: UnwindContext, } impl<'m, 'tcx> CodegenCx<'m, 'tcx> { @@ -151,7 +151,7 @@ impl<'m, 'tcx> CodegenCx<'m, 'tcx> { } } - fn finalize(self) -> (String, Option>, UnwindContext<'tcx>) { + fn finalize(self) -> (String, Option>, UnwindContext) { (self.global_asm, self.debug_context, self.unwind_context) } } diff --git a/src/main_shim.rs b/src/main_shim.rs index a6266f507765f..22dc3f2bfe64e 100644 --- a/src/main_shim.rs +++ b/src/main_shim.rs @@ -9,7 +9,7 @@ use crate::prelude::*; pub(crate) fn maybe_create_entry_wrapper( tcx: TyCtxt<'_>, module: &mut impl Module, - unwind_context: &mut UnwindContext<'_>, + unwind_context: &mut UnwindContext, ) { let (main_def_id, use_start_lang_item) = match tcx.entry_fn(LOCAL_CRATE) { Some((def_id, entry_ty)) => ( @@ -32,7 +32,7 @@ pub(crate) fn maybe_create_entry_wrapper( fn create_entry_fn( tcx: TyCtxt<'_>, m: &mut impl Module, - unwind_context: &mut UnwindContext<'_>, + unwind_context: &mut UnwindContext, rust_main_def_id: DefId, use_start_lang_item: bool, ) { From d9e9fedfe579b0062645e3be06284a9246228b8e Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 8 Apr 2021 22:01:39 +0200 Subject: [PATCH 22/52] Remove CodegenCx::finalize and pass Module separately from CodegenCx --- src/abi/mod.rs | 8 +++---- src/base.rs | 28 +++++++++++++----------- src/common.rs | 12 +++++------ src/constant.rs | 31 +++++++++++++-------------- src/driver/aot.rs | 20 +++++++++--------- src/driver/jit.rs | 48 ++++++++++++++++++++---------------------- src/driver/mod.rs | 13 ++++++------ src/inline_asm.rs | 3 +-- src/intrinsics/mod.rs | 10 ++++----- src/lib.rs | 23 ++++++-------------- src/trap.rs | 3 +-- src/value_and_place.rs | 2 +- src/vtable.rs | 12 +++++------ 13 files changed, 102 insertions(+), 111 deletions(-) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index f898348f97bc0..f450f36678713 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -71,8 +71,8 @@ pub(crate) fn import_function<'tcx>( impl<'tcx> FunctionCx<'_, '_, 'tcx> { /// Instance must be monomorphized pub(crate) fn get_function_ref(&mut self, inst: Instance<'tcx>) -> FuncRef { - let func_id = import_function(self.tcx, self.cx.module, inst); - let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func); + let func_id = import_function(self.tcx, self.module, inst); + let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func); if self.clif_comments.enabled() { self.add_comment(func_ref, format!("{:?}", inst)); @@ -89,8 +89,8 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { args: &[Value], ) -> &[Value] { let sig = Signature { params, returns, call_conv: CallConv::triple_default(self.triple()) }; - let func_id = self.cx.module.declare_function(&name, Linkage::Import, &sig).unwrap(); - let func_ref = self.cx.module.declare_func_in_func(func_id, &mut self.bcx.func); + let func_id = self.module.declare_function(&name, Linkage::Import, &sig).unwrap(); + let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func); let call_inst = self.bcx.ins().call(func_ref, args); if self.clif_comments.enabled() { self.add_comment(call_inst, format!("easy_call {}", name)); diff --git a/src/base.rs b/src/base.rs index 3a5b2be462a93..069166f7b68a0 100644 --- a/src/base.rs +++ b/src/base.rs @@ -9,7 +9,11 @@ use rustc_target::abi::call::FnAbi; use crate::constant::ConstantCx; use crate::prelude::*; -pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: Instance<'tcx>) { +pub(crate) fn codegen_fn<'tcx>( + cx: &mut crate::CodegenCx<'tcx>, + module: &mut dyn Module, + instance: Instance<'tcx>, +) { let tcx = cx.tcx; let _inst_guard = @@ -20,8 +24,8 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In // Declare function let symbol_name = tcx.symbol_name(instance); - let sig = get_function_sig(tcx, cx.module.isa().triple(), instance); - let func_id = cx.module.declare_function(symbol_name.name, Linkage::Local, &sig).unwrap(); + let sig = get_function_sig(tcx, module.isa().triple(), instance); + let func_id = module.declare_function(symbol_name.name, Linkage::Local, &sig).unwrap(); cx.cached_context.clear(); @@ -40,11 +44,12 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In (0..mir.basic_blocks().len()).map(|_| bcx.create_block()).collect(); // Make FunctionCx - let pointer_type = cx.module.target_config().pointer_type(); + let pointer_type = module.target_config().pointer_type(); let clif_comments = crate::pretty_clif::CommentWriter::new(tcx, instance); let mut fx = FunctionCx { cx, + module, tcx, pointer_type, vtables: FxHashMap::default(), @@ -94,7 +99,7 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In let source_info_set = fx.source_info_set; let local_map = fx.local_map; - fx.constants_cx.finalize(fx.tcx, &mut *fx.cx.module); + fx.constants_cx.finalize(fx.tcx, &mut *fx.module); // Store function in context let context = &mut cx.cached_context; @@ -114,8 +119,8 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In // instruction, which doesn't have an encoding. context.compute_cfg(); context.compute_domtree(); - context.eliminate_unreachable_code(cx.module.isa()).unwrap(); - context.dce(cx.module.isa()).unwrap(); + context.eliminate_unreachable_code(module.isa()).unwrap(); + context.dce(module.isa()).unwrap(); // Some Cranelift optimizations expect the domtree to not yet be computed and as such don't // invalidate it when it would change. context.domtree.clear(); @@ -123,7 +128,6 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In context.want_disasm = crate::pretty_clif::should_write_ir(tcx); // Define function - let module = &mut cx.module; tcx.sess.time("define function", || { module .define_function(func_id, context, &mut NullTrapSink {}, &mut NullStackMapSink {}) @@ -134,7 +138,7 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In crate::pretty_clif::write_clif_file( tcx, "opt", - Some(cx.module.isa()), + Some(module.isa()), instance, &context, &clif_comments, @@ -149,7 +153,7 @@ pub(crate) fn codegen_fn<'tcx>(cx: &mut crate::CodegenCx<'_, 'tcx>, instance: In } // Define debuginfo for function - let isa = cx.module.isa(); + let isa = module.isa(); let debug_context = &mut cx.debug_context; let unwind_context = &mut cx.unwind_context; tcx.sess.time("generate debug info", || { @@ -654,7 +658,7 @@ fn codegen_stmt<'tcx>( // FIXME use emit_small_memset where possible let addr = lval.to_ptr().get_addr(fx); let val = operand.load_scalar(fx); - fx.bcx.call_memset(fx.cx.module.target_config(), addr, val, times); + fx.bcx.call_memset(fx.module.target_config(), addr, val, times); } else { let loop_block = fx.bcx.create_block(); let loop_block2 = fx.bcx.create_block(); @@ -834,7 +838,7 @@ fn codegen_stmt<'tcx>( let elem_size: u64 = pointee.size.bytes(); let bytes = if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count }; - fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, bytes); + fx.bcx.call_memcpy(fx.module.target_config(), dst, src, bytes); } } } diff --git a/src/common.rs b/src/common.rs index 6752bb42dc870..92e4435565ee7 100644 --- a/src/common.rs +++ b/src/common.rs @@ -228,8 +228,9 @@ pub(crate) fn type_sign(ty: Ty<'_>) -> bool { } } -pub(crate) struct FunctionCx<'m, 'clif, 'tcx> { - pub(crate) cx: &'clif mut crate::CodegenCx<'m, 'tcx>, +pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> { + pub(crate) cx: &'clif mut crate::CodegenCx<'tcx>, + pub(crate) module: &'m mut dyn Module, pub(crate) tcx: TyCtxt<'tcx>, pub(crate) pointer_type: Type, // Cached from module pub(crate) vtables: FxHashMap<(Ty<'tcx>, Option>), DataId>, @@ -341,7 +342,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { } pub(crate) fn triple(&self) -> &target_lexicon::Triple { - self.cx.module.isa().triple() + self.module.isa().triple() } pub(crate) fn anonymous_str(&mut self, prefix: &str, msg: &str) -> Value { @@ -354,15 +355,14 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { let mut data_ctx = DataContext::new(); data_ctx.define(msg.as_bytes().to_vec().into_boxed_slice()); let msg_id = self - .cx .module .declare_data(&format!("__{}_{:08x}", prefix, msg_hash), Linkage::Local, false, false) .unwrap(); // Ignore DuplicateDefinition error, as the data will be the same - let _ = self.cx.module.define_data(msg_id, &data_ctx); + let _ = self.module.define_data(msg_id, &data_ctx); - let local_msg_id = self.cx.module.declare_data_in_func(msg_id, self.bcx.func); + let local_msg_id = self.module.declare_data_in_func(msg_id, self.bcx.func); if self.clif_comments.enabled() { self.add_comment(local_msg_id, msg); } diff --git a/src/constant.rs b/src/constant.rs index 339580eb8f4e4..0a0e02d26394e 100644 --- a/src/constant.rs +++ b/src/constant.rs @@ -13,7 +13,7 @@ use rustc_middle::ty::ConstKind; use cranelift_codegen::ir::GlobalValueData; use cranelift_module::*; -use crate::{prelude::*, CodegenCx}; +use crate::prelude::*; pub(crate) struct ConstantCx { todo: Vec, @@ -78,10 +78,10 @@ pub(crate) fn check_constants(fx: &mut FunctionCx<'_, '_, '_>) -> bool { all_constants_ok } -pub(crate) fn codegen_static(cx: &mut CodegenCx<'_, '_>, def_id: DefId) { +pub(crate) fn codegen_static(tcx: TyCtxt<'_>, module: &mut dyn Module, def_id: DefId) { let mut constants_cx = ConstantCx::new(); constants_cx.todo.push(TodoItem::Static(def_id)); - constants_cx.finalize(cx.tcx, &mut *cx.module); + constants_cx.finalize(tcx, module); } pub(crate) fn codegen_tls_ref<'tcx>( @@ -89,8 +89,8 @@ pub(crate) fn codegen_tls_ref<'tcx>( def_id: DefId, layout: TyAndLayout<'tcx>, ) -> CValue<'tcx> { - let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false); - let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false); + let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(local_data_id, format!("tls {:?}", def_id)); } @@ -103,8 +103,8 @@ fn codegen_static_ref<'tcx>( def_id: DefId, layout: TyAndLayout<'tcx>, ) -> CPlace<'tcx> { - let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false); - let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false); + let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(local_data_id, format!("{:?}", def_id)); } @@ -191,29 +191,28 @@ pub(crate) fn codegen_const_value<'tcx>( fx.constants_cx.todo.push(TodoItem::Alloc(ptr.alloc_id)); let data_id = data_id_for_alloc_id( &mut fx.constants_cx, - fx.cx.module, + fx.module, ptr.alloc_id, alloc.mutability, ); let local_data_id = - fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(local_data_id, format!("{:?}", ptr.alloc_id)); } fx.bcx.ins().global_value(fx.pointer_type, local_data_id) } Some(GlobalAlloc::Function(instance)) => { - let func_id = - crate::abi::import_function(fx.tcx, fx.cx.module, instance); + let func_id = crate::abi::import_function(fx.tcx, fx.module, instance); let local_func_id = - fx.cx.module.declare_func_in_func(func_id, &mut fx.bcx.func); + fx.module.declare_func_in_func(func_id, &mut fx.bcx.func); fx.bcx.ins().func_addr(fx.pointer_type, local_func_id) } Some(GlobalAlloc::Static(def_id)) => { assert!(fx.tcx.is_static(def_id)); - let data_id = data_id_for_static(fx.tcx, fx.cx.module, def_id, false); + let data_id = data_id_for_static(fx.tcx, fx.module, def_id, false); let local_data_id = - fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(local_data_id, format!("{:?}", def_id)); } @@ -255,9 +254,9 @@ fn pointer_for_allocation<'tcx>( let alloc_id = fx.tcx.create_memory_alloc(alloc); fx.constants_cx.todo.push(TodoItem::Alloc(alloc_id)); let data_id = - data_id_for_alloc_id(&mut fx.constants_cx, &mut *fx.cx.module, alloc_id, alloc.mutability); + data_id_for_alloc_id(&mut fx.constants_cx, &mut *fx.module, alloc_id, alloc.mutability); - let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(local_data_id, format!("{:?}", alloc_id)); } diff --git a/src/driver/aot.rs b/src/driver/aot.rs index db7d39fea4c59..fc1dd55860380 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -112,16 +112,18 @@ fn module_codegen( let mut cx = crate::CodegenCx::new( tcx, backend_config.clone(), - &mut module, + module.isa(), tcx.sess.opts.debuginfo != DebugInfo::None, ); - super::predefine_mono_items(&mut cx, &mono_items); + super::predefine_mono_items(tcx, &mut module, &mono_items); for (mono_item, _) in mono_items { match mono_item { MonoItem::Fn(inst) => { - cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst)); + cx.tcx + .sess + .time("codegen fn", || crate::base::codegen_fn(&mut cx, &mut module, inst)); } - MonoItem::Static(def_id) => crate::constant::codegen_static(&mut cx, def_id), + MonoItem::Static(def_id) => crate::constant::codegen_static(tcx, &mut module, def_id), MonoItem::GlobalAsm(item_id) => { let item = cx.tcx.hir().item(item_id); if let rustc_hir::ItemKind::GlobalAsm(rustc_hir::GlobalAsm { asm }) = item.kind { @@ -133,9 +135,7 @@ fn module_codegen( } } } - let (global_asm, debug, mut unwind_context) = - tcx.sess.time("finalize CodegenCx", || cx.finalize()); - crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut unwind_context); + crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut cx.unwind_context); let codegen_result = emit_module( tcx, @@ -143,11 +143,11 @@ fn module_codegen( cgu.name().as_str().to_string(), ModuleKind::Regular, module, - debug, - unwind_context, + cx.debug_context, + cx.unwind_context, ); - codegen_global_asm(tcx, &cgu.name().as_str(), &global_asm); + codegen_global_asm(tcx, &cgu.name().as_str(), &cx.global_asm); codegen_result } diff --git a/src/driver/jit.rs b/src/driver/jit.rs index 869b7760e8000..13cfad8df2e22 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -44,44 +44,44 @@ pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { .into_iter() .collect::>(); - let mut cx = crate::CodegenCx::new(tcx, backend_config.clone(), &mut jit_module, false); + let mut cx = crate::CodegenCx::new(tcx, backend_config.clone(), jit_module.isa(), false); super::time(tcx, backend_config.display_cg_time, "codegen mono items", || { - super::predefine_mono_items(&mut cx, &mono_items); + super::predefine_mono_items(tcx, &mut jit_module, &mono_items); for (mono_item, _) in mono_items { match mono_item { MonoItem::Fn(inst) => match backend_config.codegen_mode { CodegenMode::Aot => unreachable!(), CodegenMode::Jit => { - cx.tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, inst)); + cx.tcx.sess.time("codegen fn", || { + crate::base::codegen_fn(&mut cx, &mut jit_module, inst) + }); } - CodegenMode::JitLazy => codegen_shim(&mut cx, inst), + CodegenMode::JitLazy => codegen_shim(&mut cx, &mut jit_module, inst), }, MonoItem::Static(def_id) => { - crate::constant::codegen_static(&mut cx, def_id); + crate::constant::codegen_static(tcx, &mut jit_module, def_id); } MonoItem::GlobalAsm(item_id) => { - let item = cx.tcx.hir().item(item_id); + let item = tcx.hir().item(item_id); tcx.sess.span_fatal(item.span, "Global asm is not supported in JIT mode"); } } } }); - let (global_asm, _debug, mut unwind_context) = - tcx.sess.time("finalize CodegenCx", || cx.finalize()); jit_module.finalize_definitions(); - if !global_asm.is_empty() { + if !cx.global_asm.is_empty() { tcx.sess.fatal("Inline asm is not supported in JIT mode"); } - crate::allocator::codegen(tcx, &mut jit_module, &mut unwind_context); + crate::allocator::codegen(tcx, &mut jit_module, &mut cx.unwind_context); tcx.sess.abort_if_errors(); jit_module.finalize_definitions(); - let _unwind_register_guard = unsafe { unwind_context.register_jit(&jit_module) }; + let _unwind_register_guard = unsafe { cx.unwind_context.register_jit(&jit_module) }; println!( "Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed" @@ -171,13 +171,12 @@ extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8 let func_id = jit_module.declare_function(&name, Linkage::Export, &sig).unwrap(); jit_module.prepare_for_function_redefine(func_id).unwrap(); - let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module, false); - tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, instance)); + let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module.isa(), false); + tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, jit_module, instance)); - let (global_asm, _debug_context, unwind_context) = cx.finalize(); - assert!(global_asm.is_empty()); + assert!(cx.global_asm.is_empty()); jit_module.finalize_definitions(); - std::mem::forget(unsafe { unwind_context.register_jit(&jit_module) }); + std::mem::forget(unsafe { cx.unwind_context.register_jit(&jit_module) }); jit_module.get_finalized_function(func_id) }) }) @@ -247,24 +246,23 @@ fn load_imported_symbols_for_jit(tcx: TyCtxt<'_>) -> Vec<(String, *const u8)> { imported_symbols } -fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) { +fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: Instance<'tcx>) { let tcx = cx.tcx; - let pointer_type = cx.module.target_config().pointer_type(); + let pointer_type = module.target_config().pointer_type(); let name = tcx.symbol_name(inst).name.to_string(); - let sig = crate::abi::get_function_sig(tcx, cx.module.isa().triple(), inst); - let func_id = cx.module.declare_function(&name, Linkage::Export, &sig).unwrap(); + let sig = crate::abi::get_function_sig(tcx, module.isa().triple(), inst); + let func_id = module.declare_function(&name, Linkage::Export, &sig).unwrap(); let instance_ptr = Box::into_raw(Box::new(inst)); - let jit_fn = cx - .module + let jit_fn = module .declare_function( "__clif_jit_fn", Linkage::Import, &Signature { - call_conv: cx.module.target_config().default_call_conv, + call_conv: module.target_config().default_call_conv, params: vec![AbiParam::new(pointer_type)], returns: vec![AbiParam::new(pointer_type)], }, @@ -278,7 +276,7 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) { let mut builder_ctx = FunctionBuilderContext::new(); let mut trampoline_builder = FunctionBuilder::new(trampoline, &mut builder_ctx); - let jit_fn = cx.module.declare_func_in_func(jit_fn, trampoline_builder.func); + let jit_fn = module.declare_func_in_func(jit_fn, trampoline_builder.func); let sig_ref = trampoline_builder.func.import_signature(sig); let entry_block = trampoline_builder.create_block(); @@ -293,7 +291,7 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'_, 'tcx>, inst: Instance<'tcx>) { let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec(); trampoline_builder.ins().return_(&ret_vals); - cx.module + module .define_function( func_id, &mut cx.cached_context, diff --git a/src/driver/mod.rs b/src/driver/mod.rs index 960e10182d812..27fd7a2f2d886 100644 --- a/src/driver/mod.rs +++ b/src/driver/mod.rs @@ -40,24 +40,25 @@ pub(crate) fn codegen_crate( } fn predefine_mono_items<'tcx>( - cx: &mut crate::CodegenCx<'_, 'tcx>, + tcx: TyCtxt<'tcx>, + module: &mut dyn Module, mono_items: &[(MonoItem<'tcx>, (RLinkage, Visibility))], ) { - cx.tcx.sess.time("predefine functions", || { - let is_compiler_builtins = cx.tcx.is_compiler_builtins(LOCAL_CRATE); + tcx.sess.time("predefine functions", || { + let is_compiler_builtins = tcx.is_compiler_builtins(LOCAL_CRATE); for &(mono_item, (linkage, visibility)) in mono_items { match mono_item { MonoItem::Fn(instance) => { - let name = cx.tcx.symbol_name(instance).name.to_string(); + let name = tcx.symbol_name(instance).name.to_string(); let _inst_guard = crate::PrintOnPanic(|| format!("{:?} {}", instance, name)); - let sig = get_function_sig(cx.tcx, cx.module.isa().triple(), instance); + let sig = get_function_sig(tcx, module.isa().triple(), instance); let linkage = crate::linkage::get_clif_linkage( mono_item, linkage, visibility, is_compiler_builtins, ); - cx.module.declare_function(&name, linkage, &sig).unwrap(); + module.declare_function(&name, linkage, &sig).unwrap(); } MonoItem::Static(_) | MonoItem::GlobalAsm(_) => {} } diff --git a/src/inline_asm.rs b/src/inline_asm.rs index 7187bc785e46d..669a6d35075b4 100644 --- a/src/inline_asm.rs +++ b/src/inline_asm.rs @@ -201,7 +201,6 @@ fn call_inline_asm<'tcx>( } let inline_asm_func = fx - .cx .module .declare_function( asm_name, @@ -213,7 +212,7 @@ fn call_inline_asm<'tcx>( }, ) .unwrap(); - let inline_asm_func = fx.cx.module.declare_func_in_func(inline_asm_func, &mut fx.bcx.func); + let inline_asm_func = fx.module.declare_func_in_func(inline_asm_func, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(inline_asm_func, asm_name); } diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index 58a24c4193032..d08271f853b50 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -498,10 +498,10 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( if intrinsic.contains("nonoverlapping") { // FIXME emit_small_memcpy - fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, byte_amount); + fx.bcx.call_memcpy(fx.module.target_config(), dst, src, byte_amount); } else { // FIXME emit_small_memmove - fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount); + fx.bcx.call_memmove(fx.module.target_config(), dst, src, byte_amount); } }; // NOTE: the volatile variants have src and dst swapped @@ -517,10 +517,10 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( // FIXME make the copy actually volatile when using emit_small_mem{cpy,move} if intrinsic.contains("nonoverlapping") { // FIXME emit_small_memcpy - fx.bcx.call_memcpy(fx.cx.module.target_config(), dst, src, byte_amount); + fx.bcx.call_memcpy(fx.module.target_config(), dst, src, byte_amount); } else { // FIXME emit_small_memmove - fx.bcx.call_memmove(fx.cx.module.target_config(), dst, src, byte_amount); + fx.bcx.call_memmove(fx.module.target_config(), dst, src, byte_amount); } }; size_of_val, (c ptr) { @@ -670,7 +670,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let dst_ptr = dst.load_scalar(fx); // FIXME make the memset actually volatile when switching to emit_small_memset // FIXME use emit_small_memset - fx.bcx.call_memset(fx.cx.module.target_config(), dst_ptr, val, count); + fx.bcx.call_memset(fx.module.target_config(), dst_ptr, val, count); }; ctlz | ctlz_nonzero, (v arg) { // FIXME trap on `ctlz_nonzero` with zero arg. diff --git a/src/lib.rs b/src/lib.rs index ee556bd2281a5..1afd35af352eb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -33,6 +33,7 @@ use rustc_middle::ty::query::Providers; use rustc_session::config::OutputFilenames; use rustc_session::Session; +use cranelift_codegen::isa::TargetIsa; use cranelift_codegen::settings::{self, Configurable}; pub use crate::config::*; @@ -118,42 +119,32 @@ impl String> Drop for PrintOnPanic { } } -struct CodegenCx<'m, 'tcx: 'm> { +struct CodegenCx<'tcx> { tcx: TyCtxt<'tcx>, - module: &'m mut dyn Module, global_asm: String, cached_context: Context, debug_context: Option>, unwind_context: UnwindContext, } -impl<'m, 'tcx> CodegenCx<'m, 'tcx> { +impl<'tcx> CodegenCx<'tcx> { fn new( tcx: TyCtxt<'tcx>, backend_config: BackendConfig, - module: &'m mut dyn Module, + isa: &dyn TargetIsa, debug_info: bool, ) -> Self { - let unwind_context = UnwindContext::new( - tcx, - module.isa(), - matches!(backend_config.codegen_mode, CodegenMode::Aot), - ); - let debug_context = - if debug_info { Some(DebugContext::new(tcx, module.isa())) } else { None }; + let unwind_context = + UnwindContext::new(tcx, isa, matches!(backend_config.codegen_mode, CodegenMode::Aot)); + let debug_context = if debug_info { Some(DebugContext::new(tcx, isa)) } else { None }; CodegenCx { tcx, - module, global_asm: String::new(), cached_context: Context::new(), debug_context, unwind_context, } } - - fn finalize(self) -> (String, Option>, UnwindContext) { - (self.global_asm, self.debug_context, self.unwind_context) - } } pub struct CraneliftCodegenBackend { diff --git a/src/trap.rs b/src/trap.rs index 4e52fb161211c..819c8b51558a0 100644 --- a/src/trap.rs +++ b/src/trap.rs @@ -4,7 +4,6 @@ use crate::prelude::*; fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) { let puts = fx - .cx .module .declare_function( "puts", @@ -16,7 +15,7 @@ fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) { }, ) .unwrap(); - let puts = fx.cx.module.declare_func_in_func(puts, &mut fx.bcx.func); + let puts = fx.module.declare_func_in_func(puts, &mut fx.bcx.func); if fx.clif_comments.enabled() { fx.add_comment(puts, "puts"); } diff --git a/src/value_and_place.rs b/src/value_and_place.rs index b97d39009847a..9a572c3501f92 100644 --- a/src/value_and_place.rs +++ b/src/value_and_place.rs @@ -554,7 +554,7 @@ impl<'tcx> CPlace<'tcx> { let src_align = src_layout.align.abi.bytes() as u8; let dst_align = dst_layout.align.abi.bytes() as u8; fx.bcx.emit_small_memory_copy( - fx.cx.module.target_config(), + fx.module.target_config(), to_addr, from_addr, size, diff --git a/src/vtable.rs b/src/vtable.rs index 60bc53024a7e3..29e960bb84db7 100644 --- a/src/vtable.rs +++ b/src/vtable.rs @@ -80,7 +80,7 @@ pub(crate) fn get_vtable<'tcx>( data_id }; - let local_data_id = fx.cx.module.declare_data_in_func(data_id, &mut fx.bcx.func); + let local_data_id = fx.module.declare_data_in_func(data_id, &mut fx.bcx.func); fx.bcx.ins().global_value(fx.pointer_type, local_data_id) } @@ -94,7 +94,7 @@ fn build_vtable<'tcx>( let drop_in_place_fn = import_function( tcx, - fx.cx.module, + fx.module, Instance::resolve_drop_in_place(tcx, layout.ty).polymorphize(fx.tcx), ); @@ -111,7 +111,7 @@ fn build_vtable<'tcx>( opt_mth.map(|(def_id, substs)| { import_function( tcx, - fx.cx.module, + fx.module, Instance::resolve_for_vtable(tcx, ParamEnv::reveal_all(), def_id, substs) .unwrap() .polymorphize(fx.tcx), @@ -132,16 +132,16 @@ fn build_vtable<'tcx>( for (i, component) in components.into_iter().enumerate() { if let Some(func_id) = component { - let func_ref = fx.cx.module.declare_func_in_data(func_id, &mut data_ctx); + let func_ref = fx.module.declare_func_in_data(func_id, &mut data_ctx); data_ctx.write_function_addr((i * usize_size) as u32, func_ref); } } data_ctx.set_align(fx.tcx.data_layout.pointer_align.pref.bytes()); - let data_id = fx.cx.module.declare_anonymous_data(false, false).unwrap(); + let data_id = fx.module.declare_anonymous_data(false, false).unwrap(); - fx.cx.module.define_data(data_id, &data_ctx).unwrap(); + fx.module.define_data(data_id, &data_ctx).unwrap(); data_id } From ba8e610b269f90287aad1b5db0520bfd3118612e Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 8 Apr 2021 21:26:20 +0200 Subject: [PATCH 23/52] Inline driver::codegen_crate --- src/driver/aot.rs | 2 +- src/driver/jit.rs | 8 ++++++-- src/driver/mod.rs | 34 ++-------------------------------- src/lib.rs | 12 +++++++++++- 4 files changed, 20 insertions(+), 36 deletions(-) diff --git a/src/driver/aot.rs b/src/driver/aot.rs index fc1dd55860380..f1d30ba4bdce0 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -152,7 +152,7 @@ fn module_codegen( codegen_result } -pub(super) fn run_aot( +pub(crate) fn run_aot( tcx: TyCtxt<'_>, backend_config: BackendConfig, metadata: EncodedMetadata, diff --git a/src/driver/jit.rs b/src/driver/jit.rs index 13cfad8df2e22..bcc7a494499af 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -20,9 +20,13 @@ thread_local! { pub static CURRENT_MODULE: RefCell> = RefCell::new(None); } -pub(super) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { +pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { if !tcx.sess.opts.output_types.should_codegen() { - tcx.sess.fatal("JIT mode doesn't work with `cargo check`."); + tcx.sess.fatal("JIT mode doesn't work with `cargo check`"); + } + + if !tcx.sess.crate_types().contains(&rustc_session::config::CrateType::Executable) { + tcx.sess.fatal("can't jit non-executable crate"); } let imported_symbols = load_imported_symbols_for_jit(tcx); diff --git a/src/driver/mod.rs b/src/driver/mod.rs index 27fd7a2f2d886..8dde9992b35fb 100644 --- a/src/driver/mod.rs +++ b/src/driver/mod.rs @@ -1,43 +1,13 @@ //! Drivers are responsible for calling [`codegen_mono_item`] and performing any further actions //! like JIT executing or writing object files. -use std::any::Any; - -use rustc_middle::middle::cstore::EncodedMetadata; use rustc_middle::mir::mono::{Linkage as RLinkage, MonoItem, Visibility}; use crate::prelude::*; -use crate::CodegenMode; -mod aot; +pub(crate) mod aot; #[cfg(feature = "jit")] -mod jit; - -pub(crate) fn codegen_crate( - tcx: TyCtxt<'_>, - metadata: EncodedMetadata, - need_metadata_module: bool, - backend_config: crate::BackendConfig, -) -> Box { - tcx.sess.abort_if_errors(); - - match backend_config.codegen_mode { - CodegenMode::Aot => aot::run_aot(tcx, backend_config, metadata, need_metadata_module), - CodegenMode::Jit | CodegenMode::JitLazy => { - let is_executable = - tcx.sess.crate_types().contains(&rustc_session::config::CrateType::Executable); - if !is_executable { - tcx.sess.fatal("can't jit non-executable crate"); - } - - #[cfg(feature = "jit")] - let _: ! = jit::run_jit(tcx, backend_config); - - #[cfg(not(feature = "jit"))] - tcx.sess.fatal("jit support was disabled when compiling rustc_codegen_cranelift"); - } - } -} +pub(crate) mod jit; fn predefine_mono_items<'tcx>( tcx: TyCtxt<'tcx>, diff --git a/src/lib.rs b/src/lib.rs index 1afd35af352eb..e15d21a123d10 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -177,13 +177,23 @@ impl CodegenBackend for CraneliftCodegenBackend { metadata: EncodedMetadata, need_metadata_module: bool, ) -> Box { + tcx.sess.abort_if_errors(); let config = if let Some(config) = self.config.clone() { config } else { BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args) .unwrap_or_else(|err| tcx.sess.fatal(&err)) }; - driver::codegen_crate(tcx, metadata, need_metadata_module, config) + match config.codegen_mode { + CodegenMode::Aot => driver::aot::run_aot(tcx, config, metadata, need_metadata_module), + CodegenMode::Jit | CodegenMode::JitLazy => { + #[cfg(feature = "jit")] + let _: ! = driver::jit::run_jit(tcx, config); + + #[cfg(not(feature = "jit"))] + tcx.sess.fatal("jit support was disabled when compiling rustc_codegen_cranelift"); + } + } } fn join_codegen( From 86530f889ead9f422a621f89e888f19d42cdc34f Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 8 Apr 2021 19:03:15 +0200 Subject: [PATCH 24/52] Use maybe_create_entry_wrapper again in jit mode This simplifies the jit driver a lot --- src/driver/aot.rs | 2 +- src/driver/jit.rs | 71 +++++++++++++---------------------------------- src/main_shim.rs | 29 ++++++++++++++----- 3 files changed, 42 insertions(+), 60 deletions(-) diff --git a/src/driver/aot.rs b/src/driver/aot.rs index f1d30ba4bdce0..2ad580d633032 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -135,7 +135,7 @@ fn module_codegen( } } } - crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut cx.unwind_context); + crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut cx.unwind_context, false); let codegen_result = emit_module( tcx, diff --git a/src/driver/jit.rs b/src/driver/jit.rs index bcc7a494499af..6c15d6761238e 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -8,7 +8,6 @@ use std::os::raw::{c_char, c_int}; use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink}; use rustc_codegen_ssa::CrateInfo; use rustc_middle::mir::mono::MonoItem; -use rustc_session::config::EntryFnType; use cranelift_jit::{JITBuilder, JITModule}; @@ -81,6 +80,7 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { } crate::allocator::codegen(tcx, &mut jit_module, &mut cx.unwind_context); + crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, &mut cx.unwind_context, true); tcx.sess.abort_if_errors(); @@ -105,57 +105,24 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { assert!(tls_backend_config.borrow_mut().replace(backend_config).is_none()) }); - let (main_def_id, entry_ty) = tcx.entry_fn(LOCAL_CRATE).unwrap(); - let instance = Instance::mono(tcx, main_def_id.to_def_id()).polymorphize(tcx); - - match entry_ty { - EntryFnType::Main => { - // FIXME set program arguments somehow - - let main_sig = Signature { - params: vec![], - returns: vec![], - call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)), - }; - let main_func_id = jit_module - .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &main_sig) - .unwrap(); - let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id); - - CURRENT_MODULE.with(|current_module| { - assert!(current_module.borrow_mut().replace(jit_module).is_none()) - }); - - let f: extern "C" fn() = unsafe { ::std::mem::transmute(finalized_main) }; - f(); - std::process::exit(0); - } - EntryFnType::Start => { - let start_sig = Signature { - params: vec![ - AbiParam::new(jit_module.target_config().pointer_type()), - AbiParam::new(jit_module.target_config().pointer_type()), - ], - returns: vec![AbiParam::new( - jit_module.target_config().pointer_type(), /*isize*/ - )], - call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)), - }; - let start_func_id = jit_module - .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &start_sig) - .unwrap(); - let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id); - - CURRENT_MODULE.with(|current_module| { - assert!(current_module.borrow_mut().replace(jit_module).is_none()) - }); - - let f: extern "C" fn(c_int, *const *const c_char) -> c_int = - unsafe { ::std::mem::transmute(finalized_start) }; - let ret = f(args.len() as c_int, argv.as_ptr()); - std::process::exit(ret); - } - } + let start_sig = Signature { + params: vec![ + AbiParam::new(jit_module.target_config().pointer_type()), + AbiParam::new(jit_module.target_config().pointer_type()), + ], + returns: vec![AbiParam::new(jit_module.target_config().pointer_type() /*isize*/)], + call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)), + }; + let start_func_id = jit_module.declare_function("main", Linkage::Import, &start_sig).unwrap(); + let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id); + + CURRENT_MODULE + .with(|current_module| assert!(current_module.borrow_mut().replace(jit_module).is_none())); + + let f: extern "C" fn(c_int, *const *const c_char) -> c_int = + unsafe { ::std::mem::transmute(finalized_start) }; + let ret = f(args.len() as c_int, argv.as_ptr()); + std::process::exit(ret); } #[no_mangle] diff --git a/src/main_shim.rs b/src/main_shim.rs index 22dc3f2bfe64e..ff386710cd174 100644 --- a/src/main_shim.rs +++ b/src/main_shim.rs @@ -10,8 +10,9 @@ pub(crate) fn maybe_create_entry_wrapper( tcx: TyCtxt<'_>, module: &mut impl Module, unwind_context: &mut UnwindContext, + ignore_lang_start_wrapper: bool, ) { - let (main_def_id, use_start_lang_item) = match tcx.entry_fn(LOCAL_CRATE) { + let (main_def_id, is_main_fn) = match tcx.entry_fn(LOCAL_CRATE) { Some((def_id, entry_ty)) => ( def_id.to_def_id(), match entry_ty { @@ -27,14 +28,22 @@ pub(crate) fn maybe_create_entry_wrapper( return; } - create_entry_fn(tcx, module, unwind_context, main_def_id, use_start_lang_item); + create_entry_fn( + tcx, + module, + unwind_context, + main_def_id, + ignore_lang_start_wrapper, + is_main_fn, + ); fn create_entry_fn( tcx: TyCtxt<'_>, m: &mut impl Module, unwind_context: &mut UnwindContext, rust_main_def_id: DefId, - use_start_lang_item: bool, + ignore_lang_start_wrapper: bool, + is_main_fn: bool, ) { let main_ret_ty = tcx.fn_sig(rust_main_def_id).output(); // Given that `main()` has no arguments, @@ -74,7 +83,12 @@ pub(crate) fn maybe_create_entry_wrapper( let main_func_ref = m.declare_func_in_func(main_func_id, &mut bcx.func); - let call_inst = if use_start_lang_item { + let result = if is_main_fn && ignore_lang_start_wrapper { + // regular main fn, but ignoring #[lang = "start"] as we are running in the jit + // FIXME set program arguments somehow + bcx.ins().call(main_func_ref, &[]); + bcx.ins().iconst(m.target_config().pointer_type(), 0) + } else if is_main_fn { let start_def_id = tcx.require_lang_item(LangItem::Start, None); let start_instance = Instance::resolve( tcx, @@ -90,13 +104,14 @@ pub(crate) fn maybe_create_entry_wrapper( let main_val = bcx.ins().func_addr(m.target_config().pointer_type(), main_func_ref); let func_ref = m.declare_func_in_func(start_func_id, &mut bcx.func); - bcx.ins().call(func_ref, &[main_val, arg_argc, arg_argv]) + let call_inst = bcx.ins().call(func_ref, &[main_val, arg_argc, arg_argv]); + bcx.inst_results(call_inst)[0] } else { // using user-defined start fn - bcx.ins().call(main_func_ref, &[arg_argc, arg_argv]) + let call_inst = bcx.ins().call(main_func_ref, &[arg_argc, arg_argv]); + bcx.inst_results(call_inst)[0] }; - let result = bcx.inst_results(call_inst)[0]; bcx.ins().return_(&[result]); bcx.seal_all_blocks(); bcx.finalize(); From 6fac7f089f8e209ec2181900a322294896cfbcbf Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 8 Apr 2021 18:27:26 +0200 Subject: [PATCH 25/52] Don't unregister unwind tables after the JIT is done --- src/debuginfo/unwind.rs | 48 ++++++++--------------------------------- src/driver/jit.rs | 11 +++++++--- src/lib.rs | 2 +- 3 files changed, 18 insertions(+), 43 deletions(-) diff --git a/src/debuginfo/unwind.rs b/src/debuginfo/unwind.rs index aeafc901fa533..ca7083cccb88f 100644 --- a/src/debuginfo/unwind.rs +++ b/src/debuginfo/unwind.rs @@ -72,15 +72,12 @@ impl UnwindContext { } #[cfg(feature = "jit")] - pub(crate) unsafe fn register_jit( - self, - jit_module: &cranelift_jit::JITModule, - ) -> Option { + pub(crate) unsafe fn register_jit(self, jit_module: &cranelift_jit::JITModule) { let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(self.endian)); self.frame_table.write_eh_frame(&mut eh_frame).unwrap(); if eh_frame.0.writer.slice().is_empty() { - return None; + return; } let mut eh_frame = eh_frame.0.relocate_for_jit(jit_module); @@ -88,7 +85,10 @@ impl UnwindContext { // GCC expects a terminating "empty" length, so write a 0 length at the end of the table. eh_frame.extend(&[0, 0, 0, 0]); - let mut registrations = Vec::new(); + // FIXME support unregistering unwind tables once cranelift-jit supports deallocating + // individual functions + #[allow(unused_variables)] + let (eh_frame, eh_frame_len, _) = Vec::into_raw_parts(eh_frame); // ======================================================================= // Everything after this line up to the end of the file is loosly based on @@ -96,8 +96,8 @@ impl UnwindContext { #[cfg(target_os = "macos")] { // On macOS, `__register_frame` takes a pointer to a single FDE - let start = eh_frame.as_ptr(); - let end = start.add(eh_frame.len()); + let start = eh_frame; + let end = start.add(eh_frame_len); let mut current = start; // Walk all of the entries in the frame table and register them @@ -107,7 +107,6 @@ impl UnwindContext { // Skip over the CIE if current != start { __register_frame(current); - registrations.push(current as usize); } // Move to the next table entry (+4 because the length itself is not inclusive) @@ -117,41 +116,12 @@ impl UnwindContext { #[cfg(not(target_os = "macos"))] { // On other platforms, `__register_frame` will walk the FDEs until an entry of length 0 - let ptr = eh_frame.as_ptr(); - __register_frame(ptr); - registrations.push(ptr as usize); + __register_frame(eh_frame); } - - Some(UnwindRegistry { _frame_table: eh_frame, registrations }) } } -/// Represents a registry of function unwind information for System V ABI. -pub(crate) struct UnwindRegistry { - _frame_table: Vec, - registrations: Vec, -} - extern "C" { // libunwind import fn __register_frame(fde: *const u8); - fn __deregister_frame(fde: *const u8); -} - -impl Drop for UnwindRegistry { - fn drop(&mut self) { - unsafe { - // libgcc stores the frame entries as a linked list in decreasing sort order - // based on the PC value of the registered entry. - // - // As we store the registrations in increasing order, it would be O(N^2) to - // deregister in that order. - // - // To ensure that we just pop off the first element in the list upon every - // deregistration, walk our list of registrations backwards. - for fde in self.registrations.iter().rev() { - __deregister_frame(*fde as *const _); - } - } - } } diff --git a/src/driver/jit.rs b/src/driver/jit.rs index 6c15d6761238e..99d8cd57ba6b3 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -80,12 +80,17 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { } crate::allocator::codegen(tcx, &mut jit_module, &mut cx.unwind_context); - crate::main_shim::maybe_create_entry_wrapper(tcx, &mut jit_module, &mut cx.unwind_context, true); + crate::main_shim::maybe_create_entry_wrapper( + tcx, + &mut jit_module, + &mut cx.unwind_context, + true, + ); tcx.sess.abort_if_errors(); jit_module.finalize_definitions(); - let _unwind_register_guard = unsafe { cx.unwind_context.register_jit(&jit_module) }; + unsafe { cx.unwind_context.register_jit(&jit_module) }; println!( "Rustc codegen cranelift will JIT run the executable, because -Cllvm-args=mode=jit was passed" @@ -147,7 +152,7 @@ extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8 assert!(cx.global_asm.is_empty()); jit_module.finalize_definitions(); - std::mem::forget(unsafe { cx.unwind_context.register_jit(&jit_module) }); + unsafe { cx.unwind_context.register_jit(&jit_module) }; jit_module.get_finalized_function(func_id) }) }) diff --git a/src/lib.rs b/src/lib.rs index e15d21a123d10..12d8f9c1d73f7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,4 @@ -#![feature(rustc_private, decl_macro, never_type, hash_drain_filter)] +#![feature(rustc_private, decl_macro, never_type, hash_drain_filter, vec_into_raw_parts)] #![warn(rust_2018_idioms)] #![warn(unused_lifetimes)] #![warn(unreachable_pub)] From d4d270d5030b48d815a0ca19f28c78888847be11 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 8 Apr 2021 19:22:48 +0200 Subject: [PATCH 26/52] Merge BACKEND_CONFIG and CURRENT_MODULE thread locals --- src/driver/jit.rs | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/driver/jit.rs b/src/driver/jit.rs index 99d8cd57ba6b3..4b12c1aec9927 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -14,9 +14,13 @@ use cranelift_jit::{JITBuilder, JITModule}; use crate::{prelude::*, BackendConfig}; use crate::{CodegenCx, CodegenMode}; +struct JitState { + backend_config: BackendConfig, + jit_module: JITModule, +} + thread_local! { - pub static BACKEND_CONFIG: RefCell> = RefCell::new(None); - pub static CURRENT_MODULE: RefCell> = RefCell::new(None); + static LAZY_JIT_STATE: RefCell> = RefCell::new(None); } pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { @@ -106,10 +110,6 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { // useful as some dynamic linkers use it as a marker to jump over. argv.push(std::ptr::null()); - BACKEND_CONFIG.with(|tls_backend_config| { - assert!(tls_backend_config.borrow_mut().replace(backend_config).is_none()) - }); - let start_sig = Signature { params: vec![ AbiParam::new(jit_module.target_config().pointer_type()), @@ -121,8 +121,11 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { let start_func_id = jit_module.declare_function("main", Linkage::Import, &start_sig).unwrap(); let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id); - CURRENT_MODULE - .with(|current_module| assert!(current_module.borrow_mut().replace(jit_module).is_none())); + LAZY_JIT_STATE.with(|lazy_jit_state| { + let mut lazy_jit_state = lazy_jit_state.borrow_mut(); + assert!(lazy_jit_state.is_none()); + *lazy_jit_state = Some(JitState { backend_config, jit_module }); + }); let f: extern "C" fn(c_int, *const *const c_char) -> c_int = unsafe { ::std::mem::transmute(finalized_start) }; @@ -136,11 +139,11 @@ extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8 // lift is used to ensure the correct lifetime for instance. let instance = tcx.lift(unsafe { *instance_ptr }).unwrap(); - CURRENT_MODULE.with(|jit_module| { - let mut jit_module = jit_module.borrow_mut(); - let jit_module = jit_module.as_mut().unwrap(); - let backend_config = - BACKEND_CONFIG.with(|backend_config| backend_config.borrow().clone().unwrap()); + LAZY_JIT_STATE.with(|lazy_jit_state| { + let mut lazy_jit_state = lazy_jit_state.borrow_mut(); + let lazy_jit_state = lazy_jit_state.as_mut().unwrap(); + let jit_module = &mut lazy_jit_state.jit_module; + let backend_config = lazy_jit_state.backend_config.clone(); let name = tcx.symbol_name(instance).name.to_string(); let sig = crate::abi::get_function_sig(tcx, jit_module.isa().triple(), instance); From eed9aaa2681aec34e41f48ca4e47b3e8c6f1e490 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 8 Apr 2021 21:34:55 +0200 Subject: [PATCH 27/52] Extract create_jit_module function --- src/driver/aot.rs | 1 - src/driver/jit.rs | 53 ++++++++++++++++++++++++++++------------------- src/lib.rs | 2 ++ src/main_shim.rs | 13 +++--------- 4 files changed, 37 insertions(+), 32 deletions(-) diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 2ad580d633032..3ff96d710e58e 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -107,7 +107,6 @@ fn module_codegen( let isa = crate::build_isa(tcx.sess, &backend_config); let mut module = crate::backend::make_module(tcx.sess, isa, cgu_name.as_str().to_string()); - assert_eq!(pointer_ty(tcx), module.target_config().pointer_type()); let mut cx = crate::CodegenCx::new( tcx, diff --git a/src/driver/jit.rs b/src/driver/jit.rs index 4b12c1aec9927..f7fe02bb0160c 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -23,6 +23,33 @@ thread_local! { static LAZY_JIT_STATE: RefCell> = RefCell::new(None); } +fn create_jit_module<'tcx>( + tcx: TyCtxt<'tcx>, + backend_config: &BackendConfig, + hotswap: bool, +) -> (JITModule, CodegenCx<'tcx>) { + let imported_symbols = load_imported_symbols_for_jit(tcx); + + let isa = crate::build_isa(tcx.sess, backend_config); + let mut jit_builder = JITBuilder::with_isa(isa, cranelift_module::default_libcall_names()); + jit_builder.hotswap(hotswap); + crate::compiler_builtins::register_functions_for_jit(&mut jit_builder); + jit_builder.symbols(imported_symbols); + let mut jit_module = JITModule::new(jit_builder); + + let mut cx = crate::CodegenCx::new(tcx, backend_config.clone(), jit_module.isa(), false); + + crate::allocator::codegen(tcx, &mut jit_module, &mut cx.unwind_context); + crate::main_shim::maybe_create_entry_wrapper( + tcx, + &mut jit_module, + &mut cx.unwind_context, + true, + ); + + (jit_module, cx) +} + pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { if !tcx.sess.opts.output_types.should_codegen() { tcx.sess.fatal("JIT mode doesn't work with `cargo check`"); @@ -32,15 +59,11 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { tcx.sess.fatal("can't jit non-executable crate"); } - let imported_symbols = load_imported_symbols_for_jit(tcx); - - let isa = crate::build_isa(tcx.sess, &backend_config); - let mut jit_builder = JITBuilder::with_isa(isa, cranelift_module::default_libcall_names()); - jit_builder.hotswap(matches!(backend_config.codegen_mode, CodegenMode::JitLazy)); - crate::compiler_builtins::register_functions_for_jit(&mut jit_builder); - jit_builder.symbols(imported_symbols); - let mut jit_module = JITModule::new(jit_builder); - assert_eq!(pointer_ty(tcx), jit_module.target_config().pointer_type()); + let (mut jit_module, mut cx) = create_jit_module( + tcx, + &backend_config, + matches!(backend_config.codegen_mode, CodegenMode::JitLazy), + ); let (_, cgus) = tcx.collect_and_partition_mono_items(LOCAL_CRATE); let mono_items = cgus @@ -51,8 +74,6 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { .into_iter() .collect::>(); - let mut cx = crate::CodegenCx::new(tcx, backend_config.clone(), jit_module.isa(), false); - super::time(tcx, backend_config.display_cg_time, "codegen mono items", || { super::predefine_mono_items(tcx, &mut jit_module, &mono_items); for (mono_item, _) in mono_items { @@ -77,20 +98,10 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { } }); - jit_module.finalize_definitions(); - if !cx.global_asm.is_empty() { tcx.sess.fatal("Inline asm is not supported in JIT mode"); } - crate::allocator::codegen(tcx, &mut jit_module, &mut cx.unwind_context); - crate::main_shim::maybe_create_entry_wrapper( - tcx, - &mut jit_module, - &mut cx.unwind_context, - true, - ); - tcx.sess.abort_if_errors(); jit_module.finalize_definitions(); diff --git a/src/lib.rs b/src/lib.rs index 12d8f9c1d73f7..2c989520f6a56 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -134,6 +134,8 @@ impl<'tcx> CodegenCx<'tcx> { isa: &dyn TargetIsa, debug_info: bool, ) -> Self { + assert_eq!(pointer_ty(tcx), isa.pointer_type()); + let unwind_context = UnwindContext::new(tcx, isa, matches!(backend_config.codegen_mode, CodegenMode::Aot)); let debug_context = if debug_info { Some(DebugContext::new(tcx, isa)) } else { None }; diff --git a/src/main_shim.rs b/src/main_shim.rs index ff386710cd174..4beb6c65a7791 100644 --- a/src/main_shim.rs +++ b/src/main_shim.rs @@ -10,7 +10,7 @@ pub(crate) fn maybe_create_entry_wrapper( tcx: TyCtxt<'_>, module: &mut impl Module, unwind_context: &mut UnwindContext, - ignore_lang_start_wrapper: bool, + is_jit: bool, ) { let (main_def_id, is_main_fn) = match tcx.entry_fn(LOCAL_CRATE) { Some((def_id, entry_ty)) => ( @@ -24,18 +24,11 @@ pub(crate) fn maybe_create_entry_wrapper( }; let instance = Instance::mono(tcx, main_def_id).polymorphize(tcx); - if module.get_name(&*tcx.symbol_name(instance).name).is_none() { + if !is_jit && module.get_name(&*tcx.symbol_name(instance).name).is_none() { return; } - create_entry_fn( - tcx, - module, - unwind_context, - main_def_id, - ignore_lang_start_wrapper, - is_main_fn, - ); + create_entry_fn(tcx, module, unwind_context, main_def_id, is_jit, is_main_fn); fn create_entry_fn( tcx: TyCtxt<'_>, From ff38b37769dde7b41957157a80ab97b3e985d432 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 9 Apr 2021 11:34:58 +0200 Subject: [PATCH 28/52] Fix docs --- src/driver/jit.rs | 2 +- src/driver/mod.rs | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/driver/jit.rs b/src/driver/jit.rs index f7fe02bb0160c..63ac02f83b92c 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -1,4 +1,4 @@ -//! The JIT driver uses [`cranelift_simplejit`] to JIT execute programs without writing any object +//! The JIT driver uses [`cranelift_jit`] to JIT execute programs without writing any object //! files. use std::cell::RefCell; diff --git a/src/driver/mod.rs b/src/driver/mod.rs index 8dde9992b35fb..393cde3e9ad27 100644 --- a/src/driver/mod.rs +++ b/src/driver/mod.rs @@ -1,5 +1,8 @@ -//! Drivers are responsible for calling [`codegen_mono_item`] and performing any further actions -//! like JIT executing or writing object files. +//! Drivers are responsible for calling [`codegen_fn`] or [`codegen_static`] for each mono item and +//! performing any further actions like JIT executing or writing object files. +//! +//! [`codegen_fn`]: crate::base::codegen_fn +//! [`codegen_static`]: crate::constant::codegen_static use rustc_middle::mir::mono::{Linkage as RLinkage, MonoItem, Visibility}; From 69102dbcf732a82c4e87252bd7887f7d8065134c Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 9 Apr 2021 11:55:46 +0200 Subject: [PATCH 29/52] Add some documentation --- src/config.rs | 6 ++++++ src/lib.rs | 2 ++ src/metadata.rs | 12 ++++++++++++ 3 files changed, 20 insertions(+) diff --git a/src/config.rs b/src/config.rs index 0cf02e2bbb565..e59a0cb0a2323 100644 --- a/src/config.rs +++ b/src/config.rs @@ -5,10 +5,14 @@ fn bool_env_var(key: &str) -> bool { env::var(key).as_ref().map(|val| &**val) == Ok("1") } +/// The mode to use for compilation. #[derive(Copy, Clone, Debug)] pub enum CodegenMode { + /// AOT compile the crate. This is the default. Aot, + /// JIT compile and execute the crate. Jit, + /// JIT compile and execute the crate, but only compile functions the first time they are used. JitLazy, } @@ -25,6 +29,7 @@ impl FromStr for CodegenMode { } } +/// Configuration of cg_clif as passed in through `-Cllvm-args` and various env vars. #[derive(Clone, Debug)] pub struct BackendConfig { /// Should the crate be AOT compiled or JIT executed. @@ -76,6 +81,7 @@ impl Default for BackendConfig { } impl BackendConfig { + /// Parse the configuration passed in using `-Cllvm-args`. pub fn from_opts(opts: &[String]) -> Result { fn parse_bool(name: &str, value: &str) -> Result { value.parse().map_err(|_| format!("failed to parse value `{}` for {}", value, name)) diff --git a/src/lib.rs b/src/lib.rs index 2c989520f6a56..5a75b9be0cb69 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -119,6 +119,8 @@ impl String> Drop for PrintOnPanic { } } +/// The codegen context holds any information shared between the codegen of individual functions +/// inside a single codegen unit with the exception of the Cranelift [`Module`](cranelift_module::Module). struct CodegenCx<'tcx> { tcx: TyCtxt<'tcx>, global_asm: String, diff --git a/src/metadata.rs b/src/metadata.rs index 7bd9ebbb611c3..978a1e722ddd2 100644 --- a/src/metadata.rs +++ b/src/metadata.rs @@ -14,6 +14,18 @@ use rustc_target::spec::Target; use crate::backend::WriteMetadata; +/// The metadata loader used by cg_clif. +/// +/// The metadata is stored in the same format as cg_llvm. +/// +/// # Metadata location +/// +///
+///
rlib
+///
The metadata can be found in the `lib.rmeta` file inside of the ar archive.
+///
dylib
+///
The metadata can be found in the `.rustc` section of the shared library.
+///
pub(crate) struct CraneliftMetadataLoader; fn load_metadata_with( From 56bf873110b9fdcc62534022e9161c515bf0f949 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sat, 10 Apr 2021 13:46:28 +0200 Subject: [PATCH 30/52] Time object file writing --- src/driver/aot.rs | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/driver/aot.rs b/src/driver/aot.rs index 3ff96d710e58e..9c5cd53d8669d 100644 --- a/src/driver/aot.rs +++ b/src/driver/aot.rs @@ -136,15 +136,19 @@ fn module_codegen( } crate::main_shim::maybe_create_entry_wrapper(tcx, &mut module, &mut cx.unwind_context, false); - let codegen_result = emit_module( - tcx, - &backend_config, - cgu.name().as_str().to_string(), - ModuleKind::Regular, - module, - cx.debug_context, - cx.unwind_context, - ); + let debug_context = cx.debug_context; + let unwind_context = cx.unwind_context; + let codegen_result = tcx.sess.time("write object file", || { + emit_module( + tcx, + &backend_config, + cgu.name().as_str().to_string(), + ModuleKind::Regular, + module, + debug_context, + unwind_context, + ) + }); codegen_global_asm(tcx, &cgu.name().as_str(), &cx.global_asm); From b477a5472d4530066ad4a2205d24a3830159da6b Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sun, 11 Apr 2021 16:35:45 +0200 Subject: [PATCH 31/52] Avoid file name formatting when debug file writing is disabled --- src/base.rs | 2 +- src/pretty_clif.rs | 46 +++++++++++++++++++++++++--------------------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/src/base.rs b/src/base.rs index 069166f7b68a0..61e0206f10534 100644 --- a/src/base.rs +++ b/src/base.rs @@ -147,7 +147,7 @@ pub(crate) fn codegen_fn<'tcx>( if let Some(disasm) = &context.mach_compile_result.as_ref().unwrap().disasm { crate::pretty_clif::write_ir_file( tcx, - &format!("{}.vcode", tcx.symbol_name(instance).name), + || format!("{}.vcode", tcx.symbol_name(instance).name), |file| file.write_all(disasm.as_bytes()), ) } diff --git a/src/pretty_clif.rs b/src/pretty_clif.rs index 30e487a99f431..158811c5eaf49 100644 --- a/src/pretty_clif.rs +++ b/src/pretty_clif.rs @@ -207,7 +207,7 @@ pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool { pub(crate) fn write_ir_file( tcx: TyCtxt<'_>, - name: &str, + name: impl FnOnce() -> String, write: impl FnOnce(&mut dyn Write) -> std::io::Result<()>, ) { if !should_write_ir(tcx) { @@ -222,7 +222,7 @@ pub(crate) fn write_ir_file( res @ Err(_) => res.unwrap(), } - let clif_file_name = clif_output_dir.join(name); + let clif_file_name = clif_output_dir.join(name()); let res = std::fs::File::create(clif_file_name).and_then(|mut file| write(&mut file)); if let Err(err) = res { @@ -238,27 +238,31 @@ pub(crate) fn write_clif_file<'tcx>( context: &cranelift_codegen::Context, mut clif_comments: &CommentWriter, ) { - write_ir_file(tcx, &format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix), |file| { - let value_ranges = - isa.map(|isa| context.build_value_labels_ranges(isa).expect("value location ranges")); + write_ir_file( + tcx, + || format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix), + |file| { + let value_ranges = isa + .map(|isa| context.build_value_labels_ranges(isa).expect("value location ranges")); - let mut clif = String::new(); - cranelift_codegen::write::decorate_function( - &mut clif_comments, - &mut clif, - &context.func, - &DisplayFunctionAnnotations { isa, value_ranges: value_ranges.as_ref() }, - ) - .unwrap(); + let mut clif = String::new(); + cranelift_codegen::write::decorate_function( + &mut clif_comments, + &mut clif, + &context.func, + &DisplayFunctionAnnotations { isa, value_ranges: value_ranges.as_ref() }, + ) + .unwrap(); - writeln!(file, "test compile")?; - writeln!(file, "set is_pic")?; - writeln!(file, "set enable_simd")?; - writeln!(file, "target {} haswell", crate::target_triple(tcx.sess))?; - writeln!(file)?; - file.write_all(clif.as_bytes())?; - Ok(()) - }); + writeln!(file, "test compile")?; + writeln!(file, "set is_pic")?; + writeln!(file, "set enable_simd")?; + writeln!(file, "target {} haswell", crate::target_triple(tcx.sess))?; + writeln!(file)?; + file.write_all(clif.as_bytes())?; + Ok(()) + }, + ); } impl fmt::Debug for FunctionCx<'_, '_, '_> { From ea3398ad9856acfc3ac93991389de92597f42884 Mon Sep 17 00:00:00 2001 From: Jubilee Young Date: Wed, 14 Apr 2021 15:07:36 -0700 Subject: [PATCH 32/52] Add more SIMD math.h intrinsics LLVM supports many functions from math.h in its IR. Many of these have single-instruction variants on various platforms. So, let's add them so std::arch can use them. Yes, exact comparison is intentional: rounding must always return a valid integer-equal value, except for inf/NAN. --- src/intrinsics/simd.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs index c7ce32b385e94..27fc2abedc7e9 100644 --- a/src/intrinsics/simd.rs +++ b/src/intrinsics/simd.rs @@ -277,5 +277,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( // simd_select // simd_rem // simd_neg + // simd_trunc + // simd_floor } } From 73c0db092dd64a8830121cee04ab972fee76088e Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 16 Apr 2021 11:59:20 +0200 Subject: [PATCH 33/52] Rustup to rustc 1.53.0-nightly (132b4e5d1 2021-04-13) --- build_sysroot/Cargo.lock | 15 +++- build_sysroot/prepare_sysroot_src.sh | 2 +- ...builtins-Remove-rotate_left-from-Int.patch | 4 +- rust-toolchain | 2 +- src/base.rs | 82 ++----------------- src/inline_asm.rs | 58 +++++++++++++ src/intrinsics/cpuid.rs | 2 +- src/linkage.rs | 1 + 8 files changed, 84 insertions(+), 82 deletions(-) diff --git a/build_sysroot/Cargo.lock b/build_sysroot/Cargo.lock index d96b3e3f5ac4f..b6e27245b8f29 100644 --- a/build_sysroot/Cargo.lock +++ b/build_sysroot/Cargo.lock @@ -56,7 +56,7 @@ dependencies = [ [[package]] name = "compiler_builtins" -version = "0.1.39" +version = "0.1.40" dependencies = [ "rustc-std-workspace-core", ] @@ -167,6 +167,7 @@ dependencies = [ name = "panic_abort" version = "0.0.0" dependencies = [ + "alloc", "cfg-if", "compiler_builtins", "core", @@ -242,10 +243,22 @@ dependencies = [ "panic_abort", "panic_unwind", "rustc-demangle", + "std_detect", "unwind", "wasi", ] +[[package]] +name = "std_detect" +version = "0.1.5" +dependencies = [ + "cfg-if", + "compiler_builtins", + "libc", + "rustc-std-workspace-alloc", + "rustc-std-workspace-core", +] + [[package]] name = "sysroot" version = "0.0.0" diff --git a/build_sysroot/prepare_sysroot_src.sh b/build_sysroot/prepare_sysroot_src.sh index c90205db0fbd0..f7fcef1077410 100755 --- a/build_sysroot/prepare_sysroot_src.sh +++ b/build_sysroot/prepare_sysroot_src.sh @@ -32,7 +32,7 @@ popd git clone https://github.com/rust-lang/compiler-builtins.git || echo "rust-lang/compiler-builtins has already been cloned" pushd compiler-builtins git checkout -- . -git checkout 0.1.39 +git checkout 0.1.40 git apply ../../crate_patches/000*-compiler-builtins-*.patch popd diff --git a/crate_patches/0001-compiler-builtins-Remove-rotate_left-from-Int.patch b/crate_patches/0001-compiler-builtins-Remove-rotate_left-from-Int.patch index e14768910a9ac..b4acc4f5b7365 100644 --- a/crate_patches/0001-compiler-builtins-Remove-rotate_left-from-Int.patch +++ b/crate_patches/0001-compiler-builtins-Remove-rotate_left-from-Int.patch @@ -17,8 +17,8 @@ index 06054c8..3bea17b 100644 fn wrapping_shr(self, other: u32) -> Self; - fn rotate_left(self, other: u32) -> Self; fn overflowing_add(self, other: Self) -> (Self, bool); - fn aborting_div(self, other: Self) -> Self; - fn aborting_rem(self, other: Self) -> Self; + fn leading_zeros(self) -> u32; + } @@ -209,10 +208,6 @@ macro_rules! int_impl_common { ::wrapping_shr(self, other) } diff --git a/rust-toolchain b/rust-toolchain index fe2631bbf5a62..ef7fc7baba251 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2021-04-07" +channel = "nightly-2021-04-14" components = ["rust-src", "rustc-dev", "llvm-tools-preview"] diff --git a/src/base.rs b/src/base.rs index 61e0206f10534..3ec5c14ff17a2 100644 --- a/src/base.rs +++ b/src/base.rs @@ -744,85 +744,15 @@ fn codegen_stmt<'tcx>( | StatementKind::AscribeUserType(..) => {} StatementKind::LlvmInlineAsm(asm) => { - use rustc_span::symbol::Symbol; - let LlvmInlineAsm { asm, outputs, inputs } = &**asm; - let rustc_hir::LlvmInlineAsmInner { - asm: asm_code, // Name - outputs: output_names, // Vec - inputs: input_names, // Vec - clobbers, // Vec - volatile, // bool - alignstack, // bool - dialect: _, - asm_str_style: _, - } = asm; - match asm_code.as_str().trim() { + match asm.asm.asm.as_str().trim() { "" => { // Black box } - "mov %rbx, %rsi\n cpuid\n xchg %rbx, %rsi" => { - assert_eq!(input_names, &[Symbol::intern("{eax}"), Symbol::intern("{ecx}")]); - assert_eq!(output_names.len(), 4); - for (i, c) in (&["={eax}", "={esi}", "={ecx}", "={edx}"]).iter().enumerate() { - assert_eq!(&output_names[i].constraint.as_str(), c); - assert!(!output_names[i].is_rw); - assert!(!output_names[i].is_indirect); - } - - assert_eq!(clobbers, &[]); - - assert!(!volatile); - assert!(!alignstack); - - assert_eq!(inputs.len(), 2); - let leaf = codegen_operand(fx, &inputs[0].1).load_scalar(fx); // %eax - let subleaf = codegen_operand(fx, &inputs[1].1).load_scalar(fx); // %ecx - - let (eax, ebx, ecx, edx) = - crate::intrinsics::codegen_cpuid_call(fx, leaf, subleaf); - - assert_eq!(outputs.len(), 4); - codegen_place(fx, outputs[0]) - .write_cvalue(fx, CValue::by_val(eax, fx.layout_of(fx.tcx.types.u32))); - codegen_place(fx, outputs[1]) - .write_cvalue(fx, CValue::by_val(ebx, fx.layout_of(fx.tcx.types.u32))); - codegen_place(fx, outputs[2]) - .write_cvalue(fx, CValue::by_val(ecx, fx.layout_of(fx.tcx.types.u32))); - codegen_place(fx, outputs[3]) - .write_cvalue(fx, CValue::by_val(edx, fx.layout_of(fx.tcx.types.u32))); - } - "xgetbv" => { - assert_eq!(input_names, &[Symbol::intern("{ecx}")]); - - assert_eq!(output_names.len(), 2); - for (i, c) in (&["={eax}", "={edx}"]).iter().enumerate() { - assert_eq!(&output_names[i].constraint.as_str(), c); - assert!(!output_names[i].is_rw); - assert!(!output_names[i].is_indirect); - } - - assert_eq!(clobbers, &[]); - - assert!(!volatile); - assert!(!alignstack); - - crate::trap::trap_unimplemented(fx, "_xgetbv arch intrinsic is not supported"); - } - // ___chkstk, ___chkstk_ms and __alloca are only used on Windows - _ if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") => { - crate::trap::trap_unimplemented(fx, "Stack probes are not supported"); - } - _ if fx.tcx.symbol_name(fx.instance).name == "__alloca" => { - crate::trap::trap_unimplemented(fx, "Alloca is not supported"); - } - // Used in sys::windows::abort_internal - "int $$0x29" => { - crate::trap::trap_unimplemented(fx, "Windows abort"); - } - _ => fx - .tcx - .sess - .span_fatal(stmt.source_info.span, "Inline assembly is not supported"), + _ => fx.tcx.sess.span_fatal( + stmt.source_info.span, + "Legacy `llvm_asm!` inline assembly is not supported. \ + Try using the new `asm!` instead.", + ), } } StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"), diff --git a/src/inline_asm.rs b/src/inline_asm.rs index 669a6d35075b4..4ab4c2957ca4e 100644 --- a/src/inline_asm.rs +++ b/src/inline_asm.rs @@ -24,6 +24,64 @@ pub(crate) fn codegen_inline_asm<'tcx>( let true_ = fx.bcx.ins().iconst(types::I32, 1); fx.bcx.ins().trapnz(true_, TrapCode::User(1)); return; + } else if template[0] == InlineAsmTemplatePiece::String("mov rsi, rbx".to_string()) + && template[1] == InlineAsmTemplatePiece::String("\n".to_string()) + && template[2] == InlineAsmTemplatePiece::String("cpuid".to_string()) + && template[3] == InlineAsmTemplatePiece::String("\n".to_string()) + && template[4] == InlineAsmTemplatePiece::String("xchg rsi, rbx".to_string()) + { + assert_eq!(operands.len(), 4); + let (leaf, eax_place) = match operands[0] { + InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => { + let reg = expect_reg(reg); + assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::ax)); + ( + crate::base::codegen_operand(fx, in_value).load_scalar(fx), + crate::base::codegen_place(fx, out_place.unwrap()), + ) + } + _ => unreachable!(), + }; + let ebx_place = match operands[1] { + InlineAsmOperand::Out { reg, late: true, place } => { + let reg = expect_reg(reg); + assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::si)); + crate::base::codegen_place(fx, place.unwrap()) + } + _ => unreachable!(), + }; + let (sub_leaf, ecx_place) = match operands[2] { + InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => { + let reg = expect_reg(reg); + assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::cx)); + ( + crate::base::codegen_operand(fx, in_value).load_scalar(fx), + crate::base::codegen_place(fx, out_place.unwrap()), + ) + } + _ => unreachable!(), + }; + let edx_place = match operands[3] { + InlineAsmOperand::Out { reg, late: true, place } => { + let reg = expect_reg(reg); + assert_eq!(reg, InlineAsmReg::X86(X86InlineAsmReg::dx)); + crate::base::codegen_place(fx, place.unwrap()) + } + _ => unreachable!(), + }; + + let (eax, ebx, ecx, edx) = crate::intrinsics::codegen_cpuid_call(fx, leaf, sub_leaf); + + eax_place.write_cvalue(fx, CValue::by_val(eax, fx.layout_of(fx.tcx.types.u32))); + ebx_place.write_cvalue(fx, CValue::by_val(ebx, fx.layout_of(fx.tcx.types.u32))); + ecx_place.write_cvalue(fx, CValue::by_val(ecx, fx.layout_of(fx.tcx.types.u32))); + edx_place.write_cvalue(fx, CValue::by_val(edx, fx.layout_of(fx.tcx.types.u32))); + return; + } else if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") { + // ___chkstk, ___chkstk_ms and __alloca are only used on Windows + crate::trap::trap_unimplemented(fx, "Stack probes are not supported"); + } else if fx.tcx.symbol_name(fx.instance).name == "__alloca" { + crate::trap::trap_unimplemented(fx, "Alloca is not supported"); } let mut slot_size = Size::from_bytes(0); diff --git a/src/intrinsics/cpuid.rs b/src/intrinsics/cpuid.rs index b27b0eddfbad6..9de12e759bcc8 100644 --- a/src/intrinsics/cpuid.rs +++ b/src/intrinsics/cpuid.rs @@ -8,7 +8,7 @@ use crate::prelude::*; pub(crate) fn codegen_cpuid_call<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, leaf: Value, - _subleaf: Value, + _sub_leaf: Value, ) -> (Value, Value, Value, Value) { let leaf_0 = fx.bcx.create_block(); let leaf_1 = fx.bcx.create_block(); diff --git a/src/linkage.rs b/src/linkage.rs index a564a59f72510..ca853aac15892 100644 --- a/src/linkage.rs +++ b/src/linkage.rs @@ -13,6 +13,7 @@ pub(crate) fn get_clif_linkage( (RLinkage::External, Visibility::Default) => Linkage::Export, (RLinkage::Internal, Visibility::Default) => Linkage::Local, (RLinkage::External, Visibility::Hidden) => Linkage::Hidden, + (RLinkage::WeakAny, Visibility::Default) => Linkage::Preemptible, _ => panic!("{:?} = {:?} {:?}", mono_item, linkage, visibility), } } From 24cac8fa54005b32c29217a42f488fc4dc7ad1ef Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 16 Apr 2021 12:21:29 +0200 Subject: [PATCH 34/52] Fix rustc tests by updating compiler-builtins dep to 0.1.40 --- scripts/setup_rust_fork.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/setup_rust_fork.sh b/scripts/setup_rust_fork.sh index e8bedf625f796..4821a07ac5d5d 100644 --- a/scripts/setup_rust_fork.sh +++ b/scripts/setup_rust_fork.sh @@ -45,7 +45,7 @@ index d95b5b7f17f..00b6f0e3635 100644 [dependencies] core = { path = "../core" } -compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std'] } -+compiler_builtins = { version = "0.1.39", features = ['rustc-dep-of-std', 'no-asm'] } ++compiler_builtins = { version = "0.1.40", features = ['rustc-dep-of-std', 'no-asm'] } [dev-dependencies] rand = "0.7" From 944308089f4cc8f1384df80d00f735609358e550 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 16 Apr 2021 12:35:22 +0200 Subject: [PATCH 35/52] Disable new test --- scripts/test_rustc_tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/test_rustc_tests.sh b/scripts/test_rustc_tests.sh index 6a76f3a721445..d84bfe12d73a0 100755 --- a/scripts/test_rustc_tests.sh +++ b/scripts/test_rustc_tests.sh @@ -63,6 +63,7 @@ rm src/test/incremental/issue-54059.rs # same rm src/test/incremental/lto.rs # requires lto rm -r src/test/run-make/emit-shared-files # requires the rustdoc executable in build/bin/ +rm -r src/test/run-make/unstable-flag-required # same rm src/test/pretty/asm.rs # inline asm rm src/test/pretty/raw-str-nonexpr.rs # same From 6d6c5742896fab93788b4dabb6ba2fcf1881f5e2 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 16 Apr 2021 14:02:57 +0200 Subject: [PATCH 36/52] Fix rotate_left and rotate_right with 128bit shift amount Fixes #1114 --- example/std_example.rs | 1 + src/intrinsics/mod.rs | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/example/std_example.rs b/example/std_example.rs index 015bbdfed4648..437b7726980bf 100644 --- a/example/std_example.rs +++ b/example/std_example.rs @@ -84,6 +84,7 @@ fn main() { assert_eq!(houndred_i128 as f64, 100.0); assert_eq!(houndred_f32 as i128, 100); assert_eq!(houndred_f64 as i128, 100); + assert_eq!(1u128.rotate_left(2), 4); // Test signed 128bit comparing let max = usize::MAX as i128; diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index d08271f853b50..c42ad4337c1f2 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -632,11 +632,21 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( }; rotate_left, (v x, v y) { let layout = fx.layout_of(T); + let y = if fx.bcx.func.dfg.value_type(y) == types::I128 { + fx.bcx.ins().ireduce(types::I64, y) + } else { + y + }; let res = fx.bcx.ins().rotl(x, y); ret.write_cvalue(fx, CValue::by_val(res, layout)); }; rotate_right, (v x, v y) { let layout = fx.layout_of(T); + let y = if fx.bcx.func.dfg.value_type(y) == types::I128 { + fx.bcx.ins().ireduce(types::I64, y) + } else { + y + }; let res = fx.bcx.ins().rotr(x, y); ret.write_cvalue(fx, CValue::by_val(res, layout)); }; From 7f0e35106e3cce054551de8305f8bcbc25ef59f8 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 16 Apr 2021 14:36:07 +0200 Subject: [PATCH 37/52] Fix overflow checking when multiplying two i64 Fixes #1162 --- example/std_example.rs | 2 ++ src/num.rs | 17 ++++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/example/std_example.rs b/example/std_example.rs index 437b7726980bf..77ba72df8ef37 100644 --- a/example/std_example.rs +++ b/example/std_example.rs @@ -48,6 +48,8 @@ fn main() { assert_eq!(2.3f32.copysign(-1.0), -2.3f32); println!("{}", 2.3f32.powf(2.0)); + assert_eq!(i64::MAX.checked_mul(2), None); + assert_eq!(-128i8, (-128i8).saturating_sub(1)); assert_eq!(127i8, 127i8.saturating_sub(-128)); assert_eq!(-128i8, (-128i8).saturating_add(-128)); diff --git a/src/num.rs b/src/num.rs index 2ebf30da2d8ba..b6d378a5fe10a 100644 --- a/src/num.rs +++ b/src/num.rs @@ -271,14 +271,17 @@ pub(crate) fn codegen_checked_int_binop<'tcx>( let val_hi = fx.bcx.ins().umulhi(lhs, rhs); fx.bcx.ins().icmp_imm(IntCC::NotEqual, val_hi, 0) } else { + // Based on LLVM's instruction sequence for compiling + // a.checked_mul(b).is_some() to riscv64gc: + // mulh a2, a0, a1 + // mul a0, a0, a1 + // srai a0, a0, 63 + // xor a0, a0, a2 + // snez a0, a0 let val_hi = fx.bcx.ins().smulhi(lhs, rhs); - let not_all_zero = fx.bcx.ins().icmp_imm(IntCC::NotEqual, val_hi, 0); - let not_all_ones = fx.bcx.ins().icmp_imm( - IntCC::NotEqual, - val_hi, - u64::try_from((1u128 << ty.bits()) - 1).unwrap() as i64, - ); - fx.bcx.ins().band(not_all_zero, not_all_ones) + let val_sign = fx.bcx.ins().sshr_imm(val, i64::from(ty.bits() - 1)); + let xor = fx.bcx.ins().bxor(val_hi, val_sign); + fx.bcx.ins().icmp_imm(IntCC::NotEqual, xor, 0) }; (val, has_overflow) } From f3b5e14eca5c942c4b3d867808fa0ec99eae9c03 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 16 Apr 2021 14:39:46 +0200 Subject: [PATCH 38/52] Upload artifacts for cross compiling to MinGW Fixes #1161 --- .github/workflows/main.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2ac516381cf7a..4d45e36c956c9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -80,3 +80,10 @@ jobs: with: name: cg_clif-${{ runner.os }} path: cg_clif.tar.xz + + - name: Upload prebuilt cg_clif (cross compile) + if: matrix.env.TARGET_TRIPLE == 'x86_64-pc-windows-gnu' + uses: actions/upload-artifact@v2 + with: + name: cg_clif-${{ runner.os }}-cross-x86_64-mingw + path: cg_clif.tar.xz From 9a3d98dadeff821506bdf9cf168858587046a70e Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sun, 18 Apr 2021 10:29:20 +0200 Subject: [PATCH 39/52] Call Termination::report on main result in jit mode --- src/main_shim.rs | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/src/main_shim.rs b/src/main_shim.rs index 4beb6c65a7791..4924c4e8923cc 100644 --- a/src/main_shim.rs +++ b/src/main_shim.rs @@ -1,6 +1,9 @@ use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink}; use rustc_hir::LangItem; +use rustc_middle::ty::subst::GenericArg; +use rustc_middle::ty::AssocKind; use rustc_session::config::EntryFnType; +use rustc_span::symbol::Ident; use crate::prelude::*; @@ -79,8 +82,38 @@ pub(crate) fn maybe_create_entry_wrapper( let result = if is_main_fn && ignore_lang_start_wrapper { // regular main fn, but ignoring #[lang = "start"] as we are running in the jit // FIXME set program arguments somehow - bcx.ins().call(main_func_ref, &[]); - bcx.ins().iconst(m.target_config().pointer_type(), 0) + let call_inst = bcx.ins().call(main_func_ref, &[]); + let call_results = bcx.func.dfg.inst_results(call_inst).to_owned(); + + let termination_trait = tcx.require_lang_item(LangItem::Termination, None); + let report = tcx + .associated_items(termination_trait) + .find_by_name_and_kind( + tcx, + Ident::from_str("report"), + AssocKind::Fn, + termination_trait, + ) + .unwrap(); + let report = Instance::resolve( + tcx, + ParamEnv::reveal_all(), + report.def_id, + tcx.mk_substs([GenericArg::from(main_ret_ty)].iter()), + ) + .unwrap() + .unwrap(); + + let report_name = tcx.symbol_name(report).name; + let report_sig = get_function_sig(tcx, m.isa().triple(), report); + let report_func_id = + m.declare_function(report_name, Linkage::Import, &report_sig).unwrap(); + let report_func_ref = m.declare_func_in_func(report_func_id, &mut bcx.func); + + // FIXME do proper abi handling instead of expecting the pass mode to be identical + // for returns and arguments. + let report_call_inst = bcx.ins().call(report_func_ref, &call_results); + bcx.func.dfg.inst_results(report_call_inst)[0] } else if is_main_fn { let start_def_id = tcx.require_lang_item(LangItem::Start, None); let start_instance = Instance::resolve( From a569cb4022182b6b9c2985190ff88f7622fc8528 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sun, 18 Apr 2021 10:32:38 +0200 Subject: [PATCH 40/52] Fix test --- example/mini_core_hello_world.rs | 38 ++++++++++++++++---------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/example/mini_core_hello_world.rs b/example/mini_core_hello_world.rs index 47abe2d1de800..6570f2bf9f297 100644 --- a/example/mini_core_hello_world.rs +++ b/example/mini_core_hello_world.rs @@ -11,6 +11,22 @@ unsafe extern "C" fn my_puts(s: *const i8) { puts(s); } +macro_rules! assert { + ($e:expr) => { + if !$e { + panic(stringify!(! $e)); + } + }; +} + +macro_rules! assert_eq { + ($l:expr, $r: expr) => { + if $l != $r { + panic(stringify!($l != $r)); + } + } +} + #[lang = "termination"] trait Termination { fn report(self) -> i32; @@ -20,8 +36,9 @@ impl Termination for () { fn report(self) -> i32 { unsafe { NUM = 6 * 7 + 1 + (1u8 == 1u8) as u8; // 44 - *NUM_REF as i32 + assert_eq!(*NUM_REF as i32, 44); } + 0 } } @@ -82,29 +99,12 @@ fn start( unsafe { puts(*((argv as usize + 2 * intrinsics::size_of::<*const u8>()) as *const *const i8)); } } - main().report(); - 0 + main().report() as isize } static mut NUM: u8 = 6 * 7; static NUM_REF: &'static u8 = unsafe { &NUM }; -macro_rules! assert { - ($e:expr) => { - if !$e { - panic(stringify!(! $e)); - } - }; -} - -macro_rules! assert_eq { - ($l:expr, $r: expr) => { - if $l != $r { - panic(stringify!($l != $r)); - } - } -} - struct Unique { pointer: *const T, _marker: PhantomData, From e01de0f58df96c09d6b9da53e913af6bdb0142c0 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sun, 18 Apr 2021 10:37:02 +0200 Subject: [PATCH 41/52] Avoid .to_string() for symbol names where possible --- src/abi/mod.rs | 6 +++--- src/driver/jit.rs | 8 ++++---- src/driver/mod.rs | 4 ++-- src/main_shim.rs | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/abi/mod.rs b/src/abi/mod.rs index f450f36678713..54c8fb0e7b80b 100644 --- a/src/abi/mod.rs +++ b/src/abi/mod.rs @@ -63,9 +63,9 @@ pub(crate) fn import_function<'tcx>( module: &mut dyn Module, inst: Instance<'tcx>, ) -> FuncId { - let name = tcx.symbol_name(inst).name.to_string(); + let name = tcx.symbol_name(inst).name; let sig = get_function_sig(tcx, module.isa().triple(), inst); - module.declare_function(&name, Linkage::Import, &sig).unwrap() + module.declare_function(name, Linkage::Import, &sig).unwrap() } impl<'tcx> FunctionCx<'_, '_, 'tcx> { @@ -89,7 +89,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { args: &[Value], ) -> &[Value] { let sig = Signature { params, returns, call_conv: CallConv::triple_default(self.triple()) }; - let func_id = self.module.declare_function(&name, Linkage::Import, &sig).unwrap(); + let func_id = self.module.declare_function(name, Linkage::Import, &sig).unwrap(); let func_ref = self.module.declare_func_in_func(func_id, &mut self.bcx.func); let call_inst = self.bcx.ins().call(func_ref, args); if self.clif_comments.enabled() { diff --git a/src/driver/jit.rs b/src/driver/jit.rs index 63ac02f83b92c..526caeb4f03f5 100644 --- a/src/driver/jit.rs +++ b/src/driver/jit.rs @@ -156,9 +156,9 @@ extern "C" fn __clif_jit_fn(instance_ptr: *const Instance<'static>) -> *const u8 let jit_module = &mut lazy_jit_state.jit_module; let backend_config = lazy_jit_state.backend_config.clone(); - let name = tcx.symbol_name(instance).name.to_string(); + let name = tcx.symbol_name(instance).name; let sig = crate::abi::get_function_sig(tcx, jit_module.isa().triple(), instance); - let func_id = jit_module.declare_function(&name, Linkage::Export, &sig).unwrap(); + let func_id = jit_module.declare_function(name, Linkage::Export, &sig).unwrap(); jit_module.prepare_for_function_redefine(func_id).unwrap(); let mut cx = crate::CodegenCx::new(tcx, backend_config, jit_module.isa(), false); @@ -241,9 +241,9 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: In let pointer_type = module.target_config().pointer_type(); - let name = tcx.symbol_name(inst).name.to_string(); + let name = tcx.symbol_name(inst).name; let sig = crate::abi::get_function_sig(tcx, module.isa().triple(), inst); - let func_id = module.declare_function(&name, Linkage::Export, &sig).unwrap(); + let func_id = module.declare_function(name, Linkage::Export, &sig).unwrap(); let instance_ptr = Box::into_raw(Box::new(inst)); diff --git a/src/driver/mod.rs b/src/driver/mod.rs index 393cde3e9ad27..8f5714ecb4177 100644 --- a/src/driver/mod.rs +++ b/src/driver/mod.rs @@ -22,7 +22,7 @@ fn predefine_mono_items<'tcx>( for &(mono_item, (linkage, visibility)) in mono_items { match mono_item { MonoItem::Fn(instance) => { - let name = tcx.symbol_name(instance).name.to_string(); + let name = tcx.symbol_name(instance).name; let _inst_guard = crate::PrintOnPanic(|| format!("{:?} {}", instance, name)); let sig = get_function_sig(tcx, module.isa().triple(), instance); let linkage = crate::linkage::get_clif_linkage( @@ -31,7 +31,7 @@ fn predefine_mono_items<'tcx>( visibility, is_compiler_builtins, ); - module.declare_function(&name, linkage, &sig).unwrap(); + module.declare_function(name, linkage, &sig).unwrap(); } MonoItem::Static(_) | MonoItem::GlobalAsm(_) => {} } diff --git a/src/main_shim.rs b/src/main_shim.rs index 4924c4e8923cc..3305ca6f6eb9d 100644 --- a/src/main_shim.rs +++ b/src/main_shim.rs @@ -62,9 +62,9 @@ pub(crate) fn maybe_create_entry_wrapper( let instance = Instance::mono(tcx, rust_main_def_id).polymorphize(tcx); - let main_name = tcx.symbol_name(instance).name.to_string(); + let main_name = tcx.symbol_name(instance).name; let main_sig = get_function_sig(tcx, m.isa().triple(), instance); - let main_func_id = m.declare_function(&main_name, Linkage::Import, &main_sig).unwrap(); + let main_func_id = m.declare_function(main_name, Linkage::Import, &main_sig).unwrap(); let mut ctx = Context::new(); ctx.func = Function::with_name_signature(ExternalName::user(0, 0), cmain_sig); From bf85572f59f5b70d73869c3902964c5ca47668df Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Sun, 18 Apr 2021 10:58:42 +0200 Subject: [PATCH 42/52] Extend Termination::report return value as necessary --- src/main_shim.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main_shim.rs b/src/main_shim.rs index 3305ca6f6eb9d..713cf4ebdc00f 100644 --- a/src/main_shim.rs +++ b/src/main_shim.rs @@ -113,7 +113,12 @@ pub(crate) fn maybe_create_entry_wrapper( // FIXME do proper abi handling instead of expecting the pass mode to be identical // for returns and arguments. let report_call_inst = bcx.ins().call(report_func_ref, &call_results); - bcx.func.dfg.inst_results(report_call_inst)[0] + let res = bcx.func.dfg.inst_results(report_call_inst)[0]; + match m.target_config().pointer_type() { + types::I32 => res, + types::I64 => bcx.ins().sextend(types::I64, res), + _ => unimplemented!("16bit systems are not yet supported"), + } } else if is_main_fn { let start_def_id = tcx.require_lang_item(LangItem::Start, None); let start_instance = Instance::resolve( From 528585677139d6e9b2ae1179ae2f3c0c0c7f8d36 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Mon, 19 Apr 2021 19:41:57 +0200 Subject: [PATCH 43/52] Match on Symbol instead of &str in intrinsics handling --- src/intrinsics/llvm.rs | 10 ++-- src/intrinsics/mod.rs | 122 ++++++++++++++++++++--------------------- src/intrinsics/simd.rs | 7 +-- 3 files changed, 69 insertions(+), 70 deletions(-) diff --git a/src/intrinsics/llvm.rs b/src/intrinsics/llvm.rs index 83c91f789cd25..ba4ed2162cd5d 100644 --- a/src/intrinsics/llvm.rs +++ b/src/intrinsics/llvm.rs @@ -22,7 +22,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( }; // Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8` - llvm.x86.sse2.pmovmskb.128 | llvm.x86.avx2.pmovmskb | llvm.x86.sse2.movmsk.pd, (c a) { + "llvm.x86.sse2.pmovmskb.128" | "llvm.x86.avx2.pmovmskb" | "llvm.x86.sse2.movmsk.pd", (c a) { let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx); let lane_ty = fx.clif_type(lane_ty).unwrap(); assert!(lane_count <= 32); @@ -51,7 +51,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( let res = CValue::by_val(res, fx.layout_of(fx.tcx.types.i32)); ret.write_cvalue(fx, res); }; - llvm.x86.sse2.cmp.ps | llvm.x86.sse2.cmp.pd, (c x, c y, o kind) { + "llvm.x86.sse2.cmp.ps" | "llvm.x86.sse2.cmp.pd", (c x, c y, o kind) { let kind_const = crate::constant::mir_operand_get_const_val(fx, kind).expect("llvm.x86.sse2.cmp.* kind not const"); let flt_cc = match kind_const.try_to_bits(Size::from_bytes(1)).unwrap_or_else(|| panic!("kind not scalar: {:?}", kind_const)) { 0 => FloatCC::Equal, @@ -81,7 +81,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( bool_to_zero_or_max_uint(fx, res_lane_layout, res_lane) }); }; - llvm.x86.sse2.psrli.d, (c a, o imm8) { + "llvm.x86.sse2.psrli.d", (c a, o imm8) { let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const"); simd_for_each_lane(fx, a, ret, |fx, _lane_layout, res_lane_layout, lane| { let res_lane = match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) { @@ -91,7 +91,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( CValue::by_val(res_lane, res_lane_layout) }); }; - llvm.x86.sse2.pslli.d, (c a, o imm8) { + "llvm.x86.sse2.pslli.d", (c a, o imm8) { let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const"); simd_for_each_lane(fx, a, ret, |fx, _lane_layout, res_lane_layout, lane| { let res_lane = match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) { @@ -101,7 +101,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( CValue::by_val(res_lane, res_lane_layout) }); }; - llvm.x86.sse2.storeu.dq, (v mem_addr, c a) { + "llvm.x86.sse2.storeu.dq", (v mem_addr, c a) { // FIXME correctly handle the unalignment let dest = CPlace::for_ptr(Pointer::new(mem_addr), a.layout()); dest.write_cvalue(fx, a); diff --git a/src/intrinsics/mod.rs b/src/intrinsics/mod.rs index c42ad4337c1f2..435737f3a513b 100644 --- a/src/intrinsics/mod.rs +++ b/src/intrinsics/mod.rs @@ -8,23 +8,25 @@ mod simd; pub(crate) use cpuid::codegen_cpuid_call; pub(crate) use llvm::codegen_llvm_intrinsic_call; +use rustc_span::symbol::{sym, kw}; +use rustc_middle::ty::print::with_no_trimmed_paths; + use crate::prelude::*; use cranelift_codegen::ir::AtomicRmwOp; -use rustc_middle::ty::print::with_no_trimmed_paths; macro intrinsic_pat { (_) => { _ }, ($name:ident) => { - stringify!($name) + sym::$name + }, + (kw.$name:ident) => { + kw::$name }, ($name:literal) => { - stringify!($name) + $name }, - ($x:ident . $($xs:tt).*) => { - concat!(stringify!($x), ".", intrinsic_pat!($($xs).*)) - } } macro intrinsic_arg { @@ -87,7 +89,7 @@ macro call_intrinsic_match { )*) => { match $intrinsic { $( - stringify!($name) => { + sym::$name => { assert!($substs.is_noop()); if let [$(ref $arg),*] = *$args { let ($($arg,)*) = ( @@ -400,18 +402,17 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let def_id = instance.def_id(); let substs = instance.substs; - let intrinsic = fx.tcx.item_name(def_id).as_str(); - let intrinsic = &intrinsic[..]; + let intrinsic = fx.tcx.item_name(def_id); let ret = match destination { Some((place, _)) => place, None => { // Insert non returning intrinsics here match intrinsic { - "abort" => { + sym::abort => { trap_abort(fx, "Called intrinsic::abort."); } - "transmute" => { + sym::transmute => { crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", span); } _ => unimplemented!("unsupported instrinsic {}", intrinsic), @@ -420,7 +421,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( } }; - if intrinsic.starts_with("simd_") { + if intrinsic.as_str().starts_with("simd_") { self::simd::codegen_simd_intrinsic_call(fx, instance, args, ret, span); let ret_block = fx.get_block(destination.expect("SIMD intrinsics don't diverge").1); fx.bcx.ins().jump(ret_block, &[]); @@ -470,8 +471,6 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( sinf64(flt) -> f64 => sin, cosf32(flt) -> f32 => cosf, cosf64(flt) -> f64 => cos, - tanf32(flt) -> f32 => tanf, - tanf64(flt) -> f64 => tan, } intrinsic_match! { @@ -496,7 +495,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( count }; - if intrinsic.contains("nonoverlapping") { + if intrinsic == sym::copy_nonoverlapping { // FIXME emit_small_memcpy fx.bcx.call_memcpy(fx.module.target_config(), dst, src, byte_amount); } else { @@ -515,7 +514,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( }; // FIXME make the copy actually volatile when using emit_small_mem{cpy,move} - if intrinsic.contains("nonoverlapping") { + if intrinsic == sym::volatile_copy_nonoverlapping_memory { // FIXME emit_small_memcpy fx.bcx.call_memcpy(fx.module.target_config(), dst, src, byte_amount); } else { @@ -552,27 +551,28 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( ret.write_cvalue(fx, CValue::by_val(align, usize_layout)); }; - _ if intrinsic.starts_with("unchecked_") || intrinsic == "exact_div", (c x, c y) { + unchecked_add | unchecked_sub | unchecked_div | exact_div | unchecked_rem + | unchecked_shl | unchecked_shr, (c x, c y) { // FIXME trap on overflow let bin_op = match intrinsic { - "unchecked_add" => BinOp::Add, - "unchecked_sub" => BinOp::Sub, - "unchecked_div" | "exact_div" => BinOp::Div, - "unchecked_rem" => BinOp::Rem, - "unchecked_shl" => BinOp::Shl, - "unchecked_shr" => BinOp::Shr, - _ => unreachable!("intrinsic {}", intrinsic), + sym::unchecked_add => BinOp::Add, + sym::unchecked_sub => BinOp::Sub, + sym::unchecked_div | sym::exact_div => BinOp::Div, + sym::unchecked_rem => BinOp::Rem, + sym::unchecked_shl => BinOp::Shl, + sym::unchecked_shr => BinOp::Shr, + _ => unreachable!(), }; let res = crate::num::codegen_int_binop(fx, bin_op, x, y); ret.write_cvalue(fx, res); }; - _ if intrinsic.ends_with("_with_overflow"), (c x, c y) { + add_with_overflow | sub_with_overflow | mul_with_overflow, (c x, c y) { assert_eq!(x.layout().ty, y.layout().ty); let bin_op = match intrinsic { - "add_with_overflow" => BinOp::Add, - "sub_with_overflow" => BinOp::Sub, - "mul_with_overflow" => BinOp::Mul, - _ => unreachable!("intrinsic {}", intrinsic), + sym::add_with_overflow => BinOp::Add, + sym::sub_with_overflow => BinOp::Sub, + sym::mul_with_overflow => BinOp::Mul, + _ => unreachable!(), }; let res = crate::num::codegen_checked_int_binop( @@ -583,12 +583,12 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( ); ret.write_cvalue(fx, res); }; - _ if intrinsic.starts_with("saturating_"), (c lhs, c rhs) { + saturating_add | saturating_sub, (c lhs, c rhs) { assert_eq!(lhs.layout().ty, rhs.layout().ty); let bin_op = match intrinsic { - "saturating_add" => BinOp::Add, - "saturating_sub" => BinOp::Sub, - _ => unreachable!("intrinsic {}", intrinsic), + sym::saturating_add => BinOp::Add, + sym::saturating_sub => BinOp::Sub, + _ => unreachable!(), }; let signed = type_sign(T); @@ -609,15 +609,15 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let (min, max) = type_min_max_value(&mut fx.bcx, clif_ty, signed); let val = match (intrinsic, signed) { - ("saturating_add", false) => fx.bcx.ins().select(has_overflow, max, val), - ("saturating_sub", false) => fx.bcx.ins().select(has_overflow, min, val), - ("saturating_add", true) => { + (sym::saturating_add, false) => fx.bcx.ins().select(has_overflow, max, val), + (sym::saturating_sub, false) => fx.bcx.ins().select(has_overflow, min, val), + (sym::saturating_add, true) => { let rhs = rhs.load_scalar(fx); let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0); let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min); fx.bcx.ins().select(has_overflow, sat_val, val) } - ("saturating_sub", true) => { + (sym::saturating_sub, true) => { let rhs = rhs.load_scalar(fx); let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0); let sat_val = fx.bcx.ins().select(rhs_ge_zero, min, max); @@ -816,7 +816,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( return; } - if intrinsic == "assert_zero_valid" && !layout.might_permit_raw_init(fx, /*zero:*/ true).unwrap() { + if intrinsic == sym::assert_zero_valid && !layout.might_permit_raw_init(fx, /*zero:*/ true).unwrap() { with_no_trimmed_paths(|| crate::base::codegen_panic( fx, &format!("attempted to zero-initialize type `{}`, which is invalid", T), @@ -825,7 +825,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( return; } - if intrinsic == "assert_uninit_valid" && !layout.might_permit_raw_init(fx, /*zero:*/ false).unwrap() { + if intrinsic == sym::assert_uninit_valid && !layout.might_permit_raw_init(fx, /*zero:*/ false).unwrap() { with_no_trimmed_paths(|| crate::base::codegen_panic( fx, &format!("attempted to leave type `{}` uninitialized, which is invalid", T), @@ -886,14 +886,14 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( ret.write_cvalue(fx, caller_location); }; - _ if intrinsic.starts_with("atomic_fence"), () { + _ if intrinsic.as_str().starts_with("atomic_fence"), () { fx.bcx.ins().fence(); }; - _ if intrinsic.starts_with("atomic_singlethreadfence"), () { + _ if intrinsic.as_str().starts_with("atomic_singlethreadfence"), () { // FIXME use a compiler fence once Cranelift supports it fx.bcx.ins().fence(); }; - _ if intrinsic.starts_with("atomic_load"), (v ptr) { + _ if intrinsic.as_str().starts_with("atomic_load"), (v ptr) { validate_atomic_type!(fx, intrinsic, span, T); let ty = fx.clif_type(T).unwrap(); @@ -902,14 +902,14 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let val = CValue::by_val(val, fx.layout_of(T)); ret.write_cvalue(fx, val); }; - _ if intrinsic.starts_with("atomic_store"), (v ptr, c val) { + _ if intrinsic.as_str().starts_with("atomic_store"), (v ptr, c val) { validate_atomic_type!(fx, intrinsic, span, val.layout().ty); let val = val.load_scalar(fx); fx.bcx.ins().atomic_store(MemFlags::trusted(), val, ptr); }; - _ if intrinsic.starts_with("atomic_xchg"), (v ptr, c new) { + _ if intrinsic.as_str().starts_with("atomic_xchg"), (v ptr, c new) { let layout = new.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -921,7 +921,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_cxchg"), (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_* + _ if intrinsic.as_str().starts_with("atomic_cxchg"), (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_* let layout = new.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); @@ -935,7 +935,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( ret.write_cvalue(fx, ret_val) }; - _ if intrinsic.starts_with("atomic_xadd"), (v ptr, c amount) { + _ if intrinsic.as_str().starts_with("atomic_xadd"), (v ptr, c amount) { let layout = amount.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -947,7 +947,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_xsub"), (v ptr, c amount) { + _ if intrinsic.as_str().starts_with("atomic_xsub"), (v ptr, c amount) { let layout = amount.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -959,7 +959,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_and"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_and"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -971,7 +971,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_or"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_or"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -983,7 +983,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_xor"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_xor"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -997,7 +997,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( }; // FIXME https://github.com/bytecodealliance/wasmtime/issues/2647 - _ if intrinsic.starts_with("atomic_nand"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_nand"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -1009,7 +1009,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_max"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_max"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -1021,7 +1021,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_umax"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_umax"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -1033,7 +1033,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_min"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_min"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -1045,7 +1045,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( let old = CValue::by_val(old, layout); ret.write_cvalue(fx, old); }; - _ if intrinsic.starts_with("atomic_umin"), (v ptr, c src) { + _ if intrinsic.as_str().starts_with("atomic_umin"), (v ptr, c src) { let layout = src.layout(); validate_atomic_type!(fx, intrinsic, span, layout.ty); let ty = fx.clif_type(layout.ty).unwrap(); @@ -1079,7 +1079,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( ret.write_cvalue(fx, val); }; - try, (v f, v data, v _catch_fn) { + kw.Try, (v f, v data, v _catch_fn) { // FIXME once unwinding is supported, change this to actually catch panics let f_sig = fx.bcx.func.import_signature(Signature { call_conv: CallConv::triple_default(fx.triple()), @@ -1096,11 +1096,11 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( fadd_fast | fsub_fast | fmul_fast | fdiv_fast | frem_fast, (c x, c y) { let res = crate::num::codegen_float_binop(fx, match intrinsic { - "fadd_fast" => BinOp::Add, - "fsub_fast" => BinOp::Sub, - "fmul_fast" => BinOp::Mul, - "fdiv_fast" => BinOp::Div, - "frem_fast" => BinOp::Rem, + sym::fadd_fast => BinOp::Add, + sym::fsub_fast => BinOp::Sub, + sym::fmul_fast => BinOp::Mul, + sym::fdiv_fast => BinOp::Div, + sym::frem_fast => BinOp::Rem, _ => unreachable!(), }, x, y); ret.write_cvalue(fx, res); diff --git a/src/intrinsics/simd.rs b/src/intrinsics/simd.rs index c7ce32b385e94..e71925a0f5458 100644 --- a/src/intrinsics/simd.rs +++ b/src/intrinsics/simd.rs @@ -13,8 +13,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( let def_id = instance.def_id(); let substs = instance.substs; - let intrinsic = fx.tcx.item_name(def_id).as_str(); - let intrinsic = &intrinsic[..]; + let intrinsic = fx.tcx.item_name(def_id); intrinsic_match! { fx, intrinsic, substs, args, @@ -65,10 +64,10 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( }; // simd_shuffle32(x: T, y: T, idx: [u32; 32]) -> U - _ if intrinsic.starts_with("simd_shuffle"), (c x, c y, o idx) { + _ if intrinsic.as_str().starts_with("simd_shuffle"), (c x, c y, o idx) { validate_simd_type!(fx, intrinsic, span, x.layout().ty); - let n: u16 = intrinsic["simd_shuffle".len()..].parse().unwrap(); + let n: u16 = intrinsic.as_str()["simd_shuffle".len()..].parse().unwrap(); assert_eq!(x.layout(), y.layout()); let layout = x.layout(); From cdc0aa188e72ee100bd61943cf7c81ac28be9a81 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Wed, 21 Apr 2021 15:32:04 +0200 Subject: [PATCH 44/52] Rustup to rustc 1.53.0-nightly (6df26f897 2021-04-20) --- rust-toolchain | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust-toolchain b/rust-toolchain index ef7fc7baba251..e5f85be4436fb 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2021-04-14" +channel = "nightly-2021-04-21" components = ["rust-src", "rustc-dev", "llvm-tools-preview"] From e16ccba3945e06a9e687dd13de9053d981cc4fad Mon Sep 17 00:00:00 2001 From: Muhammad Mominul Huque Date: Fri, 23 Apr 2021 17:04:45 +0600 Subject: [PATCH 45/52] Support -Ctarget-cpu --- src/lib.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 5a75b9be0cb69..02518cc0b19dc 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -273,9 +273,14 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box Date: Fri, 23 Apr 2021 19:55:52 +0600 Subject: [PATCH 46/52] Handle native target-cpu variant and raise fatal error if the specified target cpu is not supported --- Cargo.lock | 1 + Cargo.toml | 2 ++ src/lib.rs | 29 +++++++++++++++++++++-------- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 16c2732eac9e4..e6792def56796 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -306,6 +306,7 @@ dependencies = [ "cranelift-frontend", "cranelift-jit", "cranelift-module", + "cranelift-native", "cranelift-object", "gimli", "indexmap", diff --git a/Cargo.toml b/Cargo.toml index 248540cf1a3a5..2789207c65581 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,7 @@ crate-type = ["dylib"] cranelift-codegen = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", features = ["unwind"] } cranelift-frontend = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } cranelift-module = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } +cranelift-native = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } cranelift-jit = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main", optional = true } cranelift-object = { git = "https://github.com/bytecodealliance/wasmtime/", branch = "main" } target-lexicon = "0.12.0" @@ -28,6 +29,7 @@ smallvec = "1.6.1" #cranelift-codegen = { path = "../wasmtime/cranelift/codegen" } #cranelift-frontend = { path = "../wasmtime/cranelift/frontend" } #cranelift-module = { path = "../wasmtime/cranelift/module" } +#cranelift-native = { path = ../wasmtime/cranelift/native" } #cranelift-jit = { path = "../wasmtime/cranelift/jit" } #cranelift-object = { path = "../wasmtime/cranelift/object" } diff --git a/src/lib.rs b/src/lib.rs index 02518cc0b19dc..b7a6e692e3411 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -272,15 +272,28 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box { + let builder = cranelift_native::builder_with_options(variant, true).unwrap(); + builder + } + Some(value) => { + let mut builder = cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap(); + if let Err(_) = builder.enable(value) { + sess.fatal("The target cpu isn't currently supported by Cranelift."); + } + builder + } + None => { + let mut builder = cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap(); + // Don't use "haswell" as the default, as it implies `has_lzcnt`. + // macOS CI is still at Ivy Bridge EP, so `lzcnt` is interpreted as `bsr`. + builder.enable("nehalem").unwrap(); + builder + } + }; - if let Some(target_cpu) = sess.opts.cg.target_cpu.as_ref() { - isa_builder.enable(target_cpu).unwrap(); - } else { - // Don't use "haswell" as the default, as it implies `has_lzcnt`. - // macOS CI is still at Ivy Bridge EP, so `lzcnt` is interpreted as `bsr`. - isa_builder.enable("nehalem").unwrap(); - } isa_builder.finish(flags) } From c4f50fb06ff22e2f566e7eb8dc6b7efc2d91811a Mon Sep 17 00:00:00 2001 From: Muhammad Mominul Huque Date: Fri, 23 Apr 2021 20:40:27 +0600 Subject: [PATCH 47/52] Update the error messsage Co-authored-by: bjorn3 --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index b7a6e692e3411..32f403957025a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -281,7 +281,7 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box { let mut builder = cranelift_codegen::isa::lookup_variant(target_triple, variant).unwrap(); if let Err(_) = builder.enable(value) { - sess.fatal("The target cpu isn't currently supported by Cranelift."); + sess.fatal("The specified target cpu isn't currently supported by Cranelift."); } builder } From beb4e312c8f27942c400ffcdb8454519b76a1e54 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Wed, 28 Apr 2021 19:41:10 +0200 Subject: [PATCH 48/52] Rustup to rustc 1.53.0-nightly (727d10156 2021-04-27) --- build_sysroot/Cargo.lock | 4 ++-- rust-toolchain | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build_sysroot/Cargo.lock b/build_sysroot/Cargo.lock index b6e27245b8f29..e058a972ead3c 100644 --- a/build_sysroot/Cargo.lock +++ b/build_sysroot/Cargo.lock @@ -132,9 +132,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.93" +version = "0.2.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9385f66bf6105b241aa65a61cb923ef20efc665cb9f9bb50ac2f0c4b7f378d41" +checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e" dependencies = [ "rustc-std-workspace-core", ] diff --git a/rust-toolchain b/rust-toolchain index e5f85be4436fb..5442e3345aa91 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2021-04-21" +channel = "nightly-2021-04-28" components = ["rust-src", "rustc-dev", "llvm-tools-preview"] From 88901ca9d438787cf68874c9c2a65d0a95bbc010 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Thu, 29 Apr 2021 14:16:16 +0200 Subject: [PATCH 49/52] Ignore new failing rustc test --- scripts/test_rustc_tests.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/test_rustc_tests.sh b/scripts/test_rustc_tests.sh index d84bfe12d73a0..347fb40e6f9e7 100755 --- a/scripts/test_rustc_tests.sh +++ b/scripts/test_rustc_tests.sh @@ -38,6 +38,7 @@ rm src/test/ui/threads-sendsync/task-stderr.rs rm src/test/ui/numbers-arithmetic/int-abs-overflow.rs rm src/test/ui/drop/drop-trait-enum.rs rm src/test/ui/numbers-arithmetic/issue-8460.rs +rm src/test/incremental/change_crate_dep_kind.rs # requires -Cpanic=unwind rm src/test/ui/issues/issue-28950.rs # depends on stack size optimizations rm src/test/ui/init-large-type.rs # same @@ -57,7 +58,6 @@ rm src/test/ui/intrinsics/intrinsic-nearby.rs # unimplemented nearbyintf32 and n rm src/test/incremental/hashes/inline_asm.rs # inline asm rm src/test/incremental/issue-72386.rs # same -rm src/test/incremental/change_crate_dep_kind.rs # requires -Cpanic=unwind rm src/test/incremental/issue-49482.rs # same rm src/test/incremental/issue-54059.rs # same rm src/test/incremental/lto.rs # requires lto @@ -72,6 +72,7 @@ rm -r src/test/run-pass-valgrind/unsized-locals rm src/test/ui/json-bom-plus-crlf-multifile.rs # differing warning rm src/test/ui/json-bom-plus-crlf.rs # same +rm src/test/ui/match/issue-82392.rs # differing error rm src/test/ui/type-alias-impl-trait/cross_crate_ice*.rs # requires removed aux dep rm src/test/ui/allocator/no_std-alloc-error-handler-default.rs # missing rust_oom definition From ddd4ce25535cf71203ba3700896131ce55fde795 Mon Sep 17 00:00:00 2001 From: bjorn3 Date: Fri, 30 Apr 2021 10:52:37 +0200 Subject: [PATCH 50/52] Remove unused parameter --- src/backend.rs | 4 ++-- src/metadata.rs | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/backend.rs b/src/backend.rs index 9f58db62a1fa6..05c06bac27db4 100644 --- a/src/backend.rs +++ b/src/backend.rs @@ -17,11 +17,11 @@ use gimli::SectionId; use crate::debuginfo::{DebugReloc, DebugRelocName}; pub(crate) trait WriteMetadata { - fn add_rustc_section(&mut self, symbol_name: String, data: Vec, is_like_osx: bool); + fn add_rustc_section(&mut self, symbol_name: String, data: Vec); } impl WriteMetadata for object::write::Object { - fn add_rustc_section(&mut self, symbol_name: String, data: Vec, _is_like_osx: bool) { + fn add_rustc_section(&mut self, symbol_name: String, data: Vec) { let segment = self.segment_name(object::write::StandardSegment::Data).to_vec(); let section_id = self.add_section(segment, b".rustc".to_vec(), object::SectionKind::Data); let offset = self.append_section_data(section_id, &data, 1); diff --git a/src/metadata.rs b/src/metadata.rs index 978a1e722ddd2..882232fde09d2 100644 --- a/src/metadata.rs +++ b/src/metadata.rs @@ -80,6 +80,5 @@ pub(crate) fn write_metadata(tcx: TyCtxt<'_>, object: &mut O) object.add_rustc_section( rustc_middle::middle::exported_symbols::metadata_symbol_name(tcx), compressed, - tcx.sess.target.is_like_osx, ); } From 9d07b92990401da49e99aebd815ccd0bd343a0da Mon Sep 17 00:00:00 2001 From: Erin Power Date: Fri, 30 Apr 2021 15:27:05 +0200 Subject: [PATCH 51/52] [cg_clif] Fix run_jit from sync --- .../rustc_codegen_cranelift/src/driver/jit.rs | 56 ++----------------- 1 file changed, 5 insertions(+), 51 deletions(-) diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs index 956207c5b7cfe..53c93f6a9ddca 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/jit.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs @@ -8,6 +8,7 @@ use std::os::raw::{c_char, c_int}; use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink}; use rustc_codegen_ssa::CrateInfo; use rustc_middle::mir::mono::MonoItem; +use rustc_session::config::EntryFnType; use cranelift_jit::{JITBuilder, JITModule}; @@ -138,57 +139,10 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { *lazy_jit_state = Some(JitState { backend_config, jit_module }); }); - let (main_def_id, entry_ty) = tcx.entry_fn(LOCAL_CRATE).unwrap(); - let instance = Instance::mono(tcx, main_def_id).polymorphize(tcx); - - match entry_ty { - EntryFnType::Main => { - // FIXME set program arguments somehow - - let main_sig = Signature { - params: vec![], - returns: vec![], - call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)), - }; - let main_func_id = jit_module - .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &main_sig) - .unwrap(); - let finalized_main: *const u8 = jit_module.get_finalized_function(main_func_id); - - CURRENT_MODULE.with(|current_module| { - assert!(current_module.borrow_mut().replace(jit_module).is_none()) - }); - - let f: extern "C" fn() = unsafe { ::std::mem::transmute(finalized_main) }; - f(); - std::process::exit(0); - } - EntryFnType::Start => { - let start_sig = Signature { - params: vec![ - AbiParam::new(jit_module.target_config().pointer_type()), - AbiParam::new(jit_module.target_config().pointer_type()), - ], - returns: vec![AbiParam::new( - jit_module.target_config().pointer_type(), /*isize*/ - )], - call_conv: CallConv::triple_default(&crate::target_triple(tcx.sess)), - }; - let start_func_id = jit_module - .declare_function(tcx.symbol_name(instance).name, Linkage::Import, &start_sig) - .unwrap(); - let finalized_start: *const u8 = jit_module.get_finalized_function(start_func_id); - - CURRENT_MODULE.with(|current_module| { - assert!(current_module.borrow_mut().replace(jit_module).is_none()) - }); - - let f: extern "C" fn(c_int, *const *const c_char) -> c_int = - unsafe { ::std::mem::transmute(finalized_start) }; - let ret = f(args.len() as c_int, argv.as_ptr()); - std::process::exit(ret); - } - } + let f: extern "C" fn(c_int, *const *const c_char) -> c_int = + unsafe { ::std::mem::transmute(finalized_start) }; + let ret = f(args.len() as c_int, argv.as_ptr()); + std::process::exit(ret); } #[no_mangle] From 15c8d31392b9fbab3b3368b67acc4bbe5983115a Mon Sep 17 00:00:00 2001 From: XAMPPRocky <4464295+XAMPPRocky@users.noreply.github.com> Date: Fri, 30 Apr 2021 18:44:20 +0200 Subject: [PATCH 52/52] No-op register_jit on Windows (#1170) * No-op register_jit on Windows Co-authored-by: bjorn3 --- src/debuginfo/unwind.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/debuginfo/unwind.rs b/src/debuginfo/unwind.rs index ca7083cccb88f..d1251e749f31f 100644 --- a/src/debuginfo/unwind.rs +++ b/src/debuginfo/unwind.rs @@ -71,7 +71,10 @@ impl UnwindContext { } } - #[cfg(feature = "jit")] + #[cfg(all(feature = "jit", windows))] + pub(crate) unsafe fn register_jit(self, _jit_module: &cranelift_jit::JITModule) {} + + #[cfg(all(feature = "jit", not(windows)))] pub(crate) unsafe fn register_jit(self, jit_module: &cranelift_jit::JITModule) { let mut eh_frame = EhFrame::from(super::emit::WriterRelocate::new(self.endian)); self.frame_table.write_eh_frame(&mut eh_frame).unwrap();