From bfed3c4f0d285df97c0e9cbe342665b8b37b7a96 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Wed, 16 Mar 2022 20:33:54 -0400 Subject: [PATCH 1/3] implement simd bitmask intrinsics --- src/helpers.rs | 15 ------- src/shims/intrinsics.rs | 74 ++++++++++++++++++++++++++++++++- tests/run-pass/portable-simd.rs | 15 +++++++ 3 files changed, 87 insertions(+), 17 deletions(-) diff --git a/src/helpers.rs b/src/helpers.rs index ba12e0a7e3..fe2f33ffd3 100644 --- a/src/helpers.rs +++ b/src/helpers.rs @@ -775,18 +775,3 @@ pub fn isolation_abort_error(name: &str) -> InterpResult<'static> { name, ))) } - -pub fn bool_to_simd_element(b: bool, size: Size) -> Scalar { - // SIMD uses all-1 as pattern for "true" - let val = if b { -1 } else { 0 }; - Scalar::from_int(val, size) -} - -pub fn simd_element_to_bool<'tcx>(elem: ImmTy<'tcx, Tag>) -> InterpResult<'tcx, bool> { - let val = elem.to_scalar()?.to_int(elem.layout.size)?; - Ok(match val { - 0 => false, - -1 => true, - _ => throw_ub_format!("each element of a SIMD mask must be all-0-bits or all-1-bits"), - }) -} diff --git a/src/shims/intrinsics.rs b/src/shims/intrinsics.rs index 726e6b6b96..b704004e16 100644 --- a/src/shims/intrinsics.rs +++ b/src/shims/intrinsics.rs @@ -1,3 +1,4 @@ +use std::convert::TryInto; use std::iter; use log::trace; @@ -5,10 +6,10 @@ use log::trace; use rustc_apfloat::{Float, Round}; use rustc_middle::ty::layout::{HasParamEnv, IntegerExt, LayoutOf}; use rustc_middle::{mir, mir::BinOp, ty, ty::FloatTy}; -use rustc_target::abi::{Align, Integer}; +use rustc_target::abi::{Align, Endian, HasDataLayout, Integer, Size}; use crate::*; -use helpers::{bool_to_simd_element, check_arg_count, simd_element_to_bool}; +use helpers::check_arg_count; pub enum AtomicOp { MirOp(mir::BinOp, bool), @@ -663,6 +664,35 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx this.write_immediate(*val, &dest.into())?; } } + "simd_select_bitmask" => { + let &[ref mask, ref yes, ref no] = check_arg_count(args)?; + let (yes, yes_len) = this.operand_to_simd(yes)?; + let (no, no_len) = this.operand_to_simd(no)?; + let (dest, dest_len) = this.place_to_simd(dest)?; + + assert!(mask.layout.ty.is_integral()); + assert_eq!(dest_len.max(8), mask.layout.size.bits()); + assert!(dest_len <= 64); + assert_eq!(dest_len, yes_len); + assert_eq!(dest_len, no_len); + + let mask: u64 = this + .read_scalar(mask)? + .check_init()? + .to_bits(mask.layout.size)? + .try_into() + .unwrap(); + for i in 0..dest_len { + let mask = + mask & (1 << simd_bitmask_index(i, dest_len, this.data_layout().endian)); + let yes = this.read_immediate(&this.mplace_index(&yes, i)?.into())?; + let no = this.read_immediate(&this.mplace_index(&no, i)?.into())?; + let dest = this.mplace_index(&dest, i)?; + + let val = if mask != 0 { yes } else { no }; + this.write_immediate(*val, &dest.into())?; + } + } #[rustfmt::skip] "simd_cast" | "simd_as" => { let &[ref op] = check_arg_count(args)?; @@ -787,6 +817,23 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx } } } + "simd_bitmask" => { + let &[ref op] = check_arg_count(args)?; + let (op, op_len) = this.operand_to_simd(op)?; + + assert!(dest.layout.ty.is_integral()); + assert_eq!(op_len.max(8), dest.layout.size.bits()); + assert!(op_len <= 64); + + let mut res = 0u64; + for i in 0..op_len { + let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?; + if simd_element_to_bool(op)? { + res |= 1 << simd_bitmask_index(i, op_len, this.data_layout().endian); + } + } + this.write_int(res, dest)?; + } // Atomic operations "atomic_load" => this.atomic_load(args, dest, AtomicReadOp::SeqCst)?, @@ -1307,3 +1354,26 @@ fn fmin_op<'tcx>( FloatTy::F64 => Scalar::from_f64(left.to_f64()?.min(right.to_f64()?)), }) } + +fn bool_to_simd_element(b: bool, size: Size) -> Scalar { + // SIMD uses all-1 as pattern for "true" + let val = if b { -1 } else { 0 }; + Scalar::from_int(val, size) +} + +fn simd_element_to_bool<'tcx>(elem: ImmTy<'tcx, Tag>) -> InterpResult<'tcx, bool> { + let val = elem.to_scalar()?.to_int(elem.layout.size)?; + Ok(match val { + 0 => false, + -1 => true, + _ => throw_ub_format!("each element of a SIMD mask must be all-0-bits or all-1-bits"), + }) +} + +fn simd_bitmask_index(idx: u64, len: u64, endianess: Endian) -> u64 { + assert!(idx < len); + match endianess { + Endian::Little => idx, + Endian::Big => len.max(8) - 1 - idx, // reverse order of bits + } +} diff --git a/tests/run-pass/portable-simd.rs b/tests/run-pass/portable-simd.rs index 80b0b4556c..a74559b72b 100644 --- a/tests/run-pass/portable-simd.rs +++ b/tests/run-pass/portable-simd.rs @@ -187,6 +187,21 @@ fn simd_mask() { let intmask = Mask::from_int(i32x4::from_array([0, -1, 0, 0])); assert_eq!(intmask, Mask::from_array([false, true, false, false])); assert_eq!(intmask.to_array(), [false, true, false, false]); + + let values = [ + true, false, false, true, false, false, true, false, true, true, false, false, false, true, + false, true, + ]; + let mask = Mask::::from_array(values); + let bitmask = mask.to_bitmask(); + assert_eq!(bitmask, 0b1010001101001001); + assert_eq!(Mask::::from_bitmask(bitmask), mask); + + let values = [false, false, false, true]; + let mask = Mask::::from_array(values); + let bitmask = mask.to_bitmask(); + assert_eq!(bitmask, 0b1000); + assert_eq!(Mask::::from_bitmask(bitmask), mask); } fn simd_cast() { From b5d3a25b49f97301bfeaa632d1cf432b3c0e8a71 Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Wed, 16 Mar 2022 20:52:01 -0400 Subject: [PATCH 2/3] detect when unused bits of a SIMD bitmask are non-0 --- src/shims/intrinsics.rs | 30 +++++++++++++------ .../intrinsics/simd-select-bitmask-invalid.rs | 15 ++++++++++ .../intrinsics/simd-select-invalid-bool.rs | 15 ++++++++++ 3 files changed, 51 insertions(+), 9 deletions(-) create mode 100644 tests/compile-fail/intrinsics/simd-select-bitmask-invalid.rs create mode 100644 tests/compile-fail/intrinsics/simd-select-invalid-bool.rs diff --git a/src/shims/intrinsics.rs b/src/shims/intrinsics.rs index b704004e16..495ad93951 100644 --- a/src/shims/intrinsics.rs +++ b/src/shims/intrinsics.rs @@ -669,10 +669,11 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx let (yes, yes_len) = this.operand_to_simd(yes)?; let (no, no_len) = this.operand_to_simd(no)?; let (dest, dest_len) = this.place_to_simd(dest)?; + let bitmask_len = dest_len.max(8); assert!(mask.layout.ty.is_integral()); - assert_eq!(dest_len.max(8), mask.layout.size.bits()); - assert!(dest_len <= 64); + assert!(bitmask_len <= 64); + assert_eq!(bitmask_len, mask.layout.size.bits()); assert_eq!(dest_len, yes_len); assert_eq!(dest_len, no_len); @@ -684,7 +685,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx .unwrap(); for i in 0..dest_len { let mask = - mask & (1 << simd_bitmask_index(i, dest_len, this.data_layout().endian)); + mask & (1 << simd_bitmask_index(i, bitmask_len, this.data_layout().endian)); let yes = this.read_immediate(&this.mplace_index(&yes, i)?.into())?; let no = this.read_immediate(&this.mplace_index(&no, i)?.into())?; let dest = this.mplace_index(&dest, i)?; @@ -692,6 +693,16 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx let val = if mask != 0 { yes } else { no }; this.write_immediate(*val, &dest.into())?; } + for i in dest_len..bitmask_len { + // If the mask is "padded", ensure that padding is all-zero. + let mask = + mask & (1 << simd_bitmask_index(i, bitmask_len, this.data_layout().endian)); + if mask != 0 { + throw_ub_format!( + "a SIMD bitmask less than 8 bits long must be filled with 0s for the remaining bits" + ); + } + } } #[rustfmt::skip] "simd_cast" | "simd_as" => { @@ -820,16 +831,17 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx "simd_bitmask" => { let &[ref op] = check_arg_count(args)?; let (op, op_len) = this.operand_to_simd(op)?; + let bitmask_len = op_len.max(8); assert!(dest.layout.ty.is_integral()); - assert_eq!(op_len.max(8), dest.layout.size.bits()); - assert!(op_len <= 64); + assert!(bitmask_len <= 64); + assert_eq!(bitmask_len, dest.layout.size.bits()); let mut res = 0u64; for i in 0..op_len { let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?; if simd_element_to_bool(op)? { - res |= 1 << simd_bitmask_index(i, op_len, this.data_layout().endian); + res |= 1 << simd_bitmask_index(i, bitmask_len, this.data_layout().endian); } } this.write_int(res, dest)?; @@ -1370,10 +1382,10 @@ fn simd_element_to_bool<'tcx>(elem: ImmTy<'tcx, Tag>) -> InterpResult<'tcx, bool }) } -fn simd_bitmask_index(idx: u64, len: u64, endianess: Endian) -> u64 { - assert!(idx < len); +fn simd_bitmask_index(idx: u64, bitmask_len: u64, endianess: Endian) -> u64 { + assert!(idx < bitmask_len); match endianess { Endian::Little => idx, - Endian::Big => len.max(8) - 1 - idx, // reverse order of bits + Endian::Big => bitmask_len - 1 - idx, // reverse order of bits } } diff --git a/tests/compile-fail/intrinsics/simd-select-bitmask-invalid.rs b/tests/compile-fail/intrinsics/simd-select-bitmask-invalid.rs new file mode 100644 index 0000000000..ab69072c30 --- /dev/null +++ b/tests/compile-fail/intrinsics/simd-select-bitmask-invalid.rs @@ -0,0 +1,15 @@ +#![feature(platform_intrinsics, repr_simd)] + +extern "platform-intrinsic" { + fn simd_select_bitmask(m: M, yes: T, no: T) -> T; +} + +#[repr(simd)] +#[allow(non_camel_case_types)] +#[derive(Copy, Clone)] +struct i32x2(i32, i32); + +fn main() { unsafe { + let x = i32x2(0, 1); + simd_select_bitmask(0b11111111u8, x, x); //~ERROR bitmask less than 8 bits long must be filled with 0s for the remaining bits +} } diff --git a/tests/compile-fail/intrinsics/simd-select-invalid-bool.rs b/tests/compile-fail/intrinsics/simd-select-invalid-bool.rs new file mode 100644 index 0000000000..98f67cfcd7 --- /dev/null +++ b/tests/compile-fail/intrinsics/simd-select-invalid-bool.rs @@ -0,0 +1,15 @@ +#![feature(platform_intrinsics, repr_simd)] + +extern "platform-intrinsic" { + fn simd_select(m: M, yes: T, no: T) -> T; +} + +#[repr(simd)] +#[allow(non_camel_case_types)] +#[derive(Copy, Clone)] +struct i32x2(i32, i32); + +fn main() { unsafe { + let x = i32x2(0, 1); + simd_select(x, x, x); //~ERROR must be all-0-bits or all-1-bits +} } From 1b1321a685f4d5e362786c1876dbcc9e2de866ba Mon Sep 17 00:00:00 2001 From: Ralf Jung Date: Thu, 17 Mar 2022 13:14:16 -0400 Subject: [PATCH 3/3] fix simd_bitmask shorter than a byte on big-endian --- src/shims/intrinsics.rs | 13 ++++++------- tests/run-pass/portable-simd.rs | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/shims/intrinsics.rs b/src/shims/intrinsics.rs index 495ad93951..c344d0ff9c 100644 --- a/src/shims/intrinsics.rs +++ b/src/shims/intrinsics.rs @@ -685,7 +685,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx .unwrap(); for i in 0..dest_len { let mask = - mask & (1 << simd_bitmask_index(i, bitmask_len, this.data_layout().endian)); + mask & (1 << simd_bitmask_index(i, dest_len, this.data_layout().endian)); let yes = this.read_immediate(&this.mplace_index(&yes, i)?.into())?; let no = this.read_immediate(&this.mplace_index(&no, i)?.into())?; let dest = this.mplace_index(&dest, i)?; @@ -695,8 +695,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx } for i in dest_len..bitmask_len { // If the mask is "padded", ensure that padding is all-zero. - let mask = - mask & (1 << simd_bitmask_index(i, bitmask_len, this.data_layout().endian)); + let mask = mask & (1 << i); if mask != 0 { throw_ub_format!( "a SIMD bitmask less than 8 bits long must be filled with 0s for the remaining bits" @@ -841,7 +840,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriEvalContextExt<'mir, 'tcx for i in 0..op_len { let op = this.read_immediate(&this.mplace_index(&op, i)?.into())?; if simd_element_to_bool(op)? { - res |= 1 << simd_bitmask_index(i, bitmask_len, this.data_layout().endian); + res |= 1 << simd_bitmask_index(i, op_len, this.data_layout().endian); } } this.write_int(res, dest)?; @@ -1382,10 +1381,10 @@ fn simd_element_to_bool<'tcx>(elem: ImmTy<'tcx, Tag>) -> InterpResult<'tcx, bool }) } -fn simd_bitmask_index(idx: u64, bitmask_len: u64, endianess: Endian) -> u64 { - assert!(idx < bitmask_len); +fn simd_bitmask_index(idx: u64, vec_len: u64, endianess: Endian) -> u64 { + assert!(idx < vec_len); match endianess { Endian::Little => idx, - Endian::Big => bitmask_len - 1 - idx, // reverse order of bits + Endian::Big => vec_len - 1 - idx, // reverse order of bits } } diff --git a/tests/run-pass/portable-simd.rs b/tests/run-pass/portable-simd.rs index a74559b72b..99a64ea370 100644 --- a/tests/run-pass/portable-simd.rs +++ b/tests/run-pass/portable-simd.rs @@ -200,7 +200,7 @@ fn simd_mask() { let values = [false, false, false, true]; let mask = Mask::::from_array(values); let bitmask = mask.to_bitmask(); - assert_eq!(bitmask, 0b1000); + // FIXME fails until https://github.com/rust-lang/portable-simd/pull/267 lands: assert_eq!(bitmask, 0b1000); assert_eq!(Mask::::from_bitmask(bitmask), mask); }