diff --git a/crates/core_simd/src/masks.rs b/crates/core_simd/src/masks.rs index 3bb444062ab8ee8ba61dfdf6a316b9b28c22ee07..d4a0cff4e2380a7f4b3c11ba7edcd99788be7bcb 100644 --- a/crates/core_simd/src/masks.rs +++ b/crates/core_simd/src/masks.rs @@ -12,420 +12,461 @@ )] mod mask_impl; -use crate::{SimdI16, SimdI32, SimdI64, SimdI8, SimdIsize}; - -macro_rules! define_opaque_mask { - { - $(#[$attr:meta])* - struct $name:ident($inner_ty:ty); - @bits $bits_ty:ident - } => { - $(#[$attr])* - #[allow(non_camel_case_types)] - pub struct $name($inner_ty) - where - crate::LaneCount: crate::SupportedLaneCount; - - impl_opaque_mask_reductions! { $name, $bits_ty } - - impl $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - /// Construct a mask by setting all lanes to the given value. - pub fn splat(value: bool) -> Self { - Self(<$inner_ty>::splat(value)) - } +use crate::{LaneCount, Simd, SimdElement, SupportedLaneCount}; - /// Converts an array to a SIMD vector. - pub fn from_array(array: [bool; LANES]) -> Self { - let mut vector = Self::splat(false); - let mut i = 0; - while i < $lanes { - vector.set(i, array[i]); - i += 1; - } - vector - } +/// Marker trait for types that may be used as SIMD mask elements. +pub unsafe trait MaskElement: SimdElement { + #[doc(hidden)] + fn valid(values: Simd) -> bool + where + LaneCount: SupportedLaneCount; - /// Converts a SIMD vector to an array. - pub fn to_array(self) -> [bool; LANES] { - let mut array = [false; LANES]; - let mut i = 0; - while i < $lanes { - array[i] = self.test(i); - i += 1; - } - array - } + #[doc(hidden)] + fn eq(self, other: Self) -> bool; - /// Converts a vector of integers to a mask, where 0 represents `false` and -1 - /// represents `true`. - /// - /// # Safety - /// All lanes must be either 0 or -1. - #[inline] - pub unsafe fn from_int_unchecked(value: $bits_ty) -> Self { - Self(<$inner_ty>::from_int_unchecked(value)) - } + #[doc(hidden)] + const TRUE: Self; - /// Converts a vector of integers to a mask, where 0 represents `false` and -1 - /// represents `true`. - /// - /// # Panics - /// Panics if any lane is not 0 or -1. - #[inline] - pub fn from_int(value: $bits_ty) -> Self { - assert!( - (value.lanes_eq($bits_ty::splat(0)) | value.lanes_eq($bits_ty::splat(-1))).all(), - "all values must be either 0 or -1", - ); - unsafe { Self::from_int_unchecked(value) } - } + #[doc(hidden)] + const FALSE: Self; +} - /// Converts the mask to a vector of integers, where 0 represents `false` and -1 - /// represents `true`. - #[inline] - pub fn to_int(self) -> $bits_ty { - self.0.to_int() +macro_rules! impl_element { + { $ty:ty } => { + unsafe impl MaskElement for $ty { + fn valid(value: Simd) -> bool + where + LaneCount: SupportedLaneCount, + { + (value.lanes_eq(Simd::splat(0)) | value.lanes_eq(Simd::splat(-1))).all() } - /// Tests the value of the specified lane. - /// - /// # Safety - /// `lane` must be less than `LANES`. - #[inline] - pub unsafe fn test_unchecked(&self, lane: usize) -> bool { - self.0.test_unchecked(lane) - } + fn eq(self, other: Self) -> bool { self == other } - /// Tests the value of the specified lane. - /// - /// # Panics - /// Panics if `lane` is greater than or equal to the number of lanes in the vector. - #[inline] - pub fn test(&self, lane: usize) -> bool { - assert!(lane < LANES, "lane index out of range"); - unsafe { self.test_unchecked(lane) } - } + const TRUE: Self = -1; + const FALSE: Self = 0; + } + } +} - /// Sets the value of the specified lane. - /// - /// # Safety - /// `lane` must be less than `LANES`. - #[inline] - pub unsafe fn set_unchecked(&mut self, lane: usize, value: bool) { - self.0.set_unchecked(lane, value); - } +impl_element! { i8 } +impl_element! { i16 } +impl_element! { i32 } +impl_element! { i64 } +impl_element! { isize } + +/// A SIMD vector mask for `LANES` elements of width specified by `Element`. +/// +/// The layout of this type is unspecified. +#[repr(transparent)] +pub struct Mask(mask_impl::Mask) +where + Element: MaskElement, + LaneCount: SupportedLaneCount; + +impl Copy for Mask +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ +} - /// Sets the value of the specified lane. - /// - /// # Panics - /// Panics if `lane` is greater than or equal to the number of lanes in the vector. - #[inline] - pub fn set(&mut self, lane: usize, value: bool) { - assert!(lane < LANES, "lane index out of range"); - unsafe { self.set_unchecked(lane, value); } - } +impl Clone for Mask +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + fn clone(&self) -> Self { + *self + } +} - /// Convert this mask to a bitmask, with one bit set per lane. - pub fn to_bitmask(self) -> [u8; crate::LaneCount::::BITMASK_LEN] { - self.0.to_bitmask() - } +impl Mask +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + /// Construct a mask by setting all lanes to the given value. + pub fn splat(value: bool) -> Self { + Self(mask_impl::Mask::splat(value)) + } - /// Convert a bitmask to a mask. - pub fn from_bitmask(bitmask: [u8; crate::LaneCount::::BITMASK_LEN]) -> Self { - Self(<$inner_ty>::from_bitmask(bitmask)) - } + /// Converts an array to a SIMD vector. + pub fn from_array(array: [bool; LANES]) -> Self { + let mut vector = Self::splat(false); + for (i, v) in array.iter().enumerate() { + vector.set(i, *v); } + vector + } - // vector/array conversion - impl From<[bool; LANES]> for $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - fn from(array: [bool; LANES]) -> Self { - Self::from_array(array) - } + /// Converts a SIMD vector to an array. + pub fn to_array(self) -> [bool; LANES] { + let mut array = [false; LANES]; + for (i, v) in array.iter_mut().enumerate() { + *v = self.test(i); } + array + } - impl From<$name> for [bool; LANES] - where - crate::LaneCount: crate::SupportedLaneCount, - { - fn from(vector: $name) -> Self { - vector.to_array() - } - } + /// Converts a vector of integers to a mask, where 0 represents `false` and -1 + /// represents `true`. + /// + /// # Safety + /// All lanes must be either 0 or -1. + #[inline] + pub unsafe fn from_int_unchecked(value: Simd) -> Self { + Self(mask_impl::Mask::from_int_unchecked(value)) + } - impl Copy for $name - where - crate::LaneCount: crate::SupportedLaneCount, - {} + /// Converts a vector of integers to a mask, where 0 represents `false` and -1 + /// represents `true`. + /// + /// # Panics + /// Panics if any lane is not 0 or -1. + #[inline] + pub fn from_int(value: Simd) -> Self { + assert!(Element::valid(value), "all values must be either 0 or -1",); + unsafe { Self::from_int_unchecked(value) } + } - impl Clone for $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - #[inline] - fn clone(&self) -> Self { - *self - } - } + /// Converts the mask to a vector of integers, where 0 represents `false` and -1 + /// represents `true`. + #[inline] + pub fn to_int(self) -> Simd { + self.0.to_int() + } - impl Default for $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - #[inline] - fn default() -> Self { - Self::splat(false) - } - } + /// Tests the value of the specified lane. + /// + /// # Safety + /// `lane` must be less than `LANES`. + #[inline] + pub unsafe fn test_unchecked(&self, lane: usize) -> bool { + self.0.test_unchecked(lane) + } - impl PartialEq for $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - #[inline] - fn eq(&self, other: &Self) -> bool { - self.0 == other.0 - } - } + /// Tests the value of the specified lane. + /// + /// # Panics + /// Panics if `lane` is greater than or equal to the number of lanes in the vector. + #[inline] + pub fn test(&self, lane: usize) -> bool { + assert!(lane < LANES, "lane index out of range"); + unsafe { self.test_unchecked(lane) } + } - impl PartialOrd for $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - #[inline] - fn partial_cmp(&self, other: &Self) -> Option { - self.0.partial_cmp(&other.0) - } - } + /// Sets the value of the specified lane. + /// + /// # Safety + /// `lane` must be less than `LANES`. + #[inline] + pub unsafe fn set_unchecked(&mut self, lane: usize, value: bool) { + self.0.set_unchecked(lane, value); + } - impl core::fmt::Debug for $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { - f.debug_list() - .entries((0..LANES).map(|lane| self.test(lane))) - .finish() - } + /// Sets the value of the specified lane. + /// + /// # Panics + /// Panics if `lane` is greater than or equal to the number of lanes in the vector. + #[inline] + pub fn set(&mut self, lane: usize, value: bool) { + assert!(lane < LANES, "lane index out of range"); + unsafe { + self.set_unchecked(lane, value); } + } - impl core::ops::BitAnd for $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - type Output = Self; - #[inline] - fn bitand(self, rhs: Self) -> Self { - Self(self.0 & rhs.0) - } - } + /// Convert this mask to a bitmask, with one bit set per lane. + pub fn to_bitmask(self) -> [u8; LaneCount::::BITMASK_LEN] { + self.0.to_bitmask() + } - impl core::ops::BitAnd for $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - type Output = Self; - #[inline] - fn bitand(self, rhs: bool) -> Self { - self & Self::splat(rhs) - } - } + /// Convert a bitmask to a mask. + pub fn from_bitmask(bitmask: [u8; LaneCount::::BITMASK_LEN]) -> Self { + Self(mask_impl::Mask::from_bitmask(bitmask)) + } - impl core::ops::BitAnd<$name> for bool - where - crate::LaneCount: crate::SupportedLaneCount, - { - type Output = $name; - #[inline] - fn bitand(self, rhs: $name) -> $name { - $name::::splat(self) & rhs - } - } + /// Returns true if any lane is set, or false otherwise. + #[inline] + pub fn any(self) -> bool { + self.0.any() + } - impl core::ops::BitOr for $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - type Output = Self; - #[inline] - fn bitor(self, rhs: Self) -> Self { - Self(self.0 | rhs.0) - } - } + /// Returns true if all lanes are set, or false otherwise. + #[inline] + pub fn all(self) -> bool { + self.0.all() + } +} - impl core::ops::BitOr for $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - type Output = Self; - #[inline] - fn bitor(self, rhs: bool) -> Self { - self | Self::splat(rhs) - } - } +// vector/array conversion +impl From<[bool; LANES]> for Mask +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + fn from(array: [bool; LANES]) -> Self { + Self::from_array(array) + } +} - impl core::ops::BitOr<$name> for bool - where - crate::LaneCount: crate::SupportedLaneCount, - { - type Output = $name; - #[inline] - fn bitor(self, rhs: $name) -> $name { - $name::::splat(self) | rhs - } - } +impl From> for [bool; LANES] +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + fn from(vector: Mask) -> Self { + vector.to_array() + } +} - impl core::ops::BitXor for $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - type Output = Self; - #[inline] - fn bitxor(self, rhs: Self) -> Self::Output { - Self(self.0 ^ rhs.0) - } - } +impl Default for Mask +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + #[inline] + fn default() -> Self { + Self::splat(false) + } +} - impl core::ops::BitXor for $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - type Output = Self; - #[inline] - fn bitxor(self, rhs: bool) -> Self::Output { - self ^ Self::splat(rhs) - } - } +impl PartialEq for Mask +where + Element: MaskElement + PartialEq, + LaneCount: SupportedLaneCount, +{ + #[inline] + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } +} - impl core::ops::BitXor<$name> for bool - where - crate::LaneCount: crate::SupportedLaneCount, - { - type Output = $name; - #[inline] - fn bitxor(self, rhs: $name) -> Self::Output { - $name::::splat(self) ^ rhs - } - } +impl PartialOrd for Mask +where + Element: MaskElement + PartialOrd, + LaneCount: SupportedLaneCount, +{ + #[inline] + fn partial_cmp(&self, other: &Self) -> Option { + self.0.partial_cmp(&other.0) + } +} - impl core::ops::Not for $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - type Output = $name; - #[inline] - fn not(self) -> Self::Output { - Self(!self.0) - } - } +impl core::fmt::Debug for Mask +where + Element: MaskElement + core::fmt::Debug, + LaneCount: SupportedLaneCount, +{ + fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { + f.debug_list() + .entries((0..LANES).map(|lane| self.test(lane))) + .finish() + } +} - impl core::ops::BitAndAssign for $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - #[inline] - fn bitand_assign(&mut self, rhs: Self) { - self.0 = self.0 & rhs.0; - } - } +impl core::ops::BitAnd for Mask +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Self; + #[inline] + fn bitand(self, rhs: Self) -> Self { + Self(self.0 & rhs.0) + } +} - impl core::ops::BitAndAssign for $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - #[inline] - fn bitand_assign(&mut self, rhs: bool) { - *self &= Self::splat(rhs); - } - } +impl core::ops::BitAnd for Mask +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Self; + #[inline] + fn bitand(self, rhs: bool) -> Self { + self & Self::splat(rhs) + } +} - impl core::ops::BitOrAssign for $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - #[inline] - fn bitor_assign(&mut self, rhs: Self) { - self.0 = self.0 | rhs.0; - } - } +impl core::ops::BitAnd> for bool +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Mask; + #[inline] + fn bitand(self, rhs: Mask) -> Mask { + Mask::splat(self) & rhs + } +} - impl core::ops::BitOrAssign for $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - #[inline] - fn bitor_assign(&mut self, rhs: bool) { - *self |= Self::splat(rhs); - } - } +impl core::ops::BitOr for Mask +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Self; + #[inline] + fn bitor(self, rhs: Self) -> Self { + Self(self.0 | rhs.0) + } +} - impl core::ops::BitXorAssign for $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - #[inline] - fn bitxor_assign(&mut self, rhs: Self) { - self.0 = self.0 ^ rhs.0; - } - } +impl core::ops::BitOr for Mask +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Self; + #[inline] + fn bitor(self, rhs: bool) -> Self { + self | Self::splat(rhs) + } +} - impl core::ops::BitXorAssign for $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - #[inline] - fn bitxor_assign(&mut self, rhs: bool) { - *self ^= Self::splat(rhs); - } - } - }; +impl core::ops::BitOr> for bool +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Mask; + #[inline] + fn bitor(self, rhs: Mask) -> Mask { + Mask::splat(self) | rhs + } } -define_opaque_mask! { - /// Mask for vectors with `LANES` 8-bit elements. - /// - /// The layout of this type is unspecified. - struct Mask8(mask_impl::Mask8); - @bits SimdI8 +impl core::ops::BitXor for Mask +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Self; + #[inline] + fn bitxor(self, rhs: Self) -> Self::Output { + Self(self.0 ^ rhs.0) + } } -define_opaque_mask! { - /// Mask for vectors with `LANES` 16-bit elements. - /// - /// The layout of this type is unspecified. - struct Mask16(mask_impl::Mask16); - @bits SimdI16 +impl core::ops::BitXor for Mask +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Self; + #[inline] + fn bitxor(self, rhs: bool) -> Self::Output { + self ^ Self::splat(rhs) + } } -define_opaque_mask! { - /// Mask for vectors with `LANES` 32-bit elements. - /// - /// The layout of this type is unspecified. - struct Mask32(mask_impl::Mask32); - @bits SimdI32 +impl core::ops::BitXor> for bool +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Mask; + #[inline] + fn bitxor(self, rhs: Mask) -> Self::Output { + Mask::splat(self) ^ rhs + } } -define_opaque_mask! { - /// Mask for vectors with `LANES` 64-bit elements. - /// - /// The layout of this type is unspecified. - struct Mask64(mask_impl::Mask64); - @bits SimdI64 +impl core::ops::Not for Mask +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Mask; + #[inline] + fn not(self) -> Self::Output { + Self(!self.0) + } } -define_opaque_mask! { - /// Mask for vectors with `LANES` pointer-width elements. - /// - /// The layout of this type is unspecified. - struct MaskSize(mask_impl::MaskSize); - @bits SimdIsize +impl core::ops::BitAndAssign for Mask +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + #[inline] + fn bitand_assign(&mut self, rhs: Self) { + self.0 = self.0 & rhs.0; + } +} + +impl core::ops::BitAndAssign for Mask +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + #[inline] + fn bitand_assign(&mut self, rhs: bool) { + *self &= Self::splat(rhs); + } +} + +impl core::ops::BitOrAssign for Mask +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + #[inline] + fn bitor_assign(&mut self, rhs: Self) { + self.0 = self.0 | rhs.0; + } +} + +impl core::ops::BitOrAssign for Mask +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + #[inline] + fn bitor_assign(&mut self, rhs: bool) { + *self |= Self::splat(rhs); + } } +impl core::ops::BitXorAssign for Mask +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + #[inline] + fn bitxor_assign(&mut self, rhs: Self) { + self.0 = self.0 ^ rhs.0; + } +} + +impl core::ops::BitXorAssign for Mask +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + #[inline] + fn bitxor_assign(&mut self, rhs: bool) { + *self ^= Self::splat(rhs); + } +} + +/// A SIMD mask of `LANES` 8-bit values. +pub type Mask8 = Mask; + +/// A SIMD mask of `LANES` 16-bit values. +pub type Mask16 = Mask; + +/// A SIMD mask of `LANES` 32-bit values. +pub type Mask32 = Mask; + +/// A SIMD mask of `LANES` 64-bit values. +pub type Mask64 = Mask; + +/// A SIMD mask of `LANES` pointer-width values. +pub type MaskSize = Mask; + /// Vector of eight 8-bit masks pub type mask8x8 = Mask8<8>; @@ -488,7 +529,7 @@ impl From<$from> for $to crate::LaneCount: crate::SupportedLaneCount, { fn from(value: $from) -> Self { - Self(value.0.into()) + Self(value.0.convert()) } } )* diff --git a/crates/core_simd/src/masks/bitmask.rs b/crates/core_simd/src/masks/bitmask.rs index 69edd5235872cfee978773ff39ab0e661e74e527..2b830949451051bd5d89c975e3183c9719070354 100644 --- a/crates/core_simd/src/masks/bitmask.rs +++ b/crates/core_simd/src/masks/bitmask.rs @@ -1,38 +1,26 @@ -use crate::{LaneCount, SupportedLaneCount}; - -/// Helper trait for limiting int conversion types -pub trait ConvertToInt {} -impl ConvertToInt for crate::SimdI8 where - LaneCount: SupportedLaneCount -{ -} -impl ConvertToInt for crate::SimdI16 where - LaneCount: SupportedLaneCount -{ -} -impl ConvertToInt for crate::SimdI32 where - LaneCount: SupportedLaneCount -{ -} -impl ConvertToInt for crate::SimdI64 where - LaneCount: SupportedLaneCount -{ -} -impl ConvertToInt for crate::SimdIsize where - LaneCount: SupportedLaneCount -{ -} +use crate::{LaneCount, MaskElement, Simd, SupportedLaneCount}; +use core::marker::PhantomData; /// A mask where each lane is represented by a single bit. #[repr(transparent)] -pub struct BitMask( as SupportedLaneCount>::BitMask) +pub struct Mask( + as SupportedLaneCount>::BitMask, + PhantomData, +) where + Element: MaskElement, LaneCount: SupportedLaneCount; -impl Copy for BitMask where LaneCount: SupportedLaneCount {} +impl Copy for Mask +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ +} -impl Clone for BitMask +impl Clone for Mask where + Element: MaskElement, LaneCount: SupportedLaneCount, { fn clone(&self) -> Self { @@ -40,8 +28,9 @@ fn clone(&self) -> Self { } } -impl PartialEq for BitMask +impl PartialEq for Mask where + Element: MaskElement, LaneCount: SupportedLaneCount, { fn eq(&self, other: &Self) -> bool { @@ -49,8 +38,9 @@ fn eq(&self, other: &Self) -> bool { } } -impl PartialOrd for BitMask +impl PartialOrd for Mask where + Element: MaskElement, LaneCount: SupportedLaneCount, { fn partial_cmp(&self, other: &Self) -> Option { @@ -58,10 +48,16 @@ fn partial_cmp(&self, other: &Self) -> Option { } } -impl Eq for BitMask where LaneCount: SupportedLaneCount {} +impl Eq for Mask +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ +} -impl Ord for BitMask +impl Ord for Mask where + Element: MaskElement, LaneCount: SupportedLaneCount, { fn cmp(&self, other: &Self) -> core::cmp::Ordering { @@ -69,8 +65,9 @@ fn cmp(&self, other: &Self) -> core::cmp::Ordering { } } -impl BitMask +impl Mask where + Element: MaskElement, LaneCount: SupportedLaneCount, { #[inline] @@ -84,7 +81,7 @@ pub fn splat(value: bool) -> Self { if LANES % 8 > 0 { *mask.as_mut().last_mut().unwrap() &= u8::MAX >> (8 - LANES % 8); } - Self(mask) + Self(mask, PhantomData) } #[inline] @@ -98,33 +95,28 @@ pub unsafe fn set_unchecked(&mut self, lane: usize, value: bool) { } #[inline] - pub fn to_int(self) -> V - where - V: ConvertToInt + Default + core::ops::Not, - { + pub fn to_int(self) -> Simd { unsafe { let mask: as SupportedLaneCount>::IntBitMask = core::mem::transmute_copy(&self); - crate::intrinsics::simd_select_bitmask(mask, !V::default(), V::default()) + crate::intrinsics::simd_select_bitmask( + mask, + Simd::splat(Element::TRUE), + Simd::splat(Element::FALSE), + ) } } #[inline] - pub unsafe fn from_int_unchecked(value: V) -> Self - where - V: crate::Vector, - { + pub unsafe fn from_int_unchecked(value: Simd) -> Self { // TODO remove the transmute when rustc is more flexible assert_eq!( - core::mem::size_of::< as crate::SupportedLaneCount>::BitMask>( - ), - core::mem::size_of::< - as crate::SupportedLaneCount>::IntBitMask, - >(), + core::mem::size_of::< as SupportedLaneCount>::BitMask>(), + core::mem::size_of::< as SupportedLaneCount>::IntBitMask>(), ); let mask: as SupportedLaneCount>::IntBitMask = crate::intrinsics::simd_bitmask(value); - Self(core::mem::transmute_copy(&mask)) + Self(core::mem::transmute_copy(&mask), PhantomData) } #[inline] @@ -136,7 +128,15 @@ pub unsafe fn from_int_unchecked(value: V) -> Self #[inline] pub fn from_bitmask(bitmask: [u8; LaneCount::::BITMASK_LEN]) -> Self { // Safety: these are the same type and we are laundering the generic - Self(unsafe { core::mem::transmute_copy(&bitmask) }) + Self(unsafe { core::mem::transmute_copy(&bitmask) }, PhantomData) + } + + #[inline] + pub fn convert(self) -> Mask + where + T: MaskElement, + { + unsafe { core::mem::transmute_copy(&self) } } #[inline] @@ -150,10 +150,11 @@ pub fn all(self) -> bool { } } -impl core::ops::BitAnd for BitMask +impl core::ops::BitAnd for Mask where + Element: MaskElement, LaneCount: SupportedLaneCount, - as SupportedLaneCount>::BitMask: Default + AsRef<[u8]> + AsMut<[u8]>, + as SupportedLaneCount>::BitMask: AsRef<[u8]> + AsMut<[u8]>, { type Output = Self; #[inline] @@ -165,10 +166,11 @@ fn bitand(mut self, rhs: Self) -> Self { } } -impl core::ops::BitOr for BitMask +impl core::ops::BitOr for Mask where + Element: MaskElement, LaneCount: SupportedLaneCount, - as SupportedLaneCount>::BitMask: Default + AsRef<[u8]> + AsMut<[u8]>, + as SupportedLaneCount>::BitMask: AsRef<[u8]> + AsMut<[u8]>, { type Output = Self; #[inline] @@ -180,8 +182,9 @@ fn bitor(mut self, rhs: Self) -> Self { } } -impl core::ops::BitXor for BitMask +impl core::ops::BitXor for Mask where + Element: MaskElement, LaneCount: SupportedLaneCount, { type Output = Self; @@ -194,8 +197,9 @@ fn bitxor(mut self, rhs: Self) -> Self::Output { } } -impl core::ops::Not for BitMask +impl core::ops::Not for Mask where + Element: MaskElement, LaneCount: SupportedLaneCount, { type Output = Self; @@ -210,9 +214,3 @@ fn not(mut self) -> Self::Output { self } } - -pub type Mask8 = BitMask; -pub type Mask16 = BitMask; -pub type Mask32 = BitMask; -pub type Mask64 = BitMask; -pub type MaskSize = BitMask; diff --git a/crates/core_simd/src/masks/full_masks.rs b/crates/core_simd/src/masks/full_masks.rs index 2923cf1964a0dae4f724d2164137731870a95952..858c99032a3195b8add359aa221355afdde56ac8 100644 --- a/crates/core_simd/src/masks/full_masks.rs +++ b/crates/core_simd/src/masks/full_masks.rs @@ -1,264 +1,221 @@ //! Masks that take up full SIMD vector registers. -macro_rules! define_mask { - { - $(#[$attr:meta])* - struct $name:ident( - crate::$type:ident<$lanes2:ident> - ); - } => { - $(#[$attr])* - #[repr(transparent)] - pub struct $name(crate::$type<$lanes>) - where - crate::LaneCount<$lanes>: crate::SupportedLaneCount; - - impl_full_mask_reductions! { $name, $type } - - impl Copy for $name - where - crate::LaneCount: crate::SupportedLaneCount, - {} - - impl Clone for $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - #[inline] - fn clone(&self) -> Self { - *self - } - } - - impl PartialEq for $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - fn eq(&self, other: &Self) -> bool { - self.0 == other.0 - } - } - - impl PartialOrd for $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - fn partial_cmp(&self, other: &Self) -> Option { - self.0.partial_cmp(&other.0) - } - } - - impl Eq for $name - where - crate::LaneCount: crate::SupportedLaneCount, - {} +use super::MaskElement; +use crate::{LaneCount, Simd, SupportedLaneCount}; + +#[repr(transparent)] +pub struct Mask(Simd) +where + Element: MaskElement, + LaneCount: SupportedLaneCount; + +impl Copy for Mask +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ +} - impl Ord for $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - fn cmp(&self, other: &Self) -> core::cmp::Ordering { - self.0.cmp(&other.0) - } - } +impl Clone for Mask +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + #[inline] + fn clone(&self) -> Self { + *self + } +} - impl $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - pub fn splat(value: bool) -> Self { - Self( - >::splat( - if value { - -1 - } else { - 0 - } - ), - ) - } +impl PartialEq for Mask +where + Element: MaskElement + PartialEq, + LaneCount: SupportedLaneCount, +{ + fn eq(&self, other: &Self) -> bool { + self.0.eq(&other.0) + } +} - #[inline] - pub unsafe fn test_unchecked(&self, lane: usize) -> bool { - self.0[lane] == -1 - } +impl PartialOrd for Mask +where + Element: MaskElement + PartialOrd, + LaneCount: SupportedLaneCount, +{ + fn partial_cmp(&self, other: &Self) -> Option { + self.0.partial_cmp(&other.0) + } +} - #[inline] - pub unsafe fn set_unchecked(&mut self, lane: usize, value: bool) { - self.0[lane] = if value { - -1 - } else { - 0 - } - } +impl Eq for Mask +where + Element: MaskElement + Eq, + LaneCount: SupportedLaneCount, +{ +} - #[inline] - pub fn to_int(self) -> crate::$type { - self.0 - } +impl Ord for Mask +where + Element: MaskElement + Ord, + LaneCount: SupportedLaneCount, +{ + fn cmp(&self, other: &Self) -> core::cmp::Ordering { + self.0.cmp(&other.0) + } +} - #[inline] - pub unsafe fn from_int_unchecked(value: crate::$type) -> Self { - Self(value) - } +impl Mask +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + pub fn splat(value: bool) -> Self { + Self(Simd::splat(if value { + Element::TRUE + } else { + Element::FALSE + })) + } - #[inline] - pub fn to_bitmask(self) -> [u8; crate::LaneCount::::BITMASK_LEN] { - unsafe { - // TODO remove the transmute when rustc can use arrays of u8 as bitmasks - assert_eq!( - core::mem::size_of::< as crate::SupportedLaneCount>::IntBitMask>(), - crate::LaneCount::::BITMASK_LEN, - ); - let bitmask: as crate::SupportedLaneCount>::IntBitMask = crate::intrinsics::simd_bitmask(self.0); - let mut bitmask: [u8; crate::LaneCount::::BITMASK_LEN] = core::mem::transmute_copy(&bitmask); + #[inline] + pub unsafe fn test_unchecked(&self, lane: usize) -> bool { + Element::eq(self.0[lane], Element::TRUE) + } - // There is a bug where LLVM appears to implement this operation with the wrong - // bit order. - // TODO fix this in a better way - if cfg!(any(target_arch = "mips", target_arch = "mips64")) { - for x in bitmask.as_mut() { - *x = x.reverse_bits(); - } - } + #[inline] + pub unsafe fn set_unchecked(&mut self, lane: usize, value: bool) { + self.0[lane] = if value { Element::TRUE } else { Element::FALSE } + } - bitmask - } - } + #[inline] + pub fn to_int(self) -> Simd { + self.0 + } - #[inline] - pub fn from_bitmask(mut bitmask: [u8; crate::LaneCount::::BITMASK_LEN]) -> Self { - unsafe { - // There is a bug where LLVM appears to implement this operation with the wrong - // bit order. - // TODO fix this in a better way - if cfg!(any(target_arch = "mips", target_arch = "mips64")) { - for x in bitmask.as_mut() { - *x = x.reverse_bits(); - } - } + #[inline] + pub unsafe fn from_int_unchecked(value: Simd) -> Self { + Self(value) + } - // TODO remove the transmute when rustc can use arrays of u8 as bitmasks - assert_eq!( - core::mem::size_of::< as crate::SupportedLaneCount>::IntBitMask>(), - crate::LaneCount::::BITMASK_LEN, - ); - let bitmask: as crate::SupportedLaneCount>::IntBitMask = core::mem::transmute_copy(&bitmask); + #[inline] + pub fn convert(self) -> Mask + where + T: MaskElement, + { + unsafe { Mask(crate::intrinsics::simd_cast(self.0)) } + } - Self::from_int_unchecked(crate::intrinsics::simd_select_bitmask( - bitmask, - Self::splat(true).to_int(), - Self::splat(false).to_int(), - )) + #[inline] + pub fn to_bitmask(self) -> [u8; LaneCount::::BITMASK_LEN] { + unsafe { + // TODO remove the transmute when rustc can use arrays of u8 as bitmasks + assert_eq!( + core::mem::size_of::< as SupportedLaneCount>::IntBitMask>(), + LaneCount::::BITMASK_LEN, + ); + let bitmask: as SupportedLaneCount>::IntBitMask = + crate::intrinsics::simd_bitmask(self.0); + let mut bitmask: [u8; LaneCount::::BITMASK_LEN] = + core::mem::transmute_copy(&bitmask); + + // There is a bug where LLVM appears to implement this operation with the wrong + // bit order. + // TODO fix this in a better way + if cfg!(any(target_arch = "mips", target_arch = "mips64")) { + for x in bitmask.as_mut() { + *x = x.reverse_bits(); } } - } - impl core::convert::From<$name> for crate::$type - where - crate::LaneCount: crate::SupportedLaneCount, - { - fn from(value: $name) -> Self { - value.0 - } - } - - impl core::ops::BitAnd for $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - type Output = Self; - #[inline] - fn bitand(self, rhs: Self) -> Self { - Self(self.0 & rhs.0) - } + bitmask } + } - impl core::ops::BitOr for $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - type Output = Self; - #[inline] - fn bitor(self, rhs: Self) -> Self { - Self(self.0 | rhs.0) + #[inline] + pub fn from_bitmask(mut bitmask: [u8; LaneCount::::BITMASK_LEN]) -> Self { + unsafe { + // There is a bug where LLVM appears to implement this operation with the wrong + // bit order. + // TODO fix this in a better way + if cfg!(any(target_arch = "mips", target_arch = "mips64")) { + for x in bitmask.as_mut() { + *x = x.reverse_bits(); + } } - } - impl core::ops::BitXor for $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - type Output = Self; - #[inline] - fn bitxor(self, rhs: Self) -> Self::Output { - Self(self.0 ^ rhs.0) - } - } + // TODO remove the transmute when rustc can use arrays of u8 as bitmasks + assert_eq!( + core::mem::size_of::< as SupportedLaneCount>::IntBitMask>(), + LaneCount::::BITMASK_LEN, + ); + let bitmask: as SupportedLaneCount>::IntBitMask = + core::mem::transmute_copy(&bitmask); - impl core::ops::Not for $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - type Output = Self; - #[inline] - fn not(self) -> Self::Output { - Self(!self.0) - } + Self::from_int_unchecked(crate::intrinsics::simd_select_bitmask( + bitmask, + Self::splat(true).to_int(), + Self::splat(false).to_int(), + )) } } } -define_mask! { - /// A mask equivalent to [SimdI8](crate::SimdI8), where all bits in the lane must be either set - /// or unset. - struct Mask8(crate::SimdI8); -} - -define_mask! { - /// A mask equivalent to [SimdI16](crate::SimdI16), where all bits in the lane must be either set - /// or unset. - struct Mask16(crate::SimdI16); +impl core::convert::From> for Simd +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + fn from(value: Mask) -> Self { + value.0 + } } -define_mask! { - /// A mask equivalent to [SimdI32](crate::SimdI32), where all bits in the lane must be either set - /// or unset. - struct Mask32(crate::SimdI32); +impl core::ops::BitAnd for Mask +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Self; + #[inline] + fn bitand(self, rhs: Self) -> Self { + unsafe { Self(crate::intrinsics::simd_and(self.0, rhs.0)) } + } } -define_mask! { - /// A mask equivalent to [SimdI64](crate::SimdI64), where all bits in the lane must be either set - /// or unset. - struct Mask64(crate::SimdI64); +impl core::ops::BitOr for Mask +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Self; + #[inline] + fn bitor(self, rhs: Self) -> Self { + unsafe { Self(crate::intrinsics::simd_or(self.0, rhs.0)) } + } } -define_mask! { - /// A mask equivalent to [SimdIsize](crate::SimdIsize), where all bits in the lane must be either set - /// or unset. - struct MaskSize(crate::SimdIsize); +impl core::ops::BitXor for Mask +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Self; + #[inline] + fn bitxor(self, rhs: Self) -> Self { + unsafe { Self(crate::intrinsics::simd_xor(self.0, rhs.0)) } + } } -macro_rules! impl_from { - { $from:ident ($from_inner:ident) => $($to:ident ($to_inner:ident)),* } => { - $( - impl From<$from> for $to - where - crate::LaneCount: crate::SupportedLaneCount, - { - fn from(value: $from) -> Self { - let mut new = Self::splat(false); - for i in 0..LANES { - unsafe { new.set_unchecked(i, value.test_unchecked(i)) } - } - new - } - } - )* +impl core::ops::Not for Mask +where + Element: MaskElement, + LaneCount: SupportedLaneCount, +{ + type Output = Self; + #[inline] + fn not(self) -> Self::Output { + Self::splat(true) ^ self } } -impl_from! { Mask8 (SimdI8) => Mask16 (SimdI16), Mask32 (SimdI32), Mask64 (SimdI64), MaskSize (SimdIsize) } -impl_from! { Mask16 (SimdI16) => Mask32 (SimdI32), Mask64 (SimdI64), MaskSize (SimdIsize), Mask8 (SimdI8) } -impl_from! { Mask32 (SimdI32) => Mask64 (SimdI64), MaskSize (SimdIsize), Mask8 (SimdI8), Mask16 (SimdI16) } -impl_from! { Mask64 (SimdI64) => MaskSize (SimdIsize), Mask8 (SimdI8), Mask16 (SimdI16), Mask32 (SimdI32) } -impl_from! { MaskSize (SimdIsize) => Mask8 (SimdI8), Mask16 (SimdI16), Mask32 (SimdI32), Mask64 (SimdI64) } + +impl_full_mask_reductions! {} diff --git a/crates/core_simd/src/ops.rs b/crates/core_simd/src/ops.rs index c75090aab9c5048f5d74f28040f188f1a7906560..67bafd73b144a2b0078d0469c0f7d51c36d8da27 100644 --- a/crates/core_simd/src/ops.rs +++ b/crates/core_simd/src/ops.rs @@ -1,4 +1,27 @@ -use crate::{LaneCount, SupportedLaneCount}; +use crate::{LaneCount, Simd, SimdElement, SupportedLaneCount}; + +impl core::ops::Index for Simd +where + Element: SimdElement, + LaneCount: SupportedLaneCount, + I: core::slice::SliceIndex<[Element]>, +{ + type Output = I::Output; + fn index(&self, index: I) -> &Self::Output { + &self.as_array()[index] + } +} + +impl core::ops::IndexMut for Simd +where + Element: SimdElement, + LaneCount: SupportedLaneCount, + I: core::slice::SliceIndex<[Element]>, +{ + fn index_mut(&mut self, index: I) -> &mut Self::Output { + &mut self.as_mut_array()[index] + } +} /// Checks if the right-hand side argument of a left- or right-shift would cause overflow. fn invalid_shift_rhs(rhs: T) -> bool @@ -191,31 +214,6 @@ fn neg(self) -> Self::Output { } }; - { impl Index for $type:ident, $scalar:ty } => { - impl core::ops::Index for crate::$type - where - LaneCount: SupportedLaneCount, - I: core::slice::SliceIndex<[$scalar]>, - { - type Output = I::Output; - fn index(&self, index: I) -> &Self::Output { - let slice: &[_] = self.as_ref(); - &slice[index] - } - } - - impl core::ops::IndexMut for crate::$type - where - LaneCount: SupportedLaneCount, - I: core::slice::SliceIndex<[$scalar]>, - { - fn index_mut(&mut self, index: I) -> &mut Self::Output { - let slice: &mut [_] = self.as_mut(); - &mut slice[index] - } - } - }; - // generic binary op with assignment when output is `Self` { @binary $type:ident, $scalar:ty, $trait:ident :: $trait_fn:ident, $assign_trait:ident :: $assign_trait_fn:ident, $intrinsic:ident } => { impl_ref_ops! { @@ -301,7 +299,6 @@ fn $assign_trait_fn(&mut self, rhs: $scalar) { impl_op! { impl Div for $vector, $scalar } impl_op! { impl Rem for $vector, $scalar } impl_op! { impl Neg for $vector, $scalar } - impl_op! { impl Index for $vector, $scalar } )* )* }; @@ -319,7 +316,6 @@ fn $assign_trait_fn(&mut self, rhs: $scalar) { impl_op! { impl BitOr for $vector, $scalar } impl_op! { impl BitXor for $vector, $scalar } impl_op! { impl Not for $vector, $scalar } - impl_op! { impl Index for $vector, $scalar } // Integers panic on divide by 0 impl_ref_ops! { diff --git a/crates/core_simd/src/reduction.rs b/crates/core_simd/src/reduction.rs index df227d09e3420860a1a5d1ecc76b8142ce7a105a..b9c24d027b62db199e070cf47b77b508a1812b6e 100644 --- a/crates/core_simd/src/reduction.rs +++ b/crates/core_simd/src/reduction.rs @@ -103,10 +103,11 @@ pub fn horizontal_min(self) -> $scalar { } macro_rules! impl_full_mask_reductions { - { $name:ident, $bits_ty:ident } => { - impl $name + {} => { + impl Mask where - crate::LaneCount: crate::SupportedLaneCount, + Element: MaskElement, + LaneCount: SupportedLaneCount, { #[inline] pub fn any(self) -> bool { @@ -120,24 +121,3 @@ pub fn all(self) -> bool { } } } - -macro_rules! impl_opaque_mask_reductions { - { $name:ident, $bits_ty:ident } => { - impl $name - where - crate::LaneCount: crate::SupportedLaneCount, - { - /// Returns true if any lane is set, or false otherwise. - #[inline] - pub fn any(self) -> bool { - self.0.any() - } - - /// Returns true if all lanes are set, or false otherwise. - #[inline] - pub fn all(self) -> bool { - self.0.all() - } - } - } -}