diff --git a/include/xsimd/arch/common/xsimd_common_memory.hpp b/include/xsimd/arch/common/xsimd_common_memory.hpp index 046faafad..a8dae1575 100644 --- a/include/xsimd/arch/common/xsimd_common_memory.hpp +++ b/include/xsimd/arch/common/xsimd_common_memory.hpp @@ -13,12 +13,14 @@ #define XSIMD_COMMON_MEMORY_HPP #include "../../types/xsimd_batch_constant.hpp" +#include "../../utils/bits.hpp" #include "../../utils/xsimd_type_traits.hpp" #include "./xsimd_common_details.hpp" #include #include #include +#include namespace xsimd { @@ -368,23 +370,13 @@ namespace xsimd // tag, no `enable_if` on `A`; // (b) base overloads use the `requires_arch` tag only; a generic // `requires_arch` here ties with an arch's own overload (gcc-10 ambiguity); - // (c) capability decisions go through arch-agnostic traits (see below). + // (c) preserve the memory element type through dispatch. namespace detail { - // True when an integer access can borrow the same-width float `vmaskmov*` path - // (integral type, same-size float exists, arch has that float register); - // otherwise the scalar-buffer fallback is used. Names no architecture. - template - using masked_memory_uses_fp_bitcast = std::integral_constant - && std::is_integral_v - && !std::is_void_v> - && types::has_simd_register_v, A>>; - // Scalar-buffer fallback: materialize masked-off lanes as zero, then load. template XSIMD_INLINE batch - load_masked_common(T_in const* mem, batch_bool_constant, convert, alignment, std::false_type /* uses_fp_bitcast */) noexcept + load_masked_common(T_in const* mem, batch_bool_constant, convert, alignment) noexcept { constexpr std::size_t size = batch::size; alignas(A::alignment()) std::array buffer {}; @@ -396,20 +388,9 @@ namespace xsimd return batch::load(buffer.data(), aligned_mode {}); } - // Integer-via-float path: reinterpret to the same-width float type, reuse the - // floating-point masked load (e.g. `vmaskmovps`), then bitcast the result back. - template - XSIMD_INLINE batch - load_masked_common(T const* mem, batch_bool_constant, convert, Mode, std::true_type /* uses_fp_bitcast */) noexcept - { - using fp_t = sized_fp_t; - const auto f = ::xsimd::kernel::load_masked(reinterpret_cast(mem), batch_bool_constant {}, convert {}, Mode {}, A {}); - return bitwise_cast(f); - } - template XSIMD_INLINE void - store_masked_common(T_out* mem, batch const& src, batch_bool_constant, alignment, std::false_type /* uses_fp_bitcast */) noexcept + store_masked_common(T_out* mem, batch const& src, batch_bool_constant, alignment) noexcept { constexpr std::size_t size = batch::size; constexpr bool mask[size] = { Values... }; @@ -420,21 +401,13 @@ namespace xsimd mem[i] = static_cast(src.get(i)); } } - - template - XSIMD_INLINE void - store_masked_common(T* mem, batch const& src, batch_bool_constant, Mode, std::true_type /* uses_fp_bitcast */) noexcept - { - using fp_t = sized_fp_t; - ::xsimd::kernel::store_masked(reinterpret_cast(mem), bitwise_cast(src), batch_bool_constant {}, Mode {}, A {}); - } } template XSIMD_INLINE batch load_masked(T_in const* mem, batch_bool_constant mask, convert cvt, alignment mode, requires_arch) noexcept { - return detail::load_masked_common(mem, mask, cvt, mode, detail::masked_memory_uses_fp_bitcast {}); + return detail::load_masked_common(mem, mask, cvt, mode); } template @@ -450,11 +423,36 @@ namespace xsimd return batch::load_aligned(buffer.data()); } + template + XSIMD_INLINE batch, A> + load_complex_masked(std::complex const* mem, batch_bool mask, Mode, requires_arch) noexcept + { + // Scalar fallback: only active lanes are touched. Arches with + // hardware predicated loads should override this. + constexpr std::size_t size = batch::size; + alignas(A::alignment()) std::array, size> buffer; + for (std::size_t i = 0; i < size; ++i) + buffer[i] = mask.get(i) ? mem[i] : std::complex(0); + return batch, A>::load_aligned(buffer.data()); + } + + template + XSIMD_INLINE batch, A> + load_complex_masked(std::complex const* mem, batch_bool_constant, Mode, requires_arch) noexcept + { + constexpr std::size_t size = batch::size; + alignas(A::alignment()) std::array, size> buffer; + constexpr bool mask[size] = { Values... }; + for (std::size_t i = 0; i < size; ++i) + buffer[i] = mask[i] ? mem[i] : std::complex(0); + return batch, A>::load_aligned(buffer.data()); + } + template XSIMD_INLINE void store_masked(T_out* mem, batch const& src, batch_bool_constant mask, alignment mode, requires_arch) noexcept { - detail::store_masked_common(mem, src, mask, mode, detail::masked_memory_uses_fp_bitcast {}); + detail::store_masked_common(mem, src, mask, mode); } template @@ -865,6 +863,116 @@ namespace xsimd store_complex_aligned(dst, src, A {}); } + namespace detail + { + template + XSIMD_INLINE batch, A> + load_complex_masked_chunks(std::complex const* mem, std::array, 2> masks, Mode mode) noexcept + { + using real_batch = batch; + constexpr std::size_t half = batch::size / 2; + const real_batch c0 = real_batch::load(reinterpret_cast(mem), masks[0], mode); + const real_batch c1 = real_batch::load(reinterpret_cast(mem + half), masks[1], mode); + return load_complex(bitwise_cast(c0), bitwise_cast(c1), A {}); + } + + template + XSIMD_INLINE void + store_complex_masked_chunks(std::complex* mem, batch, A> const& src, std::array, 2> masks, Mode mode) noexcept + { + constexpr std::size_t half = batch::size / 2; + bitwise_cast(complex_low(src, A {})).store(reinterpret_cast(mem), masks[0], mode); + bitwise_cast(complex_high(src, A {})).store(reinterpret_cast(mem + half), masks[1], mode); + } + + template + XSIMD_INLINE std::array, 2> zip_complex_mask_vector(batch_bool mask) noexcept + { + const batch bits(mask.to_native()); + return { batch_bool(zip_lo(bits, bits).to_native()), + batch_bool(zip_hi(bits, bits).to_native()) }; + } + + // Duplicate input bit k into output bits 2k and 2k+1. + inline constexpr std::array duplicate_bits_lut = { + 0x00, 0x03, 0x0c, 0x0f, 0x30, 0x33, 0x3c, 0x3f, + 0xc0, 0xc3, 0xcc, 0xcf, 0xf0, 0xf3, 0xfc, 0xff + }; + + // Masks for the two chunks, viewed as 64-bit lanes: one lane holds a whole + // complex, two lanes hold a complex. + template + XSIMD_INLINE std::array, 2> zip_complex_mask_kmask(batch_bool mask) noexcept + { + constexpr std::size_t half = batch::size / 2; + constexpr std::uint64_t low_bits = ::xsimd::utils::make_low_mask(static_cast(half)); + const std::uint64_t bits = mask.mask(); + std::uint64_t lo = bits & low_bits; + std::uint64_t hi = (bits >> half) & low_bits; + if constexpr (std::is_same_v) + { + lo = duplicate_bits_lut[lo]; + hi = duplicate_bits_lut[hi]; + } + return { batch_bool::from_mask(lo), batch_bool::from_mask(hi) }; + } + + // Real lane i of a chunk belongs to complex lane First + i / 2. + template + struct complex_chunk_mask + { + static constexpr bool get(std::size_t i, std::size_t) noexcept + { + return std::array { { Values... } }[First + i / 2]; + } + }; + + template + XSIMD_INLINE batch, A> + load_complex_masked_chunks(std::complex const* mem, batch_bool_constant, Mode mode) noexcept + { + using real_batch = batch; + constexpr std::size_t half = real_batch::size / 2; + const real_batch c0 = real_batch::load(reinterpret_cast(mem), make_batch_bool_constant, A>(), mode); + const real_batch c1 = real_batch::load(reinterpret_cast(mem + half), make_batch_bool_constant, A>(), mode); + return load_complex(c0, c1, A {}); + } + + template + XSIMD_INLINE void + store_complex_masked_chunks(std::complex* mem, batch, A> const& src, batch_bool_constant, Mode mode) noexcept + { + constexpr std::size_t half = batch::size / 2; + complex_low(src, A {}).store(reinterpret_cast(mem), make_batch_bool_constant, A>(), mode); + complex_high(src, A {}).store(reinterpret_cast(mem + half), make_batch_bool_constant, A>(), mode); + } + } + + template + XSIMD_INLINE void + store_complex_masked(std::complex* mem, batch, A> const& src, batch_bool mask, Mode, requires_arch) noexcept + { + constexpr std::size_t size = batch::size; + alignas(A::alignment()) std::array, size> buffer; + src.store_aligned(buffer.data()); + for (std::size_t i = 0; i < size; ++i) + if (mask.get(i)) + mem[i] = buffer[i]; + } + + template + XSIMD_INLINE void + store_complex_masked(std::complex* mem, batch, A> const& src, batch_bool_constant, Mode, requires_arch) noexcept + { + constexpr std::size_t size = batch::size; + alignas(A::alignment()) std::array, size> buffer; + constexpr bool mask[size] = { Values... }; + src.store_aligned(buffer.data()); + for (std::size_t i = 0; i < size; ++i) + if (mask[i]) + mem[i] = buffer[i]; + } + // transpose template XSIMD_INLINE void transpose(batch* matrix_begin, batch* matrix_end, requires_arch) noexcept diff --git a/include/xsimd/arch/xsimd_avx.hpp b/include/xsimd/arch/xsimd_avx.hpp index f06a603e4..9b9dfd2f7 100644 --- a/include/xsimd/arch/xsimd_avx.hpp +++ b/include/xsimd/arch/xsimd_avx.hpp @@ -959,18 +959,6 @@ namespace xsimd // AVX helpers to avoid type-based branching in the generic load_masked namespace detail { - template - XSIMD_INLINE batch maskload(float const* mem, batch, A> const& mask) noexcept - { - return _mm256_maskload_ps(mem, mask); - } - - template - XSIMD_INLINE batch maskload(double const* mem, batch, A> const& mask) noexcept - { - return _mm256_maskload_pd(mem, mask); - } - template XSIMD_INLINE batch zero_extend(batch const& hi) noexcept { @@ -1008,6 +996,18 @@ namespace xsimd { return _mm256_zextpd128_pd256(lo); } + + template >> + XSIMD_INLINE batch zero_extend(batch const& hi) noexcept + { + return ::xsimd::bitwise_cast(zero_extend(::xsimd::bitwise_cast(hi))); + } + + template >> + XSIMD_INLINE batch zero_extend_lo(batch const& lo) noexcept + { + return ::xsimd::bitwise_cast(zero_extend_lo(::xsimd::bitwise_cast(lo))); + } } // Runtime-mask load (float/double). @@ -1025,6 +1025,20 @@ namespace xsimd return _mm256_maskload_pd(mem, _mm256_castpd_si256(mask)); } + template + XSIMD_INLINE batch, A> + load_complex_masked(std::complex const* mem, batch_bool mask, Mode mode, requires_arch) noexcept + { + return detail::load_complex_masked_chunks(mem, detail::zip_complex_mask_vector(mask), mode); + } + + template + XSIMD_INLINE batch, A> + load_complex_masked(std::complex const* mem, batch_bool_constant mask, Mode mode, requires_arch) noexcept + { + return detail::load_complex_masked_chunks(mem, mask, mode); + } + // 4/8-byte ints: bitcast to same-width float, reuse the vmaskmov path. template XSIMD_INLINE std::enable_if_t && (sizeof(T) == 4 || sizeof(T) == 8), batch> @@ -1040,8 +1054,8 @@ namespace xsimd } } - // load_masked (single overload for float/double) - template >> + // load_masked (shared 4/8-byte arithmetic overload) + template && (sizeof(T) == 4 || sizeof(T) == 8)>> XSIMD_INLINE batch load_masked(T const* mem, batch_bool_constant mask, convert, Mode, requires_arch) noexcept { constexpr size_t half_size = batch::size / 2; @@ -1090,7 +1104,7 @@ namespace xsimd else { // crossing 128-bit boundary → use 256-bit masked load - return detail::maskload(mem, mask.as_batch()); + return load_masked(mem, mask.as_batch_bool(), convert {}, Mode {}, avx {}); } } @@ -1116,8 +1130,39 @@ namespace xsimd } } + // Runtime-mask store (float/double). + template + XSIMD_INLINE void + store_masked(float* mem, batch const& src, batch_bool mask, Mode, requires_arch) noexcept + { + detail::maskstore(mem, mask, src); + } + + template + XSIMD_INLINE void + store_masked(double* mem, batch const& src, batch_bool mask, Mode, requires_arch) noexcept + { + detail::maskstore(mem, mask, src); + } + + // 4/8-byte ints: bitcast to same-width float, reuse the vmaskmov path. + template + XSIMD_INLINE std::enable_if_t && (sizeof(T) == 4 || sizeof(T) == 8), void> + store_masked(T* mem, batch const& src, batch_bool mask, Mode, requires_arch) noexcept + { + if constexpr (sizeof(T) == 4) + { + _mm256_maskstore_ps(reinterpret_cast(mem), __m256i(mask), bitwise_cast(src)); + } + else + { + _mm256_maskstore_pd(reinterpret_cast(mem), __m256i(mask), bitwise_cast(src)); + } + } + + // Runtime stores must be visible before this overload delegates to them. template && detail::uses_vector_mask_v>> + typename = std::enable_if_t && (sizeof(T) == 4 || sizeof(T) == 8) && detail::uses_vector_mask_v>> XSIMD_INLINE void store_masked(T* mem, batch const& src, batch_bool_constant mask, Mode, requires_arch) noexcept { constexpr size_t half_size = batch::size / 2; @@ -1167,60 +1212,39 @@ namespace xsimd } else { - detail::maskstore(mem, mask.as_batch_bool(), src); + store_masked(mem, src, mask.as_batch_bool(), Mode {}, avx {}); } } - // Runtime-mask store (float/double). - template + template XSIMD_INLINE void - store_masked(float* mem, batch const& src, batch_bool mask, Mode, requires_arch) noexcept + store_complex_masked(std::complex* mem, batch, A> const& src, batch_bool mask, Mode mode, requires_arch) noexcept { - detail::maskstore(mem, mask, src); + detail::store_complex_masked_chunks(mem, src, detail::zip_complex_mask_vector(mask), mode); } - template + template XSIMD_INLINE void - store_masked(double* mem, batch const& src, batch_bool mask, Mode, requires_arch) noexcept - { - detail::maskstore(mem, mask, src); - } - - // 4/8-byte ints: bitcast to same-width float, reuse the vmaskmov path. - template - XSIMD_INLINE std::enable_if_t && (sizeof(T) == 4 || sizeof(T) == 8), void> - store_masked(T* mem, batch const& src, batch_bool mask, Mode, requires_arch) noexcept + store_complex_masked(std::complex* mem, batch, A> const& src, batch_bool_constant mask, Mode mode, requires_arch) noexcept { - if constexpr (sizeof(T) == 4) - { - _mm256_maskstore_ps(reinterpret_cast(mem), __m256i(mask), bitwise_cast(src)); - } - else - { - _mm256_maskstore_pd(reinterpret_cast(mem), __m256i(mask), bitwise_cast(src)); - } + detail::store_complex_masked_chunks(mem, src, mask, mode); } namespace detail { - // Reinterpret a constant-mask 4/8-byte load/store as same-width float - // and run DstArch's kernel, which lowers prefix/suffix shapes to plain - // moves. Shared by the int/EVEX 128- and 256-bit archs. + // Delegate constant-mask 4/8-byte loads/stores without changing + // the memory element type. Shared by int/EVEX 128- and 256-bit archs. template XSIMD_INLINE batch plain_move_load(T const* mem, batch_bool_constant, convert, Mode) noexcept { static_assert(sizeof(T) == 4 || sizeof(T) == 8, "plain-move delegation only supports 4/8-byte lanes"); - using F = std::conditional_t; - // same-width float has the same lane count, so the V... mask pack and the - // memory reinterpret line up one-to-one; wrong here would load the wrong lanes - static_assert(batch::size == batch::size, "same-width float must preserve lane count"); static_assert(sizeof...(V) == batch::size, "mask pack width must match the batch"); // the plain-move path emits aligned moves in aligned/stream mode, which fault // on a misaligned pointer (the old vmaskmov tolerated it) assert((std::is_same_v || ::xsimd::is_aligned(mem)) && "aligned/stream masked load needs an aligned pointer"); // qualify: an unqualified call resolves to detail::load_masked (a different // helper) under MSVC's two-phase lookup; we want the kernel-level overload - return bitwise_cast(batch(::xsimd::kernel::load_masked(reinterpret_cast(mem), batch_bool_constant {}, convert {}, Mode {}, DstArch {}))); + return ::xsimd::kernel::load_masked(mem, batch_bool_constant {}, convert {}, Mode {}, DstArch {}); } // Re-tag to DstArch so the vector-mask store kernel is used (the AVX @@ -1229,14 +1253,11 @@ namespace xsimd XSIMD_INLINE void plain_move_store(T* mem, batch const& src, batch_bool_constant, Mode) noexcept { static_assert(sizeof(T) == 4 || sizeof(T) == 8, "plain-move delegation only supports 4/8-byte lanes"); - using F = std::conditional_t; - static_assert(batch::size == batch::size, "same-width float must preserve lane count"); static_assert(sizeof...(V) == batch::size, "mask pack width must match the batch"); assert((std::is_same_v || ::xsimd::is_aligned(mem)) && "aligned/stream masked store needs an aligned pointer"); - const auto fsrc = bitwise_cast(src); // qualify: an unqualified call resolves to detail::store_masked (a different // helper) under MSVC's two-phase lookup; we want the kernel-level overload - ::xsimd::kernel::store_masked(reinterpret_cast(mem), batch(fsrc.data), batch_bool_constant {}, Mode {}, DstArch {}); + ::xsimd::kernel::store_masked(mem, batch(src.data), batch_bool_constant {}, Mode {}, DstArch {}); } } diff --git a/include/xsimd/arch/xsimd_avx512f.hpp b/include/xsimd/arch/xsimd_avx512f.hpp index 06da6218e..509813658 100644 --- a/include/xsimd/arch/xsimd_avx512f.hpp +++ b/include/xsimd/arch/xsimd_avx512f.hpp @@ -1003,6 +1003,21 @@ namespace xsimd return _mm512_permutex2var_pd(self.real(), idx, self.imag()); } } + + template + XSIMD_INLINE void + store_complex_masked(std::complex* mem, batch, A> const& src, batch_bool mask, Mode mode, requires_arch) noexcept + { + detail::store_complex_masked_chunks(mem, src, detail::zip_complex_mask_kmask(mask), mode); + } + + template + XSIMD_INLINE void + store_complex_masked(std::complex* mem, batch, A> const& src, batch_bool_constant mask, Mode mode, requires_arch) noexcept + { + detail::store_complex_masked_chunks(mem, src, mask, mode); + } + // incr_if template >> XSIMD_INLINE batch decr_if(batch const& self, batch_bool const& mask, requires_arch) noexcept @@ -1634,6 +1649,20 @@ namespace xsimd } } + template + XSIMD_INLINE batch, A> + load_complex_masked(std::complex const* mem, batch_bool mask, Mode mode, requires_arch) noexcept + { + return detail::load_complex_masked_chunks(mem, detail::zip_complex_mask_kmask(mask), mode); + } + + template + XSIMD_INLINE batch, A> + load_complex_masked(std::complex const* mem, batch_bool_constant mask, Mode mode, requires_arch) noexcept + { + return detail::load_complex_masked_chunks(mem, mask, mode); + } + // load_unaligned template >> XSIMD_INLINE batch load_unaligned(T const* mem, convert, requires_arch) noexcept diff --git a/include/xsimd/arch/xsimd_avx512vl_128.hpp b/include/xsimd/arch/xsimd_avx512vl_128.hpp index 3058bad45..5c2903941 100644 --- a/include/xsimd/arch/xsimd_avx512vl_128.hpp +++ b/include/xsimd/arch/xsimd_avx512vl_128.hpp @@ -17,6 +17,7 @@ #include "../types/xsimd_batch_constant.hpp" #include "../utils/bits.hpp" +#include #include namespace xsimd @@ -30,7 +31,7 @@ namespace xsimd // Defined in xsimd_avx512f.hpp. This header is included before it so // that avx512f.hpp's masked load/store forwarder can resolve the // avx512vl_128 overloads below by ordinary lookup; forward-declare - // the two helpers it borrows from there. + // the helpers it borrows from there. XSIMD_INLINE uint32_t morton(uint16_t x, uint16_t y) noexcept; template XSIMD_INLINE unsigned char tobitset(unsigned char unpacked[N]); @@ -349,6 +350,20 @@ namespace xsimd detail::maskstore128(mem, src, mask.mask(), Mode {}); } + template + XSIMD_INLINE batch, A> + load_complex_masked(std::complex const* mem, batch_bool mask, Mode mode, requires_arch) noexcept + { + return detail::load_complex_masked_chunks(mem, detail::zip_complex_mask_kmask(mask), mode); + } + + template + XSIMD_INLINE void + store_complex_masked(std::complex* mem, batch, A> const& src, batch_bool mask, Mode mode, requires_arch) noexcept + { + detail::store_complex_masked_chunks(mem, src, detail::zip_complex_mask_kmask(mask), mode); + } + // gather template = 0, detail::enable_sized_integral_t = 0> XSIMD_INLINE batch gather(batch const&, T const* src, batch const& index, diff --git a/include/xsimd/arch/xsimd_avx512vl_256.hpp b/include/xsimd/arch/xsimd_avx512vl_256.hpp index 2bfefcb13..b24652a38 100644 --- a/include/xsimd/arch/xsimd_avx512vl_256.hpp +++ b/include/xsimd/arch/xsimd_avx512vl_256.hpp @@ -17,6 +17,7 @@ #include "../types/xsimd_batch_constant.hpp" #include "../utils/bits.hpp" +#include #include namespace xsimd @@ -30,7 +31,7 @@ namespace xsimd // Defined in xsimd_avx512f.hpp. This header is included before it so // that avx512f.hpp's masked load/store forwarder can resolve the // avx512vl_256 overloads below by ordinary lookup; forward-declare - // the two helpers it borrows from there. + // the helpers it borrows from there. XSIMD_INLINE uint32_t morton(uint16_t x, uint16_t y) noexcept; template XSIMD_INLINE unsigned char tobitset(unsigned char unpacked[N]); @@ -332,6 +333,13 @@ namespace xsimd return detail::maskload256(mem, mask.mask(), Mode {}); } + template + XSIMD_INLINE batch, A> + load_complex_masked(std::complex const* mem, batch_bool mask, Mode mode, requires_arch) noexcept + { + return detail::load_complex_masked_chunks(mem, detail::zip_complex_mask_kmask(mask), mode); + } + template && (sizeof(T) == 4 || sizeof(T) == 8)>> XSIMD_INLINE void store_masked(T* mem, batch const& src, batch_bool_constant mask, Mode, requires_arch) noexcept @@ -357,6 +365,13 @@ namespace xsimd detail::maskstore256(mem, src, mask.mask(), Mode {}); } + template + XSIMD_INLINE void + store_complex_masked(std::complex* mem, batch, A> const& src, batch_bool mask, Mode mode, requires_arch) noexcept + { + detail::store_complex_masked_chunks(mem, src, detail::zip_complex_mask_kmask(mask), mode); + } + // gather template = 0, detail::enable_sized_integral_t = 0> XSIMD_INLINE batch gather(batch const&, T const* src, batch const& index, diff --git a/include/xsimd/arch/xsimd_avx_128.hpp b/include/xsimd/arch/xsimd_avx_128.hpp index 346f7259b..69b810ee5 100644 --- a/include/xsimd/arch/xsimd_avx_128.hpp +++ b/include/xsimd/arch/xsimd_avx_128.hpp @@ -105,7 +105,7 @@ namespace xsimd // Masks that lower to plain moves go to sse2; the rest gain nothing on a // single register, so take the runtime path. - template >> + template && (sizeof(T) == 4 || sizeof(T) == 8)>> XSIMD_INLINE batch load_masked(T const* mem, batch_bool_constant mask, convert, Mode, requires_arch) noexcept { if constexpr (detail::lowers_to_plain_moves(mask)) @@ -132,6 +132,13 @@ namespace xsimd return _mm_maskload_pd(mem, _mm_castpd_si128(mask)); } + template + XSIMD_INLINE batch, A> + load_complex_masked(std::complex const* mem, batch_bool mask, Mode mode, requires_arch) noexcept + { + return detail::load_complex_masked_chunks(mem, detail::zip_complex_mask_vector(mask), mode); + } + // 4/8-byte ints: bitcast to same-width float, reuse the vmaskmov path. template XSIMD_INLINE std::enable_if_t && (sizeof(T) == 4 || sizeof(T) == 8), batch> @@ -147,19 +154,6 @@ namespace xsimd } } - template >> - XSIMD_INLINE void store_masked(T* mem, batch const& src, batch_bool_constant mask, Mode, requires_arch) noexcept - { - if constexpr (detail::lowers_to_plain_moves(mask)) - { - store_masked(mem, src, mask, Mode {}, sse2 {}); - } - else - { - store_masked(mem, src, mask.as_batch_bool(), Mode {}, avx_128 {}); - } - } - // Runtime-mask store (float/double). template XSIMD_INLINE void @@ -174,6 +168,13 @@ namespace xsimd _mm_maskstore_pd(mem, _mm_castpd_si128(mask), src); } + template + XSIMD_INLINE void + store_complex_masked(std::complex* mem, batch, A> const& src, batch_bool mask, Mode mode, requires_arch) noexcept + { + detail::store_complex_masked_chunks(mem, src, detail::zip_complex_mask_vector(mask), mode); + } + // 4/8-byte ints: bitcast to same-width float, reuse the vmaskmov path. template XSIMD_INLINE std::enable_if_t && (sizeof(T) == 4 || sizeof(T) == 8), void> @@ -189,6 +190,19 @@ namespace xsimd } } + template && (sizeof(T) == 4 || sizeof(T) == 8)>> + XSIMD_INLINE void store_masked(T* mem, batch const& src, batch_bool_constant mask, Mode, requires_arch) noexcept + { + if constexpr (detail::lowers_to_plain_moves(mask)) + { + store_masked(mem, src, mask, Mode {}, sse2 {}); + } + else + { + store_masked(mem, src, mask.as_batch_bool(), Mode {}, avx_128 {}); + } + } + // swizzle (dynamic mask) template XSIMD_INLINE batch swizzle(batch const& self, batch mask, requires_arch) noexcept diff --git a/include/xsimd/arch/xsimd_common_fwd.hpp b/include/xsimd/arch/xsimd_common_fwd.hpp index 139b0b282..b3567d8e6 100644 --- a/include/xsimd/arch/xsimd_common_fwd.hpp +++ b/include/xsimd/arch/xsimd_common_fwd.hpp @@ -16,6 +16,8 @@ #include "../config/xsimd_macros.hpp" #include "../types/xsimd_common_arch.hpp" +#include +#include #include #include #include @@ -105,6 +107,20 @@ namespace xsimd // Forward declarations for pack-level helpers namespace detail { + template + XSIMD_INLINE std::array, 2> zip_complex_mask_vector(batch_bool mask) noexcept; + template + XSIMD_INLINE std::array, 2> zip_complex_mask_kmask(batch_bool mask) noexcept; + template + XSIMD_INLINE batch, A> load_complex_masked_chunks(std::complex const* mem, std::array, 2> masks, Mode mode) noexcept; + template + XSIMD_INLINE void store_complex_masked_chunks(std::complex* mem, batch, A> const& src, std::array, 2> masks, Mode mode) noexcept; + + template + XSIMD_INLINE batch, A> load_complex_masked_chunks(std::complex const* mem, batch_bool_constant, Mode) noexcept; + template + XSIMD_INLINE void store_complex_masked_chunks(std::complex* mem, batch, A> const& src, batch_bool_constant, Mode) noexcept; + template XSIMD_INLINE void reassociation_barrier(T& x, const char*) noexcept; template diff --git a/include/xsimd/arch/xsimd_sse2.hpp b/include/xsimd/arch/xsimd_sse2.hpp index 26e858d9c..6d11bf67b 100644 --- a/include/xsimd/arch/xsimd_sse2.hpp +++ b/include/xsimd/arch/xsimd_sse2.hpp @@ -17,6 +17,7 @@ #include "./utils/shifts.hpp" #include +#include #include #include @@ -1102,6 +1103,23 @@ namespace xsimd namespace detail { + // Keep the caller's pointer type and preserve all bits without scalar + // floating-point evaluation, which could quiet signaling NaNs on x87. + template + XSIMD_INLINE __m128 load_low_32_bits(T const* mem) noexcept + { + int32_t bits; + std::memcpy(&bits, mem, sizeof(bits)); + return _mm_castsi128_ps(_mm_cvtsi32_si128(bits)); + } + + template + XSIMD_INLINE void store_low_32_bits(T* mem, __m128 src) noexcept + { + const int32_t bits = _mm_cvtsi128_si32(_mm_castps_si128(src)); + std::memcpy(mem, &bits, sizeof(bits)); + } + // Plain moves store-forward; vmaskmov never does on Intel and its // stores are microcoded on AMD. So wider archs delegate the masks // that lower to plain moves here rather than taking their runtime path. @@ -1113,119 +1131,109 @@ namespace xsimd } } - // load_masked - template >> + template XSIMD_INLINE batch load_masked(T const* mem, batch_bool_constant mask, convert, Mode, requires_arch) noexcept { if constexpr (sizeof(T) == 2 && mask.prefix() == 1) { - return _mm_loadu_si16(mem); + uint16_t value; + std::memcpy(&value, mem, sizeof(value)); + return ::xsimd::bitwise_cast(batch(_mm_cvtsi32_si128(value))); } - else if constexpr (sizeof(T) == 4 && mask.prefix() == 1) + else if constexpr ((sizeof(T) == 4 && mask.prefix() == 1) + || (sizeof(T) == 2 && mask.prefix() == 2)) { - return _mm_loadu_si32(mem); + return ::xsimd::bitwise_cast(batch(detail::load_low_32_bits(mem))); } - else if constexpr (sizeof(T) == 8 && mask.prefix() == 1) + else if constexpr ((sizeof(T) == 8 && mask.prefix() == 1) + || (sizeof(T) == 4 && mask.prefix() == 2)) { - return _mm_loadu_si64(mem); + return ::xsimd::bitwise_cast(batch(_mm_loadl_epi64(reinterpret_cast<__m128i const*>(mem)))); } - else if constexpr (sizeof(T) == 2 && mask.prefix() == 2) + else if constexpr ((sizeof(T) == 4 && mask.suffix() == 2) + || (sizeof(T) == 8 && mask.suffix() == 1)) { - return _mm_loadu_si32(mem); + const __m128 hi = _mm_loadh_pi(_mm_setzero_ps(), + reinterpret_cast<__m64 const*>(mem + batch::size / 2)); + return ::xsimd::bitwise_cast(batch(hi)); } - else if constexpr (sizeof(T) == 4 && mask.prefix() == 2) + else if constexpr (sizeof(T) == 4 && mask.prefix() == 3) { - return _mm_loadu_si64(mem); + const __m128 lo = _mm_castsi128_ps(_mm_loadl_epi64(reinterpret_cast<__m128i const*>(mem))); + const __m128 bits = _mm_shuffle_ps(lo, detail::load_low_32_bits(mem + 2), _MM_SHUFFLE(3, 0, 1, 0)); + return ::xsimd::bitwise_cast(batch(bits)); } else { return load_masked(mem, mask, convert {}, Mode {}, common {}); } } - template - XSIMD_INLINE batch load_masked(float const* mem, batch_bool_constant mask, convert, Mode, requires_arch) noexcept + + template + XSIMD_INLINE void store_masked(T* mem, batch const& src, batch_bool_constant mask, Mode, requires_arch) noexcept { - if constexpr (mask.prefix() == 1) + const __m128 bits = ::xsimd::bitwise_cast(src); + if constexpr (sizeof(T) == 4 && mask.prefix() == 1) { - return _mm_load_ss(mem); + detail::store_low_32_bits(mem, bits); } - else if constexpr (mask.prefix() == 2) + else if constexpr ((sizeof(T) == 4 && mask.prefix() == 2) + || (sizeof(T) == 8 && mask.prefix() == 1)) { - return _mm_loadl_pi(_mm_setzero_ps(), reinterpret_cast<__m64 const*>(mem)); + _mm_storel_pi(reinterpret_cast<__m64*>(mem), bits); } - else if constexpr (mask.suffix() == 2) + else if constexpr ((sizeof(T) == 4 && mask.suffix() == 2) + || (sizeof(T) == 8 && mask.suffix() == 1)) { - return _mm_loadh_pi(_mm_setzero_ps(), reinterpret_cast<__m64 const*>(mem + 2)); + _mm_storeh_pi(reinterpret_cast<__m64*>(mem + batch::size / 2), bits); } - else if constexpr (mask.prefix() == 3) + else if constexpr (sizeof(T) == 4 && mask.prefix() == 3) { - __m128 const lo2 = _mm_castsi128_ps(_mm_loadl_epi64(reinterpret_cast<__m128i const*>(mem))); - return _mm_shuffle_ps(lo2, _mm_load_ss(mem + 2), _MM_SHUFFLE(3, 0, 1, 0)); + _mm_storel_pi(reinterpret_cast<__m64*>(mem), bits); + detail::store_low_32_bits(mem + 2, _mm_movehl_ps(bits, bits)); } else { - return load_masked(mem, mask, convert {}, Mode {}, common {}); + store_masked(mem, src, mask, Mode {}, common {}); } } - template - XSIMD_INLINE batch load_masked(double const* mem, batch_bool_constant mask, convert, Mode, requires_arch) noexcept + + // store_complex_masked + template + XSIMD_INLINE void + store_complex_masked(std::complex* mem, batch, A> const& src, batch_bool mask, Mode, requires_arch) noexcept { - if constexpr (mask.prefix() == 1) - { - return _mm_load_sd(mem); - } - else if constexpr (mask.suffix() == 1) - { - return _mm_loadh_pd(_mm_setzero_pd(), mem + 1); - } - else - { - return load_masked(mem, mask, convert {}, Mode {}, common {}); - } + const auto k = mask.mask(); + batch c0 = detail::complex_low(src, A {}); + batch c1 = detail::complex_high(src, A {}); + auto* p = reinterpret_cast<__m64*>(mem); + if (k & 1) + _mm_storel_pi(p, c0); + if (k & 2) + _mm_storeh_pi(p + 1, c0); + if (k & 4) + _mm_storel_pi(p + 2, c1); + if (k & 8) + _mm_storeh_pi(p + 3, c1); } - // store_masked - template - XSIMD_INLINE void store_masked(float* mem, batch const& src, batch_bool_constant mask, Mode, requires_arch) noexcept + template + XSIMD_INLINE void + store_complex_masked(std::complex* mem, batch, A> const& src, batch_bool mask, Mode, requires_arch) noexcept { - if constexpr (mask.prefix() == 1) - { - _mm_store_ss(mem, src); - } - else if constexpr (mask.prefix() == 2) - { - _mm_storel_pi(reinterpret_cast<__m64*>(mem), src); - } - else if constexpr (mask.suffix() == 2) - { - _mm_storeh_pi(reinterpret_cast<__m64*>(mem + 2), src); - } - else if constexpr (mask.prefix() == 3) - { - _mm_storel_pi(reinterpret_cast<__m64*>(mem), src); - _mm_store_ss(mem + 2, _mm_movehl_ps(src, src)); - } - else - { - store_masked(mem, src, mask, Mode {}, common {}); - } + const auto k = mask.mask(); + auto* p = reinterpret_cast(mem); + if (k & 1) + _mm_storeu_pd(p, detail::complex_low(src, A {})); + if (k & 2) + _mm_storeu_pd(p + 2, detail::complex_high(src, A {})); } - template - XSIMD_INLINE void store_masked(double* mem, batch const& src, batch_bool_constant mask, Mode, requires_arch) noexcept + template + XSIMD_INLINE void + store_complex_masked(std::complex* mem, batch, A> const& src, batch_bool_constant mask, Mode mode, requires_arch) noexcept { - if constexpr (mask.prefix() == 1) - { - _mm_store_sd(mem, src); - } - else if constexpr (mask.suffix() == 1) - { - _mm_storeh_pd(mem + 1, src); - } - else - { - store_masked(mem, src, mask, Mode {}, common {}); - } + detail::store_complex_masked_chunks(mem, src, mask, mode); } // load_complex @@ -1244,6 +1252,50 @@ namespace xsimd } } + // load_complex_masked + template + XSIMD_INLINE batch, A> + load_complex_masked(std::complex const* mem, batch_bool mask, Mode, requires_arch) noexcept + { + const auto k = mask.mask(); + batch c0 = _mm_setzero_ps(); + batch c1 = _mm_setzero_ps(); + auto const* p = reinterpret_cast<__m64 const*>(mem); + if (k & 1) + c0 = _mm_loadl_pi(c0, p); + if (k & 2) + c0 = _mm_loadh_pi(c0, p + 1); + if (k & 4) + c1 = _mm_loadl_pi(c1, p + 2); + if (k & 8) + c1 = _mm_loadh_pi(c1, p + 3); + + return detail::load_complex(c0, c1, A {}); + } + + template + XSIMD_INLINE batch, A> + load_complex_masked(std::complex const* mem, batch_bool mask, Mode, requires_arch) noexcept + { + const auto k = mask.mask(); + batch c0 = _mm_setzero_pd(); + batch c1 = _mm_setzero_pd(); + auto const* p = reinterpret_cast(mem); + if (k & 1) + c0 = _mm_loadu_pd(p); + if (k & 2) + c1 = _mm_loadu_pd(p + 2); + + return detail::load_complex(c0, c1, A {}); + } + + template + XSIMD_INLINE batch, A> + load_complex_masked(std::complex const* mem, batch_bool_constant mask, Mode mode, requires_arch) noexcept + { + return detail::load_complex_masked_chunks(mem, mask, mode); + } + // le template XSIMD_INLINE batch_bool le(batch const& self, batch const& other, requires_arch) noexcept diff --git a/include/xsimd/types/xsimd_api.hpp b/include/xsimd/types/xsimd_api.hpp index 31a850aef..7448218a9 100644 --- a/include/xsimd/types/xsimd_api.hpp +++ b/include/xsimd/types/xsimd_api.hpp @@ -1565,6 +1565,16 @@ namespace xsimd return batch::load(ptr, mask, aligned_mode {}); } + /// \overload + template + XSIMD_INLINE batch, A> load(std::complex const* ptr, + batch_bool_constant const& mask, + aligned_mode = {}) noexcept + { + detail::static_check_supported_config(); + return batch, A>::load(ptr, mask, aligned_mode {}); + } + /** * @ingroup batch_data_transfer * @@ -1586,6 +1596,26 @@ namespace xsimd return batch::load(ptr, mask, aligned_mode {}); } + /** + * @ingroup batch_data_transfer + * + * Creates a complex batch from the buffer \c ptr using a runtime mask. + * The mask is a \c batch_bool of the underlying real type, with one lane + * per complex element. Elements corresponding to \c false in the mask are + * not accessed in memory and are zero-initialized in the resulting batch. + * @param ptr the memory buffer to read + * @param mask runtime selection mask for the elements to load + * @return a new batch instance + */ + template + XSIMD_INLINE batch, A> load(std::complex const* ptr, + batch_bool mask, + aligned_mode = {}) noexcept + { + detail::static_check_supported_config(); + return batch, A>::load(ptr, mask, aligned_mode {}); + } + /** * @ingroup batch_data_transfer * @@ -1606,6 +1636,16 @@ namespace xsimd return batch::load(ptr, mask, unaligned_mode {}); } + /// \overload + template + XSIMD_INLINE batch, A> load(std::complex const* ptr, + batch_bool_constant const& mask, + unaligned_mode) noexcept + { + detail::static_check_supported_config(); + return batch, A>::load(ptr, mask, unaligned_mode {}); + } + /// \overload template XSIMD_INLINE batch load(T const* ptr, @@ -1616,6 +1656,16 @@ namespace xsimd return batch::load(ptr, mask, unaligned_mode {}); } + /// \overload + template + XSIMD_INLINE batch, A> load(std::complex const* ptr, + batch_bool mask, + unaligned_mode) noexcept + { + detail::static_check_supported_config(); + return batch, A>::load(ptr, mask, unaligned_mode {}); + } + /** * @ingroup batch_data_transfer * @@ -2759,6 +2809,39 @@ namespace xsimd val.store(mem, mask, aligned_mode {}); } + /// \overload + template + XSIMD_INLINE void store(std::complex* mem, + batch, A> const& val, + batch_bool_constant const& mask, + aligned_mode = {}) noexcept + { + detail::static_check_supported_config(); + val.store(mem, mask, aligned_mode {}); + } + + /** + * @ingroup batch_data_transfer + * + * Copy selected elements of complex batch \c val to the buffer \c mem + * using a runtime mask. The mask is a \c batch_bool of the underlying real + * type, with one lane per complex element. Elements corresponding to + * \c false in the mask are not written and leave the contents of \c mem + * untouched. + * @param mem the memory buffer to write to + * @param val the batch to copy from + * @param mask runtime selection mask for the elements to store + */ + template + XSIMD_INLINE void store(std::complex* mem, + batch, A> const& val, + batch_bool mask, + aligned_mode = {}) noexcept + { + detail::static_check_supported_config(); + val.store(mem, mask, aligned_mode {}); + } + /** * @ingroup batch_data_transfer * @@ -2801,6 +2884,17 @@ namespace xsimd val.store(mem, mask, unaligned_mode {}); } + /// \overload + template + XSIMD_INLINE void store(std::complex* mem, + batch, A> const& val, + batch_bool_constant const& mask, + unaligned_mode) noexcept + { + detail::static_check_supported_config(); + val.store(mem, mask, unaligned_mode {}); + } + /// \overload template XSIMD_INLINE void store(T* mem, @@ -2812,6 +2906,17 @@ namespace xsimd val.store(mem, mask, unaligned_mode {}); } + /// \overload + template + XSIMD_INLINE void store(std::complex* mem, + batch, A> const& val, + batch_bool mask, + unaligned_mode) noexcept + { + detail::static_check_supported_config(); + val.store(mem, mask, unaligned_mode {}); + } + /** * @ingroup batch_data_transfer * diff --git a/include/xsimd/types/xsimd_batch.hpp b/include/xsimd/types/xsimd_batch.hpp index e784d720f..448928f02 100644 --- a/include/xsimd/types/xsimd_batch.hpp +++ b/include/xsimd/types/xsimd_batch.hpp @@ -542,7 +542,7 @@ namespace xsimd [[nodiscard]] static XSIMD_INLINE batch load(U const* mem, unaligned_mode) noexcept; // Compile-time mask overloads template - [[nodiscard]] static XSIMD_INLINE batch load(U const* mem, batch_bool_constant mask, Mode = {}) noexcept; + [[nodiscard]] static XSIMD_INLINE batch load(U const* mem, batch_bool_constant mask, Mode = {}) noexcept; template [[nodiscard]] static XSIMD_INLINE batch load(value_type const* mem, batch_bool mask, Mode = {}) noexcept; template @@ -553,7 +553,7 @@ namespace xsimd XSIMD_INLINE void store(U* mem, unaligned_mode) const noexcept; // Compile-time mask overloads template - XSIMD_INLINE void store(U* mem, batch_bool_constant mask, Mode = {}) const noexcept; + XSIMD_INLINE void store(U* mem, batch_bool_constant mask, Mode = {}) const noexcept; template XSIMD_INLINE void store(value_type* mem, batch_bool mask, Mode = {}) const noexcept; template @@ -1490,20 +1490,33 @@ namespace xsimd // Compile-time mask overloads for complex store template template - XSIMD_INLINE void batch, A>::store(U* mem, batch_bool_constant mask, Mode mode) const noexcept + XSIMD_INLINE void batch, A>::store(U* mem, batch_bool_constant mask, Mode mode) const noexcept { - kernel::store_masked(mem, *this, mask, mode, A {}); + detail::static_check_supported_config(); + static_assert(std::is_same_v || std::is_same_v, + "supported store mode"); + if constexpr (mask.all()) + { + store(mem, mode); + } + else if constexpr (mask.none()) + { + return; + } + else + { + kernel::store_complex_masked(mem, *this, mask, mode, A {}); + } } template template - XSIMD_INLINE void batch, A>::store(value_type* mem, batch_bool mask, Mode) const noexcept + XSIMD_INLINE void batch, A>::store(value_type* mem, batch_bool mask, Mode mode) const noexcept { - alignas(A::alignment()) std::array buffer; - store_aligned(buffer.data()); - for (std::size_t i = 0; i < size; ++i) - if (mask.get(i)) - mem[i] = buffer[i]; + detail::static_check_supported_config(); + static_assert(std::is_same_v || std::is_same_v, + "supported store mode"); + kernel::store_complex_masked(mem, *this, mask, mode, A {}); } template @@ -1538,20 +1551,34 @@ namespace xsimd template template XSIMD_INLINE batch, A> batch, A>::load(U const* mem, - batch_bool_constant mask, + batch_bool_constant mask, Mode mode) noexcept { - return kernel::load_masked(mem, mask, kernel::convert {}, mode, A {}); + detail::static_check_supported_config(); + static_assert(std::is_same_v || std::is_same_v, + "supported load mode"); + if constexpr (mask.all()) + { + return load(mem, mode); + } + else if constexpr (mask.none()) + { + return broadcast(0); + } + else + { + return kernel::load_complex_masked(mem, mask, mode, A {}); + } } template template - XSIMD_INLINE batch, A> batch, A>::load(value_type const* mem, batch_bool mask, Mode) noexcept + XSIMD_INLINE batch, A> batch, A>::load(value_type const* mem, batch_bool mask, Mode mode) noexcept { - alignas(A::alignment()) std::array buffer {}; - for (std::size_t i = 0; i < size; ++i) - buffer[i] = mask.get(i) ? mem[i] : value_type(0); - return load_aligned(buffer.data()); + detail::static_check_supported_config(); + static_assert(std::is_same_v || std::is_same_v, + "supported load mode"); + return kernel::load_complex_masked(mem, mask, mode, A {}); } template diff --git a/test/test_batch_complex.cpp b/test/test_batch_complex.cpp index 94bb799fd..ab75bcfda 100644 --- a/test/test_batch_complex.cpp +++ b/test/test_batch_complex.cpp @@ -115,40 +115,6 @@ struct batch_complex_test CHECK_EQ(ares_real, areal); CHECK_EQ(ares_imag, aimag); } - { - // Runtime-masked load / store - alignas(arch_type::alignment()) array_type amem; - for (size_t i = 0; i < size; ++i) - amem[i] = lhs[i]; - - uint64_t bits = 0; - for (size_t i = 0; i < size; ++i) - if (i % 2 == 0) - bits |= uint64_t(1) << i; - using mask_type = typename batch_type::batch_bool_type; - auto mask = mask_type::from_mask(bits); - - batch_type loaded = batch_type::load(amem.data(), mask, xsimd::aligned_mode()); - for (size_t i = 0; i < size; ++i) - { - if (i % 2 == 0) - CHECK_EQ(loaded.get(i), amem[i]); - else - CHECK_EQ(loaded.get(i), value_type(0, 0)); - } - - batch_type to_store = batch_type::load_unaligned(rhs.data()); - alignas(arch_type::alignment()) array_type adest; - std::fill(adest.begin(), adest.end(), value_type(0, 0)); - to_store.store(adest.data(), mask, xsimd::aligned_mode()); - for (size_t i = 0; i < size; ++i) - { - if (i % 2 == 0) - CHECK_EQ(adest[i], rhs[i]); - else - CHECK_EQ(adest[i], value_type(0, 0)); - } - } } #ifdef XSIMD_ENABLE_XTL_COMPLEX void test_load_store_xtl() const diff --git a/test/test_load_store.cpp b/test/test_load_store.cpp index 08b34572f..cb9f20f9e 100644 --- a/test/test_load_store.cpp +++ b/test/test_load_store.cpp @@ -59,7 +59,6 @@ struct load_store_test { using batch_type = B; using value_type = typename B::value_type; - using batch_bool_type = typename B::batch_bool_type; using index_type = typename xsimd::as_integer_t; template using allocator = xsimd::default_allocator; @@ -82,99 +81,6 @@ struct load_store_test using double_vector_type = std::vector>; #endif - struct mask_none - { - static constexpr bool get(std::size_t, std::size_t) noexcept { return false; } - }; - - struct mask_first - { - static constexpr bool get(std::size_t index, std::size_t) noexcept { return index == 0; } - }; - - struct mask_first_half - { - static constexpr bool get(std::size_t index, std::size_t size) noexcept { return index < (size / 2); } - }; - - struct mask_last_half - { - static constexpr bool get(std::size_t index, std::size_t size) noexcept { return index >= (size / 2); } - }; - - struct mask_even - { - static constexpr bool get(std::size_t index, std::size_t) noexcept { return (index % 2) == 0; } - }; - - struct mask_odd - { - static constexpr bool get(std::size_t index, std::size_t) noexcept { return (index % 2) == 1; } - }; - - struct mask_pseudo_random - { - static constexpr bool get(std::size_t index, std::size_t size) noexcept - { - return ((index * 7) + 3) % size < (size > 2 ? size / 3 : std::size_t(1)); - } - }; - - struct mask_all - { - static constexpr bool get(std::size_t, std::size_t) noexcept { return true; } - }; - - struct mask_all_but_last - { - static constexpr bool get(std::size_t index, std::size_t size) noexcept { return index + 1 < size; } - }; - - struct mask_all_but_first - { - static constexpr bool get(std::size_t index, std::size_t) noexcept { return index != 0; } - }; - - struct mask_last - { - static constexpr bool get(std::size_t index, std::size_t size) noexcept { return index + 1 == size; } - }; - - template - static batch_bool_type make_runtime_mask() noexcept - { - uint64_t bits = 0; - for (std::size_t i = 0; i < size; ++i) - { - if (Generator::get(i, size)) - { - bits |= uint64_t(1) << i; - } - } - return batch_bool_type::from_mask(bits); - } - - struct compile_time_mask - { - static constexpr const char* tag = ""; - template - static auto make() noexcept - -> decltype(xsimd::make_batch_bool_constant()) - { - return xsimd::make_batch_bool_constant(); - } - }; - - struct runtime_mask - { - static constexpr const char* tag = " runtime"; - template - static batch_bool_type make() noexcept - { - return make_runtime_mask(); - } - }; - int8_vector_type i8_vec; uint8_vector_type ui8_vec; int16_vector_type i16_vec; @@ -430,154 +336,6 @@ struct load_store_test stream_load_if_same(v.data(), b, expected, name, std::integral_constant> {}); - - run_mask_tests(v, name, b, expected, std::is_same {}); - } - - template - void run_mask_tests(const V& v, const std::string& name, batch_type& b, const array_type& expected, std::true_type) - { - run_load_mask_patterns(v, name, b, expected); - run_load_mask_patterns(v, name, b, expected); - } - - template - void run_load_mask_patterns(const V& v, const std::string& name, batch_type& b, const array_type& expected) - { - const std::string p = std::string(MaskKind::tag) + " masked"; - run_load_mask_pattern(v, name, b, expected, p + " none"); - run_load_mask_pattern(v, name, b, expected, p + " first element"); - run_load_mask_pattern(v, name, b, expected, p + " first half"); - run_load_mask_pattern(v, name, b, expected, p + " last half"); - run_load_mask_pattern(v, name, b, expected, p + " even elements"); - run_load_mask_pattern(v, name, b, expected, p + " odd elements"); - run_load_mask_pattern(v, name, b, expected, p + " last element"); - run_load_mask_pattern(v, name, b, expected, p + " all but last"); - run_load_mask_pattern(v, name, b, expected, p + " all but first"); - run_load_mask_pattern(v, name, b, expected, p + " pseudo random"); - run_load_mask_pattern(v, name, b, expected, p + " all elements"); - } - - template - void run_mask_tests(const V&, const std::string&, batch_type&, const array_type&, std::false_type) - { - } - - template - void run_load_mask_pattern(const V& v, const std::string& name, batch_type& b, const array_type& expected, const std::string& label) - { - const auto mask = MaskKind::template make(); - array_type expected_masked { 0 }; - - for (std::size_t i = 0; i < size; ++i) - { - const bool active = Generator::get(i, size); - expected_masked[i] = active ? expected[i] : value_type(); - } - - b = xsimd::load(v.data(), mask, xsimd::aligned_mode()); - INFO(name, label + " aligned"); - CHECK_BATCH_EQ(b, expected_masked); - b = xsimd::load(v.data(), mask, xsimd::unaligned_mode()); - INFO(name, label + " unaligned"); - CHECK_BATCH_EQ(b, expected_masked); - } - - template - void run_store_mask_pattern(const V& v, const std::string& name, batch_type& b, V& res, V& expected_masked, const std::string& label) - { - const auto mask = MaskKind::template make(); - for (std::size_t i = 0; i < size; ++i) - { - expected_masked[i] = Generator::get(i, size) ? v[i] : value_type(); - } - std::fill(res.begin(), res.end(), value_type()); - b.store(res.data(), mask, xsimd::aligned_mode()); - INFO(name, label + " aligned"); - CHECK_VECTOR_EQ(res, expected_masked); - std::fill(res.begin(), res.end(), value_type()); - b.store(res.data(), mask, xsimd::unaligned_mode()); - INFO(name, label + " unaligned"); - CHECK_VECTOR_EQ(res, expected_masked); - } - - template - void run_store_mask_tests(const V& v, const std::string& name, batch_type& b, V& res, V& expected_masked, std::true_type) - { - run_store_mask_patterns(v, name, b, res, expected_masked); - run_store_mask_patterns(v, name, b, res, expected_masked); - } - - template - void run_store_mask_patterns(const V& v, const std::string& name, batch_type& b, V& res, V& expected_masked) - { - const std::string p = std::string(MaskKind::tag) + " masked"; - run_store_mask_pattern(v, name, b, res, expected_masked, p + " first element"); - run_store_mask_pattern(v, name, b, res, expected_masked, p + " first half"); - run_store_mask_pattern(v, name, b, res, expected_masked, p + " last half"); - run_store_mask_pattern(v, name, b, res, expected_masked, p + " even elements"); - run_store_mask_pattern(v, name, b, res, expected_masked, p + " odd elements"); - run_store_mask_pattern(v, name, b, res, expected_masked, p + " last element"); - run_store_mask_pattern(v, name, b, res, expected_masked, p + " all but last"); - run_store_mask_pattern(v, name, b, res, expected_masked, p + " all but first"); - run_store_mask_pattern(v, name, b, res, expected_masked, p + " pseudo random"); - run_store_mask_pattern(v, name, b, res, expected_masked, p + " all elements"); - } - - template - void run_store_mask_tests(const V&, const std::string&, batch_type&, V&, V&, std::false_type) - { - } - - template - void run_store_mask_section(const V& v, - const std::string& name, - batch_type& b, - V& res, - V& expected_masked, - std::true_type) - { - static constexpr auto sentinel = static_cast(37); - V sentinel_expected(size, sentinel); - - auto zero_mask = xsimd::make_batch_bool_constant(); - auto runtime_zero_mask = make_runtime_mask(); - std::fill(res.begin(), res.end(), sentinel); - b.store(res.data(), zero_mask, xsimd::aligned_mode()); - INFO(name, " masked none aligned store"); - CHECK_VECTOR_EQ(res, sentinel_expected); - - V scratch(res.size() + size); - std::fill(scratch.begin(), scratch.end(), sentinel); - auto* scratch_ptr = scratch.data() + 1; - b.store(scratch_ptr, zero_mask, xsimd::unaligned_mode()); - INFO(name, " masked none unaligned store"); - - V scratch_slice(res.size()); - std::copy(scratch_ptr, scratch_ptr + scratch_slice.size(), scratch_slice.begin()); - CHECK_VECTOR_EQ(scratch_slice, sentinel_expected); - CHECK(std::all_of(scratch.begin(), scratch.end(), [](const value_type v) - { return v == sentinel; })); - - std::fill(res.begin(), res.end(), sentinel); - xsimd::store(res.data(), b, runtime_zero_mask, xsimd::aligned_mode()); - INFO(name, " runtime masked none aligned store"); - CHECK_VECTOR_EQ(res, sentinel_expected); - - std::fill(scratch.begin(), scratch.end(), sentinel); - xsimd::store(scratch_ptr, b, runtime_zero_mask, xsimd::unaligned_mode()); - INFO(name, " runtime masked none unaligned store"); - std::copy(scratch_ptr, scratch_ptr + scratch_slice.size(), scratch_slice.begin()); - CHECK_VECTOR_EQ(scratch_slice, sentinel_expected); - CHECK(std::all_of(scratch.begin(), scratch.end(), [](const value_type v) - { return v == sentinel; })); - - run_store_mask_tests(v, name, b, res, expected_masked, std::true_type {}); - } - - template - void run_store_mask_section(const V&, const std::string&, batch_type&, V&, V&, std::false_type) - { } struct test_load_char @@ -623,10 +381,6 @@ struct load_store_test stream_store_if_same(res, b, v, name, std::integral_constant> {}); - - V expected_masked(size); - - run_store_mask_section(v, name, b, res, expected_masked, std::is_same {}); } template @@ -707,34 +461,153 @@ TEST_CASE_TEMPLATE("[load store]", B, BATCH_TYPES) SUBCASE("masked") { Test.test_masked(); } } -TEST_CASE_TEMPLATE("store_masked respects Mode", B, BATCH_TYPES) +template +struct masked_load_store_test { - using T = typename B::value_type; - using A = typename B::arch_type; - constexpr std::size_t N = B::size; + using value_type = typename B::value_type; + using arch_type = typename B::arch_type; + using mask_type = typename B::batch_bool_type; + using mask_value_type = typename mask_type::operand_type; + static constexpr std::size_t size = B::size; + using storage_type = std::array; + + struct mask_none + { + static constexpr bool get(std::size_t, std::size_t) noexcept { return false; } + }; - // Unaligned-mode + unaligned pointer: must not fault. - alignas(A::alignment()) T big[2 * N + 1] = {}; - T* unaligned_ptr = big + 1; // sizeof(T)-aligned only + struct mask_first + { + static constexpr bool get(std::size_t index, std::size_t) noexcept { return index == 0; } + }; - struct AllTrue + struct mask_first_half { - static constexpr bool get(std::size_t, std::size_t) { return true; } + static constexpr bool get(std::size_t index, std::size_t size) noexcept { return index < (size / 2); } }; - auto cst_mask = xsimd::make_batch_bool_constant(); - - B v(T(7)); - v.store(unaligned_ptr, cst_mask, xsimd::unaligned_mode {}); - for (std::size_t i = 0; i < N; ++i) - CHECK_EQ(unaligned_ptr[i], T(7)); - - // Overload resolution: store_masked with same-typed mask must compile. - // If C3 regresses, this becomes a compile error. - auto signed_cst_mask = xsimd::make_batch_bool_constant(); - alignas(A::alignment()) T aligned_buf[N] = {}; - v.store(aligned_buf, signed_cst_mask, xsimd::aligned_mode {}); - for (std::size_t i = 0; i < N; ++i) - CHECK_EQ(aligned_buf[i], T(7)); + + struct mask_last_half + { + static constexpr bool get(std::size_t index, std::size_t size) noexcept { return index >= (size / 2); } + }; + + struct mask_even + { + static constexpr bool get(std::size_t index, std::size_t) noexcept { return (index % 2) == 0; } + }; + + struct mask_odd + { + static constexpr bool get(std::size_t index, std::size_t) noexcept { return (index % 2) == 1; } + }; + + struct mask_pseudo_random + { + static constexpr bool get(std::size_t index, std::size_t size) noexcept + { + return ((index * 7) + 3) % size < (size > 2 ? size / 3 : std::size_t(1)); + } + }; + + struct mask_all + { + static constexpr bool get(std::size_t, std::size_t) noexcept { return true; } + }; + + struct mask_all_but_last + { + static constexpr bool get(std::size_t index, std::size_t size) noexcept { return index + 1 < size; } + }; + + struct mask_all_but_first + { + static constexpr bool get(std::size_t index, std::size_t) noexcept { return index != 0; } + }; + + struct mask_last + { + static constexpr bool get(std::size_t index, std::size_t size) noexcept { return index + 1 == size; } + }; + + static value_type make_value(int i) + { + if constexpr (xsimd::is_batch_complex::value) + return value_type(mask_value_type(i + 1), mask_value_type(-2 * i - 3)); + else + return value_type(i + 1); + } + + template + void check(Mask mask, Mode mode, std::size_t offset) const + { + INFO("offset = ", offset, ", runtime mask = ", xsimd::is_batch_bool::value); + alignas(arch_type::alignment()) storage_type input; + alignas(arch_type::alignment()) storage_type output; + for (std::size_t i = 0; i < input.size(); ++i) + input[i] = make_value(static_cast(i)); + + const value_type sentinel = make_value(100); + const B source = B::load_unaligned(input.data() + offset); + auto check_results = [&](const auto& loaded) + { + static_assert(std::is_same_v, B>, + "masked load preserves batch type"); + for (std::size_t i = 0; i < size; ++i) + { + const value_type expected = Generator::get(i, size) ? input[i + offset] : value_type {}; + CHECK_EQ(loaded.get(i), expected); + } + for (std::size_t i = 0; i < output.size(); ++i) + { + const bool active = i >= offset && i < offset + size && Generator::get(i - offset, size); + CHECK_EQ(output[i], active ? input[i] : sentinel); + } + }; + + output.fill(sentinel); + xsimd::store(output.data() + offset, source, mask, mode); + check_results(xsimd::load(input.data() + offset, mask, mode)); + + if (offset == 0) + { + INFO("default mode"); + output.fill(sentinel); + xsimd::store(output.data(), source, mask); + check_results(xsimd::load(input.data(), mask)); + } + } + + template + void check_pattern(const char* name) const + { + INFO(std::string(name)); + constexpr auto constant_mask = xsimd::make_batch_bool_constant(); + const auto runtime_mask = constant_mask.as_batch_bool(); + check(constant_mask, xsimd::aligned_mode {}, 0); + check(constant_mask, xsimd::unaligned_mode {}, 1); + check(runtime_mask, xsimd::aligned_mode {}, 0); + check(runtime_mask, xsimd::unaligned_mode {}, 1); + } + + void run() const + { + check_pattern("none"); + check_pattern("all"); + check_pattern("first element"); + check_pattern("last element"); + check_pattern("first half"); + check_pattern("last half"); + check_pattern("even elements"); + check_pattern("odd elements"); + check_pattern("all but first"); + check_pattern("all but last"); + check_pattern("pseudo random"); + } +}; + +TEST_CASE_TEMPLATE("[masked load store]", B, BATCH_TYPES, BATCH_COMPLEX_TYPES) +{ + masked_load_store_test {}.run(); } #endif