diff --git a/modules/core/include/opencv2/core/hal/intrin_neon.hpp b/modules/core/include/opencv2/core/hal/intrin_neon.hpp index f3e47ca8ba3b6f8f33587b41eb28f46d08e84093..f989b3a8ad657b9877e0cd0e37d04a203ded67c9 100644 --- a/modules/core/include/opencv2/core/hal/intrin_neon.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_neon.hpp @@ -53,6 +53,28 @@ namespace cv //! @cond IGNORED #define CV_SIMD128 1 +#if defined(__aarch64__) +#define CV_SIMD128_64F 1 +#else +#define CV_SIMD128_64F 0 +#endif + +#if CV_SIMD128_64F +#define OPENCV_HAL_IMPL_NEON_REINTERPRET(_Tpv, suffix) \ +template static inline \ +_Tpv vreinterpretq_##suffix##_f64(T a) { return (_Tpv) a; } \ +template static inline \ +float64x2_t vreinterpretq_f64_##suffix(T a) { return (float64x2_t) a; } +OPENCV_HAL_IMPL_NEON_REINTERPRET(uint8x16_t, u8) +OPENCV_HAL_IMPL_NEON_REINTERPRET(int8x16_t, s8) +OPENCV_HAL_IMPL_NEON_REINTERPRET(uint16x8_t, u16) +OPENCV_HAL_IMPL_NEON_REINTERPRET(int16x8_t, s16) +OPENCV_HAL_IMPL_NEON_REINTERPRET(uint32x4_t, u32) +OPENCV_HAL_IMPL_NEON_REINTERPRET(int32x4_t, s32) +OPENCV_HAL_IMPL_NEON_REINTERPRET(uint64x2_t, u64) +OPENCV_HAL_IMPL_NEON_REINTERPRET(int64x2_t, s64) +OPENCV_HAL_IMPL_NEON_REINTERPRET(float32x4_t, f32) +#endif struct v_uint8x16 { @@ -232,6 +254,27 @@ struct v_int64x2 int64x2_t val; }; +#if CV_SIMD128_64F +struct v_float64x2 +{ + typedef double lane_type; + enum { nlanes = 2 }; + + v_float64x2() {} + explicit v_float64x2(float64x2_t v) : val(v) {} + v_float64x2(double v0, double v1) + { + double v[] = {v0, v1}; + val = vld1q_f64(v); + } + double get0() const + { + return vgetq_lane_f64(val, 0); + } + float64x2_t val; +}; +#endif + #define OPENCV_HAL_IMPL_NEON_INIT(_Tpv, _Tp, suffix) \ inline v_##_Tpv v_setzero_##suffix() { return v_##_Tpv(vdupq_n_##suffix((_Tp)0)); } \ inline v_##_Tpv v_setall_##suffix(_Tp v) { return v_##_Tpv(vdupq_n_##suffix(v)); } \ @@ -255,6 +298,21 @@ OPENCV_HAL_IMPL_NEON_INIT(int32x4, int, s32) OPENCV_HAL_IMPL_NEON_INIT(uint64x2, uint64, u64) OPENCV_HAL_IMPL_NEON_INIT(int64x2, int64, s64) OPENCV_HAL_IMPL_NEON_INIT(float32x4, float, f32) +#if CV_SIMD128_64F +#define OPENCV_HAL_IMPL_NEON_INIT_64(_Tpv, suffix) \ +inline v_float64x2 v_reinterpret_as_f64(const v_##_Tpv& v) { return v_float64x2(vreinterpretq_f64_##suffix(v.val)); } +OPENCV_HAL_IMPL_NEON_INIT(float64x2, double, f64) +OPENCV_HAL_IMPL_NEON_INIT_64(uint8x16, u8) +OPENCV_HAL_IMPL_NEON_INIT_64(int8x16, s8) +OPENCV_HAL_IMPL_NEON_INIT_64(uint16x8, u16) +OPENCV_HAL_IMPL_NEON_INIT_64(int16x8, s16) +OPENCV_HAL_IMPL_NEON_INIT_64(uint32x4, u32) +OPENCV_HAL_IMPL_NEON_INIT_64(int32x4, s32) +OPENCV_HAL_IMPL_NEON_INIT_64(uint64x2, u64) +OPENCV_HAL_IMPL_NEON_INIT_64(int64x2, s64) +OPENCV_HAL_IMPL_NEON_INIT_64(float32x4, f32) +OPENCV_HAL_IMPL_NEON_INIT_64(float64x2, f64) +#endif #define OPENCV_HAL_IMPL_NEON_PACK(_Tpvec, _Tp, hreg, suffix, _Tpwvec, wsuffix, pack, op) \ inline _Tpvec v_##pack(const _Tpwvec& a, const _Tpwvec& b) \ @@ -337,7 +395,13 @@ OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_int64x2, vaddq_s64) OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_int64x2, vsubq_s64) OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_uint64x2, vaddq_u64) OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_uint64x2, vsubq_u64) - +#if CV_SIMD128_64F +OPENCV_HAL_IMPL_NEON_BIN_OP(/, v_float32x4, vdivq_f32) +OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_float64x2, vaddq_f64) +OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_float64x2, vsubq_f64) +OPENCV_HAL_IMPL_NEON_BIN_OP(*, v_float64x2, vmulq_f64) +OPENCV_HAL_IMPL_NEON_BIN_OP(/, v_float64x2, vdivq_f64) +#else inline v_float32x4 operator / (const v_float32x4& a, const v_float32x4& b) { float32x4_t reciprocal = vrecpeq_f32(b.val); @@ -353,6 +417,7 @@ inline v_float32x4& operator /= (v_float32x4& a, const v_float32x4& b) a.val = vmulq_f32(a.val, reciprocal); return a; } +#endif inline void v_mul_expand(const v_int16x8& a, const v_int16x8& b, v_int32x4& c, v_int32x4& d) @@ -421,6 +486,18 @@ inline v_float32x4 operator ~ (const v_float32x4& a) return v_float32x4(vreinterpretq_f32_s32(vmvnq_s32(vreinterpretq_s32_f32(a.val)))); } +#if CV_SIMD128_64F +inline v_float32x4 v_sqrt(const v_float32x4& x) +{ + return v_float32x4(vsqrtq_f32(x.val)); +} + +inline v_float32x4 v_invsqrt(const v_float32x4& x) +{ + v_float32x4 one = v_setall_f32(1.0f); + return one / v_sqrt(x); +} +#else inline v_float32x4 v_sqrt(const v_float32x4& x) { float32x4_t x1 = vmaxq_f32(x.val, vdupq_n_f32(FLT_MIN)); @@ -437,10 +514,47 @@ inline v_float32x4 v_invsqrt(const v_float32x4& x) e = vmulq_f32(vrsqrtsq_f32(vmulq_f32(x.val, e), e), e); return v_float32x4(e); } +#endif inline v_float32x4 v_abs(v_float32x4 x) { return v_float32x4(vabsq_f32(x.val)); } +#if CV_SIMD128_64F +#define OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(bin_op, intrin) \ +inline v_float64x2 operator bin_op (const v_float64x2& a, const v_float64x2& b) \ +{ \ + return v_float64x2(vreinterpretq_f64_s64(intrin(vreinterpretq_s64_f64(a.val), vreinterpretq_s64_f64(b.val)))); \ +} \ +inline v_float64x2& operator bin_op##= (v_float64x2& a, const v_float64x2& b) \ +{ \ + a.val = vreinterpretq_f64_s64(intrin(vreinterpretq_s64_f64(a.val), vreinterpretq_s64_f64(b.val))); \ + return a; \ +} + +OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(&, vandq_s64) +OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(|, vorrq_s64) +OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(^, veorq_s64) + +inline v_float64x2 operator ~ (const v_float64x2& a) +{ + return v_float64x2(vreinterpretq_f64_s32(vmvnq_s32(vreinterpretq_s32_f64(a.val)))); +} + +inline v_float64x2 v_sqrt(const v_float64x2& x) +{ + return v_float64x2(vsqrtq_f64(x.val)); +} + +inline v_float64x2 v_invsqrt(const v_float64x2& x) +{ + v_float64x2 one = v_setall_f64(1.0f); + return one / v_sqrt(x); +} + +inline v_float64x2 v_abs(v_float64x2 x) +{ return v_float64x2(vabsq_f64(x.val)); } +#endif + // TODO: exp, log, sin, cos #define OPENCV_HAL_IMPL_NEON_BIN_FUNC(_Tpvec, func, intrin) \ @@ -463,8 +577,23 @@ OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_int32x4, v_min, vminq_s32) OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_int32x4, v_max, vmaxq_s32) OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_float32x4, v_min, vminq_f32) OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_float32x4, v_max, vmaxq_f32) +#if CV_SIMD128_64F +OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_float64x2, v_min, vminq_f64) +OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_float64x2, v_max, vmaxq_f64) +#endif - +#if CV_SIMD128_64F +inline int64x2_t vmvnq_s64(int64x2_t a) +{ + int64x2_t vx = vreinterpretq_s64_u32(vdupq_n_u32(0xFFFFFFFF)); + return veorq_s64(a, vx); +} +inline uint64x2_t vmvnq_u64(uint64x2_t a) +{ + uint64x2_t vx = vreinterpretq_u64_u32(vdupq_n_u32(0xFFFFFFFF)); + return veorq_u64(a, vx); +} +#endif #define OPENCV_HAL_IMPL_NEON_INT_CMP_OP(_Tpvec, cast, suffix, not_suffix) \ inline _Tpvec operator == (const _Tpvec& a, const _Tpvec& b) \ { return _Tpvec(cast(vceqq_##suffix(a.val, b.val))); } \ @@ -486,6 +615,11 @@ OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_int16x8, vreinterpretq_s16_u16, s16, u16) OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_uint32x4, OPENCV_HAL_NOP, u32, u32) OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_int32x4, vreinterpretq_s32_u32, s32, u32) OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_float32x4, vreinterpretq_f32_u32, f32, u32) +#if CV_SIMD128_64F +OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_uint64x2, OPENCV_HAL_NOP, u64, u64) +OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_int64x2, vreinterpretq_s64_u64, s64, u64) +OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_float64x2, vreinterpretq_f64_u64, f64, u64) +#endif OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_uint8x16, v_add_wrap, vaddq_u8) OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_int8x16, v_add_wrap, vaddq_s8) @@ -501,6 +635,9 @@ OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_uint8x16, v_absdiff, vabdq_u8) OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_uint16x8, v_absdiff, vabdq_u16) OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_uint32x4, v_absdiff, vabdq_u32) OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_float32x4, v_absdiff, vabdq_f32) +#if CV_SIMD128_64F +OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_float64x2, v_absdiff, vabdq_f64) +#endif #define OPENCV_HAL_IMPL_NEON_BIN_FUNC2(_Tpvec, _Tpvec2, cast, func, intrin) \ inline _Tpvec2 func(const _Tpvec& a, const _Tpvec& b) \ @@ -528,6 +665,24 @@ inline v_float32x4 v_muladd(const v_float32x4& a, const v_float32x4& b, const v_ return v_float32x4(vmlaq_f32(c.val, a.val, b.val)); } +#if CV_SIMD128_64F +inline v_float64x2 v_magnitude(const v_float64x2& a, const v_float64x2& b) +{ + v_float64x2 x(vaddq_f64(vmulq_f64(a.val, a.val), vmulq_f64(b.val, b.val))); + return v_sqrt(x); +} + +inline v_float64x2 v_sqr_magnitude(const v_float64x2& a, const v_float64x2& b) +{ + return v_float64x2(vaddq_f64(vmulq_f64(a.val, a.val), vmulq_f64(b.val, b.val))); +} + +inline v_float64x2 v_muladd(const v_float64x2& a, const v_float64x2& b, const v_float64x2& c) +{ + return v_float64x2(vaddq_f64(c.val, vmulq_f64(a.val, b.val))); +} +#endif + // trade efficiency for convenience #define OPENCV_HAL_IMPL_NEON_SHIFT_OP(_Tpvec, suffix, _Tps, ssuffix) \ inline _Tpvec operator << (const _Tpvec& a, int n) \ @@ -575,6 +730,9 @@ OPENCV_HAL_IMPL_NEON_LOADSTORE_OP(v_int32x4, int, s32) OPENCV_HAL_IMPL_NEON_LOADSTORE_OP(v_uint64x2, uint64, u64) OPENCV_HAL_IMPL_NEON_LOADSTORE_OP(v_int64x2, int64, s64) OPENCV_HAL_IMPL_NEON_LOADSTORE_OP(v_float32x4, float, f32) +#if CV_SIMD128_64F +OPENCV_HAL_IMPL_NEON_LOADSTORE_OP(v_float64x2, double, f64) +#endif #define OPENCV_HAL_IMPL_NEON_REDUCE_OP_4(_Tpvec, scalartype, func, scalar_func) \ inline scalartype v_reduce_##func(const _Tpvec& a) \ @@ -627,6 +785,16 @@ inline int v_signmask(const v_int32x4& a) { return v_signmask(v_reinterpret_as_u32(a)); } inline int v_signmask(const v_float32x4& a) { return v_signmask(v_reinterpret_as_u32(a)); } +#if CV_SIMD128_64F +inline int v_signmask(const v_uint64x2& a) +{ + int64x1_t m0 = vdup_n_s64(0); + uint64x2_t v0 = vshlq_u64(vshrq_n_u64(a.val, 63), vcombine_s64(m0, m0)); + return (int)vgetq_lane_u64(v0, 0) + ((int)vgetq_lane_u64(v0, 1) << 1); +} +inline int v_signmask(const v_float64x2& a) +{ return v_signmask(v_reinterpret_as_u64(a)); } +#endif #define OPENCV_HAL_IMPL_NEON_CHECK_ALLANY(_Tpvec, suffix, shift) \ inline bool v_check_all(const v_##_Tpvec& a) \ @@ -645,6 +813,9 @@ inline bool v_check_any(const v_##_Tpvec& a) \ OPENCV_HAL_IMPL_NEON_CHECK_ALLANY(uint8x16, u8, 7) OPENCV_HAL_IMPL_NEON_CHECK_ALLANY(uint16x8, u16, 15) OPENCV_HAL_IMPL_NEON_CHECK_ALLANY(uint32x4, u32, 31) +#if CV_SIMD128_64F +OPENCV_HAL_IMPL_NEON_CHECK_ALLANY(uint64x2, u64, 63) +#endif inline bool v_check_all(const v_int8x16& a) { return v_check_all(v_reinterpret_as_u8(a)); } @@ -664,6 +835,17 @@ inline bool v_check_any(const v_int32x4& a) inline bool v_check_any(const v_float32x4& a) { return v_check_any(v_reinterpret_as_u32(a)); } +#if CV_SIMD128_64F +inline bool v_check_all(const v_int64x2& a) +{ return v_check_all(v_reinterpret_as_u64(a)); } +inline bool v_check_all(const v_float64x2& a) +{ return v_check_all(v_reinterpret_as_u64(a)); } +inline bool v_check_any(const v_int64x2& a) +{ return v_check_any(v_reinterpret_as_u64(a)); } +inline bool v_check_any(const v_float64x2& a) +{ return v_check_any(v_reinterpret_as_u64(a)); } +#endif + #define OPENCV_HAL_IMPL_NEON_SELECT(_Tpvec, suffix, usuffix) \ inline _Tpvec v_select(const _Tpvec& mask, const _Tpvec& a, const _Tpvec& b) \ { \ @@ -677,6 +859,9 @@ OPENCV_HAL_IMPL_NEON_SELECT(v_int16x8, s16, u16) OPENCV_HAL_IMPL_NEON_SELECT(v_uint32x4, u32, u32) OPENCV_HAL_IMPL_NEON_SELECT(v_int32x4, s32, u32) OPENCV_HAL_IMPL_NEON_SELECT(v_float32x4, f32, u32) +#if CV_SIMD128_64F +OPENCV_HAL_IMPL_NEON_SELECT(v_float64x2, f64, u64) +#endif #define OPENCV_HAL_IMPL_NEON_EXPAND(_Tpvec, _Tpwvec, _Tp, suffix) \ inline void v_expand(const _Tpvec& a, _Tpwvec& b0, _Tpwvec& b1) \ @@ -710,6 +895,27 @@ inline v_int32x4 v_load_expand_q(const schar* ptr) return v_int32x4(vmovl_s16(v1)); } +#if defined(__aarch64__) +#define OPENCV_HAL_IMPL_NEON_UNPACKS(_Tpvec, suffix) \ +inline void v_zip(const v_##_Tpvec& a0, const v_##_Tpvec& a1, v_##_Tpvec& b0, v_##_Tpvec& b1) \ +{ \ + b0.val = vzip1q_##suffix(a0.val, a1.val); \ + b1.val = vzip2q_##suffix(a0.val, a1.val); \ +} \ +inline v_##_Tpvec v_combine_low(const v_##_Tpvec& a, const v_##_Tpvec& b) \ +{ \ + return v_##_Tpvec(vcombine_##suffix(vget_low_##suffix(a.val), vget_low_##suffix(b.val))); \ +} \ +inline v_##_Tpvec v_combine_high(const v_##_Tpvec& a, const v_##_Tpvec& b) \ +{ \ + return v_##_Tpvec(vcombine_##suffix(vget_high_##suffix(a.val), vget_high_##suffix(b.val))); \ +} \ +inline void v_recombine(const v_##_Tpvec& a, const v_##_Tpvec& b, v_##_Tpvec& c, v_##_Tpvec& d) \ +{ \ + c.val = vcombine_##suffix(vget_low_##suffix(a.val), vget_low_##suffix(b.val)); \ + d.val = vcombine_##suffix(vget_high_##suffix(a.val), vget_high_##suffix(b.val)); \ +} +#else #define OPENCV_HAL_IMPL_NEON_UNPACKS(_Tpvec, suffix) \ inline void v_zip(const v_##_Tpvec& a0, const v_##_Tpvec& a1, v_##_Tpvec& b0, v_##_Tpvec& b1) \ { \ @@ -730,6 +936,7 @@ inline void v_recombine(const v_##_Tpvec& a, const v_##_Tpvec& b, v_##_Tpvec& c, c.val = vcombine_##suffix(vget_low_##suffix(a.val), vget_low_##suffix(b.val)); \ d.val = vcombine_##suffix(vget_high_##suffix(a.val), vget_high_##suffix(b.val)); \ } +#endif OPENCV_HAL_IMPL_NEON_UNPACKS(uint8x16, u8) OPENCV_HAL_IMPL_NEON_UNPACKS(int8x16, s8) @@ -738,6 +945,9 @@ OPENCV_HAL_IMPL_NEON_UNPACKS(int16x8, s16) OPENCV_HAL_IMPL_NEON_UNPACKS(uint32x4, u32) OPENCV_HAL_IMPL_NEON_UNPACKS(int32x4, s32) OPENCV_HAL_IMPL_NEON_UNPACKS(float32x4, f32) +#if CV_SIMD128_64F +OPENCV_HAL_IMPL_NEON_UNPACKS(float64x2, f64) +#endif #define OPENCV_HAL_IMPL_NEON_EXTRACT(_Tpvec, suffix) \ template \ @@ -755,6 +965,9 @@ OPENCV_HAL_IMPL_NEON_EXTRACT(int32x4, s32) OPENCV_HAL_IMPL_NEON_EXTRACT(uint64x2, u64) OPENCV_HAL_IMPL_NEON_EXTRACT(int64x2, s64) OPENCV_HAL_IMPL_NEON_EXTRACT(float32x4, f32) +#if CV_SIMD128_64F +OPENCV_HAL_IMPL_NEON_EXTRACT(float64x2, f64) +#endif inline v_int32x4 v_round(const v_float32x4& a) { @@ -782,6 +995,38 @@ inline v_int32x4 v_ceil(const v_float32x4& a) inline v_int32x4 v_trunc(const v_float32x4& a) { return v_int32x4(vcvtq_s32_f32(a.val)); } +#if CV_SIMD128_64F +inline v_int32x4 v_round(const v_float64x2& a) +{ + static const int32x2_t zero = vdup_n_s32(0); + return v_int32x4(vcombine_s32(vmovn_s64(vcvtaq_s64_f64(a.val)), zero)); +} + +inline v_int32x4 v_floor(const v_float64x2& a) +{ + static const int32x2_t zero = vdup_n_s32(0); + int64x2_t a1 = vcvtq_s64_f64(a.val); + uint64x2_t mask = vcgtq_f64(vcvtq_f64_s64(a1), a.val); + a1 = vaddq_s64(a1, vreinterpretq_s64_u64(mask)); + return v_int32x4(vcombine_s32(vmovn_s64(a1), zero)); +} + +inline v_int32x4 v_ceil(const v_float64x2& a) +{ + static const int32x2_t zero = vdup_n_s32(0); + int64x2_t a1 = vcvtq_s64_f64(a.val); + uint64x2_t mask = vcgtq_f64(a.val, vcvtq_f64_s64(a1)); + a1 = vsubq_s64(a1, vreinterpretq_s64_u64(mask)); + return v_int32x4(vcombine_s32(vmovn_s64(a1), zero)); +} + +inline v_int32x4 v_trunc(const v_float64x2& a) +{ + static const int32x2_t zero = vdup_n_s32(0); + return v_int32x4(vcombine_s32(vmovn_s64(vcvtaq_s64_f64(a.val)), zero)); +} +#endif + #define OPENCV_HAL_IMPL_NEON_TRANSPOSE4x4(_Tpvec, suffix) \ inline void v_transpose4x4(const v_##_Tpvec& a0, const v_##_Tpvec& a1, \ const v_##_Tpvec& a2, const v_##_Tpvec& a3, \ @@ -851,12 +1096,45 @@ OPENCV_HAL_IMPL_NEON_INTERLEAVED(int16x8, short, s16) OPENCV_HAL_IMPL_NEON_INTERLEAVED(uint32x4, unsigned, u32) OPENCV_HAL_IMPL_NEON_INTERLEAVED(int32x4, int, s32) OPENCV_HAL_IMPL_NEON_INTERLEAVED(float32x4, float, f32) +#if CV_SIMD128_64F +OPENCV_HAL_IMPL_NEON_INTERLEAVED(float64x2, double, f64) +#endif inline v_float32x4 v_cvt_f32(const v_int32x4& a) { return v_float32x4(vcvtq_f32_s32(a.val)); } +#if CV_SIMD128_64F +inline v_float32x4 v_cvt_f32(const v_float64x2& a) +{ + float32x2_t zero = vdup_n_f32(0.0f); + return v_float32x4(vcombine_f32(vcvt_f32_f64(a.val), zero)); +} + +inline v_float64x2 v_cvt_f64(const v_int32x4& a) +{ + return v_float64x2(vcvt_f64_f32(vcvt_f32_s32(vget_low_s32(a.val)))); +} + +inline v_float64x2 v_cvt_f64_high(const v_int32x4& a) +{ + return v_float64x2(vcvt_f64_f32(vcvt_f32_s32(vget_high_s32(a.val)))); +} + +inline v_float64x2 v_cvt_f64(const v_float32x4& a) +{ + return v_float64x2(vcvt_f64_f32(vget_low_f32(a.val))); +} + +inline v_float64x2 v_cvt_f64_high(const v_float32x4& a) +{ + return v_float64x2(vcvt_f64_f32(vget_high_f32(a.val))); +} +#endif + + + //! @endcond } diff --git a/modules/core/include/opencv2/core/hal/intrin_sse.hpp b/modules/core/include/opencv2/core/hal/intrin_sse.hpp index 1840e0305d307427d80eecb4f121d2e4cc7c7eac..8b49bbb46577f1663e93b26e1a9f747325f6a50b 100644 --- a/modules/core/include/opencv2/core/hal/intrin_sse.hpp +++ b/modules/core/include/opencv2/core/hal/intrin_sse.hpp @@ -1587,11 +1587,21 @@ inline v_float64x2 v_cvt_f64(const v_int32x4& a) return v_float64x2(_mm_cvtepi32_pd(a.val)); } +inline v_float64x2 v_cvt_f64_high(const v_int32x4& a) +{ + return v_float64x2(_mm_cvtepi32_pd(_mm_srli_si128(a.val,8))); +} + inline v_float64x2 v_cvt_f64(const v_float32x4& a) { return v_float64x2(_mm_cvtps_pd(a.val)); } +inline v_float64x2 v_cvt_f64_high(const v_float32x4& a) +{ + return v_float64x2(_mm_cvtps_pd(_mm_castsi128_ps(_mm_srli_si128(_mm_castps_si128(a.val),8)))); +} + //! @endcond } diff --git a/modules/core/test/test_intrin.cpp b/modules/core/test/test_intrin.cpp index ca9d3dc7b7e4f802e73944f4d195610b498db945..c1f2a31c0543907bc4a85aeca36839034896e5c4 100644 --- a/modules/core/test/test_intrin.cpp +++ b/modules/core/test/test_intrin.cpp @@ -626,12 +626,18 @@ template struct TheTest dataA *= 1.1; R a = dataA; Rt b = v_cvt_f64(a); + Rt c = v_cvt_f64_high(a); Data resB = b; + Data resC = c; int n = std::min(Rt::nlanes, R::nlanes); for (int i = 0; i < n; ++i) { EXPECT_EQ((typename Rt::lane_type)dataA[i], resB[i]); } + for (int i = 0; i < n; ++i) + { + EXPECT_EQ((typename Rt::lane_type)dataA[i+n], resC[i]); + } #endif return *this; } diff --git a/modules/imgproc/src/accum.cpp b/modules/imgproc/src/accum.cpp index 8626187fd889da28543b13b2438a773a716282dc..eaf85a2654fdba443e2fad2402fa39a9c89018c7 100644 --- a/modules/imgproc/src/accum.cpp +++ b/modules/imgproc/src/accum.cpp @@ -395,7 +395,7 @@ struct AccW_SIMD return x; } }; -#elif CV_SSE2 +#elif CV_SIMD128 template <> struct Acc_SIMD { @@ -408,8 +408,8 @@ struct Acc_SIMD len *= cn; for ( ; x <= len - 8; x += 8) { - _mm_storeu_ps(dst + x, _mm_add_ps(_mm_loadu_ps(dst + x), _mm_loadu_ps(src + x))); - _mm_storeu_ps(dst + x + 4, _mm_add_ps(_mm_loadu_ps(dst + x + 4), _mm_loadu_ps(src + x + 4))); + v_store(dst + x, v_load(dst + x) + v_load(src + x)); + v_store(dst + x + 4, v_load(dst + x + 4) + v_load(src + x + 4)); } } @@ -417,6 +417,7 @@ struct Acc_SIMD } }; +#if CV_SIMD128_64F template <> struct Acc_SIMD { @@ -429,17 +430,12 @@ struct Acc_SIMD len *= cn; for ( ; x <= len - 4; x += 4) { - __m128 v_src = _mm_loadu_ps(src + x); - __m128d v_src0 = _mm_cvtps_pd(v_src); - __m128d v_src1 = _mm_cvtps_pd(_mm_shuffle_ps(v_src, v_src, _MM_SHUFFLE(1, 0, 3, 2))); + v_float32x4 v_src = v_load(src + x); + v_float64x2 v_src0 = v_cvt_f64(v_src); + v_float64x2 v_src1 = v_cvt_f64_high(v_src); - __m128d v_dst0 = _mm_loadu_pd(dst + x); - __m128d v_dst1 = _mm_loadu_pd(dst + x + 2); - - v_dst0 = _mm_add_pd(v_dst0, v_src0); - v_dst1 = _mm_add_pd(v_dst1, v_src1); - _mm_storeu_pd(dst + x, v_dst0); - _mm_storeu_pd(dst + x + 2, v_dst1); + v_store(dst + x, v_load(dst + x) + v_src0); + v_store(dst + x + 2, v_load(dst + x + 2) + v_src1); } } return x; @@ -458,21 +454,17 @@ struct Acc_SIMD len *= cn; for ( ; x <= len - 4; x += 4) { - __m128d v_src0 = _mm_loadu_pd(src + x); - __m128d v_src1 = _mm_loadu_pd(src + x + 2); + v_float64x2 v_src0 = v_load(src + x); + v_float64x2 v_src1 = v_load(src + x + 2); - __m128d v_dst0 = _mm_loadu_pd(dst + x); - __m128d v_dst1 = _mm_loadu_pd(dst + x + 2); - - v_dst0 = _mm_add_pd(v_dst0, v_src0); - v_dst1 = _mm_add_pd(v_dst1, v_src1); - _mm_storeu_pd(dst + x, v_dst0); - _mm_storeu_pd(dst + x + 2, v_dst1); + v_store(dst + x, v_load(dst + x) + v_src0); + v_store(dst + x + 2, v_load(dst + x + 2) + v_src1); } } return x; } }; +#endif //CV_SIMD128_64F template <> struct AccSqr_SIMD @@ -486,12 +478,13 @@ struct AccSqr_SIMD len *= cn; for ( ; x <= len - 8; x += 8) { - __m128 v_src0 = _mm_loadu_ps(src + x); - __m128 v_src1 = _mm_loadu_ps(src + x + 4); - v_src0 = _mm_mul_ps(v_src0, v_src0); - v_src1 = _mm_mul_ps(v_src1, v_src1); - _mm_storeu_ps(dst + x, _mm_add_ps(_mm_loadu_ps(dst + x), v_src0)); - _mm_storeu_ps(dst + x + 4, _mm_add_ps(_mm_loadu_ps(dst + x + 4), v_src1)); + v_float32x4 v_src0 = v_load(src + x); + v_float32x4 v_src1 = v_load(src + x + 4); + v_src0 = v_src0 * v_src0; + v_src1 = v_src1 * v_src1; + + v_store(dst + x, v_load(dst + x) + v_src0); + v_store(dst + x + 4, v_load(dst + x + 4) + v_src1); } } @@ -499,6 +492,7 @@ struct AccSqr_SIMD } }; +#if CV_SIMD128_64F template <> struct AccSqr_SIMD { @@ -511,19 +505,14 @@ struct AccSqr_SIMD len *= cn; for ( ; x <= len - 4; x += 4) { - __m128 v_src = _mm_loadu_ps(src + x); - __m128d v_src0 = _mm_cvtps_pd(v_src); - __m128d v_src1 = _mm_cvtps_pd(_mm_shuffle_ps(v_src, v_src, _MM_SHUFFLE(1, 0, 3, 2))); - v_src0 = _mm_mul_pd(v_src0, v_src0); - v_src1 = _mm_mul_pd(v_src1, v_src1); - - __m128d v_dst0 = _mm_loadu_pd(dst + x); - __m128d v_dst1 = _mm_loadu_pd(dst + x + 2); - v_dst0 = _mm_add_pd(v_dst0, v_src0); - v_dst1 = _mm_add_pd(v_dst1, v_src1); - - _mm_storeu_pd(dst + x, v_dst0); - _mm_storeu_pd(dst + x + 2, v_dst1); + v_float32x4 v_src = v_load(src + x); + v_float64x2 v_src0 = v_cvt_f64(v_src); + v_float64x2 v_src1 = v_cvt_f64_high(v_src); + v_src0 = v_src0 * v_src0; + v_src1 = v_src1 * v_src1; + + v_store(dst + x, v_load(dst + x) + v_src0); + v_store(dst + x + 2, v_load(dst + x + 2) + v_src1); } } return x; @@ -542,23 +531,19 @@ struct AccSqr_SIMD len *= cn; for ( ; x <= len - 4; x += 4) { - __m128d v_src0 = _mm_loadu_pd(src + x); - __m128d v_src1 = _mm_loadu_pd(src + x + 2); - v_src0 = _mm_mul_pd(v_src0, v_src0); - v_src1 = _mm_mul_pd(v_src1, v_src1); - - __m128d v_dst0 = _mm_loadu_pd(dst + x); - __m128d v_dst1 = _mm_loadu_pd(dst + x + 2); - - v_dst0 = _mm_add_pd(v_dst0, v_src0); - v_dst1 = _mm_add_pd(v_dst1, v_src1); - _mm_storeu_pd(dst + x, v_dst0); - _mm_storeu_pd(dst + x + 2, v_dst1); + v_float64x2 v_src0 = v_load(src + x); + v_float64x2 v_src1 = v_load(src + x + 2); + v_src0 = v_src0 * v_src0; + v_src1 = v_src1 * v_src1; + + v_store(dst + x, v_load(dst + x) + v_src0); + v_store(dst + x + 2, v_load(dst + x + 2) + v_src1); } } return x; } }; +#endif //CV_SIMD128_64F template <> struct AccProd_SIMD @@ -572,8 +557,8 @@ struct AccProd_SIMD len *= cn; for ( ; x <= len - 8; x += 8) { - _mm_storeu_ps(dst + x, _mm_add_ps(_mm_loadu_ps(dst + x), _mm_mul_ps(_mm_loadu_ps(src1 + x), _mm_loadu_ps(src2 + x)))); - _mm_storeu_ps(dst + x + 4, _mm_add_ps(_mm_loadu_ps(dst + x + 4), _mm_mul_ps(_mm_loadu_ps(src1 + x + 4), _mm_loadu_ps(src2 + x + 4)))); + v_store(dst + x, v_load(dst + x) + v_load(src1 + x) * v_load(src2 + x)); + v_store(dst + x + 4, v_load(dst + x + 4) + v_load(src1 + x + 4) * v_load(src2 + x + 4)); } } @@ -581,6 +566,7 @@ struct AccProd_SIMD } }; +#if CV_SIMD128_64F template <> struct AccProd_SIMD { @@ -593,22 +579,16 @@ struct AccProd_SIMD len *= cn; for ( ; x <= len - 4; x += 4) { - __m128 v_1src = _mm_loadu_ps(src1 + x); - __m128 v_2src = _mm_loadu_ps(src2 + x); - - __m128d v_1src0 = _mm_cvtps_pd(v_1src); - __m128d v_1src1 = _mm_cvtps_pd(_mm_shuffle_ps(v_1src, v_1src, _MM_SHUFFLE(0, 0, 3, 2))); - __m128d v_2src0 = _mm_cvtps_pd(v_2src); - __m128d v_2src1 = _mm_cvtps_pd(_mm_shuffle_ps(v_2src, v_2src, _MM_SHUFFLE(0, 0, 3, 2))); + v_float32x4 v_1src = v_load(src1 + x); + v_float32x4 v_2src = v_load(src2 + x); - __m128d v_dst0 = _mm_loadu_pd(dst + x); - __m128d v_dst1 = _mm_loadu_pd(dst + x + 2); + v_float64x2 v_1src0 = v_cvt_f64(v_1src); + v_float64x2 v_1src1 = v_cvt_f64_high(v_1src); + v_float64x2 v_2src0 = v_cvt_f64(v_2src); + v_float64x2 v_2src1 = v_cvt_f64_high(v_2src); - v_dst0 = _mm_add_pd(v_dst0, _mm_mul_pd(v_1src0, v_2src0)); - v_dst1 = _mm_add_pd(v_dst1, _mm_mul_pd(v_1src1, v_2src1)); - - _mm_storeu_pd(dst + x, v_dst0); - _mm_storeu_pd(dst + x + 2, v_dst1); + v_store(dst + x, v_load(dst + x) + (v_1src0 * v_2src0)); + v_store(dst + x + 2, v_load(dst + x + 2) + (v_1src1 * v_2src1)); } } return x; @@ -627,25 +607,19 @@ struct AccProd_SIMD len *= cn; for ( ; x <= len - 4; x += 4) { - __m128d v_src00 = _mm_loadu_pd(src1 + x); - __m128d v_src01 = _mm_loadu_pd(src1 + x + 2); - __m128d v_src10 = _mm_loadu_pd(src2 + x); - __m128d v_src11 = _mm_loadu_pd(src2 + x + 2); - __m128d v_src0 = _mm_mul_pd(v_src00, v_src10); - __m128d v_src1 = _mm_mul_pd(v_src01, v_src11); - - __m128d v_dst0 = _mm_loadu_pd(dst + x); - __m128d v_dst1 = _mm_loadu_pd(dst + x + 2); - - v_dst0 = _mm_add_pd(v_dst0, v_src0); - v_dst1 = _mm_add_pd(v_dst1, v_src1); - _mm_storeu_pd(dst + x, v_dst0); - _mm_storeu_pd(dst + x + 2, v_dst1); + v_float64x2 v_src00 = v_load(src1 + x); + v_float64x2 v_src01 = v_load(src1 + x + 2); + v_float64x2 v_src10 = v_load(src2 + x); + v_float64x2 v_src11 = v_load(src2 + x + 2); + + v_store(dst + x, v_load(dst + x) + (v_src00 * v_src10)); + v_store(dst + x + 2, v_load(dst + x + 2) + (v_src01 * v_src11)); } } return x; } }; +#endif //CV_SIMD128_64F template <> struct AccW_SIMD @@ -653,16 +627,16 @@ struct AccW_SIMD int operator() (const float * src, float * dst, const uchar * mask, int len, int cn, float alpha) const { int x = 0; - __m128 v_alpha = _mm_set1_ps(alpha); - __m128 v_beta = _mm_set1_ps(1.0f - alpha); + v_float32x4 v_alpha = v_setall_f32(alpha); + v_float32x4 v_beta = v_setall_f32(1.0f - alpha); if (!mask) { len *= cn; for ( ; x <= len - 8; x += 8) { - _mm_storeu_ps(dst + x, _mm_add_ps(_mm_mul_ps(_mm_loadu_ps(dst + x), v_beta), _mm_mul_ps(_mm_loadu_ps(src + x), v_alpha))); - _mm_storeu_ps(dst + x + 4, _mm_add_ps(_mm_mul_ps(_mm_loadu_ps(dst + x + 4), v_beta), _mm_mul_ps(_mm_loadu_ps(src + x + 4), v_alpha))); + v_store(dst + x, ((v_load(dst + x) * v_beta) + (v_load(src + x) * v_alpha))); + v_store(dst + x + 4, ((v_load(dst + x + 4) * v_beta) + (v_load(src + x + 4) * v_alpha))); } } @@ -670,31 +644,32 @@ struct AccW_SIMD } }; +#if CV_SIMD128_64F template <> struct AccW_SIMD { int operator() (const float * src, double * dst, const uchar * mask, int len, int cn, double alpha) const { int x = 0; - __m128d v_alpha = _mm_set1_pd(alpha); - __m128d v_beta = _mm_set1_pd(1.0f - alpha); + v_float64x2 v_alpha = v_setall_f64(alpha); + v_float64x2 v_beta = v_setall_f64(1.0f - alpha); if (!mask) { len *= cn; for ( ; x <= len - 8; x += 8) { - __m128 v_src0 = _mm_loadu_ps(src + x); - __m128 v_src1 = _mm_loadu_ps(src + x + 4); - __m128d v_src00 = _mm_cvtps_pd(v_src0); - __m128d v_src01 = _mm_cvtps_pd(_mm_shuffle_ps(v_src0, v_src0, _MM_SHUFFLE(0, 0, 3, 2))); - __m128d v_src10 = _mm_cvtps_pd(v_src1); - __m128d v_src11 = _mm_cvtps_pd(_mm_shuffle_ps(v_src1, v_src1, _MM_SHUFFLE(0, 0, 3, 2))); - - _mm_storeu_pd(dst + x, _mm_add_pd(_mm_mul_pd(_mm_loadu_pd(dst + x), v_beta), _mm_mul_pd(v_src00, v_alpha))); - _mm_storeu_pd(dst + x + 2, _mm_add_pd(_mm_mul_pd(_mm_loadu_pd(dst + x + 2), v_beta), _mm_mul_pd(v_src01, v_alpha))); - _mm_storeu_pd(dst + x + 4, _mm_add_pd(_mm_mul_pd(_mm_loadu_pd(dst + x + 4), v_beta), _mm_mul_pd(v_src10, v_alpha))); - _mm_storeu_pd(dst + x + 6, _mm_add_pd(_mm_mul_pd(_mm_loadu_pd(dst + x + 6), v_beta), _mm_mul_pd(v_src11, v_alpha))); + v_float32x4 v_src0 = v_load(src + x); + v_float32x4 v_src1 = v_load(src + x + 4); + v_float64x2 v_src00 = v_cvt_f64(v_src0); + v_float64x2 v_src01 = v_cvt_f64_high(v_src0); + v_float64x2 v_src10 = v_cvt_f64(v_src1); + v_float64x2 v_src11 = v_cvt_f64_high(v_src1); + + v_store(dst + x, ((v_load(dst + x) * v_beta) + (v_src00 * v_alpha))); + v_store(dst + x + 2, ((v_load(dst + x + 2) * v_beta) + (v_src01 * v_alpha))); + v_store(dst + x + 4, ((v_load(dst + x + 4) * v_beta) + (v_src10 * v_alpha))); + v_store(dst + x + 6, ((v_load(dst + x + 6) * v_beta) + (v_src11 * v_alpha))); } } @@ -708,26 +683,27 @@ struct AccW_SIMD int operator() (const double * src, double * dst, const uchar * mask, int len, int cn, double alpha) const { int x = 0; - __m128d v_alpha = _mm_set1_pd(alpha); - __m128d v_beta = _mm_set1_pd(1.0f - alpha); + v_float64x2 v_alpha = v_setall_f64(alpha); + v_float64x2 v_beta = v_setall_f64(1.0f - alpha); if (!mask) { len *= cn; for ( ; x <= len - 4; x += 4) { - __m128d v_src0 = _mm_loadu_pd(src + x); - __m128d v_src1 = _mm_loadu_pd(src + x + 2); + v_float64x2 v_src0 = v_load(src + x); + v_float64x2 v_src1 = v_load(src + x + 2); - _mm_storeu_pd(dst + x, _mm_add_pd(_mm_mul_pd(_mm_loadu_pd(dst + x), v_beta), _mm_mul_pd(v_src0, v_alpha))); - _mm_storeu_pd(dst + x + 2, _mm_add_pd(_mm_mul_pd(_mm_loadu_pd(dst + x + 2), v_beta), _mm_mul_pd(v_src1, v_alpha))); + v_store(dst + x, ((v_load(dst + x) * v_beta) + (v_src0 * v_alpha))); + v_store(dst + x + 2, ((v_load(dst + x + 2) * v_beta) + (v_src1 * v_alpha))); } } return x; } }; -#endif +#endif //CV_SIMD128_64F +#endif //CV_SIMD128 #if CV_SIMD128 template <> @@ -742,7 +718,7 @@ struct Acc_SIMD len *= cn; for ( ; x <= len - 16; x += 16) { - v_uint8x16 v_src = v_load((const uchar*)(src + x)); + v_uint8x16 v_src = v_load(src + x); v_uint16x8 v_src0, v_src1; v_expand(v_src, v_src0, v_src1); @@ -762,9 +738,9 @@ struct Acc_SIMD for ( ; x <= len - 16; x += 16) { - v_uint8x16 v_mask = v_load((const uchar*)(mask + x)); + v_uint8x16 v_mask = v_load(mask + x); v_mask = ~(v_0 == v_mask); - v_uint8x16 v_src = v_load((const uchar*)(src + x)); + v_uint8x16 v_src = v_load(src + x); v_src = v_src & v_mask; v_uint16x8 v_src0, v_src1; v_expand(v_src, v_src0, v_src1); @@ -795,7 +771,7 @@ struct Acc_SIMD len *= cn; for ( ; x <= len - 8; x += 8) { - v_uint16x8 v_src = v_load((const ushort*)(src + x)); + v_uint16x8 v_src = v_load(src + x); v_uint32x4 v_src0, v_src1; v_expand(v_src, v_src0, v_src1); @@ -808,7 +784,7 @@ struct Acc_SIMD } }; -#if CV_SSE2 +#if CV_SIMD128_64F template <> struct Acc_SIMD { @@ -818,52 +794,52 @@ struct Acc_SIMD if (!mask) { - __m128i v_0 = _mm_setzero_si128(); len *= cn; for ( ; x <= len - 16; x += 16) { - __m128i v_src = _mm_loadu_si128((const __m128i*)(src + x)); - __m128i v_int0 = _mm_unpacklo_epi8(v_src, v_0); - __m128i v_int1 = _mm_unpackhi_epi8(v_src, v_0); - __m128i v_int00 = _mm_unpacklo_epi16(v_int0, v_0); - __m128i v_int01 = _mm_unpackhi_epi16(v_int0, v_0); - __m128i v_int10 = _mm_unpacklo_epi16(v_int1, v_0); - __m128i v_int11 = _mm_unpackhi_epi16(v_int1, v_0); - __m128d v_src0 = _mm_cvtepi32_pd(v_int00); - __m128d v_src1 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int00, _MM_SHUFFLE(0, 0, 3, 2))); - __m128d v_src2 = _mm_cvtepi32_pd(v_int01); - __m128d v_src3 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int01, _MM_SHUFFLE(0, 0, 3, 2))); - __m128d v_src4 = _mm_cvtepi32_pd(v_int10); - __m128d v_src5 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int10, _MM_SHUFFLE(0, 0, 3, 2))); - __m128d v_src6 = _mm_cvtepi32_pd(v_int11); - __m128d v_src7 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int11, _MM_SHUFFLE(0, 0, 3, 2))); - - __m128d v_dst0 = _mm_loadu_pd(dst + x); - __m128d v_dst1 = _mm_loadu_pd(dst + x + 2); - __m128d v_dst2 = _mm_loadu_pd(dst + x + 4); - __m128d v_dst3 = _mm_loadu_pd(dst + x + 6); - __m128d v_dst4 = _mm_loadu_pd(dst + x + 8); - __m128d v_dst5 = _mm_loadu_pd(dst + x + 10); - __m128d v_dst6 = _mm_loadu_pd(dst + x + 12); - __m128d v_dst7 = _mm_loadu_pd(dst + x + 14); - - v_dst0 = _mm_add_pd(v_dst0, v_src0); - v_dst1 = _mm_add_pd(v_dst1, v_src1); - v_dst2 = _mm_add_pd(v_dst2, v_src2); - v_dst3 = _mm_add_pd(v_dst3, v_src3); - v_dst4 = _mm_add_pd(v_dst4, v_src4); - v_dst5 = _mm_add_pd(v_dst5, v_src5); - v_dst6 = _mm_add_pd(v_dst6, v_src6); - v_dst7 = _mm_add_pd(v_dst7, v_src7); - - _mm_storeu_pd(dst + x, v_dst0); - _mm_storeu_pd(dst + x + 2, v_dst1); - _mm_storeu_pd(dst + x + 4, v_dst2); - _mm_storeu_pd(dst + x + 6, v_dst3); - _mm_storeu_pd(dst + x + 8, v_dst4); - _mm_storeu_pd(dst + x + 10, v_dst5); - _mm_storeu_pd(dst + x + 12, v_dst6); - _mm_storeu_pd(dst + x + 14, v_dst7); + v_uint8x16 v_src = v_load(src + x); + v_uint16x8 v_int0, v_int1; + v_expand(v_src, v_int0, v_int1); + + v_uint32x4 v_int00, v_int01, v_int10, v_int11; + v_expand(v_int0, v_int00, v_int01); + v_expand(v_int1, v_int10, v_int11); + + v_float64x2 v_src0 = v_cvt_f64(v_reinterpret_as_s32(v_int00)); + v_float64x2 v_src1 = v_cvt_f64_high(v_reinterpret_as_s32(v_int00)); + v_float64x2 v_src2 = v_cvt_f64(v_reinterpret_as_s32(v_int01)); + v_float64x2 v_src3 = v_cvt_f64_high(v_reinterpret_as_s32(v_int01)); + v_float64x2 v_src4 = v_cvt_f64(v_reinterpret_as_s32(v_int10)); + v_float64x2 v_src5 = v_cvt_f64_high(v_reinterpret_as_s32(v_int10)); + v_float64x2 v_src6 = v_cvt_f64(v_reinterpret_as_s32(v_int11)); + v_float64x2 v_src7 = v_cvt_f64_high(v_reinterpret_as_s32(v_int11)); + + v_float64x2 v_dst0 = v_load(dst + x); + v_float64x2 v_dst1 = v_load(dst + x + 2); + v_float64x2 v_dst2 = v_load(dst + x + 4); + v_float64x2 v_dst3 = v_load(dst + x + 6); + v_float64x2 v_dst4 = v_load(dst + x + 8); + v_float64x2 v_dst5 = v_load(dst + x + 10); + v_float64x2 v_dst6 = v_load(dst + x + 12); + v_float64x2 v_dst7 = v_load(dst + x + 14); + + v_dst0 = v_dst0 + v_src0; + v_dst1 = v_dst1 + v_src1; + v_dst2 = v_dst2 + v_src2; + v_dst3 = v_dst3 + v_src3; + v_dst4 = v_dst4 + v_src4; + v_dst5 = v_dst5 + v_src5; + v_dst6 = v_dst6 + v_src6; + v_dst7 = v_dst7 + v_src7; + + v_store(dst + x, v_dst0); + v_store(dst + x + 2, v_dst1); + v_store(dst + x + 4, v_dst2); + v_store(dst + x + 6, v_dst3); + v_store(dst + x + 8, v_dst4); + v_store(dst + x + 10, v_dst5); + v_store(dst + x + 12, v_dst6); + v_store(dst + x + 14, v_dst7); } } return x; @@ -879,32 +855,32 @@ struct Acc_SIMD if (!mask) { - __m128i v_0 = _mm_setzero_si128(); len *= cn; for ( ; x <= len - 8; x += 8) { - __m128i v_src = _mm_loadu_si128((const __m128i*)(src + x)); - __m128i v_int0 = _mm_unpacklo_epi16(v_src, v_0); - __m128i v_int1 = _mm_unpackhi_epi16(v_src, v_0); - __m128d v_src0 = _mm_cvtepi32_pd(v_int0); - __m128d v_src1 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int0, _MM_SHUFFLE(0, 0, 3, 2))); - __m128d v_src2 = _mm_cvtepi32_pd(v_int1); - __m128d v_src3 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int1, _MM_SHUFFLE(0, 0, 3, 2))); - - __m128d v_dst0 = _mm_loadu_pd(dst + x); - __m128d v_dst1 = _mm_loadu_pd(dst + x + 2); - __m128d v_dst2 = _mm_loadu_pd(dst + x + 4); - __m128d v_dst3 = _mm_loadu_pd(dst + x + 6); - - v_dst0 = _mm_add_pd(v_dst0, v_src0); - v_dst1 = _mm_add_pd(v_dst1, v_src1); - v_dst2 = _mm_add_pd(v_dst2, v_src2); - v_dst3 = _mm_add_pd(v_dst3, v_src3); - - _mm_storeu_pd(dst + x, v_dst0); - _mm_storeu_pd(dst + x + 2, v_dst1); - _mm_storeu_pd(dst + x + 4, v_dst2); - _mm_storeu_pd(dst + x + 6, v_dst3); + v_uint16x8 v_src = v_load(src + x); + v_uint32x4 v_int0, v_int1; + v_expand(v_src, v_int0, v_int1); + + v_float64x2 v_src0 = v_cvt_f64(v_reinterpret_as_s32(v_int0)); + v_float64x2 v_src1 = v_cvt_f64_high(v_reinterpret_as_s32(v_int0)); + v_float64x2 v_src2 = v_cvt_f64(v_reinterpret_as_s32(v_int1)); + v_float64x2 v_src3 = v_cvt_f64_high(v_reinterpret_as_s32(v_int1)); + + v_float64x2 v_dst0 = v_load(dst + x); + v_float64x2 v_dst1 = v_load(dst + x + 2); + v_float64x2 v_dst2 = v_load(dst + x + 4); + v_float64x2 v_dst3 = v_load(dst + x + 6); + + v_dst0 = v_dst0 + v_src0; + v_dst1 = v_dst1 + v_src1; + v_dst2 = v_dst2 + v_src2; + v_dst3 = v_dst3 + v_src3; + + v_store(dst + x, v_dst0); + v_store(dst + x + 2, v_dst1); + v_store(dst + x + 4, v_dst2); + v_store(dst + x + 6, v_dst3); } } return x; @@ -924,7 +900,7 @@ struct AccSqr_SIMD len *= cn; for ( ; x <= len - 16; x += 16) { - v_uint8x16 v_src = v_load((const uchar*)(src + x)); + v_uint8x16 v_src = v_load(src + x); v_uint16x8 v_src0, v_src1; v_expand(v_src, v_src0, v_src1); v_src0 = v_src0 * v_src0; @@ -945,9 +921,9 @@ struct AccSqr_SIMD v_uint8x16 v_0 = v_setall_u8(0); for ( ; x <= len - 16; x += 16) { - v_uint8x16 v_mask = v_load((const uchar*)(mask + x)); + v_uint8x16 v_mask = v_load(mask + x); v_mask = ~(v_0 == v_mask); - v_uint8x16 v_src = v_load((const uchar*)(src + x)); + v_uint8x16 v_src = v_load(src + x); v_src = v_src & v_mask; v_uint16x8 v_src0, v_src1; v_expand(v_src, v_src0, v_src1); @@ -981,7 +957,7 @@ struct AccSqr_SIMD len *= cn; for ( ; x <= len - 8; x += 8) { - v_uint16x8 v_src = v_load((const ushort*)(src + x)); + v_uint16x8 v_src = v_load(src + x); v_uint32x4 v_src0, v_src1; v_expand(v_src, v_src0, v_src1); @@ -1000,7 +976,7 @@ struct AccSqr_SIMD } }; -#if CV_SSE2 +#if CV_SIMD128_64F template <> struct AccSqr_SIMD { @@ -1010,37 +986,39 @@ struct AccSqr_SIMD if (!mask) { - __m128i v_0 = _mm_setzero_si128(); len *= cn; for ( ; x <= len - 8; x += 8) { - __m128i v_src = _mm_loadl_epi64((const __m128i*)(src + x)); - __m128i v_int = _mm_unpacklo_epi8(v_src, v_0); - __m128i v_int0 = _mm_unpacklo_epi16(v_int, v_0); - __m128i v_int1 = _mm_unpackhi_epi16(v_int, v_0); - __m128d v_src0 = _mm_cvtepi32_pd(v_int0); - __m128d v_src1 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int0, _MM_SHUFFLE(0, 0, 3, 2))); - __m128d v_src2 = _mm_cvtepi32_pd(v_int1); - __m128d v_src3 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int1, _MM_SHUFFLE(0, 0, 3, 2))); - v_src0 = _mm_mul_pd(v_src0, v_src0); - v_src1 = _mm_mul_pd(v_src1, v_src1); - v_src2 = _mm_mul_pd(v_src2, v_src2); - v_src3 = _mm_mul_pd(v_src3, v_src3); - - __m128d v_dst0 = _mm_loadu_pd(dst + x); - __m128d v_dst1 = _mm_loadu_pd(dst + x + 2); - __m128d v_dst2 = _mm_loadu_pd(dst + x + 4); - __m128d v_dst3 = _mm_loadu_pd(dst + x + 6); - - v_dst0 = _mm_add_pd(v_dst0, v_src0); - v_dst1 = _mm_add_pd(v_dst1, v_src1); - v_dst2 = _mm_add_pd(v_dst2, v_src2); - v_dst3 = _mm_add_pd(v_dst3, v_src3); - - _mm_storeu_pd(dst + x, v_dst0); - _mm_storeu_pd(dst + x + 2, v_dst1); - _mm_storeu_pd(dst + x + 4, v_dst2); - _mm_storeu_pd(dst + x + 6, v_dst3); + v_uint8x16 v_src = v_load(src + x); + v_uint16x8 v_int, dummy; + v_expand(v_src, v_int, dummy); + + v_uint32x4 v_int0, v_int1; + v_expand(v_int, v_int0, v_int1); + + v_float64x2 v_src0 = v_cvt_f64(v_reinterpret_as_s32(v_int0)); + v_float64x2 v_src1 = v_cvt_f64_high(v_reinterpret_as_s32(v_int0)); + v_float64x2 v_src2 = v_cvt_f64(v_reinterpret_as_s32(v_int1)); + v_float64x2 v_src3 = v_cvt_f64_high(v_reinterpret_as_s32(v_int1)); + v_src0 = v_src0 * v_src0; + v_src1 = v_src1 * v_src1; + v_src2 = v_src2 * v_src2; + v_src3 = v_src3 * v_src3; + + v_float64x2 v_dst0 = v_load(dst + x); + v_float64x2 v_dst1 = v_load(dst + x + 2); + v_float64x2 v_dst2 = v_load(dst + x + 4); + v_float64x2 v_dst3 = v_load(dst + x + 6); + + v_dst0 += v_src0; + v_dst1 += v_src1; + v_dst2 += v_src2; + v_dst3 += v_src3; + + v_store(dst + x, v_dst0); + v_store(dst + x + 2, v_dst1); + v_store(dst + x + 4, v_dst2); + v_store(dst + x + 6, v_dst3); } } return x; @@ -1056,36 +1034,39 @@ struct AccSqr_SIMD if (!mask) { - __m128i v_0 = _mm_setzero_si128(); len *= cn; for ( ; x <= len - 8; x += 8) { - __m128i v_src = _mm_loadu_si128((const __m128i*)(src + x)); - __m128i v_int0 = _mm_unpacklo_epi16(v_src, v_0); - __m128i v_int1 = _mm_unpackhi_epi16(v_src, v_0); - __m128d v_src0 = _mm_cvtepi32_pd(v_int0); - __m128d v_src1 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int0, _MM_SHUFFLE(0, 0, 3, 2))); - __m128d v_src2 = _mm_cvtepi32_pd(v_int1); - __m128d v_src3 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int1, _MM_SHUFFLE(0, 0, 3, 2))); - v_src0 = _mm_mul_pd(v_src0, v_src0); - v_src1 = _mm_mul_pd(v_src1, v_src1); - v_src2 = _mm_mul_pd(v_src2, v_src2); - v_src3 = _mm_mul_pd(v_src3, v_src3); - - __m128d v_dst0 = _mm_loadu_pd(dst + x); - __m128d v_dst1 = _mm_loadu_pd(dst + x + 2); - __m128d v_dst2 = _mm_loadu_pd(dst + x + 4); - __m128d v_dst3 = _mm_loadu_pd(dst + x + 6); - - v_dst0 = _mm_add_pd(v_dst0, v_src0); - v_dst1 = _mm_add_pd(v_dst1, v_src1); - v_dst2 = _mm_add_pd(v_dst2, v_src2); - v_dst3 = _mm_add_pd(v_dst3, v_src3); - - _mm_storeu_pd(dst + x, v_dst0); - _mm_storeu_pd(dst + x + 2, v_dst1); - _mm_storeu_pd(dst + x + 4, v_dst2); - _mm_storeu_pd(dst + x + 6, v_dst3); + v_uint16x8 v_src = v_load(src + x); + v_uint32x4 v_int_0, v_int_1; + v_expand(v_src, v_int_0, v_int_1); + + v_int32x4 v_int0 = v_reinterpret_as_s32(v_int_0); + v_int32x4 v_int1 = v_reinterpret_as_s32(v_int_1); + + v_float64x2 v_src0 = v_cvt_f64(v_int0); + v_float64x2 v_src1 = v_cvt_f64_high(v_int0); + v_float64x2 v_src2 = v_cvt_f64(v_int1); + v_float64x2 v_src3 = v_cvt_f64_high(v_int1); + v_src0 = v_src0 * v_src0; + v_src1 = v_src1 * v_src1; + v_src2 = v_src2 * v_src2; + v_src3 = v_src3 * v_src3; + + v_float64x2 v_dst0 = v_load(dst + x); + v_float64x2 v_dst1 = v_load(dst + x + 2); + v_float64x2 v_dst2 = v_load(dst + x + 4); + v_float64x2 v_dst3 = v_load(dst + x + 6); + + v_dst0 += v_src0; + v_dst1 += v_src1; + v_dst2 += v_src2; + v_dst3 += v_src3; + + v_store(dst + x, v_dst0); + v_store(dst + x + 2, v_dst1); + v_store(dst + x + 4, v_dst2); + v_store(dst + x + 6, v_dst3); } } return x; @@ -1227,7 +1208,7 @@ struct AccProd_SIMD } }; -#if CV_SSE2 +#if CV_SIMD128_64F template <> struct AccProd_SIMD { @@ -1237,38 +1218,44 @@ struct AccProd_SIMD if (!mask) { - __m128i v_0 = _mm_setzero_si128(); len *= cn; for ( ; x <= len - 8; x += 8) { - __m128i v_1src = _mm_loadl_epi64((const __m128i*)(src1 + x)); - __m128i v_2src = _mm_loadl_epi64((const __m128i*)(src2 + x)); - - __m128i v_1int = _mm_unpacklo_epi8(v_1src, v_0); - __m128i v_2int = _mm_unpacklo_epi8(v_2src, v_0); - __m128i v_1int0 = _mm_unpacklo_epi16(v_1int, v_0); - __m128i v_1int1 = _mm_unpackhi_epi16(v_1int, v_0); - __m128i v_2int0 = _mm_unpacklo_epi16(v_2int, v_0); - __m128i v_2int1 = _mm_unpackhi_epi16(v_2int, v_0); - __m128d v_src0 = _mm_mul_pd(_mm_cvtepi32_pd(v_1int0),_mm_cvtepi32_pd(v_2int0)); - __m128d v_src1 = _mm_mul_pd(_mm_cvtepi32_pd(_mm_shuffle_epi32(v_1int0, _MM_SHUFFLE(0, 0, 3, 2))),_mm_cvtepi32_pd(_mm_shuffle_epi32(v_2int0, _MM_SHUFFLE(0, 0, 3, 2)))); - __m128d v_src2 = _mm_mul_pd(_mm_cvtepi32_pd(v_1int1),_mm_cvtepi32_pd(v_2int1)); - __m128d v_src3 = _mm_mul_pd(_mm_cvtepi32_pd(_mm_shuffle_epi32(v_1int1, _MM_SHUFFLE(0, 0, 3, 2))),_mm_cvtepi32_pd(_mm_shuffle_epi32(v_2int1, _MM_SHUFFLE(0, 0, 3, 2)))); - - __m128d v_dst0 = _mm_loadu_pd(dst + x); - __m128d v_dst1 = _mm_loadu_pd(dst + x + 2); - __m128d v_dst2 = _mm_loadu_pd(dst + x + 4); - __m128d v_dst3 = _mm_loadu_pd(dst + x + 6); - - v_dst0 = _mm_add_pd(v_dst0, v_src0); - v_dst1 = _mm_add_pd(v_dst1, v_src1); - v_dst2 = _mm_add_pd(v_dst2, v_src2); - v_dst3 = _mm_add_pd(v_dst3, v_src3); - - _mm_storeu_pd(dst + x, v_dst0); - _mm_storeu_pd(dst + x + 2, v_dst1); - _mm_storeu_pd(dst + x + 4, v_dst2); - _mm_storeu_pd(dst + x + 6, v_dst3); + v_uint8x16 v_1src = v_load(src1 + x); + v_uint8x16 v_2src = v_load(src2 + x); + + v_uint16x8 v_1int, v_2int, dummy; + v_expand(v_1src, v_1int, dummy); + v_expand(v_2src, v_2int, dummy); + + v_uint32x4 v_1int_0, v_1int_1, v_2int_0, v_2int_1; + v_expand(v_1int, v_1int_0, v_1int_1); + v_expand(v_2int, v_2int_0, v_2int_1); + + v_int32x4 v_1int0 = v_reinterpret_as_s32(v_1int_0); + v_int32x4 v_1int1 = v_reinterpret_as_s32(v_1int_1); + v_int32x4 v_2int0 = v_reinterpret_as_s32(v_2int_0); + v_int32x4 v_2int1 = v_reinterpret_as_s32(v_2int_1); + + v_float64x2 v_src0 = v_cvt_f64(v_1int0) * v_cvt_f64(v_2int0); + v_float64x2 v_src1 = v_cvt_f64_high(v_1int0) * v_cvt_f64_high(v_2int0); + v_float64x2 v_src2 = v_cvt_f64(v_1int1) * v_cvt_f64(v_2int1); + v_float64x2 v_src3 = v_cvt_f64_high(v_1int1) * v_cvt_f64_high(v_2int1); + + v_float64x2 v_dst0 = v_load(dst + x); + v_float64x2 v_dst1 = v_load(dst + x + 2); + v_float64x2 v_dst2 = v_load(dst + x + 4); + v_float64x2 v_dst3 = v_load(dst + x + 6); + + v_dst0 += v_src0; + v_dst1 += v_src1; + v_dst2 += v_src2; + v_dst3 += v_src3; + + v_store(dst + x, v_dst0); + v_store(dst + x + 2, v_dst1); + v_store(dst + x + 4, v_dst2); + v_store(dst + x + 6, v_dst3); } } return x; @@ -1284,35 +1271,40 @@ struct AccProd_SIMD if (!mask) { - __m128i v_0 = _mm_setzero_si128(); len *= cn; for ( ; x <= len - 8; x += 8) { - __m128i v_1src = _mm_loadu_si128((const __m128i*)(src1 + x)); - __m128i v_2src = _mm_loadu_si128((const __m128i*)(src2 + x)); - __m128i v_1int0 = _mm_unpacklo_epi16(v_1src, v_0); - __m128i v_1int1 = _mm_unpackhi_epi16(v_1src, v_0); - __m128i v_2int0 = _mm_unpacklo_epi16(v_2src, v_0); - __m128i v_2int1 = _mm_unpackhi_epi16(v_2src, v_0); - __m128d v_src0 = _mm_mul_pd(_mm_cvtepi32_pd(v_1int0),_mm_cvtepi32_pd(v_2int0)); - __m128d v_src1 = _mm_mul_pd(_mm_cvtepi32_pd(_mm_shuffle_epi32(v_1int0, _MM_SHUFFLE(0, 0, 3, 2))),_mm_cvtepi32_pd(_mm_shuffle_epi32(v_2int0, _MM_SHUFFLE(0, 0, 3, 2)))); - __m128d v_src2 = _mm_mul_pd(_mm_cvtepi32_pd(v_1int1),_mm_cvtepi32_pd(v_2int1)); - __m128d v_src3 = _mm_mul_pd(_mm_cvtepi32_pd(_mm_shuffle_epi32(v_1int1, _MM_SHUFFLE(0, 0, 3, 2))),_mm_cvtepi32_pd(_mm_shuffle_epi32(v_2int1, _MM_SHUFFLE(0, 0, 3, 2)))); - - __m128d v_dst0 = _mm_loadu_pd(dst + x); - __m128d v_dst1 = _mm_loadu_pd(dst + x + 2); - __m128d v_dst2 = _mm_loadu_pd(dst + x + 4); - __m128d v_dst3 = _mm_loadu_pd(dst + x + 6); - - v_dst0 = _mm_add_pd(v_dst0, v_src0); - v_dst1 = _mm_add_pd(v_dst1, v_src1); - v_dst2 = _mm_add_pd(v_dst2, v_src2); - v_dst3 = _mm_add_pd(v_dst3, v_src3); - - _mm_storeu_pd(dst + x, v_dst0); - _mm_storeu_pd(dst + x + 2, v_dst1); - _mm_storeu_pd(dst + x + 4, v_dst2); - _mm_storeu_pd(dst + x + 6, v_dst3); + v_uint16x8 v_1src = v_load(src1 + x); + v_uint16x8 v_2src = v_load(src2 + x); + + v_uint32x4 v_1int_0, v_1int_1, v_2int_0, v_2int_1; + v_expand(v_1src, v_1int_0, v_1int_1); + v_expand(v_2src, v_2int_0, v_2int_1); + + v_int32x4 v_1int0 = v_reinterpret_as_s32(v_1int_0); + v_int32x4 v_1int1 = v_reinterpret_as_s32(v_1int_1); + v_int32x4 v_2int0 = v_reinterpret_as_s32(v_2int_0); + v_int32x4 v_2int1 = v_reinterpret_as_s32(v_2int_1); + + v_float64x2 v_src0 = v_cvt_f64(v_1int0) * v_cvt_f64(v_2int0); + v_float64x2 v_src1 = v_cvt_f64_high(v_1int0) * v_cvt_f64_high(v_2int0); + v_float64x2 v_src2 = v_cvt_f64(v_1int1) * v_cvt_f64(v_2int1); + v_float64x2 v_src3 = v_cvt_f64_high(v_1int1) * v_cvt_f64_high(v_2int1); + + v_float64x2 v_dst0 = v_load(dst + x); + v_float64x2 v_dst1 = v_load(dst + x + 2); + v_float64x2 v_dst2 = v_load(dst + x + 4); + v_float64x2 v_dst3 = v_load(dst + x + 6); + + v_dst0 = v_dst0 + v_src0; + v_dst1 = v_dst1 + v_src1; + v_dst2 = v_dst2 + v_src2; + v_dst3 = v_dst3 + v_src3; + + v_store(dst + x, v_dst0); + v_store(dst + x + 2, v_dst1); + v_store(dst + x + 4, v_dst2); + v_store(dst + x + 6, v_dst3); } } return x; @@ -1399,45 +1391,50 @@ struct AccW_SIMD } }; -#if CV_SSE2 +#if CV_SIMD128_64F template <> struct AccW_SIMD { int operator() (const uchar * src, double * dst, const uchar * mask, int len, int cn, double alpha) const { int x = 0; - __m128d v_alpha = _mm_set1_pd(alpha); - __m128d v_beta = _mm_set1_pd(1.0f - alpha); - __m128i v_0 = _mm_setzero_si128(); + v_float64x2 v_alpha = v_setall_f64(alpha); + v_float64x2 v_beta = v_setall_f64(1.0f - alpha); if (!mask) { len *= cn; for ( ; x <= len - 8; x += 8) { - __m128i v_src = _mm_loadl_epi64((const __m128i*)(src + x)); - __m128i v_int = _mm_unpacklo_epi8(v_src, v_0); - __m128i v_int0 = _mm_unpacklo_epi16(v_int, v_0); - __m128i v_int1 = _mm_unpackhi_epi16(v_int, v_0); - __m128d v_src0 = _mm_cvtepi32_pd(v_int0); - __m128d v_src1 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int0, _MM_SHUFFLE(0, 0, 3, 2))); - __m128d v_src2 = _mm_cvtepi32_pd(v_int1); - __m128d v_src3 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int1, _MM_SHUFFLE(0, 0, 3, 2))); - - __m128d v_dst0 = _mm_loadu_pd(dst + x); - __m128d v_dst1 = _mm_loadu_pd(dst + x + 2); - __m128d v_dst2 = _mm_loadu_pd(dst + x + 4); - __m128d v_dst3 = _mm_loadu_pd(dst + x + 6); - - v_dst0 = _mm_add_pd(_mm_mul_pd(v_dst0, v_beta), _mm_mul_pd(v_src0, v_alpha)); - v_dst1 = _mm_add_pd(_mm_mul_pd(v_dst1, v_beta), _mm_mul_pd(v_src1, v_alpha)); - v_dst2 = _mm_add_pd(_mm_mul_pd(v_dst2, v_beta), _mm_mul_pd(v_src2, v_alpha)); - v_dst3 = _mm_add_pd(_mm_mul_pd(v_dst3, v_beta), _mm_mul_pd(v_src3, v_alpha)); - - _mm_storeu_pd(dst + x, v_dst0); - _mm_storeu_pd(dst + x + 2, v_dst1); - _mm_storeu_pd(dst + x + 4, v_dst2); - _mm_storeu_pd(dst + x + 6, v_dst3); + v_uint8x16 v_src = v_load(src + x); + v_uint16x8 v_int, dummy; + v_expand(v_src, v_int, dummy); + + v_uint32x4 v_int_0, v_int_1; + v_expand(v_int, v_int_0, v_int_1); + + v_int32x4 v_int0 = v_reinterpret_as_s32(v_int_0); + v_int32x4 v_int1 = v_reinterpret_as_s32(v_int_1); + + v_float64x2 v_src0 = v_cvt_f64(v_int0); + v_float64x2 v_src1 = v_cvt_f64_high(v_int0); + v_float64x2 v_src2 = v_cvt_f64(v_int1); + v_float64x2 v_src3 = v_cvt_f64_high(v_int1); + + v_float64x2 v_dst0 = v_load(dst + x); + v_float64x2 v_dst1 = v_load(dst + x + 2); + v_float64x2 v_dst2 = v_load(dst + x + 4); + v_float64x2 v_dst3 = v_load(dst + x + 6); + + v_dst0 = (v_dst0 * v_beta) + (v_src0 * v_alpha); + v_dst1 = (v_dst1 * v_beta) + (v_src1 * v_alpha); + v_dst2 = (v_dst2 * v_beta) + (v_src2 * v_alpha); + v_dst3 = (v_dst3 * v_beta) + (v_src3 * v_alpha); + + v_store(dst + x, v_dst0); + v_store(dst + x + 2, v_dst1); + v_store(dst + x + 4, v_dst2); + v_store(dst + x + 6, v_dst3); } } @@ -1451,44 +1448,47 @@ struct AccW_SIMD int operator() (const ushort * src, double * dst, const uchar * mask, int len, int cn, double alpha) const { int x = 0; - __m128d v_alpha = _mm_set1_pd(alpha); - __m128d v_beta = _mm_set1_pd(1.0f - alpha); - __m128i v_0 = _mm_setzero_si128(); + v_float64x2 v_alpha = v_setall_f64(alpha); + v_float64x2 v_beta = v_setall_f64(1.0f - alpha); if (!mask) { len *= cn; for ( ; x <= len - 8; x += 8) { - __m128i v_src = _mm_loadu_si128((const __m128i*)(src + x)); - __m128i v_int0 = _mm_unpacklo_epi16(v_src, v_0); - __m128i v_int1 = _mm_unpackhi_epi16(v_src, v_0); - __m128d v_src00 = _mm_cvtepi32_pd(v_int0); - __m128d v_src01 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int0, _MM_SHUFFLE(0, 0, 3, 2))); - __m128d v_src10 = _mm_cvtepi32_pd(v_int1); - __m128d v_src11 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int1, _MM_SHUFFLE(0, 0, 3, 2))); - - __m128d v_dst00 = _mm_loadu_pd(dst + x); - __m128d v_dst01 = _mm_loadu_pd(dst + x + 2); - __m128d v_dst10 = _mm_loadu_pd(dst + x + 4); - __m128d v_dst11 = _mm_loadu_pd(dst + x + 6); - - v_dst00 = _mm_add_pd(_mm_mul_pd(v_dst00, v_beta), _mm_mul_pd(v_src00, v_alpha)); - v_dst01 = _mm_add_pd(_mm_mul_pd(v_dst01, v_beta), _mm_mul_pd(v_src01, v_alpha)); - v_dst10 = _mm_add_pd(_mm_mul_pd(v_dst10, v_beta), _mm_mul_pd(v_src10, v_alpha)); - v_dst11 = _mm_add_pd(_mm_mul_pd(v_dst11, v_beta), _mm_mul_pd(v_src11, v_alpha)); - - _mm_storeu_pd(dst + x, v_dst00); - _mm_storeu_pd(dst + x + 2, v_dst01); - _mm_storeu_pd(dst + x + 4, v_dst10); - _mm_storeu_pd(dst + x + 6, v_dst11); + v_uint16x8 v_src = v_load(src + x); + v_uint32x4 v_int_0, v_int_1; + v_expand(v_src, v_int_0, v_int_1); + + v_int32x4 v_int0 = v_reinterpret_as_s32(v_int_0); + v_int32x4 v_int1 = v_reinterpret_as_s32(v_int_1); + + v_float64x2 v_src00 = v_cvt_f64(v_int0); + v_float64x2 v_src01 = v_cvt_f64_high(v_int0); + v_float64x2 v_src10 = v_cvt_f64(v_int1); + v_float64x2 v_src11 = v_cvt_f64_high(v_int1); + + v_float64x2 v_dst00 = v_load(dst + x); + v_float64x2 v_dst01 = v_load(dst + x + 2); + v_float64x2 v_dst10 = v_load(dst + x + 4); + v_float64x2 v_dst11 = v_load(dst + x + 6); + + v_dst00 = (v_dst00 * v_beta) + (v_src00 * v_alpha); + v_dst01 = (v_dst01 * v_beta) + (v_src01 * v_alpha); + v_dst10 = (v_dst10 * v_beta) + (v_src10 * v_alpha); + v_dst11 = (v_dst11 * v_beta) + (v_src11 * v_alpha); + + v_store(dst + x, v_dst00); + v_store(dst + x + 2, v_dst01); + v_store(dst + x + 4, v_dst10); + v_store(dst + x + 6, v_dst11); } } return x; } }; -#endif //CV_SSE2 +#endif //CV_SIMD128_64F #endif //CV_SIMD128 template void