Skip to content

O
Opencv

Greenplum / Opencv
大约 1 年 前同步成功

通知 7
Star 0
Fork 0
O
Opencv

体验新版 GitCode，发现更多精彩内容 >>

提交 7fef96be 编写于 作者: T Tomoaki Teshima
浏览文件
操作

add 64F intrinsic in HAL NEON 

  * use universal intrinsic for accumulate series using float/double
  * accumulate, accumulateSquare, accumulateProduct and accumulateWeighted
  * add v_cvt_f64_high in both SSE/NEON
  * add test for conversion v_cvt_f64_high in test_intrin.cpp
  * improve some existing universal intrinsic by using new instructions in Aarch64
  * add workaround for Android build in intrin_neon.hpp
上级 da5ead2c
隐藏空白更改
内联 并排
Showing with 643 addition and 349 deletion
modules/core/include/opencv2/core/hal/intrin_neon.hpp
浏览文件 @ 7fef96be
......@@ -53,6 +53,28 @@ namespace cv
//! @cond IGNORED
#define CV_SIMD128 1
#if defined(__aarch64__)
#define CV_SIMD128_64F 1
#else
#define CV_SIMD128_64F 0
#endif
#if CV_SIMD128_64F
#define OPENCV_HAL_IMPL_NEON_REINTERPRET(_Tpv, suffix) \
template <typename T> static inline \
_Tpv vreinterpretq_##suffix##_f64(T a) { return (_Tpv) a; } \
template <typename T> static inline \
float64x2_t vreinterpretq_f64_##suffix(T a) { return (float64x2_t) a; }
OPENCV_HAL_IMPL_NEON_REINTERPRET(uint8x16_t, u8)
OPENCV_HAL_IMPL_NEON_REINTERPRET(int8x16_t, s8)
OPENCV_HAL_IMPL_NEON_REINTERPRET(uint16x8_t, u16)
OPENCV_HAL_IMPL_NEON_REINTERPRET(int16x8_t, s16)
OPENCV_HAL_IMPL_NEON_REINTERPRET(uint32x4_t, u32)
OPENCV_HAL_IMPL_NEON_REINTERPRET(int32x4_t, s32)
OPENCV_HAL_IMPL_NEON_REINTERPRET(uint64x2_t, u64)
OPENCV_HAL_IMPL_NEON_REINTERPRET(int64x2_t, s64)
OPENCV_HAL_IMPL_NEON_REINTERPRET(float32x4_t, f32)
#endif
struct v_uint8x16
{
......@@ -232,6 +254,27 @@ struct v_int64x2
int64x2_t val;
};
#if CV_SIMD128_64F
struct v_float64x2
{
typedef double lane_type;
enum { nlanes = 2 };
v_float64x2() {}
explicit v_float64x2(float64x2_t v) : val(v) {}
v_float64x2(double v0, double v1)
{
double v[] = {v0, v1};
val = vld1q_f64(v);
}
double get0() const
{
return vgetq_lane_f64(val, 0);
}
float64x2_t val;
};
#endif
#define OPENCV_HAL_IMPL_NEON_INIT(_Tpv, _Tp, suffix) \
inline v_##_Tpv v_setzero_##suffix() { return v_##_Tpv(vdupq_n_##suffix((_Tp)0)); } \
inline v_##_Tpv v_setall_##suffix(_Tp v) { return v_##_Tpv(vdupq_n_##suffix(v)); } \
......@@ -255,6 +298,21 @@ OPENCV_HAL_IMPL_NEON_INIT(int32x4, int, s32)
OPENCV_HAL_IMPL_NEON_INIT(uint64x2, uint64, u64)
OPENCV_HAL_IMPL_NEON_INIT(int64x2, int64, s64)
OPENCV_HAL_IMPL_NEON_INIT(float32x4, float, f32)
#if CV_SIMD128_64F
#define OPENCV_HAL_IMPL_NEON_INIT_64(_Tpv, suffix) \
inline v_float64x2 v_reinterpret_as_f64(const v_##_Tpv& v) { return v_float64x2(vreinterpretq_f64_##suffix(v.val)); }
OPENCV_HAL_IMPL_NEON_INIT(float64x2, double, f64)
OPENCV_HAL_IMPL_NEON_INIT_64(uint8x16, u8)
OPENCV_HAL_IMPL_NEON_INIT_64(int8x16, s8)
OPENCV_HAL_IMPL_NEON_INIT_64(uint16x8, u16)
OPENCV_HAL_IMPL_NEON_INIT_64(int16x8, s16)
OPENCV_HAL_IMPL_NEON_INIT_64(uint32x4, u32)
OPENCV_HAL_IMPL_NEON_INIT_64(int32x4, s32)
OPENCV_HAL_IMPL_NEON_INIT_64(uint64x2, u64)
OPENCV_HAL_IMPL_NEON_INIT_64(int64x2, s64)
OPENCV_HAL_IMPL_NEON_INIT_64(float32x4, f32)
OPENCV_HAL_IMPL_NEON_INIT_64(float64x2, f64)
#endif
#define OPENCV_HAL_IMPL_NEON_PACK(_Tpvec, _Tp, hreg, suffix, _Tpwvec, wsuffix, pack, op) \
inline _Tpvec v_##pack(const _Tpwvec& a, const _Tpwvec& b) \
......@@ -337,7 +395,13 @@ OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_int64x2, vaddq_s64)
OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_int64x2, vsubq_s64)
OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_uint64x2, vaddq_u64)
OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_uint64x2, vsubq_u64)
#if CV_SIMD128_64F
OPENCV_HAL_IMPL_NEON_BIN_OP(/, v_float32x4, vdivq_f32)
OPENCV_HAL_IMPL_NEON_BIN_OP(+, v_float64x2, vaddq_f64)
OPENCV_HAL_IMPL_NEON_BIN_OP(-, v_float64x2, vsubq_f64)
OPENCV_HAL_IMPL_NEON_BIN_OP(*, v_float64x2, vmulq_f64)
OPENCV_HAL_IMPL_NEON_BIN_OP(/, v_float64x2, vdivq_f64)
#else
inline v_float32x4 operator / (const v_float32x4& a, const v_float32x4& b)
{
float32x4_t reciprocal = vrecpeq_f32(b.val);
......@@ -353,6 +417,7 @@ inline v_float32x4& operator /= (v_float32x4& a, const v_float32x4& b)
a.val = vmulq_f32(a.val, reciprocal);
return a;
}
#endif
inline void v_mul_expand(const v_int16x8& a, const v_int16x8& b,
v_int32x4& c, v_int32x4& d)
......@@ -421,6 +486,18 @@ inline v_float32x4 operator ~ (const v_float32x4& a)
return v_float32x4(vreinterpretq_f32_s32(vmvnq_s32(vreinterpretq_s32_f32(a.val))));
}
#if CV_SIMD128_64F
inline v_float32x4 v_sqrt(const v_float32x4& x)
{
return v_float32x4(vsqrtq_f32(x.val));
}
inline v_float32x4 v_invsqrt(const v_float32x4& x)
{
v_float32x4 one = v_setall_f32(1.0f);
return one / v_sqrt(x);
}
#else
inline v_float32x4 v_sqrt(const v_float32x4& x)
{
float32x4_t x1 = vmaxq_f32(x.val, vdupq_n_f32(FLT_MIN));
......@@ -437,10 +514,47 @@ inline v_float32x4 v_invsqrt(const v_float32x4& x)
e = vmulq_f32(vrsqrtsq_f32(vmulq_f32(x.val, e), e), e);
return v_float32x4(e);
}
#endif
inline v_float32x4 v_abs(v_float32x4 x)
{ return v_float32x4(vabsq_f32(x.val)); }
#if CV_SIMD128_64F
#define OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(bin_op, intrin) \
inline v_float64x2 operator bin_op (const v_float64x2& a, const v_float64x2& b) \
{ \
return v_float64x2(vreinterpretq_f64_s64(intrin(vreinterpretq_s64_f64(a.val), vreinterpretq_s64_f64(b.val)))); \
} \
inline v_float64x2& operator bin_op##= (v_float64x2& a, const v_float64x2& b) \
{ \
a.val = vreinterpretq_f64_s64(intrin(vreinterpretq_s64_f64(a.val), vreinterpretq_s64_f64(b.val))); \
return a; \
}
OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(&, vandq_s64)
OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(|, vorrq_s64)
OPENCV_HAL_IMPL_NEON_DBL_BIT_OP(^, veorq_s64)
inline v_float64x2 operator ~ (const v_float64x2& a)
{
return v_float64x2(vreinterpretq_f64_s32(vmvnq_s32(vreinterpretq_s32_f64(a.val))));
}
inline v_float64x2 v_sqrt(const v_float64x2& x)
{
return v_float64x2(vsqrtq_f64(x.val));
}
inline v_float64x2 v_invsqrt(const v_float64x2& x)
{
v_float64x2 one = v_setall_f64(1.0f);
return one / v_sqrt(x);
}
inline v_float64x2 v_abs(v_float64x2 x)
{ return v_float64x2(vabsq_f64(x.val)); }
#endif
// TODO: exp, log, sin, cos
#define OPENCV_HAL_IMPL_NEON_BIN_FUNC(_Tpvec, func, intrin) \
......@@ -463,8 +577,23 @@ OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_int32x4, v_min, vminq_s32)
OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_int32x4, v_max, vmaxq_s32)
OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_float32x4, v_min, vminq_f32)
OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_float32x4, v_max, vmaxq_f32)
#if CV_SIMD128_64F
OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_float64x2, v_min, vminq_f64)
OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_float64x2, v_max, vmaxq_f64)
#endif
#if CV_SIMD128_64F
inline int64x2_t vmvnq_s64(int64x2_t a)
{
int64x2_t vx = vreinterpretq_s64_u32(vdupq_n_u32(0xFFFFFFFF));
return veorq_s64(a, vx);
}
inline uint64x2_t vmvnq_u64(uint64x2_t a)
{
uint64x2_t vx = vreinterpretq_u64_u32(vdupq_n_u32(0xFFFFFFFF));
return veorq_u64(a, vx);
}
#endif
#define OPENCV_HAL_IMPL_NEON_INT_CMP_OP(_Tpvec, cast, suffix, not_suffix) \
inline _Tpvec operator == (const _Tpvec& a, const _Tpvec& b) \
{ return _Tpvec(cast(vceqq_##suffix(a.val, b.val))); } \
......@@ -486,6 +615,11 @@ OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_int16x8, vreinterpretq_s16_u16, s16, u16)
OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_uint32x4, OPENCV_HAL_NOP, u32, u32)
OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_int32x4, vreinterpretq_s32_u32, s32, u32)
OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_float32x4, vreinterpretq_f32_u32, f32, u32)
#if CV_SIMD128_64F
OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_uint64x2, OPENCV_HAL_NOP, u64, u64)
OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_int64x2, vreinterpretq_s64_u64, s64, u64)
OPENCV_HAL_IMPL_NEON_INT_CMP_OP(v_float64x2, vreinterpretq_f64_u64, f64, u64)
#endif
OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_uint8x16, v_add_wrap, vaddq_u8)
OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_int8x16, v_add_wrap, vaddq_s8)
......@@ -501,6 +635,9 @@ OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_uint8x16, v_absdiff, vabdq_u8)
OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_uint16x8, v_absdiff, vabdq_u16)
OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_uint32x4, v_absdiff, vabdq_u32)
OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_float32x4, v_absdiff, vabdq_f32)
#if CV_SIMD128_64F
OPENCV_HAL_IMPL_NEON_BIN_FUNC(v_float64x2, v_absdiff, vabdq_f64)
#endif
#define OPENCV_HAL_IMPL_NEON_BIN_FUNC2(_Tpvec, _Tpvec2, cast, func, intrin) \
inline _Tpvec2 func(const _Tpvec& a, const _Tpvec& b) \
......@@ -528,6 +665,24 @@ inline v_float32x4 v_muladd(const v_float32x4& a, const v_float32x4& b, const v_
return v_float32x4(vmlaq_f32(c.val, a.val, b.val));
}
#if CV_SIMD128_64F
inline v_float64x2 v_magnitude(const v_float64x2& a, const v_float64x2& b)
{
v_float64x2 x(vaddq_f64(vmulq_f64(a.val, a.val), vmulq_f64(b.val, b.val)));
return v_sqrt(x);
}
inline v_float64x2 v_sqr_magnitude(const v_float64x2& a, const v_float64x2& b)
{
return v_float64x2(vaddq_f64(vmulq_f64(a.val, a.val), vmulq_f64(b.val, b.val)));
}
inline v_float64x2 v_muladd(const v_float64x2& a, const v_float64x2& b, const v_float64x2& c)
{
return v_float64x2(vaddq_f64(c.val, vmulq_f64(a.val, b.val)));
}
#endif
// trade efficiency for convenience
#define OPENCV_HAL_IMPL_NEON_SHIFT_OP(_Tpvec, suffix, _Tps, ssuffix) \
inline _Tpvec operator << (const _Tpvec& a, int n) \
......@@ -575,6 +730,9 @@ OPENCV_HAL_IMPL_NEON_LOADSTORE_OP(v_int32x4, int, s32)
OPENCV_HAL_IMPL_NEON_LOADSTORE_OP(v_uint64x2, uint64, u64)
OPENCV_HAL_IMPL_NEON_LOADSTORE_OP(v_int64x2, int64, s64)
OPENCV_HAL_IMPL_NEON_LOADSTORE_OP(v_float32x4, float, f32)
#if CV_SIMD128_64F
OPENCV_HAL_IMPL_NEON_LOADSTORE_OP(v_float64x2, double, f64)
#endif
#define OPENCV_HAL_IMPL_NEON_REDUCE_OP_4(_Tpvec, scalartype, func, scalar_func) \
inline scalartype v_reduce_##func(const _Tpvec& a) \
......@@ -627,6 +785,16 @@ inline int v_signmask(const v_int32x4& a)
{ return v_signmask(v_reinterpret_as_u32(a)); }
inline int v_signmask(const v_float32x4& a)
{ return v_signmask(v_reinterpret_as_u32(a)); }
#if CV_SIMD128_64F
inline int v_signmask(const v_uint64x2& a)
{
int64x1_t m0 = vdup_n_s64(0);
uint64x2_t v0 = vshlq_u64(vshrq_n_u64(a.val, 63), vcombine_s64(m0, m0));
return (int)vgetq_lane_u64(v0, 0) + ((int)vgetq_lane_u64(v0, 1) << 1);
}
inline int v_signmask(const v_float64x2& a)
{ return v_signmask(v_reinterpret_as_u64(a)); }
#endif
#define OPENCV_HAL_IMPL_NEON_CHECK_ALLANY(_Tpvec, suffix, shift) \
inline bool v_check_all(const v_##_Tpvec& a) \
......@@ -645,6 +813,9 @@ inline bool v_check_any(const v_##_Tpvec& a) \
OPENCV_HAL_IMPL_NEON_CHECK_ALLANY(uint8x16, u8, 7)
OPENCV_HAL_IMPL_NEON_CHECK_ALLANY(uint16x8, u16, 15)
OPENCV_HAL_IMPL_NEON_CHECK_ALLANY(uint32x4, u32, 31)
#if CV_SIMD128_64F
OPENCV_HAL_IMPL_NEON_CHECK_ALLANY(uint64x2, u64, 63)
#endif
inline bool v_check_all(const v_int8x16& a)
{ return v_check_all(v_reinterpret_as_u8(a)); }
......@@ -664,6 +835,17 @@ inline bool v_check_any(const v_int32x4& a)
inline bool v_check_any(const v_float32x4& a)
{ return v_check_any(v_reinterpret_as_u32(a)); }
#if CV_SIMD128_64F
inline bool v_check_all(const v_int64x2& a)
{ return v_check_all(v_reinterpret_as_u64(a)); }
inline bool v_check_all(const v_float64x2& a)
{ return v_check_all(v_reinterpret_as_u64(a)); }
inline bool v_check_any(const v_int64x2& a)
{ return v_check_any(v_reinterpret_as_u64(a)); }
inline bool v_check_any(const v_float64x2& a)
{ return v_check_any(v_reinterpret_as_u64(a)); }
#endif
#define OPENCV_HAL_IMPL_NEON_SELECT(_Tpvec, suffix, usuffix) \
inline _Tpvec v_select(const _Tpvec& mask, const _Tpvec& a, const _Tpvec& b) \
{ \
......@@ -677,6 +859,9 @@ OPENCV_HAL_IMPL_NEON_SELECT(v_int16x8, s16, u16)
OPENCV_HAL_IMPL_NEON_SELECT(v_uint32x4, u32, u32)
OPENCV_HAL_IMPL_NEON_SELECT(v_int32x4, s32, u32)
OPENCV_HAL_IMPL_NEON_SELECT(v_float32x4, f32, u32)
#if CV_SIMD128_64F
OPENCV_HAL_IMPL_NEON_SELECT(v_float64x2, f64, u64)
#endif
#define OPENCV_HAL_IMPL_NEON_EXPAND(_Tpvec, _Tpwvec, _Tp, suffix) \
inline void v_expand(const _Tpvec& a, _Tpwvec& b0, _Tpwvec& b1) \
......@@ -710,6 +895,27 @@ inline v_int32x4 v_load_expand_q(const schar* ptr)
return v_int32x4(vmovl_s16(v1));
}
#if defined(__aarch64__)
#define OPENCV_HAL_IMPL_NEON_UNPACKS(_Tpvec, suffix) \
inline void v_zip(const v_##_Tpvec& a0, const v_##_Tpvec& a1, v_##_Tpvec& b0, v_##_Tpvec& b1) \
{ \
b0.val = vzip1q_##suffix(a0.val, a1.val); \
b1.val = vzip2q_##suffix(a0.val, a1.val); \
} \
inline v_##_Tpvec v_combine_low(const v_##_Tpvec& a, const v_##_Tpvec& b) \
{ \
return v_##_Tpvec(vcombine_##suffix(vget_low_##suffix(a.val), vget_low_##suffix(b.val))); \
} \
inline v_##_Tpvec v_combine_high(const v_##_Tpvec& a, const v_##_Tpvec& b) \
{ \
return v_##_Tpvec(vcombine_##suffix(vget_high_##suffix(a.val), vget_high_##suffix(b.val))); \
} \
inline void v_recombine(const v_##_Tpvec& a, const v_##_Tpvec& b, v_##_Tpvec& c, v_##_Tpvec& d) \
{ \
c.val = vcombine_##suffix(vget_low_##suffix(a.val), vget_low_##suffix(b.val)); \
d.val = vcombine_##suffix(vget_high_##suffix(a.val), vget_high_##suffix(b.val)); \
}
#else
#define OPENCV_HAL_IMPL_NEON_UNPACKS(_Tpvec, suffix) \
inline void v_zip(const v_##_Tpvec& a0, const v_##_Tpvec& a1, v_##_Tpvec& b0, v_##_Tpvec& b1) \
{ \
......@@ -730,6 +936,7 @@ inline void v_recombine(const v_##_Tpvec& a, const v_##_Tpvec& b, v_##_Tpvec& c,
c.val = vcombine_##suffix(vget_low_##suffix(a.val), vget_low_##suffix(b.val)); \
d.val = vcombine_##suffix(vget_high_##suffix(a.val), vget_high_##suffix(b.val)); \
}
#endif
OPENCV_HAL_IMPL_NEON_UNPACKS(uint8x16, u8)
OPENCV_HAL_IMPL_NEON_UNPACKS(int8x16, s8)
......@@ -738,6 +945,9 @@ OPENCV_HAL_IMPL_NEON_UNPACKS(int16x8, s16)
OPENCV_HAL_IMPL_NEON_UNPACKS(uint32x4, u32)
OPENCV_HAL_IMPL_NEON_UNPACKS(int32x4, s32)
OPENCV_HAL_IMPL_NEON_UNPACKS(float32x4, f32)
#if CV_SIMD128_64F
OPENCV_HAL_IMPL_NEON_UNPACKS(float64x2, f64)
#endif
#define OPENCV_HAL_IMPL_NEON_EXTRACT(_Tpvec, suffix) \
template <int s> \
......@@ -755,6 +965,9 @@ OPENCV_HAL_IMPL_NEON_EXTRACT(int32x4, s32)
OPENCV_HAL_IMPL_NEON_EXTRACT(uint64x2, u64)
OPENCV_HAL_IMPL_NEON_EXTRACT(int64x2, s64)
OPENCV_HAL_IMPL_NEON_EXTRACT(float32x4, f32)
#if CV_SIMD128_64F
OPENCV_HAL_IMPL_NEON_EXTRACT(float64x2, f64)
#endif
inline v_int32x4 v_round(const v_float32x4& a)
{
......@@ -782,6 +995,38 @@ inline v_int32x4 v_ceil(const v_float32x4& a)
inline v_int32x4 v_trunc(const v_float32x4& a)
{ return v_int32x4(vcvtq_s32_f32(a.val)); }
#if CV_SIMD128_64F
inline v_int32x4 v_round(const v_float64x2& a)
{
static const int32x2_t zero = vdup_n_s32(0);
return v_int32x4(vcombine_s32(vmovn_s64(vcvtaq_s64_f64(a.val)), zero));
}
inline v_int32x4 v_floor(const v_float64x2& a)
{
static const int32x2_t zero = vdup_n_s32(0);
int64x2_t a1 = vcvtq_s64_f64(a.val);
uint64x2_t mask = vcgtq_f64(vcvtq_f64_s64(a1), a.val);
a1 = vaddq_s64(a1, vreinterpretq_s64_u64(mask));
return v_int32x4(vcombine_s32(vmovn_s64(a1), zero));
}
inline v_int32x4 v_ceil(const v_float64x2& a)
{
static const int32x2_t zero = vdup_n_s32(0);
int64x2_t a1 = vcvtq_s64_f64(a.val);
uint64x2_t mask = vcgtq_f64(a.val, vcvtq_f64_s64(a1));
a1 = vsubq_s64(a1, vreinterpretq_s64_u64(mask));
return v_int32x4(vcombine_s32(vmovn_s64(a1), zero));
}
inline v_int32x4 v_trunc(const v_float64x2& a)
{
static const int32x2_t zero = vdup_n_s32(0);
return v_int32x4(vcombine_s32(vmovn_s64(vcvtaq_s64_f64(a.val)), zero));
}
#endif
#define OPENCV_HAL_IMPL_NEON_TRANSPOSE4x4(_Tpvec, suffix) \
inline void v_transpose4x4(const v_##_Tpvec& a0, const v_##_Tpvec& a1, \
const v_##_Tpvec& a2, const v_##_Tpvec& a3, \
......@@ -851,12 +1096,45 @@ OPENCV_HAL_IMPL_NEON_INTERLEAVED(int16x8, short, s16)
OPENCV_HAL_IMPL_NEON_INTERLEAVED(uint32x4, unsigned, u32)
OPENCV_HAL_IMPL_NEON_INTERLEAVED(int32x4, int, s32)
OPENCV_HAL_IMPL_NEON_INTERLEAVED(float32x4, float, f32)
#if CV_SIMD128_64F
OPENCV_HAL_IMPL_NEON_INTERLEAVED(float64x2, double, f64)
#endif
inline v_float32x4 v_cvt_f32(const v_int32x4& a)
{
return v_float32x4(vcvtq_f32_s32(a.val));
}
#if CV_SIMD128_64F
inline v_float32x4 v_cvt_f32(const v_float64x2& a)
{
float32x2_t zero = vdup_n_f32(0.0f);
return v_float32x4(vcombine_f32(vcvt_f32_f64(a.val), zero));
}
inline v_float64x2 v_cvt_f64(const v_int32x4& a)
{
return v_float64x2(vcvt_f64_f32(vcvt_f32_s32(vget_low_s32(a.val))));
}
inline v_float64x2 v_cvt_f64_high(const v_int32x4& a)
{
return v_float64x2(vcvt_f64_f32(vcvt_f32_s32(vget_high_s32(a.val))));
}
inline v_float64x2 v_cvt_f64(const v_float32x4& a)
{
return v_float64x2(vcvt_f64_f32(vget_low_f32(a.val)));
}
inline v_float64x2 v_cvt_f64_high(const v_float32x4& a)
{
return v_float64x2(vcvt_f64_f32(vget_high_f32(a.val)));
}
#endif
//! @endcond
}
......
modules/core/include/opencv2/core/hal/intrin_sse.hpp
浏览文件 @ 7fef96be
......@@ -1587,11 +1587,21 @@ inline v_float64x2 v_cvt_f64(const v_int32x4& a)
return v_float64x2(_mm_cvtepi32_pd(a.val));
}
inline v_float64x2 v_cvt_f64_high(const v_int32x4& a)
{
return v_float64x2(_mm_cvtepi32_pd(_mm_srli_si128(a.val,8)));
}
inline v_float64x2 v_cvt_f64(const v_float32x4& a)
{
return v_float64x2(_mm_cvtps_pd(a.val));
}
inline v_float64x2 v_cvt_f64_high(const v_float32x4& a)
{
return v_float64x2(_mm_cvtps_pd(_mm_castsi128_ps(_mm_srli_si128(_mm_castps_si128(a.val),8))));
}
//! @endcond
}
......
modules/core/test/test_intrin.cpp
浏览文件 @ 7fef96be
......@@ -626,12 +626,18 @@ template<typename R> struct TheTest
dataA *= 1.1;
R a = dataA;
Rt b = v_cvt_f64(a);
Rt c = v_cvt_f64_high(a);
Data<Rt> resB = b;
Data<Rt> resC = c;
int n = std::min<int>(Rt::nlanes, R::nlanes);
for (int i = 0; i < n; ++i)
{
EXPECT_EQ((typename Rt::lane_type)dataA[i], resB[i]);
}
for (int i = 0; i < n; ++i)
{
EXPECT_EQ((typename Rt::lane_type)dataA[i+n], resC[i]);
}
#endif
return *this;
}
......
modules/imgproc/src/accum.cpp
浏览文件 @ 7fef96be
......@@ -395,7 +395,7 @@ struct AccW_SIMD<double, double>
return x;
}
};
#elif CV_SSE2
#elif CV_SIMD128
template <>
struct Acc_SIMD<float, float>
{
......@@ -408,8 +408,8 @@ struct Acc_SIMD<float, float>
len *= cn;
for ( ; x <= len - 8; x += 8)
{
_mm_storeu_ps(dst + x, _mm_add_ps(_mm_loadu_ps(dst + x), _mm_loadu_ps(src + x)));
_mm_storeu_ps(dst + x + 4, _mm_add_ps(_mm_loadu_ps(dst + x + 4), _mm_loadu_ps(src + x + 4)));
v_store(dst + x, v_load(dst + x) + v_load(src + x));
v_store(dst + x + 4, v_load(dst + x + 4) + v_load(src + x + 4));
}
}
......@@ -417,6 +417,7 @@ struct Acc_SIMD<float, float>
}
};
#if CV_SIMD128_64F
template <>
struct Acc_SIMD<float, double>
{
......@@ -429,17 +430,12 @@ struct Acc_SIMD<float, double>
len *= cn;
for ( ; x <= len - 4; x += 4)
{
__m128 v_src = _mm_loadu_ps(src + x);
__m128d v_src0 = _mm_cvtps_pd(v_src);
__m128d v_src1 = _mm_cvtps_pd(_mm_shuffle_ps(v_src, v_src, _MM_SHUFFLE(1, 0, 3, 2)));
v_float32x4 v_src = v_load(src + x);
v_float64x2 v_src0 = v_cvt_f64(v_src);
v_float64x2 v_src1 = v_cvt_f64_high(v_src);
__m128d v_dst0 = _mm_loadu_pd(dst + x);
__m128d v_dst1 = _mm_loadu_pd(dst + x + 2);
v_dst0 = _mm_add_pd(v_dst0, v_src0);
v_dst1 = _mm_add_pd(v_dst1, v_src1);
_mm_storeu_pd(dst + x, v_dst0);
_mm_storeu_pd(dst + x + 2, v_dst1);
v_store(dst + x, v_load(dst + x) + v_src0);
v_store(dst + x + 2, v_load(dst + x + 2) + v_src1);
}
}
return x;
......@@ -458,21 +454,17 @@ struct Acc_SIMD<double, double>
len *= cn;
for ( ; x <= len - 4; x += 4)
{
__m128d v_src0 = _mm_loadu_pd(src + x);
__m128d v_src1 = _mm_loadu_pd(src + x + 2);
v_float64x2 v_src0 = v_load(src + x);
v_float64x2 v_src1 = v_load(src + x + 2);
__m128d v_dst0 = _mm_loadu_pd(dst + x);
__m128d v_dst1 = _mm_loadu_pd(dst + x + 2);
v_dst0 = _mm_add_pd(v_dst0, v_src0);
v_dst1 = _mm_add_pd(v_dst1, v_src1);
_mm_storeu_pd(dst + x, v_dst0);
_mm_storeu_pd(dst + x + 2, v_dst1);
v_store(dst + x, v_load(dst + x) + v_src0);
v_store(dst + x + 2, v_load(dst + x + 2) + v_src1);
}
}
return x;
}
};
#endif //CV_SIMD128_64F
template <>
struct AccSqr_SIMD<float, float>
......@@ -486,12 +478,13 @@ struct AccSqr_SIMD<float, float>
len *= cn;
for ( ; x <= len - 8; x += 8)
{
__m128 v_src0 = _mm_loadu_ps(src + x);
__m128 v_src1 = _mm_loadu_ps(src + x + 4);
v_src0 = _mm_mul_ps(v_src0, v_src0);
v_src1 = _mm_mul_ps(v_src1, v_src1);
_mm_storeu_ps(dst + x, _mm_add_ps(_mm_loadu_ps(dst + x), v_src0));
_mm_storeu_ps(dst + x + 4, _mm_add_ps(_mm_loadu_ps(dst + x + 4), v_src1));
v_float32x4 v_src0 = v_load(src + x);
v_float32x4 v_src1 = v_load(src + x + 4);
v_src0 = v_src0 * v_src0;
v_src1 = v_src1 * v_src1;
v_store(dst + x, v_load(dst + x) + v_src0);
v_store(dst + x + 4, v_load(dst + x + 4) + v_src1);
}
}
......@@ -499,6 +492,7 @@ struct AccSqr_SIMD<float, float>
}
};
#if CV_SIMD128_64F
template <>
struct AccSqr_SIMD<float, double>
{
......@@ -511,19 +505,14 @@ struct AccSqr_SIMD<float, double>
len *= cn;
for ( ; x <= len - 4; x += 4)
{
__m128 v_src = _mm_loadu_ps(src + x);
__m128d v_src0 = _mm_cvtps_pd(v_src);
__m128d v_src1 = _mm_cvtps_pd(_mm_shuffle_ps(v_src, v_src, _MM_SHUFFLE(1, 0, 3, 2)));
v_src0 = _mm_mul_pd(v_src0, v_src0);
v_src1 = _mm_mul_pd(v_src1, v_src1);
__m128d v_dst0 = _mm_loadu_pd(dst + x);
__m128d v_dst1 = _mm_loadu_pd(dst + x + 2);
v_dst0 = _mm_add_pd(v_dst0, v_src0);
v_dst1 = _mm_add_pd(v_dst1, v_src1);
_mm_storeu_pd(dst + x, v_dst0);
_mm_storeu_pd(dst + x + 2, v_dst1);
v_float32x4 v_src = v_load(src + x);
v_float64x2 v_src0 = v_cvt_f64(v_src);
v_float64x2 v_src1 = v_cvt_f64_high(v_src);
v_src0 = v_src0 * v_src0;
v_src1 = v_src1 * v_src1;
v_store(dst + x, v_load(dst + x) + v_src0);
v_store(dst + x + 2, v_load(dst + x + 2) + v_src1);
}
}
return x;
......@@ -542,23 +531,19 @@ struct AccSqr_SIMD<double, double>
len *= cn;
for ( ; x <= len - 4; x += 4)
{
__m128d v_src0 = _mm_loadu_pd(src + x);
__m128d v_src1 = _mm_loadu_pd(src + x + 2);
v_src0 = _mm_mul_pd(v_src0, v_src0);
v_src1 = _mm_mul_pd(v_src1, v_src1);
__m128d v_dst0 = _mm_loadu_pd(dst + x);
__m128d v_dst1 = _mm_loadu_pd(dst + x + 2);
v_dst0 = _mm_add_pd(v_dst0, v_src0);
v_dst1 = _mm_add_pd(v_dst1, v_src1);
_mm_storeu_pd(dst + x, v_dst0);
_mm_storeu_pd(dst + x + 2, v_dst1);
v_float64x2 v_src0 = v_load(src + x);
v_float64x2 v_src1 = v_load(src + x + 2);
v_src0 = v_src0 * v_src0;
v_src1 = v_src1 * v_src1;
v_store(dst + x, v_load(dst + x) + v_src0);
v_store(dst + x + 2, v_load(dst + x + 2) + v_src1);
}
}
return x;
}
};
#endif //CV_SIMD128_64F
template <>
struct AccProd_SIMD<float, float>
......@@ -572,8 +557,8 @@ struct AccProd_SIMD<float, float>
len *= cn;
for ( ; x <= len - 8; x += 8)
{
_mm_storeu_ps(dst + x, _mm_add_ps(_mm_loadu_ps(dst + x), _mm_mul_ps(_mm_loadu_ps(src1 + x), _mm_loadu_ps(src2 + x))));
_mm_storeu_ps(dst + x + 4, _mm_add_ps(_mm_loadu_ps(dst + x + 4), _mm_mul_ps(_mm_loadu_ps(src1 + x + 4), _mm_loadu_ps(src2 + x + 4))));
v_store(dst + x, v_load(dst + x) + v_load(src1 + x) * v_load(src2 + x));
v_store(dst + x + 4, v_load(dst + x + 4) + v_load(src1 + x + 4) * v_load(src2 + x + 4));
}
}
......@@ -581,6 +566,7 @@ struct AccProd_SIMD<float, float>
}
};
#if CV_SIMD128_64F
template <>
struct AccProd_SIMD<float, double>
{
......@@ -593,22 +579,16 @@ struct AccProd_SIMD<float, double>
len *= cn;
for ( ; x <= len - 4; x += 4)
{
__m128 v_1src = _mm_loadu_ps(src1 + x);
__m128 v_2src = _mm_loadu_ps(src2 + x);
__m128d v_1src0 = _mm_cvtps_pd(v_1src);
__m128d v_1src1 = _mm_cvtps_pd(_mm_shuffle_ps(v_1src, v_1src, _MM_SHUFFLE(0, 0, 3, 2)));
__m128d v_2src0 = _mm_cvtps_pd(v_2src);
__m128d v_2src1 = _mm_cvtps_pd(_mm_shuffle_ps(v_2src, v_2src, _MM_SHUFFLE(0, 0, 3, 2)));
v_float32x4 v_1src = v_load(src1 + x);
v_float32x4 v_2src = v_load(src2 + x);
__m128d v_dst0 = _mm_loadu_pd(dst + x);
__m128d v_dst1 = _mm_loadu_pd(dst + x + 2);
v_float64x2 v_1src0 = v_cvt_f64(v_1src);
v_float64x2 v_1src1 = v_cvt_f64_high(v_1src);
v_float64x2 v_2src0 = v_cvt_f64(v_2src);
v_float64x2 v_2src1 = v_cvt_f64_high(v_2src);
v_dst0 = _mm_add_pd(v_dst0, _mm_mul_pd(v_1src0, v_2src0));
v_dst1 = _mm_add_pd(v_dst1, _mm_mul_pd(v_1src1, v_2src1));
_mm_storeu_pd(dst + x, v_dst0);
_mm_storeu_pd(dst + x + 2, v_dst1);
v_store(dst + x, v_load(dst + x) + (v_1src0 * v_2src0));
v_store(dst + x + 2, v_load(dst + x + 2) + (v_1src1 * v_2src1));
}
}
return x;
......@@ -627,25 +607,19 @@ struct AccProd_SIMD<double, double>
len *= cn;
for ( ; x <= len - 4; x += 4)
{
__m128d v_src00 = _mm_loadu_pd(src1 + x);
__m128d v_src01 = _mm_loadu_pd(src1 + x + 2);
__m128d v_src10 = _mm_loadu_pd(src2 + x);
__m128d v_src11 = _mm_loadu_pd(src2 + x + 2);
__m128d v_src0 = _mm_mul_pd(v_src00, v_src10);
__m128d v_src1 = _mm_mul_pd(v_src01, v_src11);
__m128d v_dst0 = _mm_loadu_pd(dst + x);
__m128d v_dst1 = _mm_loadu_pd(dst + x + 2);
v_dst0 = _mm_add_pd(v_dst0, v_src0);
v_dst1 = _mm_add_pd(v_dst1, v_src1);
_mm_storeu_pd(dst + x, v_dst0);
_mm_storeu_pd(dst + x + 2, v_dst1);
v_float64x2 v_src00 = v_load(src1 + x);
v_float64x2 v_src01 = v_load(src1 + x + 2);
v_float64x2 v_src10 = v_load(src2 + x);
v_float64x2 v_src11 = v_load(src2 + x + 2);
v_store(dst + x, v_load(dst + x) + (v_src00 * v_src10));
v_store(dst + x + 2, v_load(dst + x + 2) + (v_src01 * v_src11));
}
}
return x;
}
};
#endif //CV_SIMD128_64F
template <>
struct AccW_SIMD<float, float>
......@@ -653,16 +627,16 @@ struct AccW_SIMD<float, float>
int operator() (const float * src, float * dst, const uchar * mask, int len, int cn, float alpha) const
{
int x = 0;
__m128 v_alpha = _mm_set1_ps(alpha);
__m128 v_beta = _mm_set1_ps(1.0f - alpha);
v_float32x4 v_alpha = v_setall_f32(alpha);
v_float32x4 v_beta = v_setall_f32(1.0f - alpha);
if (!mask)
{
len *= cn;
for ( ; x <= len - 8; x += 8)
{
_mm_storeu_ps(dst + x, _mm_add_ps(_mm_mul_ps(_mm_loadu_ps(dst + x), v_beta), _mm_mul_ps(_mm_loadu_ps(src + x), v_alpha)));
_mm_storeu_ps(dst + x + 4, _mm_add_ps(_mm_mul_ps(_mm_loadu_ps(dst + x + 4), v_beta), _mm_mul_ps(_mm_loadu_ps(src + x + 4), v_alpha)));
v_store(dst + x, ((v_load(dst + x) * v_beta) + (v_load(src + x) * v_alpha)));
v_store(dst + x + 4, ((v_load(dst + x + 4) * v_beta) + (v_load(src + x + 4) * v_alpha)));
}
}
......@@ -670,31 +644,32 @@ struct AccW_SIMD<float, float>
}
};
#if CV_SIMD128_64F
template <>
struct AccW_SIMD<float, double>
{
int operator() (const float * src, double * dst, const uchar * mask, int len, int cn, double alpha) const
{
int x = 0;
__m128d v_alpha = _mm_set1_pd(alpha);
__m128d v_beta = _mm_set1_pd(1.0f - alpha);
v_float64x2 v_alpha = v_setall_f64(alpha);
v_float64x2 v_beta = v_setall_f64(1.0f - alpha);
if (!mask)
{
len *= cn;
for ( ; x <= len - 8; x += 8)
{
__m128 v_src0 = _mm_loadu_ps(src + x);
__m128 v_src1 = _mm_loadu_ps(src + x + 4);
__m128d v_src00 = _mm_cvtps_pd(v_src0);
__m128d v_src01 = _mm_cvtps_pd(_mm_shuffle_ps(v_src0, v_src0, _MM_SHUFFLE(0, 0, 3, 2)));
__m128d v_src10 = _mm_cvtps_pd(v_src1);
__m128d v_src11 = _mm_cvtps_pd(_mm_shuffle_ps(v_src1, v_src1, _MM_SHUFFLE(0, 0, 3, 2)));
_mm_storeu_pd(dst + x, _mm_add_pd(_mm_mul_pd(_mm_loadu_pd(dst + x), v_beta), _mm_mul_pd(v_src00, v_alpha)));
_mm_storeu_pd(dst + x + 2, _mm_add_pd(_mm_mul_pd(_mm_loadu_pd(dst + x + 2), v_beta), _mm_mul_pd(v_src01, v_alpha)));
_mm_storeu_pd(dst + x + 4, _mm_add_pd(_mm_mul_pd(_mm_loadu_pd(dst + x + 4), v_beta), _mm_mul_pd(v_src10, v_alpha)));
_mm_storeu_pd(dst + x + 6, _mm_add_pd(_mm_mul_pd(_mm_loadu_pd(dst + x + 6), v_beta), _mm_mul_pd(v_src11, v_alpha)));
v_float32x4 v_src0 = v_load(src + x);
v_float32x4 v_src1 = v_load(src + x + 4);
v_float64x2 v_src00 = v_cvt_f64(v_src0);
v_float64x2 v_src01 = v_cvt_f64_high(v_src0);
v_float64x2 v_src10 = v_cvt_f64(v_src1);
v_float64x2 v_src11 = v_cvt_f64_high(v_src1);
v_store(dst + x, ((v_load(dst + x) * v_beta) + (v_src00 * v_alpha)));
v_store(dst + x + 2, ((v_load(dst + x + 2) * v_beta) + (v_src01 * v_alpha)));
v_store(dst + x + 4, ((v_load(dst + x + 4) * v_beta) + (v_src10 * v_alpha)));
v_store(dst + x + 6, ((v_load(dst + x + 6) * v_beta) + (v_src11 * v_alpha)));
}
}
......@@ -708,26 +683,27 @@ struct AccW_SIMD<double, double>
int operator() (const double * src, double * dst, const uchar * mask, int len, int cn, double alpha) const
{
int x = 0;
__m128d v_alpha = _mm_set1_pd(alpha);
__m128d v_beta = _mm_set1_pd(1.0f - alpha);
v_float64x2 v_alpha = v_setall_f64(alpha);
v_float64x2 v_beta = v_setall_f64(1.0f - alpha);
if (!mask)
{
len *= cn;
for ( ; x <= len - 4; x += 4)
{
__m128d v_src0 = _mm_loadu_pd(src + x);
__m128d v_src1 = _mm_loadu_pd(src + x + 2);
v_float64x2 v_src0 = v_load(src + x);
v_float64x2 v_src1 = v_load(src + x + 2);
_mm_storeu_pd(dst + x, _mm_add_pd(_mm_mul_pd(_mm_loadu_pd(dst + x), v_beta), _mm_mul_pd(v_src0, v_alpha)));
_mm_storeu_pd(dst + x + 2, _mm_add_pd(_mm_mul_pd(_mm_loadu_pd(dst + x + 2), v_beta), _mm_mul_pd(v_src1, v_alpha)));
v_store(dst + x, ((v_load(dst + x) * v_beta) + (v_src0 * v_alpha)));
v_store(dst + x + 2, ((v_load(dst + x + 2) * v_beta) + (v_src1 * v_alpha)));
}
}
return x;
}
};
#endif
#endif //CV_SIMD128_64F
#endif //CV_SIMD128
#if CV_SIMD128
template <>
......@@ -742,7 +718,7 @@ struct Acc_SIMD<uchar, float>
len *= cn;
for ( ; x <= len - 16; x += 16)
{
v_uint8x16 v_src = v_load((const uchar*)(src + x));
v_uint8x16 v_src = v_load(src + x);
v_uint16x8 v_src0, v_src1;
v_expand(v_src, v_src0, v_src1);
......@@ -762,9 +738,9 @@ struct Acc_SIMD<uchar, float>
for ( ; x <= len - 16; x += 16)
{
v_uint8x16 v_mask = v_load((const uchar*)(mask + x));
v_uint8x16 v_mask = v_load(mask + x);
v_mask = ~(v_0 == v_mask);
v_uint8x16 v_src = v_load((const uchar*)(src + x));
v_uint8x16 v_src = v_load(src + x);
v_src = v_src & v_mask;
v_uint16x8 v_src0, v_src1;
v_expand(v_src, v_src0, v_src1);
......@@ -795,7 +771,7 @@ struct Acc_SIMD<ushort, float>
len *= cn;
for ( ; x <= len - 8; x += 8)
{
v_uint16x8 v_src = v_load((const ushort*)(src + x));
v_uint16x8 v_src = v_load(src + x);
v_uint32x4 v_src0, v_src1;
v_expand(v_src, v_src0, v_src1);
......@@ -808,7 +784,7 @@ struct Acc_SIMD<ushort, float>
}
};
#if CV_SSE2
#if CV_SIMD128_64F
template <>
struct Acc_SIMD<uchar, double>
{
......@@ -818,52 +794,52 @@ struct Acc_SIMD<uchar, double>
if (!mask)
{
__m128i v_0 = _mm_setzero_si128();
len *= cn;
for ( ; x <= len - 16; x += 16)
{
__m128i v_src = _mm_loadu_si128((const __m128i*)(src + x));
__m128i v_int0 = _mm_unpacklo_epi8(v_src, v_0);
__m128i v_int1 = _mm_unpackhi_epi8(v_src, v_0);
__m128i v_int00 = _mm_unpacklo_epi16(v_int0, v_0);
__m128i v_int01 = _mm_unpackhi_epi16(v_int0, v_0);
__m128i v_int10 = _mm_unpacklo_epi16(v_int1, v_0);
__m128i v_int11 = _mm_unpackhi_epi16(v_int1, v_0);
__m128d v_src0 = _mm_cvtepi32_pd(v_int00);
__m128d v_src1 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int00, _MM_SHUFFLE(0, 0, 3, 2)));
__m128d v_src2 = _mm_cvtepi32_pd(v_int01);
__m128d v_src3 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int01, _MM_SHUFFLE(0, 0, 3, 2)));
__m128d v_src4 = _mm_cvtepi32_pd(v_int10);
__m128d v_src5 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int10, _MM_SHUFFLE(0, 0, 3, 2)));
__m128d v_src6 = _mm_cvtepi32_pd(v_int11);
__m128d v_src7 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int11, _MM_SHUFFLE(0, 0, 3, 2)));
__m128d v_dst0 = _mm_loadu_pd(dst + x);
__m128d v_dst1 = _mm_loadu_pd(dst + x + 2);
__m128d v_dst2 = _mm_loadu_pd(dst + x + 4);
__m128d v_dst3 = _mm_loadu_pd(dst + x + 6);
__m128d v_dst4 = _mm_loadu_pd(dst + x + 8);
__m128d v_dst5 = _mm_loadu_pd(dst + x + 10);
__m128d v_dst6 = _mm_loadu_pd(dst + x + 12);
__m128d v_dst7 = _mm_loadu_pd(dst + x + 14);
v_dst0 = _mm_add_pd(v_dst0, v_src0);
v_dst1 = _mm_add_pd(v_dst1, v_src1);
v_dst2 = _mm_add_pd(v_dst2, v_src2);
v_dst3 = _mm_add_pd(v_dst3, v_src3);
v_dst4 = _mm_add_pd(v_dst4, v_src4);
v_dst5 = _mm_add_pd(v_dst5, v_src5);
v_dst6 = _mm_add_pd(v_dst6, v_src6);
v_dst7 = _mm_add_pd(v_dst7, v_src7);
_mm_storeu_pd(dst + x, v_dst0);
_mm_storeu_pd(dst + x + 2, v_dst1);
_mm_storeu_pd(dst + x + 4, v_dst2);
_mm_storeu_pd(dst + x + 6, v_dst3);
_mm_storeu_pd(dst + x + 8, v_dst4);
_mm_storeu_pd(dst + x + 10, v_dst5);
_mm_storeu_pd(dst + x + 12, v_dst6);
_mm_storeu_pd(dst + x + 14, v_dst7);
v_uint8x16 v_src = v_load(src + x);
v_uint16x8 v_int0, v_int1;
v_expand(v_src, v_int0, v_int1);
v_uint32x4 v_int00, v_int01, v_int10, v_int11;
v_expand(v_int0, v_int00, v_int01);
v_expand(v_int1, v_int10, v_int11);
v_float64x2 v_src0 = v_cvt_f64(v_reinterpret_as_s32(v_int00));
v_float64x2 v_src1 = v_cvt_f64_high(v_reinterpret_as_s32(v_int00));
v_float64x2 v_src2 = v_cvt_f64(v_reinterpret_as_s32(v_int01));
v_float64x2 v_src3 = v_cvt_f64_high(v_reinterpret_as_s32(v_int01));
v_float64x2 v_src4 = v_cvt_f64(v_reinterpret_as_s32(v_int10));
v_float64x2 v_src5 = v_cvt_f64_high(v_reinterpret_as_s32(v_int10));
v_float64x2 v_src6 = v_cvt_f64(v_reinterpret_as_s32(v_int11));
v_float64x2 v_src7 = v_cvt_f64_high(v_reinterpret_as_s32(v_int11));
v_float64x2 v_dst0 = v_load(dst + x);
v_float64x2 v_dst1 = v_load(dst + x + 2);
v_float64x2 v_dst2 = v_load(dst + x + 4);
v_float64x2 v_dst3 = v_load(dst + x + 6);
v_float64x2 v_dst4 = v_load(dst + x + 8);
v_float64x2 v_dst5 = v_load(dst + x + 10);
v_float64x2 v_dst6 = v_load(dst + x + 12);
v_float64x2 v_dst7 = v_load(dst + x + 14);
v_dst0 = v_dst0 + v_src0;
v_dst1 = v_dst1 + v_src1;
v_dst2 = v_dst2 + v_src2;
v_dst3 = v_dst3 + v_src3;
v_dst4 = v_dst4 + v_src4;
v_dst5 = v_dst5 + v_src5;
v_dst6 = v_dst6 + v_src6;
v_dst7 = v_dst7 + v_src7;
v_store(dst + x, v_dst0);
v_store(dst + x + 2, v_dst1);
v_store(dst + x + 4, v_dst2);
v_store(dst + x + 6, v_dst3);
v_store(dst + x + 8, v_dst4);
v_store(dst + x + 10, v_dst5);
v_store(dst + x + 12, v_dst6);
v_store(dst + x + 14, v_dst7);
}
}
return x;
......@@ -879,32 +855,32 @@ struct Acc_SIMD<ushort, double>
if (!mask)
{
__m128i v_0 = _mm_setzero_si128();
len *= cn;
for ( ; x <= len - 8; x += 8)
{
__m128i v_src = _mm_loadu_si128((const __m128i*)(src + x));
__m128i v_int0 = _mm_unpacklo_epi16(v_src, v_0);
__m128i v_int1 = _mm_unpackhi_epi16(v_src, v_0);
__m128d v_src0 = _mm_cvtepi32_pd(v_int0);
__m128d v_src1 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int0, _MM_SHUFFLE(0, 0, 3, 2)));
__m128d v_src2 = _mm_cvtepi32_pd(v_int1);
__m128d v_src3 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int1, _MM_SHUFFLE(0, 0, 3, 2)));
__m128d v_dst0 = _mm_loadu_pd(dst + x);
__m128d v_dst1 = _mm_loadu_pd(dst + x + 2);
__m128d v_dst2 = _mm_loadu_pd(dst + x + 4);
__m128d v_dst3 = _mm_loadu_pd(dst + x + 6);
v_dst0 = _mm_add_pd(v_dst0, v_src0);
v_dst1 = _mm_add_pd(v_dst1, v_src1);
v_dst2 = _mm_add_pd(v_dst2, v_src2);
v_dst3 = _mm_add_pd(v_dst3, v_src3);
_mm_storeu_pd(dst + x, v_dst0);
_mm_storeu_pd(dst + x + 2, v_dst1);
_mm_storeu_pd(dst + x + 4, v_dst2);
_mm_storeu_pd(dst + x + 6, v_dst3);
v_uint16x8 v_src = v_load(src + x);
v_uint32x4 v_int0, v_int1;
v_expand(v_src, v_int0, v_int1);
v_float64x2 v_src0 = v_cvt_f64(v_reinterpret_as_s32(v_int0));
v_float64x2 v_src1 = v_cvt_f64_high(v_reinterpret_as_s32(v_int0));
v_float64x2 v_src2 = v_cvt_f64(v_reinterpret_as_s32(v_int1));
v_float64x2 v_src3 = v_cvt_f64_high(v_reinterpret_as_s32(v_int1));
v_float64x2 v_dst0 = v_load(dst + x);
v_float64x2 v_dst1 = v_load(dst + x + 2);
v_float64x2 v_dst2 = v_load(dst + x + 4);
v_float64x2 v_dst3 = v_load(dst + x + 6);
v_dst0 = v_dst0 + v_src0;
v_dst1 = v_dst1 + v_src1;
v_dst2 = v_dst2 + v_src2;
v_dst3 = v_dst3 + v_src3;
v_store(dst + x, v_dst0);
v_store(dst + x + 2, v_dst1);
v_store(dst + x + 4, v_dst2);
v_store(dst + x + 6, v_dst3);
}
}
return x;
......@@ -924,7 +900,7 @@ struct AccSqr_SIMD<uchar, float>
len *= cn;
for ( ; x <= len - 16; x += 16)
{
v_uint8x16 v_src = v_load((const uchar*)(src + x));
v_uint8x16 v_src = v_load(src + x);
v_uint16x8 v_src0, v_src1;
v_expand(v_src, v_src0, v_src1);
v_src0 = v_src0 * v_src0;
......@@ -945,9 +921,9 @@ struct AccSqr_SIMD<uchar, float>
v_uint8x16 v_0 = v_setall_u8(0);
for ( ; x <= len - 16; x += 16)
{
v_uint8x16 v_mask = v_load((const uchar*)(mask + x));
v_uint8x16 v_mask = v_load(mask + x);
v_mask = ~(v_0 == v_mask);
v_uint8x16 v_src = v_load((const uchar*)(src + x));
v_uint8x16 v_src = v_load(src + x);
v_src = v_src & v_mask;
v_uint16x8 v_src0, v_src1;
v_expand(v_src, v_src0, v_src1);
......@@ -981,7 +957,7 @@ struct AccSqr_SIMD<ushort, float>
len *= cn;
for ( ; x <= len - 8; x += 8)
{
v_uint16x8 v_src = v_load((const ushort*)(src + x));
v_uint16x8 v_src = v_load(src + x);
v_uint32x4 v_src0, v_src1;
v_expand(v_src, v_src0, v_src1);
......@@ -1000,7 +976,7 @@ struct AccSqr_SIMD<ushort, float>
}
};
#if CV_SSE2
#if CV_SIMD128_64F
template <>
struct AccSqr_SIMD<uchar, double>
{
......@@ -1010,37 +986,39 @@ struct AccSqr_SIMD<uchar, double>
if (!mask)
{
__m128i v_0 = _mm_setzero_si128();
len *= cn;
for ( ; x <= len - 8; x += 8)
{
__m128i v_src = _mm_loadl_epi64((const __m128i*)(src + x));
__m128i v_int = _mm_unpacklo_epi8(v_src, v_0);
__m128i v_int0 = _mm_unpacklo_epi16(v_int, v_0);
__m128i v_int1 = _mm_unpackhi_epi16(v_int, v_0);
__m128d v_src0 = _mm_cvtepi32_pd(v_int0);
__m128d v_src1 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int0, _MM_SHUFFLE(0, 0, 3, 2)));
__m128d v_src2 = _mm_cvtepi32_pd(v_int1);
__m128d v_src3 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int1, _MM_SHUFFLE(0, 0, 3, 2)));
v_src0 = _mm_mul_pd(v_src0, v_src0);
v_src1 = _mm_mul_pd(v_src1, v_src1);
v_src2 = _mm_mul_pd(v_src2, v_src2);
v_src3 = _mm_mul_pd(v_src3, v_src3);
__m128d v_dst0 = _mm_loadu_pd(dst + x);
__m128d v_dst1 = _mm_loadu_pd(dst + x + 2);
__m128d v_dst2 = _mm_loadu_pd(dst + x + 4);
__m128d v_dst3 = _mm_loadu_pd(dst + x + 6);
v_dst0 = _mm_add_pd(v_dst0, v_src0);
v_dst1 = _mm_add_pd(v_dst1, v_src1);
v_dst2 = _mm_add_pd(v_dst2, v_src2);
v_dst3 = _mm_add_pd(v_dst3, v_src3);
_mm_storeu_pd(dst + x, v_dst0);
_mm_storeu_pd(dst + x + 2, v_dst1);
_mm_storeu_pd(dst + x + 4, v_dst2);
_mm_storeu_pd(dst + x + 6, v_dst3);
v_uint8x16 v_src = v_load(src + x);
v_uint16x8 v_int, dummy;
v_expand(v_src, v_int, dummy);
v_uint32x4 v_int0, v_int1;
v_expand(v_int, v_int0, v_int1);
v_float64x2 v_src0 = v_cvt_f64(v_reinterpret_as_s32(v_int0));
v_float64x2 v_src1 = v_cvt_f64_high(v_reinterpret_as_s32(v_int0));
v_float64x2 v_src2 = v_cvt_f64(v_reinterpret_as_s32(v_int1));
v_float64x2 v_src3 = v_cvt_f64_high(v_reinterpret_as_s32(v_int1));
v_src0 = v_src0 * v_src0;
v_src1 = v_src1 * v_src1;
v_src2 = v_src2 * v_src2;
v_src3 = v_src3 * v_src3;
v_float64x2 v_dst0 = v_load(dst + x);
v_float64x2 v_dst1 = v_load(dst + x + 2);
v_float64x2 v_dst2 = v_load(dst + x + 4);
v_float64x2 v_dst3 = v_load(dst + x + 6);
v_dst0 += v_src0;
v_dst1 += v_src1;
v_dst2 += v_src2;
v_dst3 += v_src3;
v_store(dst + x, v_dst0);
v_store(dst + x + 2, v_dst1);
v_store(dst + x + 4, v_dst2);
v_store(dst + x + 6, v_dst3);
}
}
return x;
......@@ -1056,36 +1034,39 @@ struct AccSqr_SIMD<ushort, double>
if (!mask)
{
__m128i v_0 = _mm_setzero_si128();
len *= cn;
for ( ; x <= len - 8; x += 8)
{
__m128i v_src = _mm_loadu_si128((const __m128i*)(src + x));
__m128i v_int0 = _mm_unpacklo_epi16(v_src, v_0);
__m128i v_int1 = _mm_unpackhi_epi16(v_src, v_0);
__m128d v_src0 = _mm_cvtepi32_pd(v_int0);
__m128d v_src1 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int0, _MM_SHUFFLE(0, 0, 3, 2)));
__m128d v_src2 = _mm_cvtepi32_pd(v_int1);
__m128d v_src3 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int1, _MM_SHUFFLE(0, 0, 3, 2)));
v_src0 = _mm_mul_pd(v_src0, v_src0);
v_src1 = _mm_mul_pd(v_src1, v_src1);
v_src2 = _mm_mul_pd(v_src2, v_src2);
v_src3 = _mm_mul_pd(v_src3, v_src3);
__m128d v_dst0 = _mm_loadu_pd(dst + x);
__m128d v_dst1 = _mm_loadu_pd(dst + x + 2);
__m128d v_dst2 = _mm_loadu_pd(dst + x + 4);
__m128d v_dst3 = _mm_loadu_pd(dst + x + 6);
v_dst0 = _mm_add_pd(v_dst0, v_src0);
v_dst1 = _mm_add_pd(v_dst1, v_src1);
v_dst2 = _mm_add_pd(v_dst2, v_src2);
v_dst3 = _mm_add_pd(v_dst3, v_src3);
_mm_storeu_pd(dst + x, v_dst0);
_mm_storeu_pd(dst + x + 2, v_dst1);
_mm_storeu_pd(dst + x + 4, v_dst2);
_mm_storeu_pd(dst + x + 6, v_dst3);
v_uint16x8 v_src = v_load(src + x);
v_uint32x4 v_int_0, v_int_1;
v_expand(v_src, v_int_0, v_int_1);
v_int32x4 v_int0 = v_reinterpret_as_s32(v_int_0);
v_int32x4 v_int1 = v_reinterpret_as_s32(v_int_1);
v_float64x2 v_src0 = v_cvt_f64(v_int0);
v_float64x2 v_src1 = v_cvt_f64_high(v_int0);
v_float64x2 v_src2 = v_cvt_f64(v_int1);
v_float64x2 v_src3 = v_cvt_f64_high(v_int1);
v_src0 = v_src0 * v_src0;
v_src1 = v_src1 * v_src1;
v_src2 = v_src2 * v_src2;
v_src3 = v_src3 * v_src3;
v_float64x2 v_dst0 = v_load(dst + x);
v_float64x2 v_dst1 = v_load(dst + x + 2);
v_float64x2 v_dst2 = v_load(dst + x + 4);
v_float64x2 v_dst3 = v_load(dst + x + 6);
v_dst0 += v_src0;
v_dst1 += v_src1;
v_dst2 += v_src2;
v_dst3 += v_src3;
v_store(dst + x, v_dst0);
v_store(dst + x + 2, v_dst1);
v_store(dst + x + 4, v_dst2);
v_store(dst + x + 6, v_dst3);
}
}
return x;
......@@ -1227,7 +1208,7 @@ struct AccProd_SIMD<ushort, float>
}
};
#if CV_SSE2
#if CV_SIMD128_64F
template <>
struct AccProd_SIMD<uchar, double>
{
......@@ -1237,38 +1218,44 @@ struct AccProd_SIMD<uchar, double>
if (!mask)
{
__m128i v_0 = _mm_setzero_si128();
len *= cn;
for ( ; x <= len - 8; x += 8)
{
__m128i v_1src = _mm_loadl_epi64((const __m128i*)(src1 + x));
__m128i v_2src = _mm_loadl_epi64((const __m128i*)(src2 + x));
__m128i v_1int = _mm_unpacklo_epi8(v_1src, v_0);
__m128i v_2int = _mm_unpacklo_epi8(v_2src, v_0);
__m128i v_1int0 = _mm_unpacklo_epi16(v_1int, v_0);
__m128i v_1int1 = _mm_unpackhi_epi16(v_1int, v_0);
__m128i v_2int0 = _mm_unpacklo_epi16(v_2int, v_0);
__m128i v_2int1 = _mm_unpackhi_epi16(v_2int, v_0);
__m128d v_src0 = _mm_mul_pd(_mm_cvtepi32_pd(v_1int0),_mm_cvtepi32_pd(v_2int0));
__m128d v_src1 = _mm_mul_pd(_mm_cvtepi32_pd(_mm_shuffle_epi32(v_1int0, _MM_SHUFFLE(0, 0, 3, 2))),_mm_cvtepi32_pd(_mm_shuffle_epi32(v_2int0, _MM_SHUFFLE(0, 0, 3, 2))));
__m128d v_src2 = _mm_mul_pd(_mm_cvtepi32_pd(v_1int1),_mm_cvtepi32_pd(v_2int1));
__m128d v_src3 = _mm_mul_pd(_mm_cvtepi32_pd(_mm_shuffle_epi32(v_1int1, _MM_SHUFFLE(0, 0, 3, 2))),_mm_cvtepi32_pd(_mm_shuffle_epi32(v_2int1, _MM_SHUFFLE(0, 0, 3, 2))));
__m128d v_dst0 = _mm_loadu_pd(dst + x);
__m128d v_dst1 = _mm_loadu_pd(dst + x + 2);
__m128d v_dst2 = _mm_loadu_pd(dst + x + 4);
__m128d v_dst3 = _mm_loadu_pd(dst + x + 6);
v_dst0 = _mm_add_pd(v_dst0, v_src0);
v_dst1 = _mm_add_pd(v_dst1, v_src1);
v_dst2 = _mm_add_pd(v_dst2, v_src2);
v_dst3 = _mm_add_pd(v_dst3, v_src3);
_mm_storeu_pd(dst + x, v_dst0);
_mm_storeu_pd(dst + x + 2, v_dst1);
_mm_storeu_pd(dst + x + 4, v_dst2);
_mm_storeu_pd(dst + x + 6, v_dst3);
v_uint8x16 v_1src = v_load(src1 + x);
v_uint8x16 v_2src = v_load(src2 + x);
v_uint16x8 v_1int, v_2int, dummy;
v_expand(v_1src, v_1int, dummy);
v_expand(v_2src, v_2int, dummy);
v_uint32x4 v_1int_0, v_1int_1, v_2int_0, v_2int_1;
v_expand(v_1int, v_1int_0, v_1int_1);
v_expand(v_2int, v_2int_0, v_2int_1);
v_int32x4 v_1int0 = v_reinterpret_as_s32(v_1int_0);
v_int32x4 v_1int1 = v_reinterpret_as_s32(v_1int_1);
v_int32x4 v_2int0 = v_reinterpret_as_s32(v_2int_0);
v_int32x4 v_2int1 = v_reinterpret_as_s32(v_2int_1);
v_float64x2 v_src0 = v_cvt_f64(v_1int0) * v_cvt_f64(v_2int0);
v_float64x2 v_src1 = v_cvt_f64_high(v_1int0) * v_cvt_f64_high(v_2int0);
v_float64x2 v_src2 = v_cvt_f64(v_1int1) * v_cvt_f64(v_2int1);
v_float64x2 v_src3 = v_cvt_f64_high(v_1int1) * v_cvt_f64_high(v_2int1);
v_float64x2 v_dst0 = v_load(dst + x);
v_float64x2 v_dst1 = v_load(dst + x + 2);
v_float64x2 v_dst2 = v_load(dst + x + 4);
v_float64x2 v_dst3 = v_load(dst + x + 6);
v_dst0 += v_src0;
v_dst1 += v_src1;
v_dst2 += v_src2;
v_dst3 += v_src3;
v_store(dst + x, v_dst0);
v_store(dst + x + 2, v_dst1);
v_store(dst + x + 4, v_dst2);
v_store(dst + x + 6, v_dst3);
}
}
return x;
......@@ -1284,35 +1271,40 @@ struct AccProd_SIMD<ushort, double>
if (!mask)
{
__m128i v_0 = _mm_setzero_si128();
len *= cn;
for ( ; x <= len - 8; x += 8)
{
__m128i v_1src = _mm_loadu_si128((const __m128i*)(src1 + x));
__m128i v_2src = _mm_loadu_si128((const __m128i*)(src2 + x));
__m128i v_1int0 = _mm_unpacklo_epi16(v_1src, v_0);
__m128i v_1int1 = _mm_unpackhi_epi16(v_1src, v_0);
__m128i v_2int0 = _mm_unpacklo_epi16(v_2src, v_0);
__m128i v_2int1 = _mm_unpackhi_epi16(v_2src, v_0);
__m128d v_src0 = _mm_mul_pd(_mm_cvtepi32_pd(v_1int0),_mm_cvtepi32_pd(v_2int0));
__m128d v_src1 = _mm_mul_pd(_mm_cvtepi32_pd(_mm_shuffle_epi32(v_1int0, _MM_SHUFFLE(0, 0, 3, 2))),_mm_cvtepi32_pd(_mm_shuffle_epi32(v_2int0, _MM_SHUFFLE(0, 0, 3, 2))));
__m128d v_src2 = _mm_mul_pd(_mm_cvtepi32_pd(v_1int1),_mm_cvtepi32_pd(v_2int1));
__m128d v_src3 = _mm_mul_pd(_mm_cvtepi32_pd(_mm_shuffle_epi32(v_1int1, _MM_SHUFFLE(0, 0, 3, 2))),_mm_cvtepi32_pd(_mm_shuffle_epi32(v_2int1, _MM_SHUFFLE(0, 0, 3, 2))));
__m128d v_dst0 = _mm_loadu_pd(dst + x);
__m128d v_dst1 = _mm_loadu_pd(dst + x + 2);
__m128d v_dst2 = _mm_loadu_pd(dst + x + 4);
__m128d v_dst3 = _mm_loadu_pd(dst + x + 6);
v_dst0 = _mm_add_pd(v_dst0, v_src0);
v_dst1 = _mm_add_pd(v_dst1, v_src1);
v_dst2 = _mm_add_pd(v_dst2, v_src2);
v_dst3 = _mm_add_pd(v_dst3, v_src3);
_mm_storeu_pd(dst + x, v_dst0);
_mm_storeu_pd(dst + x + 2, v_dst1);
_mm_storeu_pd(dst + x + 4, v_dst2);
_mm_storeu_pd(dst + x + 6, v_dst3);
v_uint16x8 v_1src = v_load(src1 + x);
v_uint16x8 v_2src = v_load(src2 + x);
v_uint32x4 v_1int_0, v_1int_1, v_2int_0, v_2int_1;
v_expand(v_1src, v_1int_0, v_1int_1);
v_expand(v_2src, v_2int_0, v_2int_1);
v_int32x4 v_1int0 = v_reinterpret_as_s32(v_1int_0);
v_int32x4 v_1int1 = v_reinterpret_as_s32(v_1int_1);
v_int32x4 v_2int0 = v_reinterpret_as_s32(v_2int_0);
v_int32x4 v_2int1 = v_reinterpret_as_s32(v_2int_1);
v_float64x2 v_src0 = v_cvt_f64(v_1int0) * v_cvt_f64(v_2int0);
v_float64x2 v_src1 = v_cvt_f64_high(v_1int0) * v_cvt_f64_high(v_2int0);
v_float64x2 v_src2 = v_cvt_f64(v_1int1) * v_cvt_f64(v_2int1);
v_float64x2 v_src3 = v_cvt_f64_high(v_1int1) * v_cvt_f64_high(v_2int1);
v_float64x2 v_dst0 = v_load(dst + x);
v_float64x2 v_dst1 = v_load(dst + x + 2);
v_float64x2 v_dst2 = v_load(dst + x + 4);
v_float64x2 v_dst3 = v_load(dst + x + 6);
v_dst0 = v_dst0 + v_src0;
v_dst1 = v_dst1 + v_src1;
v_dst2 = v_dst2 + v_src2;
v_dst3 = v_dst3 + v_src3;
v_store(dst + x, v_dst0);
v_store(dst + x + 2, v_dst1);
v_store(dst + x + 4, v_dst2);
v_store(dst + x + 6, v_dst3);
}
}
return x;
......@@ -1399,45 +1391,50 @@ struct AccW_SIMD<ushort, float>
}
};
#if CV_SSE2
#if CV_SIMD128_64F
template <>
struct AccW_SIMD<uchar, double>
{
int operator() (const uchar * src, double * dst, const uchar * mask, int len, int cn, double alpha) const
{
int x = 0;
__m128d v_alpha = _mm_set1_pd(alpha);
__m128d v_beta = _mm_set1_pd(1.0f - alpha);
__m128i v_0 = _mm_setzero_si128();
v_float64x2 v_alpha = v_setall_f64(alpha);
v_float64x2 v_beta = v_setall_f64(1.0f - alpha);
if (!mask)
{
len *= cn;
for ( ; x <= len - 8; x += 8)
{
__m128i v_src = _mm_loadl_epi64((const __m128i*)(src + x));
__m128i v_int = _mm_unpacklo_epi8(v_src, v_0);
__m128i v_int0 = _mm_unpacklo_epi16(v_int, v_0);
__m128i v_int1 = _mm_unpackhi_epi16(v_int, v_0);
__m128d v_src0 = _mm_cvtepi32_pd(v_int0);
__m128d v_src1 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int0, _MM_SHUFFLE(0, 0, 3, 2)));
__m128d v_src2 = _mm_cvtepi32_pd(v_int1);
__m128d v_src3 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int1, _MM_SHUFFLE(0, 0, 3, 2)));
__m128d v_dst0 = _mm_loadu_pd(dst + x);
__m128d v_dst1 = _mm_loadu_pd(dst + x + 2);
__m128d v_dst2 = _mm_loadu_pd(dst + x + 4);
__m128d v_dst3 = _mm_loadu_pd(dst + x + 6);
v_dst0 = _mm_add_pd(_mm_mul_pd(v_dst0, v_beta), _mm_mul_pd(v_src0, v_alpha));
v_dst1 = _mm_add_pd(_mm_mul_pd(v_dst1, v_beta), _mm_mul_pd(v_src1, v_alpha));
v_dst2 = _mm_add_pd(_mm_mul_pd(v_dst2, v_beta), _mm_mul_pd(v_src2, v_alpha));
v_dst3 = _mm_add_pd(_mm_mul_pd(v_dst3, v_beta), _mm_mul_pd(v_src3, v_alpha));
_mm_storeu_pd(dst + x, v_dst0);
_mm_storeu_pd(dst + x + 2, v_dst1);
_mm_storeu_pd(dst + x + 4, v_dst2);
_mm_storeu_pd(dst + x + 6, v_dst3);
v_uint8x16 v_src = v_load(src + x);
v_uint16x8 v_int, dummy;
v_expand(v_src, v_int, dummy);
v_uint32x4 v_int_0, v_int_1;
v_expand(v_int, v_int_0, v_int_1);
v_int32x4 v_int0 = v_reinterpret_as_s32(v_int_0);
v_int32x4 v_int1 = v_reinterpret_as_s32(v_int_1);
v_float64x2 v_src0 = v_cvt_f64(v_int0);
v_float64x2 v_src1 = v_cvt_f64_high(v_int0);
v_float64x2 v_src2 = v_cvt_f64(v_int1);
v_float64x2 v_src3 = v_cvt_f64_high(v_int1);
v_float64x2 v_dst0 = v_load(dst + x);
v_float64x2 v_dst1 = v_load(dst + x + 2);
v_float64x2 v_dst2 = v_load(dst + x + 4);
v_float64x2 v_dst3 = v_load(dst + x + 6);
v_dst0 = (v_dst0 * v_beta) + (v_src0 * v_alpha);
v_dst1 = (v_dst1 * v_beta) + (v_src1 * v_alpha);
v_dst2 = (v_dst2 * v_beta) + (v_src2 * v_alpha);
v_dst3 = (v_dst3 * v_beta) + (v_src3 * v_alpha);
v_store(dst + x, v_dst0);
v_store(dst + x + 2, v_dst1);
v_store(dst + x + 4, v_dst2);
v_store(dst + x + 6, v_dst3);
}
}
......@@ -1451,44 +1448,47 @@ struct AccW_SIMD<ushort, double>
int operator() (const ushort * src, double * dst, const uchar * mask, int len, int cn, double alpha) const
{
int x = 0;
__m128d v_alpha = _mm_set1_pd(alpha);
__m128d v_beta = _mm_set1_pd(1.0f - alpha);
__m128i v_0 = _mm_setzero_si128();
v_float64x2 v_alpha = v_setall_f64(alpha);
v_float64x2 v_beta = v_setall_f64(1.0f - alpha);
if (!mask)
{
len *= cn;
for ( ; x <= len - 8; x += 8)
{
__m128i v_src = _mm_loadu_si128((const __m128i*)(src + x));
__m128i v_int0 = _mm_unpacklo_epi16(v_src, v_0);
__m128i v_int1 = _mm_unpackhi_epi16(v_src, v_0);
__m128d v_src00 = _mm_cvtepi32_pd(v_int0);
__m128d v_src01 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int0, _MM_SHUFFLE(0, 0, 3, 2)));
__m128d v_src10 = _mm_cvtepi32_pd(v_int1);
__m128d v_src11 = _mm_cvtepi32_pd(_mm_shuffle_epi32(v_int1, _MM_SHUFFLE(0, 0, 3, 2)));
__m128d v_dst00 = _mm_loadu_pd(dst + x);
__m128d v_dst01 = _mm_loadu_pd(dst + x + 2);
__m128d v_dst10 = _mm_loadu_pd(dst + x + 4);
__m128d v_dst11 = _mm_loadu_pd(dst + x + 6);
v_dst00 = _mm_add_pd(_mm_mul_pd(v_dst00, v_beta), _mm_mul_pd(v_src00, v_alpha));
v_dst01 = _mm_add_pd(_mm_mul_pd(v_dst01, v_beta), _mm_mul_pd(v_src01, v_alpha));
v_dst10 = _mm_add_pd(_mm_mul_pd(v_dst10, v_beta), _mm_mul_pd(v_src10, v_alpha));
v_dst11 = _mm_add_pd(_mm_mul_pd(v_dst11, v_beta), _mm_mul_pd(v_src11, v_alpha));
_mm_storeu_pd(dst + x, v_dst00);
_mm_storeu_pd(dst + x + 2, v_dst01);
_mm_storeu_pd(dst + x + 4, v_dst10);
_mm_storeu_pd(dst + x + 6, v_dst11);
v_uint16x8 v_src = v_load(src + x);
v_uint32x4 v_int_0, v_int_1;
v_expand(v_src, v_int_0, v_int_1);
v_int32x4 v_int0 = v_reinterpret_as_s32(v_int_0);
v_int32x4 v_int1 = v_reinterpret_as_s32(v_int_1);
v_float64x2 v_src00 = v_cvt_f64(v_int0);
v_float64x2 v_src01 = v_cvt_f64_high(v_int0);
v_float64x2 v_src10 = v_cvt_f64(v_int1);
v_float64x2 v_src11 = v_cvt_f64_high(v_int1);
v_float64x2 v_dst00 = v_load(dst + x);
v_float64x2 v_dst01 = v_load(dst + x + 2);
v_float64x2 v_dst10 = v_load(dst + x + 4);
v_float64x2 v_dst11 = v_load(dst + x + 6);
v_dst00 = (v_dst00 * v_beta) + (v_src00 * v_alpha);
v_dst01 = (v_dst01 * v_beta) + (v_src01 * v_alpha);
v_dst10 = (v_dst10 * v_beta) + (v_src10 * v_alpha);
v_dst11 = (v_dst11 * v_beta) + (v_src11 * v_alpha);
v_store(dst + x, v_dst00);
v_store(dst + x + 2, v_dst01);
v_store(dst + x + 4, v_dst10);
v_store(dst + x + 6, v_dst11);
}
}
return x;
}
};
#endif //CV_SSE2
#endif //CV_SIMD128_64F
#endif //CV_SIMD128
template<typename T, typename AT> void
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或