28 template<
typename Func,
typename Evaluator>
32 typedef typename find_best_packet<typename Evaluator::Scalar,Evaluator::SizeAtCompileTime>::type PacketType;
35 InnerMaxSize = int(Evaluator::IsRowMajor)
36 ? Evaluator::MaxColsAtCompileTime
37 : Evaluator::MaxRowsAtCompileTime,
38 OuterMaxSize = int(Evaluator::IsRowMajor)
39 ? Evaluator::MaxRowsAtCompileTime
40 : Evaluator::MaxColsAtCompileTime,
42 : int(OuterMaxSize)==
Dynamic ? (int(InnerMaxSize)>=int(PacketSize) ?
Dynamic : 0)
43 : (
int(InnerMaxSize)/int(PacketSize)) *
int(OuterMaxSize)
49 && (functor_traits<Func>::PacketAccess),
50 MayLinearVectorize =
bool(MightVectorize) &&
bool(MayLinearize),
51 MaySliceVectorize =
bool(MightVectorize) && (int(SliceVectorizedWork)==
Dynamic || int(SliceVectorizedWork)>=3)
65 : int(Evaluator::SizeAtCompileTime) * int(Evaluator::CoeffReadCost) + (Evaluator::SizeAtCompileTime-1) * functor_traits<Func>::Cost,
74 #ifdef EIGEN_DEBUG_ASSIGN
77 std::cerr <<
"Xpr: " <<
typeid(
typename Evaluator::XprType).name() << std::endl;
78 std::cerr.setf(std::ios::hex, std::ios::basefield);
80 std::cerr.unsetf(std::ios::hex);
88 std::cerr <<
"Traversal" <<
" = " << Traversal <<
" (" << demangle_traversal(Traversal) <<
")" << std::endl;
90 std::cerr <<
"Unrolling" <<
" = " << Unrolling <<
" (" << demangle_unrolling(Unrolling) <<
")" << std::endl;
91 std::cerr << std::endl;
102 template<
typename Func,
typename Evaluator, Index Start, Index Length>
103 struct redux_novec_unroller
105 static constexpr
Index HalfLength = Length/2;
107 typedef typename Evaluator::Scalar Scalar;
110 static EIGEN_STRONG_INLINE Scalar run(
const Evaluator &eval,
const Func& func)
112 return func(redux_novec_unroller<Func, Evaluator, Start, HalfLength>::run(eval,func),
113 redux_novec_unroller<Func, Evaluator, Start+HalfLength, Length-HalfLength>::run(eval,func));
117 template<
typename Func,
typename Evaluator, Index Start>
118 struct redux_novec_unroller<Func, Evaluator, Start, 1>
120 static constexpr
Index outer = Start / Evaluator::InnerSizeAtCompileTime;
121 static constexpr
Index inner = Start % Evaluator::InnerSizeAtCompileTime;
123 typedef typename Evaluator::Scalar Scalar;
126 static EIGEN_STRONG_INLINE Scalar run(
const Evaluator &eval,
const Func&)
128 return eval.coeffByOuterInner(outer, inner);
135 template<
typename Func,
typename Evaluator, Index Start>
136 struct redux_novec_unroller<Func, Evaluator, Start, 0>
138 typedef typename Evaluator::Scalar Scalar;
140 static EIGEN_STRONG_INLINE Scalar run(
const Evaluator&,
const Func&) {
return Scalar(); }
143 template<
typename Func,
typename Evaluator, Index Start, Index Length>
144 struct redux_novec_linear_unroller
146 static constexpr
Index HalfLength = Length/2;
148 typedef typename Evaluator::Scalar Scalar;
151 static EIGEN_STRONG_INLINE Scalar run(
const Evaluator &eval,
const Func& func)
153 return func(redux_novec_linear_unroller<Func, Evaluator, Start, HalfLength>::run(eval,func),
154 redux_novec_linear_unroller<Func, Evaluator, Start+HalfLength, Length-HalfLength>::run(eval,func));
158 template<
typename Func,
typename Evaluator, Index Start>
159 struct redux_novec_linear_unroller<Func, Evaluator, Start, 1>
161 typedef typename Evaluator::Scalar Scalar;
164 static EIGEN_STRONG_INLINE Scalar run(
const Evaluator &eval,
const Func&)
166 return eval.coeff(Start);
173 template<
typename Func,
typename Evaluator, Index Start>
174 struct redux_novec_linear_unroller<Func, Evaluator, Start, 0>
176 typedef typename Evaluator::Scalar Scalar;
178 static EIGEN_STRONG_INLINE Scalar run(
const Evaluator&,
const Func&) {
return Scalar(); }
183 template<
typename Func,
typename Evaluator, Index Start, Index Length>
184 struct redux_vec_unroller
186 template<
typename PacketType>
188 static EIGEN_STRONG_INLINE PacketType run(
const Evaluator &eval,
const Func& func)
190 constexpr
Index HalfLength = Length/2;
192 return func.packetOp(
193 redux_vec_unroller<Func, Evaluator, Start, HalfLength>::template run<PacketType>(eval,func),
194 redux_vec_unroller<Func, Evaluator, Start+HalfLength, Length-HalfLength>::template run<PacketType>(eval,func) );
198 template<
typename Func,
typename Evaluator, Index Start>
199 struct redux_vec_unroller<Func, Evaluator, Start, 1>
201 template<
typename PacketType>
203 static EIGEN_STRONG_INLINE PacketType run(
const Evaluator &eval,
const Func&)
206 constexpr
Index index = Start * PacketSize;
207 constexpr
Index outer = index / int(Evaluator::InnerSizeAtCompileTime);
208 constexpr
Index inner = index % int(Evaluator::InnerSizeAtCompileTime);
209 constexpr
int alignment = Evaluator::Alignment;
211 return eval.template packetByOuterInner<alignment,PacketType>(outer, inner);
215 template<
typename Func,
typename Evaluator, Index Start, Index Length>
216 struct redux_vec_linear_unroller
218 template<
typename PacketType>
220 static EIGEN_STRONG_INLINE PacketType run(
const Evaluator &eval,
const Func& func)
222 constexpr
Index HalfLength = Length/2;
224 return func.packetOp(
225 redux_vec_linear_unroller<Func, Evaluator, Start, HalfLength>::template run<PacketType>(eval,func),
226 redux_vec_linear_unroller<Func, Evaluator, Start+HalfLength, Length-HalfLength>::template run<PacketType>(eval,func) );
230 template<
typename Func,
typename Evaluator, Index Start>
231 struct redux_vec_linear_unroller<Func, Evaluator, Start, 1>
233 template<
typename PacketType>
235 static EIGEN_STRONG_INLINE PacketType run(
const Evaluator &eval,
const Func&)
238 constexpr
Index index = (Start * PacketSize);
239 constexpr
int alignment = Evaluator::Alignment;
240 return eval.template packet<alignment,PacketType>(index);
248 template<
typename Func,
typename Evaluator,
249 int Traversal = redux_traits<Func, Evaluator>::Traversal,
250 int Unrolling = redux_traits<Func, Evaluator>::Unrolling
254 template<
typename Func,
typename Evaluator>
257 typedef typename Evaluator::Scalar Scalar;
259 template<
typename XprType>
261 Scalar run(
const Evaluator &eval,
const Func& func,
const XprType& xpr)
263 eigen_assert(xpr.rows()>0 && xpr.cols()>0 &&
"you are using an empty matrix");
264 Scalar
res = eval.coeffByOuterInner(0, 0);
265 for(
Index i = 1;
i < xpr.innerSize(); ++
i)
266 res = func(
res, eval.coeffByOuterInner(0,
i));
267 for(
Index i = 1;
i < xpr.outerSize(); ++
i)
268 for(
Index j = 0;
j < xpr.innerSize(); ++
j)
269 res = func(
res, eval.coeffByOuterInner(
i,
j));
274 template<
typename Func,
typename Evaluator>
277 typedef typename Evaluator::Scalar Scalar;
279 template<
typename XprType>
281 Scalar run(
const Evaluator &eval,
const Func& func,
const XprType& xpr)
283 eigen_assert(xpr.size()>0 &&
"you are using an empty matrix");
284 Scalar
res = eval.coeff(0);
285 for(
Index k = 1; k < xpr.size(); ++k)
286 res = func(
res, eval.coeff(k));
291 template<
typename Func,
typename Evaluator>
293 : redux_novec_unroller<Func,Evaluator, 0, Evaluator::SizeAtCompileTime>
295 typedef redux_novec_unroller<Func,Evaluator, 0, Evaluator::SizeAtCompileTime> Base;
296 typedef typename Evaluator::Scalar Scalar;
297 template<
typename XprType>
299 Scalar run(
const Evaluator &eval,
const Func& func,
const XprType& )
301 return Base::run(eval,func);
305 template<
typename Func,
typename Evaluator>
307 : redux_novec_linear_unroller<Func,Evaluator, 0, Evaluator::SizeAtCompileTime>
309 typedef redux_novec_linear_unroller<Func,Evaluator, 0, Evaluator::SizeAtCompileTime> Base;
310 typedef typename Evaluator::Scalar Scalar;
311 template<
typename XprType>
313 Scalar run(
const Evaluator &eval,
const Func& func,
const XprType& )
315 return Base::run(eval,func);
319 template<
typename Func,
typename Evaluator>
322 typedef typename Evaluator::Scalar Scalar;
323 typedef typename redux_traits<Func, Evaluator>::PacketType PacketScalar;
325 template<
typename XprType>
326 static Scalar run(
const Evaluator &eval,
const Func& func,
const XprType& xpr)
330 constexpr
Index packetSize = redux_traits<Func, Evaluator>::PacketSize;
331 constexpr
int packetAlignment = unpacket_traits<PacketScalar>::alignment;
332 constexpr
int alignment0 = (
bool(Evaluator::Flags &
DirectAccessBit) &&
bool(packet_traits<Scalar>::AlignedOnScalar)) ?
int(packetAlignment) : int(
Unaligned);
333 constexpr
int alignment =
plain_enum_max(alignment0, Evaluator::Alignment);
335 const Index alignedSize2 = ((
size-alignedStart)/(2*packetSize))*(2*packetSize);
336 const Index alignedSize = ((
size-alignedStart)/(packetSize))*(packetSize);
337 const Index alignedEnd2 = alignedStart + alignedSize2;
338 const Index alignedEnd = alignedStart + alignedSize;
342 PacketScalar packet_res0 = eval.template packet<alignment,PacketScalar>(alignedStart);
343 if(alignedSize>packetSize)
345 PacketScalar packet_res1 = eval.template packet<alignment,PacketScalar>(alignedStart+packetSize);
346 for(
Index index = alignedStart + 2*packetSize; index < alignedEnd2; index += 2*packetSize)
348 packet_res0 = func.packetOp(packet_res0, eval.template packet<alignment,PacketScalar>(index));
349 packet_res1 = func.packetOp(packet_res1, eval.template packet<alignment,PacketScalar>(index+packetSize));
352 packet_res0 = func.packetOp(packet_res0,packet_res1);
353 if(alignedEnd>alignedEnd2)
354 packet_res0 = func.packetOp(packet_res0, eval.template packet<alignment,PacketScalar>(alignedEnd2));
356 res = func.predux(packet_res0);
358 for(
Index index = 0; index < alignedStart; ++index)
359 res = func(
res,eval.coeff(index));
361 for(
Index index = alignedEnd; index <
size; ++index)
362 res = func(
res,eval.coeff(index));
368 for(
Index index = 1; index <
size; ++index)
369 res = func(
res,eval.coeff(index));
377 template<
typename Func,
typename Evaluator,
int Unrolling>
380 typedef typename Evaluator::Scalar Scalar;
381 typedef typename redux_traits<Func, Evaluator>::PacketType PacketType;
383 template<
typename XprType>
384 EIGEN_DEVICE_FUNC static Scalar run(
const Evaluator &eval,
const Func& func,
const XprType& xpr)
386 eigen_assert(xpr.rows()>0 && xpr.cols()>0 &&
"you are using an empty matrix");
387 constexpr
Index packetSize = redux_traits<Func, Evaluator>::PacketSize;
388 const Index innerSize = xpr.innerSize();
389 const Index outerSize = xpr.outerSize();
390 const Index packetedInnerSize = ((innerSize)/packetSize)*packetSize;
392 if(packetedInnerSize)
394 PacketType packet_res = eval.template packet<Unaligned,PacketType>(0,0);
396 for(
Index i=(
j==0?packetSize:0);
i<packetedInnerSize;
i+=
Index(packetSize))
397 packet_res = func.packetOp(packet_res, eval.template packetByOuterInner<Unaligned,PacketType>(
j,
i));
399 res = func.predux(packet_res);
401 for(
Index i=packetedInnerSize;
i<innerSize; ++
i)
402 res = func(
res, eval.coeffByOuterInner(
j,
i));
407 res = redux_impl<Func, Evaluator, DefaultTraversal, NoUnrolling>::run(eval, func, xpr);
414 template<
typename Func,
typename Evaluator>
417 typedef typename Evaluator::Scalar Scalar;
419 typedef typename redux_traits<Func, Evaluator>::PacketType PacketType;
420 static constexpr
Index PacketSize = redux_traits<Func, Evaluator>::PacketSize;
421 static constexpr
Index Size = Evaluator::SizeAtCompileTime;
422 static constexpr
Index VectorizedSize = (int(Size) / int(PacketSize)) *
int(PacketSize);
424 template<
typename XprType>
426 Scalar run(
const Evaluator &eval,
const Func& func,
const XprType &xpr)
429 eigen_assert(xpr.rows()>0 && xpr.cols()>0 &&
"you are using an empty matrix");
430 if (VectorizedSize > 0) {
431 Scalar
res = func.predux(redux_vec_linear_unroller<Func, Evaluator, 0, Size / PacketSize>::template run<PacketType>(eval,func));
432 if (VectorizedSize != Size)
433 res = func(
res,redux_novec_linear_unroller<Func, Evaluator, VectorizedSize, Size-VectorizedSize>::run(eval,func));
437 return redux_novec_linear_unroller<Func, Evaluator, 0, Size>::run(eval,func);
443 template<
typename XprType_>
444 class redux_evaluator :
public internal::evaluator<XprType_>
446 typedef internal::evaluator<XprType_>
Base;
448 typedef XprType_ XprType;
450 explicit redux_evaluator(
const XprType &xpr) :
Base(xpr) {}
452 typedef typename XprType::Scalar
Scalar;
457 MaxRowsAtCompileTime = XprType::MaxRowsAtCompileTime,
458 MaxColsAtCompileTime = XprType::MaxColsAtCompileTime,
461 IsRowMajor = XprType::IsRowMajor,
462 SizeAtCompileTime = XprType::SizeAtCompileTime,
463 InnerSizeAtCompileTime = XprType::InnerSizeAtCompileTime
467 CoeffReturnType coeffByOuterInner(
Index outer,
Index inner)
const
468 {
return Base::coeff(IsRowMajor ? outer : inner, IsRowMajor ? inner : outer); }
470 template<
int LoadMode,
typename PacketType>
472 PacketType packetByOuterInner(
Index outer,
Index inner)
const
473 {
return Base::template packet<LoadMode,PacketType>(IsRowMajor ? outer : inner, IsRowMajor ? inner : outer); }
493 template<
typename Derived>
494 template<
typename Func>
500 typedef typename internal::redux_evaluator<Derived> ThisEvaluator;
501 ThisEvaluator thisEval(derived());
505 return internal::redux_impl<Func, ThisEvaluator>::run(thisEval, func, derived());
515 template<
typename Derived>
516 template<
int NaNPropagation>
520 return derived().redux(Eigen::internal::scalar_min_op<Scalar,Scalar, NaNPropagation>());
530 template<
typename Derived>
531 template<
int NaNPropagation>
535 return derived().redux(Eigen::internal::scalar_max_op<Scalar,Scalar, NaNPropagation>());
544 template<
typename Derived>
548 if(SizeAtCompileTime==0 || (SizeAtCompileTime==
Dynamic &&
size()==0))
550 return derived().redux(Eigen::internal::scalar_sum_op<Scalar,Scalar>());
557 template<
typename Derived>
561 #ifdef __INTEL_COMPILER
563 #pragma warning ( disable : 2259 )
565 return Scalar(derived().redux(Eigen::internal::scalar_sum_op<Scalar,Scalar>())) /
Scalar(this->
size());
566 #ifdef __INTEL_COMPILER
578 template<
typename Derived>
582 if(SizeAtCompileTime==0 || (SizeAtCompileTime==
Dynamic &&
size()==0))
584 return derived().redux(Eigen::internal::scalar_product_op<Scalar>());
593 template<
typename Derived>
597 return derived().diagonal().sum();
#define EIGEN_DEBUG_VAR(x)
#define EIGEN_DEVICE_FUNC
#define EIGEN_ONLY_USED_FOR_DEBUG(x)
cout<< "Here is the matrix m:"<< endl<< m<< endl;Matrix< ptrdiff_t, 3, 1 > res
#define EIGEN_UNROLLING_LIMIT
internal::traits< Derived >::Scalar minCoeff() const
Base::CoeffReturnType CoeffReturnType
internal::traits< Homogeneous< MatrixType, Direction_ > >::Scalar Scalar
internal::traits< Derived >::Scalar maxCoeff() const
Scalar redux(const BinaryOp &func) const
internal::find_best_packet< Scalar, SizeAtCompileTime >::type PacketScalar
const unsigned int ActualPacketAccessBit
const unsigned int LinearAccessBit
const unsigned int DirectAccessBit
constexpr int plain_enum_max(A a, B b)
static Index first_default_aligned(const DenseBase< Derived > &m)
@ LinearVectorizedTraversal
@ SliceVectorizedTraversal
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.