NEON/PacketMath.h
Go to the documentation of this file.
1 // This file is part of Eigen, a lightweight C++ template library
2 // for linear algebra.
3 //
4 // Copyright (C) 2008-2009 Gael Guennebaud <gael.guennebaud@inria.fr>
5 // Copyright (C) 2010 Konstantinos Margaritis <markos@freevec.org>
6 // Heavily based on Gael's SSE version.
7 //
8 // This Source Code Form is subject to the terms of the Mozilla
9 // Public License v. 2.0. If a copy of the MPL was not distributed
10 // with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
11 
12 #ifndef EIGEN_PACKET_MATH_NEON_H
13 #define EIGEN_PACKET_MATH_NEON_H
14 
15 #include "../../InternalHeaderCheck.h"
16 
17 namespace Eigen {
18 
19 namespace internal {
20 
21 #ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
22 #define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
23 #endif
24 
25 #ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
26 #define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
27 #endif
28 
29 #ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
30 #if EIGEN_ARCH_ARM64
31 #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32
32 #else
33 #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 16
34 #endif
35 #endif
36 
37 #if EIGEN_COMP_MSVC_STRICT
38 
39 // In MSVC's arm_neon.h header file, all NEON vector types
40 // are aliases to the same underlying type __n128.
41 // We thus have to wrap them to make them different C++ types.
42 // (See also bug 1428)
43 typedef eigen_packet_wrapper<float32x2_t,0> Packet2f;
44 typedef eigen_packet_wrapper<float32x4_t,1> Packet4f;
45 typedef eigen_packet_wrapper<int32_t ,2> Packet4c;
46 typedef eigen_packet_wrapper<int8x8_t ,3> Packet8c;
47 typedef eigen_packet_wrapper<int8x16_t ,4> Packet16c;
48 typedef eigen_packet_wrapper<uint32_t ,5> Packet4uc;
49 typedef eigen_packet_wrapper<uint8x8_t ,6> Packet8uc;
50 typedef eigen_packet_wrapper<uint8x16_t ,7> Packet16uc;
51 typedef eigen_packet_wrapper<int16x4_t ,8> Packet4s;
52 typedef eigen_packet_wrapper<int16x8_t ,9> Packet8s;
53 typedef eigen_packet_wrapper<uint16x4_t ,10> Packet4us;
54 typedef eigen_packet_wrapper<uint16x8_t ,11> Packet8us;
55 typedef eigen_packet_wrapper<int32x2_t ,12> Packet2i;
56 typedef eigen_packet_wrapper<int32x4_t ,13> Packet4i;
57 typedef eigen_packet_wrapper<uint32x2_t ,14> Packet2ui;
58 typedef eigen_packet_wrapper<uint32x4_t ,15> Packet4ui;
59 typedef eigen_packet_wrapper<int64x2_t ,16> Packet2l;
60 typedef eigen_packet_wrapper<uint64x2_t ,17> Packet2ul;
61 
62 EIGEN_ALWAYS_INLINE Packet4f make_packet4f(float a, float b, float c, float d) {
63  float from[4] = {a, b, c, d};
64  return vld1q_f32(from);
65 }
66 
67 EIGEN_ALWAYS_INLINE Packet2f make_packet2f(float a, float b) {
68  float from[2] = {a, b};
69  return vld1_f32(from);
70 }
71 
72 #else
73 
74 typedef float32x2_t Packet2f;
75 typedef float32x4_t Packet4f;
76 typedef eigen_packet_wrapper<int32_t ,2> Packet4c;
77 typedef int8x8_t Packet8c;
78 typedef int8x16_t Packet16c;
79 typedef eigen_packet_wrapper<uint32_t ,5> Packet4uc;
80 typedef uint8x8_t Packet8uc;
81 typedef uint8x16_t Packet16uc;
82 typedef int16x4_t Packet4s;
83 typedef int16x8_t Packet8s;
84 typedef uint16x4_t Packet4us;
85 typedef uint16x8_t Packet8us;
86 typedef int32x2_t Packet2i;
87 typedef int32x4_t Packet4i;
88 typedef uint32x2_t Packet2ui;
89 typedef uint32x4_t Packet4ui;
90 typedef int64x2_t Packet2l;
91 typedef uint64x2_t Packet2ul;
92 
93 EIGEN_ALWAYS_INLINE Packet4f make_packet4f(float a, float b, float c, float d) { return Packet4f{a, b, c, d}; }
94 EIGEN_ALWAYS_INLINE Packet2f make_packet2f(float a, float b) { return Packet2f{a, b}; }
95 
96 #endif // EIGEN_COMP_MSVC_STRICT
97 
98 EIGEN_STRONG_INLINE Packet4f shuffle1(const Packet4f& m, int mask){
99  const float* a = reinterpret_cast<const float*>(&m);
100  Packet4f res = make_packet4f(*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3 )), *(a + ((mask >> 6) & 3)));
101  return res;
102 }
103 
104 // fuctionally equivalent to _mm_shuffle_ps in SSE when interleave
105 // == false (i.e. shuffle<false>(m, n, mask) equals _mm_shuffle_ps(m, n, mask)),
106 // interleave m and n when interleave == true. Currently used in LU/arch/InverseSize4.h
107 // to enable a shared implementation for fast inversion of matrices of size 4.
108 template<bool interleave>
109 EIGEN_STRONG_INLINE Packet4f shuffle2(const Packet4f &m, const Packet4f &n, int mask)
110 {
111  const float* a = reinterpret_cast<const float*>(&m);
112  const float* b = reinterpret_cast<const float*>(&n);
113  Packet4f res = make_packet4f(*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(b + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3)));
114  return res;
115 }
116 
117 template<>
118 EIGEN_STRONG_INLINE Packet4f shuffle2<true>(const Packet4f &m, const Packet4f &n, int mask)
119 {
120  const float* a = reinterpret_cast<const float*>(&m);
121  const float* b = reinterpret_cast<const float*>(&n);
122  Packet4f res = make_packet4f(*(a + (mask & 3)), *(b + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3)));
123  return res;
124 }
125 
126 EIGEN_STRONG_INLINE static int eigen_neon_shuffle_mask(int p, int q, int r, int s) {return ((s)<<6|(r)<<4|(q)<<2|(p));}
127 
128 EIGEN_STRONG_INLINE Packet4f vec4f_swizzle1(const Packet4f& a, int p, int q, int r, int s)
129 {
130  return shuffle1(a, eigen_neon_shuffle_mask(p, q, r, s));
131 }
132 EIGEN_STRONG_INLINE Packet4f vec4f_swizzle2(const Packet4f& a, const Packet4f& b, int p, int q, int r, int s)
133 {
134  return shuffle2<false>(a,b,eigen_neon_shuffle_mask(p, q, r, s));
135 }
136 EIGEN_STRONG_INLINE Packet4f vec4f_movelh(const Packet4f& a, const Packet4f& b)
137 {
138  return shuffle2<false>(a,b,eigen_neon_shuffle_mask(0, 1, 0, 1));
139 }
140 EIGEN_STRONG_INLINE Packet4f vec4f_movehl(const Packet4f& a, const Packet4f& b)
141 {
142  return shuffle2<false>(b,a,eigen_neon_shuffle_mask(2, 3, 2, 3));
143 }
144 EIGEN_STRONG_INLINE Packet4f vec4f_unpacklo(const Packet4f& a, const Packet4f& b)
145 {
146  return shuffle2<true>(a,b,eigen_neon_shuffle_mask(0, 0, 1, 1));
147 }
148 EIGEN_STRONG_INLINE Packet4f vec4f_unpackhi(const Packet4f& a, const Packet4f& b)
149 {
150  return shuffle2<true>(a,b,eigen_neon_shuffle_mask(2, 2, 3, 3));
151 }
152 #define vec4f_duplane(a, p) \
153  Packet4f(vdupq_lane_f32(vget_low_f32(a), p))
154 
155 #define EIGEN_DECLARE_CONST_Packet4f(NAME,X) \
156  const Packet4f p4f_##NAME = pset1<Packet4f>(X)
157 
158 #define EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \
159  const Packet4f p4f_##NAME = vreinterpretq_f32_u32(pset1<int32_t>(X))
160 
161 #define EIGEN_DECLARE_CONST_Packet4i(NAME,X) \
162  const Packet4i p4i_##NAME = pset1<Packet4i>(X)
163 
164 #if EIGEN_ARCH_ARM64 && EIGEN_COMP_GNUC
165  // __builtin_prefetch tends to do nothing on ARM64 compilers because the
166  // prefetch instructions there are too detailed for __builtin_prefetch to map
167  // meaningfully to them.
168  #define EIGEN_ARM_PREFETCH(ADDR) __asm__ __volatile__("prfm pldl1keep, [%[addr]]\n" ::[addr] "r"(ADDR) : );
169 #elif EIGEN_HAS_BUILTIN(__builtin_prefetch) || EIGEN_COMP_GNUC
170  #define EIGEN_ARM_PREFETCH(ADDR) __builtin_prefetch(ADDR);
171 #elif defined __pld
172  #define EIGEN_ARM_PREFETCH(ADDR) __pld(ADDR)
173 #elif EIGEN_ARCH_ARM
174  #define EIGEN_ARM_PREFETCH(ADDR) __asm__ __volatile__ ("pld [%[addr]]\n" :: [addr] "r" (ADDR) : );
175 #else
176  // by default no explicit prefetching
177  #define EIGEN_ARM_PREFETCH(ADDR)
178 #endif
179 
180 template <>
181 struct packet_traits<float> : default_packet_traits
182 {
183  typedef Packet4f type;
184  typedef Packet2f half;
185  enum
186  {
187  Vectorizable = 1,
188  AlignedOnScalar = 1,
189  size = 4,
190 
191  HasCmp = 1,
192  HasAdd = 1,
193  HasSub = 1,
194  HasShift = 1,
195  HasMul = 1,
196  HasNegate = 1,
197  HasAbs = 1,
198  HasArg = 0,
199  HasAbs2 = 1,
200  HasAbsDiff = 1,
201  HasMin = 1,
202  HasMax = 1,
203  HasConj = 1,
204  HasSetLinear = 1,
205  HasBlend = 0,
206 
207  HasDiv = 1,
208  HasFloor = 1,
209  HasCeil = 1,
210  HasRint = 1,
211 
212  HasSin = EIGEN_FAST_MATH,
213  HasCos = EIGEN_FAST_MATH,
214  HasACos = 1,
215  HasASin = 1,
216  HasATan = 1,
217  HasATanh = 1,
218  HasLog = 1,
219  HasExp = 1,
220  HasSqrt = 1,
221  HasRsqrt = 1,
222  HasTanh = EIGEN_FAST_MATH,
223  HasErf = EIGEN_FAST_MATH,
224  HasBessel = 0, // Issues with accuracy.
225  HasNdtri = 0
226  };
227 };
228 
229 template <>
230 struct packet_traits<int8_t> : default_packet_traits
231 {
232  typedef Packet16c type;
233  typedef Packet8c half;
234  enum
235  {
236  Vectorizable = 1,
237  AlignedOnScalar = 1,
238  size = 16,
239 
240  HasCmp = 1,
241  HasAdd = 1,
242  HasSub = 1,
243  HasShift = 1,
244  HasMul = 1,
245  HasNegate = 1,
246  HasAbs = 1,
247  HasAbsDiff = 1,
248  HasArg = 0,
249  HasAbs2 = 1,
250  HasMin = 1,
251  HasMax = 1,
252  HasConj = 1,
253  HasSetLinear = 1,
254  HasBlend = 0
255  };
256 };
257 
258 template <>
259 struct packet_traits<uint8_t> : default_packet_traits
260 {
261  typedef Packet16uc type;
262  typedef Packet8uc half;
263  enum
264  {
265  Vectorizable = 1,
266  AlignedOnScalar = 1,
267  size = 16,
268 
269  HasCmp = 1,
270  HasAdd = 1,
271  HasSub = 1,
272  HasShift = 1,
273  HasMul = 1,
274  HasNegate = 0,
275  HasAbs = 1,
276  HasAbsDiff = 1,
277  HasArg = 0,
278  HasAbs2 = 1,
279  HasMin = 1,
280  HasMax = 1,
281  HasConj = 1,
282  HasSetLinear = 1,
283  HasBlend = 0,
284 
285  HasSqrt = 1
286  };
287 };
288 
289 template <>
290 struct packet_traits<int16_t> : default_packet_traits
291 {
292  typedef Packet8s type;
293  typedef Packet4s half;
294  enum
295  {
296  Vectorizable = 1,
297  AlignedOnScalar = 1,
298  size = 8,
299 
300  HasCmp = 1,
301  HasAdd = 1,
302  HasSub = 1,
303  HasShift = 1,
304  HasMul = 1,
305  HasNegate = 1,
306  HasAbs = 1,
307  HasAbsDiff = 1,
308  HasArg = 0,
309  HasAbs2 = 1,
310  HasMin = 1,
311  HasMax = 1,
312  HasConj = 1,
313  HasSetLinear = 1,
314  HasBlend = 0
315  };
316 };
317 
318 template <>
319 struct packet_traits<uint16_t> : default_packet_traits
320 {
321  typedef Packet8us type;
322  typedef Packet4us half;
323  enum
324  {
325  Vectorizable = 1,
326  AlignedOnScalar = 1,
327  size = 8,
328 
329  HasCmp = 1,
330  HasAdd = 1,
331  HasSub = 1,
332  HasShift = 1,
333  HasMul = 1,
334  HasNegate = 0,
335  HasAbs = 1,
336  HasAbsDiff = 1,
337  HasArg = 0,
338  HasAbs2 = 1,
339  HasMin = 1,
340  HasMax = 1,
341  HasConj = 1,
342  HasSetLinear = 1,
343  HasBlend = 0,
344  HasSqrt = 1
345  };
346 };
347 
348 template <>
349 struct packet_traits<int32_t> : default_packet_traits
350 {
351  typedef Packet4i type;
352  typedef Packet2i half;
353  enum
354  {
355  Vectorizable = 1,
356  AlignedOnScalar = 1,
357  size = 4,
358 
359  HasCmp = 1,
360  HasAdd = 1,
361  HasSub = 1,
362  HasShift = 1,
363  HasMul = 1,
364  HasNegate = 1,
365  HasAbs = 1,
366  HasArg = 0,
367  HasAbs2 = 1,
368  HasAbsDiff = 1,
369  HasMin = 1,
370  HasMax = 1,
371  HasConj = 1,
372  HasSetLinear = 1,
373  HasBlend = 0
374  };
375 };
376 
377 template <>
378 struct packet_traits<uint32_t> : default_packet_traits
379 {
380  typedef Packet4ui type;
381  typedef Packet2ui half;
382  enum
383  {
384  Vectorizable = 1,
385  AlignedOnScalar = 1,
386  size = 4,
387 
388  HasCmp = 1,
389  HasAdd = 1,
390  HasSub = 1,
391  HasShift = 1,
392  HasMul = 1,
393  HasNegate = 0,
394  HasAbs = 1,
395  HasArg = 0,
396  HasAbs2 = 1,
397  HasAbsDiff = 1,
398  HasMin = 1,
399  HasMax = 1,
400  HasConj = 1,
401  HasSetLinear = 1,
402  HasBlend = 0,
403 
404  HasSqrt = 1
405  };
406 };
407 
408 template <>
409 struct packet_traits<int64_t> : default_packet_traits
410 {
411  typedef Packet2l type;
412  typedef Packet2l half;
413  enum
414  {
415  Vectorizable = 1,
416  AlignedOnScalar = 1,
417  size = 2,
418 
419  HasCmp = 1,
420  HasAdd = 1,
421  HasSub = 1,
422  HasShift = 1,
423  HasMul = 1,
424  HasNegate = 1,
425  HasAbs = 1,
426  HasArg = 0,
427  HasAbs2 = 1,
428  HasAbsDiff = 1,
429  HasMin = 1,
430  HasMax = 1,
431  HasConj = 1,
432  HasSetLinear = 1,
433  HasBlend = 0
434  };
435 };
436 
437 template <>
438 struct packet_traits<uint64_t> : default_packet_traits
439 {
440  typedef Packet2ul type;
441  typedef Packet2ul half;
442  enum
443  {
444  Vectorizable = 1,
445  AlignedOnScalar = 1,
446  size = 2,
447 
448  HasCmp = 1,
449  HasAdd = 1,
450  HasSub = 1,
451  HasShift = 1,
452  HasMul = 1,
453  HasNegate = 0,
454  HasAbs = 1,
455  HasArg = 0,
456  HasAbs2 = 1,
457  HasAbsDiff = 1,
458  HasMin = 1,
459  HasMax = 1,
460  HasConj = 1,
461  HasSetLinear = 1,
462  HasBlend = 0
463  };
464 };
465 
466 template<> struct unpacket_traits<Packet2f>
467 {
468  typedef float type;
469  typedef Packet2f half;
470  typedef Packet2i integer_packet;
471  enum
472  {
473  size = 2,
474  alignment = Aligned16,
475  vectorizable = true,
476  masked_load_available = false,
477  masked_store_available = false
478  };
479 };
480 template<> struct unpacket_traits<Packet4f>
481 {
482  typedef float type;
483  typedef Packet2f half;
484  typedef Packet4i integer_packet;
485  enum
486  {
487  size = 4,
488  alignment = Aligned16,
489  vectorizable = true,
490  masked_load_available = false,
491  masked_store_available = false
492  };
493 };
494 template<> struct unpacket_traits<Packet4c>
495 {
496  typedef int8_t type;
497  typedef Packet4c half;
498  enum
499  {
500  size = 4,
501  alignment = Unaligned,
502  vectorizable = true,
503  masked_load_available = false,
504  masked_store_available = false
505  };
506 };
507 template<> struct unpacket_traits<Packet8c>
508 {
509  typedef int8_t type;
510  typedef Packet4c half;
511  enum
512  {
513  size = 8,
514  alignment = Aligned16,
515  vectorizable = true,
516  masked_load_available = false,
517  masked_store_available = false
518  };
519 };
520 template<> struct unpacket_traits<Packet16c>
521 {
522  typedef int8_t type;
523  typedef Packet8c half;
524  enum
525  {
526  size = 16,
527  alignment = Aligned16,
528  vectorizable = true,
529  masked_load_available = false,
530  masked_store_available = false
531  };
532 };
533 template<> struct unpacket_traits<Packet4uc>
534 {
535  typedef uint8_t type;
536  typedef Packet4uc half;
537  enum
538  {
539  size = 4,
540  alignment = Unaligned,
541  vectorizable = true,
542  masked_load_available = false,
543  masked_store_available = false
544  };
545 };
546 template<> struct unpacket_traits<Packet8uc>
547 {
548  typedef uint8_t type;
549  typedef Packet4uc half;
550  enum
551  {
552  size = 8,
553  alignment = Aligned16,
554  vectorizable = true,
555  masked_load_available = false,
556  masked_store_available = false
557  };
558 };
559 template<> struct unpacket_traits<Packet16uc>
560 {
561  typedef uint8_t type;
562  typedef Packet8uc half;
563  enum
564  {
565  size = 16,
566  alignment = Aligned16,
567  vectorizable = true,
568  masked_load_available = false,
569  masked_store_available = false};
570 };
571 template<> struct unpacket_traits<Packet4s>
572 {
573  typedef int16_t type;
574  typedef Packet4s half;
575  enum
576  {
577  size = 4,
578  alignment = Aligned16,
579  vectorizable = true,
580  masked_load_available = false,
581  masked_store_available = false
582  };
583 };
584 template<> struct unpacket_traits<Packet8s>
585 {
586  typedef int16_t type;
587  typedef Packet4s half;
588  enum
589  {
590  size = 8,
591  alignment = Aligned16,
592  vectorizable = true,
593  masked_load_available = false,
594  masked_store_available = false
595  };
596 };
597 template<> struct unpacket_traits<Packet4us>
598 {
599  typedef uint16_t type;
600  typedef Packet4us half;
601  enum
602  {
603  size = 4,
604  alignment = Aligned16,
605  vectorizable = true,
606  masked_load_available = false,
607  masked_store_available = false
608  };
609 };
610 template<> struct unpacket_traits<Packet8us>
611 {
612  typedef uint16_t type;
613  typedef Packet4us half;
614  enum
615  {
616  size = 8,
617  alignment = Aligned16,
618  vectorizable = true,
619  masked_load_available = false,
620  masked_store_available = false
621  };
622 };
623 template<> struct unpacket_traits<Packet2i>
624 {
625  typedef int32_t type;
626  typedef Packet2i half;
627  enum
628  {
629  size = 2,
630  alignment = Aligned16,
631  vectorizable = true,
632  masked_load_available = false,
633  masked_store_available = false
634  };
635 };
636 template<> struct unpacket_traits<Packet4i>
637 {
638  typedef int32_t type;
639  typedef Packet2i half;
640  enum
641  {
642  size = 4,
643  alignment = Aligned16,
644  vectorizable = true,
645  masked_load_available = false,
646  masked_store_available = false
647  };
648 };
649 template<> struct unpacket_traits<Packet2ui>
650 {
651  typedef uint32_t type;
652  typedef Packet2ui half;
653  enum
654  {
655  size = 2,
656  alignment = Aligned16,
657  vectorizable = true,
658  masked_load_available = false,
659  masked_store_available = false
660  };
661 };
662 template<> struct unpacket_traits<Packet4ui>
663 {
664  typedef uint32_t type;
665  typedef Packet2ui half;
666  enum
667  {
668  size = 4,
669  alignment = Aligned16,
670  vectorizable = true,
671  masked_load_available = false,
672  masked_store_available = false
673  };
674 };
675 template<> struct unpacket_traits<Packet2l>
676 {
677  typedef int64_t type;
678  typedef Packet2l half;
679  enum
680  {
681  size = 2,
682  alignment = Aligned16,
683  vectorizable = true,
684  masked_load_available = false,
685  masked_store_available = false
686  };
687 };
688 template<> struct unpacket_traits<Packet2ul>
689 {
690  typedef uint64_t type;
691  typedef Packet2ul half;
692  enum
693  {
694  size = 2,
695  alignment = Aligned16,
696  vectorizable = true,
697  masked_load_available = false,
698  masked_store_available = false
699  };
700 };
701 
702 template<> EIGEN_STRONG_INLINE Packet2f pset1<Packet2f>(const float& from) { return vdup_n_f32(from); }
703 template<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(const float& from) { return vdupq_n_f32(from); }
704 template<> EIGEN_STRONG_INLINE Packet4c pset1<Packet4c>(const int8_t& from)
705 { return vget_lane_s32(vreinterpret_s32_s8(vdup_n_s8(from)), 0); }
706 template<> EIGEN_STRONG_INLINE Packet8c pset1<Packet8c>(const int8_t& from) { return vdup_n_s8(from); }
707 template<> EIGEN_STRONG_INLINE Packet16c pset1<Packet16c>(const int8_t& from) { return vdupq_n_s8(from); }
708 template<> EIGEN_STRONG_INLINE Packet4uc pset1<Packet4uc>(const uint8_t& from)
709 { return vget_lane_u32(vreinterpret_u32_u8(vdup_n_u8(from)), 0); }
710 template<> EIGEN_STRONG_INLINE Packet8uc pset1<Packet8uc>(const uint8_t& from) { return vdup_n_u8(from); }
711 template<> EIGEN_STRONG_INLINE Packet16uc pset1<Packet16uc>(const uint8_t& from) { return vdupq_n_u8(from); }
712 template<> EIGEN_STRONG_INLINE Packet4s pset1<Packet4s>(const int16_t& from) { return vdup_n_s16(from); }
713 template<> EIGEN_STRONG_INLINE Packet8s pset1<Packet8s>(const int16_t& from) { return vdupq_n_s16(from); }
714 template<> EIGEN_STRONG_INLINE Packet4us pset1<Packet4us>(const uint16_t& from) { return vdup_n_u16(from); }
715 template<> EIGEN_STRONG_INLINE Packet8us pset1<Packet8us>(const uint16_t& from) { return vdupq_n_u16(from); }
716 template<> EIGEN_STRONG_INLINE Packet2i pset1<Packet2i>(const int32_t& from) { return vdup_n_s32(from); }
717 template<> EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(const int32_t& from) { return vdupq_n_s32(from); }
718 template<> EIGEN_STRONG_INLINE Packet2ui pset1<Packet2ui>(const uint32_t& from) { return vdup_n_u32(from); }
719 template<> EIGEN_STRONG_INLINE Packet4ui pset1<Packet4ui>(const uint32_t& from) { return vdupq_n_u32(from); }
720 template<> EIGEN_STRONG_INLINE Packet2l pset1<Packet2l>(const int64_t& from) { return vdupq_n_s64(from); }
721 template<> EIGEN_STRONG_INLINE Packet2ul pset1<Packet2ul>(const uint64_t& from) { return vdupq_n_u64(from); }
722 
723 template<> EIGEN_STRONG_INLINE Packet2f pset1frombits<Packet2f>(uint32_t from)
724 { return vreinterpret_f32_u32(vdup_n_u32(from)); }
725 template<> EIGEN_STRONG_INLINE Packet4f pset1frombits<Packet4f>(uint32_t from)
726 { return vreinterpretq_f32_u32(vdupq_n_u32(from)); }
727 
728 template<> EIGEN_STRONG_INLINE Packet2f plset<Packet2f>(const float& a)
729 {
730  const float c[] = {0.0f,1.0f};
731  return vadd_f32(pset1<Packet2f>(a), vld1_f32(c));
732 }
733 template<> EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(const float& a)
734 {
735  const float c[] = {0.0f,1.0f,2.0f,3.0f};
736  return vaddq_f32(pset1<Packet4f>(a), vld1q_f32(c));
737 }
738 template<> EIGEN_STRONG_INLINE Packet4c plset<Packet4c>(const int8_t& a)
739 { return vget_lane_s32(vreinterpret_s32_s8(vadd_s8(vreinterpret_s8_u32(vdup_n_u32(0x03020100)), vdup_n_s8(a))), 0); }
740 template<> EIGEN_STRONG_INLINE Packet8c plset<Packet8c>(const int8_t& a)
741 {
742  const int8_t c[] = {0,1,2,3,4,5,6,7};
743  return vadd_s8(pset1<Packet8c>(a), vld1_s8(c));
744 }
745 template<> EIGEN_STRONG_INLINE Packet16c plset<Packet16c>(const int8_t& a)
746 {
747  const int8_t c[] = {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15};
748  return vaddq_s8(pset1<Packet16c>(a), vld1q_s8(c));
749 }
750 template<> EIGEN_STRONG_INLINE Packet4uc plset<Packet4uc>(const uint8_t& a)
751 { return vget_lane_u32(vreinterpret_u32_u8(vadd_u8(vreinterpret_u8_u32(vdup_n_u32(0x03020100)), vdup_n_u8(a))), 0); }
752 template<> EIGEN_STRONG_INLINE Packet8uc plset<Packet8uc>(const uint8_t& a)
753 {
754  const uint8_t c[] = {0,1,2,3,4,5,6,7};
755  return vadd_u8(pset1<Packet8uc>(a), vld1_u8(c));
756 }
757 template<> EIGEN_STRONG_INLINE Packet16uc plset<Packet16uc>(const uint8_t& a)
758 {
759  const uint8_t c[] = {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15};
760  return vaddq_u8(pset1<Packet16uc>(a), vld1q_u8(c));
761 }
762 template<> EIGEN_STRONG_INLINE Packet4s plset<Packet4s>(const int16_t& a)
763 {
764  const int16_t c[] = {0,1,2,3};
765  return vadd_s16(pset1<Packet4s>(a), vld1_s16(c));
766 }
767 template<> EIGEN_STRONG_INLINE Packet4us plset<Packet4us>(const uint16_t& a)
768 {
769  const uint16_t c[] = {0,1,2,3};
770  return vadd_u16(pset1<Packet4us>(a), vld1_u16(c));
771 }
772 template<> EIGEN_STRONG_INLINE Packet8s plset<Packet8s>(const int16_t& a)
773 {
774  const int16_t c[] = {0,1,2,3,4,5,6,7};
775  return vaddq_s16(pset1<Packet8s>(a), vld1q_s16(c));
776 }
777 template<> EIGEN_STRONG_INLINE Packet8us plset<Packet8us>(const uint16_t& a)
778 {
779  const uint16_t c[] = {0,1,2,3,4,5,6,7};
780  return vaddq_u16(pset1<Packet8us>(a), vld1q_u16(c));
781 }
782 template<> EIGEN_STRONG_INLINE Packet2i plset<Packet2i>(const int32_t& a)
783 {
784  const int32_t c[] = {0,1};
785  return vadd_s32(pset1<Packet2i>(a), vld1_s32(c));
786 }
787 template<> EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(const int32_t& a)
788 {
789  const int32_t c[] = {0,1,2,3};
790  return vaddq_s32(pset1<Packet4i>(a), vld1q_s32(c));
791 }
792 template<> EIGEN_STRONG_INLINE Packet2ui plset<Packet2ui>(const uint32_t& a)
793 {
794  const uint32_t c[] = {0,1};
795  return vadd_u32(pset1<Packet2ui>(a), vld1_u32(c));
796 }
797 template<> EIGEN_STRONG_INLINE Packet4ui plset<Packet4ui>(const uint32_t& a)
798 {
799  const uint32_t c[] = {0,1,2,3};
800  return vaddq_u32(pset1<Packet4ui>(a), vld1q_u32(c));
801 }
802 template<> EIGEN_STRONG_INLINE Packet2l plset<Packet2l>(const int64_t& a)
803 {
804  const int64_t c[] = {0,1};
805  return vaddq_s64(pset1<Packet2l>(a), vld1q_s64(c));
806 }
807 template<> EIGEN_STRONG_INLINE Packet2ul plset<Packet2ul>(const uint64_t& a)
808 {
809  const uint64_t c[] = {0,1};
810  return vaddq_u64(pset1<Packet2ul>(a), vld1q_u64(c));
811 }
812 
813 template<> EIGEN_STRONG_INLINE Packet2f padd<Packet2f>(const Packet2f& a, const Packet2f& b) { return vadd_f32(a,b); }
814 template<> EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(const Packet4f& a, const Packet4f& b) { return vaddq_f32(a,b); }
815 template<> EIGEN_STRONG_INLINE Packet4c padd<Packet4c>(const Packet4c& a, const Packet4c& b)
816 {
817  return vget_lane_s32(vreinterpret_s32_s8(vadd_s8(
818  vreinterpret_s8_s32(vdup_n_s32(a)),
819  vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
820 }
821 template<> EIGEN_STRONG_INLINE Packet8c padd<Packet8c>(const Packet8c& a, const Packet8c& b) { return vadd_s8(a,b); }
822 template<> EIGEN_STRONG_INLINE Packet16c padd<Packet16c>(const Packet16c& a, const Packet16c& b) { return vaddq_s8(a,b); }
823 template<> EIGEN_STRONG_INLINE Packet4uc padd<Packet4uc>(const Packet4uc& a, const Packet4uc& b)
824 {
825  return vget_lane_u32(vreinterpret_u32_u8(vadd_u8(
826  vreinterpret_u8_u32(vdup_n_u32(a)),
827  vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
828 }
829 template<> EIGEN_STRONG_INLINE Packet8uc padd<Packet8uc>(const Packet8uc& a, const Packet8uc& b) { return vadd_u8(a,b); }
830 template<> EIGEN_STRONG_INLINE Packet16uc padd<Packet16uc>(const Packet16uc& a, const Packet16uc& b) { return vaddq_u8(a,b); }
831 template<> EIGEN_STRONG_INLINE Packet4s padd<Packet4s>(const Packet4s& a, const Packet4s& b) { return vadd_s16(a,b); }
832 template<> EIGEN_STRONG_INLINE Packet8s padd<Packet8s>(const Packet8s& a, const Packet8s& b) { return vaddq_s16(a,b); }
833 template<> EIGEN_STRONG_INLINE Packet4us padd<Packet4us>(const Packet4us& a, const Packet4us& b) { return vadd_u16(a,b); }
834 template<> EIGEN_STRONG_INLINE Packet8us padd<Packet8us>(const Packet8us& a, const Packet8us& b) { return vaddq_u16(a,b); }
835 template<> EIGEN_STRONG_INLINE Packet2i padd<Packet2i>(const Packet2i& a, const Packet2i& b) { return vadd_s32(a,b); }
836 template<> EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(const Packet4i& a, const Packet4i& b) { return vaddq_s32(a,b); }
837 template<> EIGEN_STRONG_INLINE Packet2ui padd<Packet2ui>(const Packet2ui& a, const Packet2ui& b) { return vadd_u32(a,b); }
838 template<> EIGEN_STRONG_INLINE Packet4ui padd<Packet4ui>(const Packet4ui& a, const Packet4ui& b) { return vaddq_u32(a,b); }
839 template<> EIGEN_STRONG_INLINE Packet2l padd<Packet2l>(const Packet2l& a, const Packet2l& b) { return vaddq_s64(a,b); }
840 template<> EIGEN_STRONG_INLINE Packet2ul padd<Packet2ul>(const Packet2ul& a, const Packet2ul& b) { return vaddq_u64(a,b); }
841 
842 template<> EIGEN_STRONG_INLINE Packet2f psub<Packet2f>(const Packet2f& a, const Packet2f& b) { return vsub_f32(a,b); }
843 template<> EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(const Packet4f& a, const Packet4f& b) { return vsubq_f32(a,b); }
844 template<> EIGEN_STRONG_INLINE Packet4c psub<Packet4c>(const Packet4c& a, const Packet4c& b)
845 {
846  return vget_lane_s32(vreinterpret_s32_s8(vsub_s8(
847  vreinterpret_s8_s32(vdup_n_s32(a)),
848  vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
849 }
850 template<> EIGEN_STRONG_INLINE Packet8c psub<Packet8c>(const Packet8c& a, const Packet8c& b) { return vsub_s8(a,b); }
851 template<> EIGEN_STRONG_INLINE Packet16c psub<Packet16c>(const Packet16c& a, const Packet16c& b) { return vsubq_s8(a,b); }
852 template<> EIGEN_STRONG_INLINE Packet4uc psub<Packet4uc>(const Packet4uc& a, const Packet4uc& b)
853 {
854  return vget_lane_u32(vreinterpret_u32_u8(vsub_u8(
855  vreinterpret_u8_u32(vdup_n_u32(a)),
856  vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
857 }
858 template<> EIGEN_STRONG_INLINE Packet8uc psub<Packet8uc>(const Packet8uc& a, const Packet8uc& b) { return vsub_u8(a,b); }
859 template<> EIGEN_STRONG_INLINE Packet16uc psub<Packet16uc>(const Packet16uc& a, const Packet16uc& b) { return vsubq_u8(a,b); }
860 template<> EIGEN_STRONG_INLINE Packet4s psub<Packet4s>(const Packet4s& a, const Packet4s& b) { return vsub_s16(a,b); }
861 template<> EIGEN_STRONG_INLINE Packet8s psub<Packet8s>(const Packet8s& a, const Packet8s& b) { return vsubq_s16(a,b); }
862 template<> EIGEN_STRONG_INLINE Packet4us psub<Packet4us>(const Packet4us& a, const Packet4us& b) { return vsub_u16(a,b); }
863 template<> EIGEN_STRONG_INLINE Packet8us psub<Packet8us>(const Packet8us& a, const Packet8us& b) { return vsubq_u16(a,b); }
864 template<> EIGEN_STRONG_INLINE Packet2i psub<Packet2i>(const Packet2i& a, const Packet2i& b) { return vsub_s32(a,b); }
865 template<> EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(const Packet4i& a, const Packet4i& b) { return vsubq_s32(a,b); }
866 template<> EIGEN_STRONG_INLINE Packet2ui psub<Packet2ui>(const Packet2ui& a, const Packet2ui& b) { return vsub_u32(a,b); }
867 template<> EIGEN_STRONG_INLINE Packet4ui psub<Packet4ui>(const Packet4ui& a, const Packet4ui& b) { return vsubq_u32(a,b); }
868 template<> EIGEN_STRONG_INLINE Packet2l psub<Packet2l>(const Packet2l& a, const Packet2l& b) { return vsubq_s64(a,b); }
869 template<> EIGEN_STRONG_INLINE Packet2ul psub<Packet2ul>(const Packet2ul& a, const Packet2ul& b) { return vsubq_u64(a,b); }
870 
871 template<> EIGEN_STRONG_INLINE Packet2f pxor<Packet2f>(const Packet2f& a, const Packet2f& b);
872 template<> EIGEN_STRONG_INLINE Packet2f paddsub<Packet2f>(const Packet2f& a, const Packet2f & b) {
873  Packet2f mask = make_packet2f(numext::bit_cast<float>(0x80000000u), 0.0f);
874  return padd(a, pxor(mask, b));
875 }
876 template<> EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b);
877 template<> EIGEN_STRONG_INLINE Packet4f paddsub<Packet4f>(const Packet4f& a, const Packet4f& b) {
878  Packet4f mask = make_packet4f(numext::bit_cast<float>(0x80000000u), 0.0f, numext::bit_cast<float>(0x80000000u), 0.0f);
879  return padd(a, pxor(mask, b));
880 }
881 
882 template<> EIGEN_STRONG_INLINE Packet2f pnegate(const Packet2f& a) { return vneg_f32(a); }
883 template<> EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) { return vnegq_f32(a); }
884 template<> EIGEN_STRONG_INLINE Packet4c pnegate(const Packet4c& a)
885 { return vget_lane_s32(vreinterpret_s32_s8(vneg_s8(vreinterpret_s8_s32(vdup_n_s32(a)))), 0); }
886 template<> EIGEN_STRONG_INLINE Packet8c pnegate(const Packet8c& a) { return vneg_s8(a); }
887 template<> EIGEN_STRONG_INLINE Packet16c pnegate(const Packet16c& a) { return vnegq_s8(a); }
888 template<> EIGEN_STRONG_INLINE Packet4s pnegate(const Packet4s& a) { return vneg_s16(a); }
889 template<> EIGEN_STRONG_INLINE Packet8s pnegate(const Packet8s& a) { return vnegq_s16(a); }
890 template<> EIGEN_STRONG_INLINE Packet2i pnegate(const Packet2i& a) { return vneg_s32(a); }
891 template<> EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) { return vnegq_s32(a); }
892 template<> EIGEN_STRONG_INLINE Packet2l pnegate(const Packet2l& a) {
893 #if EIGEN_ARCH_ARM64
894  return vnegq_s64(a);
895 #else
896  return vcombine_s64(
897  vdup_n_s64(-vgetq_lane_s64(a, 0)),
898  vdup_n_s64(-vgetq_lane_s64(a, 1)));
899 #endif
900 }
901 
902 template<> EIGEN_STRONG_INLINE Packet2f pconj(const Packet2f& a) { return a; }
903 template<> EIGEN_STRONG_INLINE Packet4f pconj(const Packet4f& a) { return a; }
904 template<> EIGEN_STRONG_INLINE Packet4c pconj(const Packet4c& a) { return a; }
905 template<> EIGEN_STRONG_INLINE Packet8c pconj(const Packet8c& a) { return a; }
906 template<> EIGEN_STRONG_INLINE Packet16c pconj(const Packet16c& a) { return a; }
907 template<> EIGEN_STRONG_INLINE Packet4uc pconj(const Packet4uc& a) { return a; }
908 template<> EIGEN_STRONG_INLINE Packet8uc pconj(const Packet8uc& a) { return a; }
909 template<> EIGEN_STRONG_INLINE Packet16uc pconj(const Packet16uc& a) { return a; }
910 template<> EIGEN_STRONG_INLINE Packet4s pconj(const Packet4s& a) { return a; }
911 template<> EIGEN_STRONG_INLINE Packet8s pconj(const Packet8s& a) { return a; }
912 template<> EIGEN_STRONG_INLINE Packet4us pconj(const Packet4us& a) { return a; }
913 template<> EIGEN_STRONG_INLINE Packet8us pconj(const Packet8us& a) { return a; }
914 template<> EIGEN_STRONG_INLINE Packet2i pconj(const Packet2i& a) { return a; }
915 template<> EIGEN_STRONG_INLINE Packet4i pconj(const Packet4i& a) { return a; }
916 template<> EIGEN_STRONG_INLINE Packet2ui pconj(const Packet2ui& a) { return a; }
917 template<> EIGEN_STRONG_INLINE Packet4ui pconj(const Packet4ui& a) { return a; }
918 template<> EIGEN_STRONG_INLINE Packet2l pconj(const Packet2l& a) { return a; }
919 template<> EIGEN_STRONG_INLINE Packet2ul pconj(const Packet2ul& a) { return a; }
920 
921 template<> EIGEN_STRONG_INLINE Packet2f pmul<Packet2f>(const Packet2f& a, const Packet2f& b) { return vmul_f32(a,b); }
922 template<> EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(const Packet4f& a, const Packet4f& b) { return vmulq_f32(a,b); }
923 template<> EIGEN_STRONG_INLINE Packet4c pmul<Packet4c>(const Packet4c& a, const Packet4c& b)
924 {
925  return vget_lane_s32(vreinterpret_s32_s8(vmul_s8(
926  vreinterpret_s8_s32(vdup_n_s32(a)),
927  vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
928 }
929 template<> EIGEN_STRONG_INLINE Packet8c pmul<Packet8c>(const Packet8c& a, const Packet8c& b) { return vmul_s8(a,b); }
930 template<> EIGEN_STRONG_INLINE Packet16c pmul<Packet16c>(const Packet16c& a, const Packet16c& b) { return vmulq_s8(a,b); }
931 template<> EIGEN_STRONG_INLINE Packet4uc pmul<Packet4uc>(const Packet4uc& a, const Packet4uc& b)
932 {
933  return vget_lane_u32(vreinterpret_u32_u8(vmul_u8(
934  vreinterpret_u8_u32(vdup_n_u32(a)),
935  vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
936 }
937 template<> EIGEN_STRONG_INLINE Packet8uc pmul<Packet8uc>(const Packet8uc& a, const Packet8uc& b) { return vmul_u8(a,b); }
938 template<> EIGEN_STRONG_INLINE Packet16uc pmul<Packet16uc>(const Packet16uc& a, const Packet16uc& b) { return vmulq_u8(a,b); }
939 template<> EIGEN_STRONG_INLINE Packet4s pmul<Packet4s>(const Packet4s& a, const Packet4s& b) { return vmul_s16(a,b); }
940 template<> EIGEN_STRONG_INLINE Packet8s pmul<Packet8s>(const Packet8s& a, const Packet8s& b) { return vmulq_s16(a,b); }
941 template<> EIGEN_STRONG_INLINE Packet4us pmul<Packet4us>(const Packet4us& a, const Packet4us& b) { return vmul_u16(a,b); }
942 template<> EIGEN_STRONG_INLINE Packet8us pmul<Packet8us>(const Packet8us& a, const Packet8us& b) { return vmulq_u16(a,b); }
943 template<> EIGEN_STRONG_INLINE Packet2i pmul<Packet2i>(const Packet2i& a, const Packet2i& b) { return vmul_s32(a,b); }
944 template<> EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(const Packet4i& a, const Packet4i& b) { return vmulq_s32(a,b); }
945 template<> EIGEN_STRONG_INLINE Packet2ui pmul<Packet2ui>(const Packet2ui& a, const Packet2ui& b) { return vmul_u32(a,b); }
946 template<> EIGEN_STRONG_INLINE Packet4ui pmul<Packet4ui>(const Packet4ui& a, const Packet4ui& b) { return vmulq_u32(a,b); }
947 template<> EIGEN_STRONG_INLINE Packet2l pmul<Packet2l>(const Packet2l& a, const Packet2l& b) {
948  return vcombine_s64(
949  vdup_n_s64(vgetq_lane_s64(a, 0)*vgetq_lane_s64(b, 0)),
950  vdup_n_s64(vgetq_lane_s64(a, 1)*vgetq_lane_s64(b, 1)));
951 }
952 template<> EIGEN_STRONG_INLINE Packet2ul pmul<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
953  return vcombine_u64(
954  vdup_n_u64(vgetq_lane_u64(a, 0)*vgetq_lane_u64(b, 0)),
955  vdup_n_u64(vgetq_lane_u64(a, 1)*vgetq_lane_u64(b, 1)));
956 }
957 
958 template<> EIGEN_STRONG_INLINE Packet2f pdiv<Packet2f>(const Packet2f& a, const Packet2f& b)
959 {
960 #if EIGEN_ARCH_ARM64
961  return vdiv_f32(a,b);
962 #else
963  Packet2f inv, restep, div;
964 
965  // NEON does not offer a divide instruction, we have to do a reciprocal approximation
966  // However NEON in contrast to other SIMD engines (AltiVec/SSE), offers
967  // a reciprocal estimate AND a reciprocal step -which saves a few instructions
968  // vrecpeq_f32() returns an estimate to 1/b, which we will finetune with
969  // Newton-Raphson and vrecpsq_f32()
970  inv = vrecpe_f32(b);
971 
972  // This returns a differential, by which we will have to multiply inv to get a better
973  // approximation of 1/b.
974  restep = vrecps_f32(b, inv);
975  inv = vmul_f32(restep, inv);
976 
977  // Finally, multiply a by 1/b and get the wanted result of the division.
978  div = vmul_f32(a, inv);
979 
980  return div;
981 #endif
982 }
983 template<> EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(const Packet4f& a, const Packet4f& b)
984 {
985 #if EIGEN_ARCH_ARM64
986  return vdivq_f32(a,b);
987 #else
988  Packet4f inv, restep, div;
989 
990  // NEON does not offer a divide instruction, we have to do a reciprocal approximation
991  // However NEON in contrast to other SIMD engines (AltiVec/SSE), offers
992  // a reciprocal estimate AND a reciprocal step -which saves a few instructions
993  // vrecpeq_f32() returns an estimate to 1/b, which we will finetune with
994  // Newton-Raphson and vrecpsq_f32()
995  inv = vrecpeq_f32(b);
996 
997  // This returns a differential, by which we will have to multiply inv to get a better
998  // approximation of 1/b.
999  restep = vrecpsq_f32(b, inv);
1000  inv = vmulq_f32(restep, inv);
1001 
1002  // Finally, multiply a by 1/b and get the wanted result of the division.
1003  div = vmulq_f32(a, inv);
1004 
1005  return div;
1006 #endif
1007 }
1008 
1009 template<> EIGEN_STRONG_INLINE Packet4c pdiv<Packet4c>(const Packet4c& /*a*/, const Packet4c& /*b*/)
1010 {
1011  eigen_assert(false && "packet integer division are not supported by NEON");
1012  return pset1<Packet4c>(0);
1013 }
1014 template<> EIGEN_STRONG_INLINE Packet8c pdiv<Packet8c>(const Packet8c& /*a*/, const Packet8c& /*b*/)
1015 {
1016  eigen_assert(false && "packet integer division are not supported by NEON");
1017  return pset1<Packet8c>(0);
1018 }
1019 template<> EIGEN_STRONG_INLINE Packet16c pdiv<Packet16c>(const Packet16c& /*a*/, const Packet16c& /*b*/)
1020 {
1021  eigen_assert(false && "packet integer division are not supported by NEON");
1022  return pset1<Packet16c>(0);
1023 }
1024 template<> EIGEN_STRONG_INLINE Packet4uc pdiv<Packet4uc>(const Packet4uc& /*a*/, const Packet4uc& /*b*/)
1025 {
1026  eigen_assert(false && "packet integer division are not supported by NEON");
1027  return pset1<Packet4uc>(0);
1028 }
1029 template<> EIGEN_STRONG_INLINE Packet8uc pdiv<Packet8uc>(const Packet8uc& /*a*/, const Packet8uc& /*b*/)
1030 {
1031  eigen_assert(false && "packet integer division are not supported by NEON");
1032  return pset1<Packet8uc>(0);
1033 }
1034 template<> EIGEN_STRONG_INLINE Packet16uc pdiv<Packet16uc>(const Packet16uc& /*a*/, const Packet16uc& /*b*/)
1035 {
1036  eigen_assert(false && "packet integer division are not supported by NEON");
1037  return pset1<Packet16uc>(0);
1038 }
1039 template<> EIGEN_STRONG_INLINE Packet4s pdiv<Packet4s>(const Packet4s& /*a*/, const Packet4s& /*b*/)
1040 {
1041  eigen_assert(false && "packet integer division are not supported by NEON");
1042  return pset1<Packet4s>(0);
1043 }
1044 template<> EIGEN_STRONG_INLINE Packet8s pdiv<Packet8s>(const Packet8s& /*a*/, const Packet8s& /*b*/)
1045 {
1046  eigen_assert(false && "packet integer division are not supported by NEON");
1047  return pset1<Packet8s>(0);
1048 }
1049 template<> EIGEN_STRONG_INLINE Packet4us pdiv<Packet4us>(const Packet4us& /*a*/, const Packet4us& /*b*/)
1050 {
1051  eigen_assert(false && "packet integer division are not supported by NEON");
1052  return pset1<Packet4us>(0);
1053 }
1054 template<> EIGEN_STRONG_INLINE Packet8us pdiv<Packet8us>(const Packet8us& /*a*/, const Packet8us& /*b*/)
1055 {
1056  eigen_assert(false && "packet integer division are not supported by NEON");
1057  return pset1<Packet8us>(0);
1058 }
1059 template<> EIGEN_STRONG_INLINE Packet2i pdiv<Packet2i>(const Packet2i& /*a*/, const Packet2i& /*b*/)
1060 {
1061  eigen_assert(false && "packet integer division are not supported by NEON");
1062  return pset1<Packet2i>(0);
1063 }
1064 template<> EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(const Packet4i& /*a*/, const Packet4i& /*b*/)
1065 {
1066  eigen_assert(false && "packet integer division are not supported by NEON");
1067  return pset1<Packet4i>(0);
1068 }
1069 template<> EIGEN_STRONG_INLINE Packet2ui pdiv<Packet2ui>(const Packet2ui& /*a*/, const Packet2ui& /*b*/)
1070 {
1071  eigen_assert(false && "packet integer division are not supported by NEON");
1072  return pset1<Packet2ui>(0);
1073 }
1074 template<> EIGEN_STRONG_INLINE Packet4ui pdiv<Packet4ui>(const Packet4ui& /*a*/, const Packet4ui& /*b*/)
1075 {
1076  eigen_assert(false && "packet integer division are not supported by NEON");
1077  return pset1<Packet4ui>(0);
1078 }
1079 template<> EIGEN_STRONG_INLINE Packet2l pdiv<Packet2l>(const Packet2l& /*a*/, const Packet2l& /*b*/)
1080 {
1081  eigen_assert(false && "packet integer division are not supported by NEON");
1082  return pset1<Packet2l>(0LL);
1083 }
1084 template<> EIGEN_STRONG_INLINE Packet2ul pdiv<Packet2ul>(const Packet2ul& /*a*/, const Packet2ul& /*b*/)
1085 {
1086  eigen_assert(false && "packet integer division are not supported by NEON");
1087  return pset1<Packet2ul>(0ULL);
1088 }
1089 
1090 
1091 #ifdef __ARM_FEATURE_FMA
1092 template<> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c)
1093 { return vfmaq_f32(c,a,b); }
1094 template<> EIGEN_STRONG_INLINE Packet2f pmadd(const Packet2f& a, const Packet2f& b, const Packet2f& c)
1095 { return vfma_f32(c,a,b); }
1096 #else
1097 template<> EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c)
1098 {
1099  return vmlaq_f32(c,a,b);
1100 }
1101 template<> EIGEN_STRONG_INLINE Packet2f pmadd(const Packet2f& a, const Packet2f& b, const Packet2f& c)
1102 {
1103  return vmla_f32(c,a,b);
1104 }
1105 #endif
1106 
1107 // No FMA instruction for int, so use MLA unconditionally.
1108 template<> EIGEN_STRONG_INLINE Packet4c pmadd(const Packet4c& a, const Packet4c& b, const Packet4c& c)
1109 {
1110  return vget_lane_s32(vreinterpret_s32_s8(vmla_s8(
1111  vreinterpret_s8_s32(vdup_n_s32(c)),
1112  vreinterpret_s8_s32(vdup_n_s32(a)),
1113  vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
1114 }
1115 template<> EIGEN_STRONG_INLINE Packet8c pmadd(const Packet8c& a, const Packet8c& b, const Packet8c& c)
1116 { return vmla_s8(c,a,b); }
1117 template<> EIGEN_STRONG_INLINE Packet16c pmadd(const Packet16c& a, const Packet16c& b, const Packet16c& c)
1118 { return vmlaq_s8(c,a,b); }
1119 template<> EIGEN_STRONG_INLINE Packet4uc pmadd(const Packet4uc& a, const Packet4uc& b, const Packet4uc& c)
1120 {
1121  return vget_lane_u32(vreinterpret_u32_u8(vmla_u8(
1122  vreinterpret_u8_u32(vdup_n_u32(c)),
1123  vreinterpret_u8_u32(vdup_n_u32(a)),
1124  vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
1125 }
1126 template<> EIGEN_STRONG_INLINE Packet8uc pmadd(const Packet8uc& a, const Packet8uc& b, const Packet8uc& c)
1127 { return vmla_u8(c,a,b); }
1128 template<> EIGEN_STRONG_INLINE Packet16uc pmadd(const Packet16uc& a, const Packet16uc& b, const Packet16uc& c)
1129 { return vmlaq_u8(c,a,b); }
1130 template<> EIGEN_STRONG_INLINE Packet4s pmadd(const Packet4s& a, const Packet4s& b, const Packet4s& c)
1131 { return vmla_s16(c,a,b); }
1132 template<> EIGEN_STRONG_INLINE Packet8s pmadd(const Packet8s& a, const Packet8s& b, const Packet8s& c)
1133 { return vmlaq_s16(c,a,b); }
1134 template<> EIGEN_STRONG_INLINE Packet4us pmadd(const Packet4us& a, const Packet4us& b, const Packet4us& c)
1135 { return vmla_u16(c,a,b); }
1136 template<> EIGEN_STRONG_INLINE Packet8us pmadd(const Packet8us& a, const Packet8us& b, const Packet8us& c)
1137 { return vmlaq_u16(c,a,b); }
1138 template<> EIGEN_STRONG_INLINE Packet2i pmadd(const Packet2i& a, const Packet2i& b, const Packet2i& c)
1139 { return vmla_s32(c,a,b); }
1140 template<> EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c)
1141 { return vmlaq_s32(c,a,b); }
1142 template<> EIGEN_STRONG_INLINE Packet2ui pmadd(const Packet2ui& a, const Packet2ui& b, const Packet2ui& c)
1143 { return vmla_u32(c,a,b); }
1144 template<> EIGEN_STRONG_INLINE Packet4ui pmadd(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c)
1145 { return vmlaq_u32(c,a,b); }
1146 
1147 template<> EIGEN_STRONG_INLINE Packet2f pabsdiff<Packet2f>(const Packet2f& a, const Packet2f& b)
1148 { return vabd_f32(a,b); }
1149 template<> EIGEN_STRONG_INLINE Packet4f pabsdiff<Packet4f>(const Packet4f& a, const Packet4f& b)
1150 { return vabdq_f32(a,b); }
1151 template<> EIGEN_STRONG_INLINE Packet4c pabsdiff<Packet4c>(const Packet4c& a, const Packet4c& b)
1152 {
1153  return vget_lane_s32(vreinterpret_s32_s8(vabd_s8(
1154  vreinterpret_s8_s32(vdup_n_s32(a)),
1155  vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
1156 }
1157 template<> EIGEN_STRONG_INLINE Packet8c pabsdiff<Packet8c>(const Packet8c& a, const Packet8c& b)
1158 { return vabd_s8(a,b); }
1159 template<> EIGEN_STRONG_INLINE Packet16c pabsdiff<Packet16c>(const Packet16c& a, const Packet16c& b)
1160 { return vabdq_s8(a,b); }
1161 template<> EIGEN_STRONG_INLINE Packet4uc pabsdiff<Packet4uc>(const Packet4uc& a, const Packet4uc& b)
1162 {
1163  return vget_lane_u32(vreinterpret_u32_u8(vabd_u8(
1164  vreinterpret_u8_u32(vdup_n_u32(a)),
1165  vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
1166 }
1167 template<> EIGEN_STRONG_INLINE Packet8uc pabsdiff<Packet8uc>(const Packet8uc& a, const Packet8uc& b)
1168 { return vabd_u8(a,b); }
1169 template<> EIGEN_STRONG_INLINE Packet16uc pabsdiff<Packet16uc>(const Packet16uc& a, const Packet16uc& b)
1170 { return vabdq_u8(a,b); }
1171 template<> EIGEN_STRONG_INLINE Packet4s pabsdiff<Packet4s>(const Packet4s& a, const Packet4s& b)
1172 { return vabd_s16(a,b); }
1173 template<> EIGEN_STRONG_INLINE Packet8s pabsdiff<Packet8s>(const Packet8s& a, const Packet8s& b)
1174 { return vabdq_s16(a,b); }
1175 template<> EIGEN_STRONG_INLINE Packet4us pabsdiff<Packet4us>(const Packet4us& a, const Packet4us& b)
1176 { return vabd_u16(a,b); }
1177 template<> EIGEN_STRONG_INLINE Packet8us pabsdiff<Packet8us>(const Packet8us& a, const Packet8us& b)
1178 { return vabdq_u16(a,b); }
1179 template<> EIGEN_STRONG_INLINE Packet2i pabsdiff<Packet2i>(const Packet2i& a, const Packet2i& b)
1180 { return vabd_s32(a,b); }
1181 template<> EIGEN_STRONG_INLINE Packet4i pabsdiff<Packet4i>(const Packet4i& a, const Packet4i& b)
1182 { return vabdq_s32(a,b); }
1183 template<> EIGEN_STRONG_INLINE Packet2ui pabsdiff<Packet2ui>(const Packet2ui& a, const Packet2ui& b)
1184 { return vabd_u32(a,b); }
1185 template<> EIGEN_STRONG_INLINE Packet4ui pabsdiff<Packet4ui>(const Packet4ui& a, const Packet4ui& b)
1186 { return vabdq_u32(a,b); }
1187 
1188 template<> EIGEN_STRONG_INLINE Packet2f pmin<Packet2f>(const Packet2f& a, const Packet2f& b) { return vmin_f32(a,b); }
1189 template<> EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(const Packet4f& a, const Packet4f& b) { return vminq_f32(a,b); }
1190 
1191 #ifdef __ARM_FEATURE_NUMERIC_MAXMIN
1192 // numeric max and min are only available if ARM_FEATURE_NUMERIC_MAXMIN is defined (which can only be the case for Armv8 systems).
1193 template<> EIGEN_STRONG_INLINE Packet4f pmin<PropagateNumbers, Packet4f>(const Packet4f& a, const Packet4f& b) { return vminnmq_f32(a, b); }
1194 template<> EIGEN_STRONG_INLINE Packet2f pmin<PropagateNumbers, Packet2f>(const Packet2f& a, const Packet2f& b) { return vminnm_f32(a, b); }
1195 #endif
1196 
1197 template<> EIGEN_STRONG_INLINE Packet4f pmin<PropagateNaN, Packet4f>(const Packet4f& a, const Packet4f& b) { return pmin<Packet4f>(a, b); }
1198 
1199 template<> EIGEN_STRONG_INLINE Packet2f pmin<PropagateNaN, Packet2f>(const Packet2f& a, const Packet2f& b) { return pmin<Packet2f>(a, b); }
1200 
1201 template<> EIGEN_STRONG_INLINE Packet4c pmin<Packet4c>(const Packet4c& a, const Packet4c& b)
1202 {
1203  return vget_lane_s32(vreinterpret_s32_s8(vmin_s8(
1204  vreinterpret_s8_s32(vdup_n_s32(a)),
1205  vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
1206 }
1207 template<> EIGEN_STRONG_INLINE Packet8c pmin<Packet8c>(const Packet8c& a, const Packet8c& b) { return vmin_s8(a,b); }
1208 template<> EIGEN_STRONG_INLINE Packet16c pmin<Packet16c>(const Packet16c& a, const Packet16c& b) { return vminq_s8(a,b); }
1209 template<> EIGEN_STRONG_INLINE Packet4uc pmin<Packet4uc>(const Packet4uc& a, const Packet4uc& b)
1210 {
1211  return vget_lane_u32(vreinterpret_u32_u8(vmin_u8(
1212  vreinterpret_u8_u32(vdup_n_u32(a)),
1213  vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
1214 }
1215 template<> EIGEN_STRONG_INLINE Packet8uc pmin<Packet8uc>(const Packet8uc& a, const Packet8uc& b) { return vmin_u8(a,b); }
1216 template<> EIGEN_STRONG_INLINE Packet16uc pmin<Packet16uc>(const Packet16uc& a, const Packet16uc& b) { return vminq_u8(a,b); }
1217 template<> EIGEN_STRONG_INLINE Packet4s pmin<Packet4s>(const Packet4s& a, const Packet4s& b) { return vmin_s16(a,b); }
1218 template<> EIGEN_STRONG_INLINE Packet8s pmin<Packet8s>(const Packet8s& a, const Packet8s& b) { return vminq_s16(a,b); }
1219 template<> EIGEN_STRONG_INLINE Packet4us pmin<Packet4us>(const Packet4us& a, const Packet4us& b) { return vmin_u16(a,b); }
1220 template<> EIGEN_STRONG_INLINE Packet8us pmin<Packet8us>(const Packet8us& a, const Packet8us& b) { return vminq_u16(a,b); }
1221 template<> EIGEN_STRONG_INLINE Packet2i pmin<Packet2i>(const Packet2i& a, const Packet2i& b) { return vmin_s32(a,b); }
1222 template<> EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(const Packet4i& a, const Packet4i& b) { return vminq_s32(a,b); }
1223 template<> EIGEN_STRONG_INLINE Packet2ui pmin<Packet2ui>(const Packet2ui& a, const Packet2ui& b) { return vmin_u32(a,b); }
1224 template<> EIGEN_STRONG_INLINE Packet4ui pmin<Packet4ui>(const Packet4ui& a, const Packet4ui& b) { return vminq_u32(a,b); }
1225 template<> EIGEN_STRONG_INLINE Packet2l pmin<Packet2l>(const Packet2l& a, const Packet2l& b) {
1226  return vcombine_s64(
1227  vdup_n_s64((std::min)(vgetq_lane_s64(a, 0), vgetq_lane_s64(b, 0))),
1228  vdup_n_s64((std::min)(vgetq_lane_s64(a, 1), vgetq_lane_s64(b, 1))));
1229 }
1230 template<> EIGEN_STRONG_INLINE Packet2ul pmin<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
1231  return vcombine_u64(
1232  vdup_n_u64((std::min)(vgetq_lane_u64(a, 0), vgetq_lane_u64(b, 0))),
1233  vdup_n_u64((std::min)(vgetq_lane_u64(a, 1), vgetq_lane_u64(b, 1))));
1234 }
1235 
1236 template<> EIGEN_STRONG_INLINE Packet2f pmax<Packet2f>(const Packet2f& a, const Packet2f& b) { return vmax_f32(a,b); }
1237 template<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(const Packet4f& a, const Packet4f& b) { return vmaxq_f32(a,b); }
1238 
1239 #ifdef __ARM_FEATURE_NUMERIC_MAXMIN
1240 // numeric max and min are only available if ARM_FEATURE_NUMERIC_MAXMIN is defined (which can only be the case for Armv8 systems).
1241 template<> EIGEN_STRONG_INLINE Packet4f pmax<PropagateNumbers, Packet4f>(const Packet4f& a, const Packet4f& b) { return vmaxnmq_f32(a, b); }
1242 template<> EIGEN_STRONG_INLINE Packet2f pmax<PropagateNumbers, Packet2f>(const Packet2f& a, const Packet2f& b) { return vmaxnm_f32(a, b); }
1243 #endif
1244 
1245 template<> EIGEN_STRONG_INLINE Packet4f pmax<PropagateNaN, Packet4f>(const Packet4f& a, const Packet4f& b) { return pmax<Packet4f>(a, b); }
1246 
1247 template<> EIGEN_STRONG_INLINE Packet2f pmax<PropagateNaN, Packet2f>(const Packet2f& a, const Packet2f& b) { return pmax<Packet2f>(a, b); }
1248 
1249 template<> EIGEN_STRONG_INLINE Packet4c pmax<Packet4c>(const Packet4c& a, const Packet4c& b)
1250 {
1251  return vget_lane_s32(vreinterpret_s32_s8(vmax_s8(
1252  vreinterpret_s8_s32(vdup_n_s32(a)),
1253  vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
1254 }
1255 template<> EIGEN_STRONG_INLINE Packet8c pmax<Packet8c>(const Packet8c& a, const Packet8c& b) { return vmax_s8(a,b); }
1256 template<> EIGEN_STRONG_INLINE Packet16c pmax<Packet16c>(const Packet16c& a, const Packet16c& b) { return vmaxq_s8(a,b); }
1257 template<> EIGEN_STRONG_INLINE Packet4uc pmax<Packet4uc>(const Packet4uc& a, const Packet4uc& b)
1258 {
1259  return vget_lane_u32(vreinterpret_u32_u8(vmax_u8(
1260  vreinterpret_u8_u32(vdup_n_u32(a)),
1261  vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
1262 }
1263 template<> EIGEN_STRONG_INLINE Packet8uc pmax<Packet8uc>(const Packet8uc& a, const Packet8uc& b) { return vmax_u8(a,b); }
1264 template<> EIGEN_STRONG_INLINE Packet16uc pmax<Packet16uc>(const Packet16uc& a, const Packet16uc& b) { return vmaxq_u8(a,b); }
1265 template<> EIGEN_STRONG_INLINE Packet4s pmax<Packet4s>(const Packet4s& a, const Packet4s& b) { return vmax_s16(a,b); }
1266 template<> EIGEN_STRONG_INLINE Packet8s pmax<Packet8s>(const Packet8s& a, const Packet8s& b) { return vmaxq_s16(a,b); }
1267 template<> EIGEN_STRONG_INLINE Packet4us pmax<Packet4us>(const Packet4us& a, const Packet4us& b) { return vmax_u16(a,b); }
1268 template<> EIGEN_STRONG_INLINE Packet8us pmax<Packet8us>(const Packet8us& a, const Packet8us& b) { return vmaxq_u16(a,b); }
1269 template<> EIGEN_STRONG_INLINE Packet2i pmax<Packet2i>(const Packet2i& a, const Packet2i& b) { return vmax_s32(a,b); }
1270 template<> EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(const Packet4i& a, const Packet4i& b) { return vmaxq_s32(a,b); }
1271 template<> EIGEN_STRONG_INLINE Packet2ui pmax<Packet2ui>(const Packet2ui& a, const Packet2ui& b) { return vmax_u32(a,b); }
1272 template<> EIGEN_STRONG_INLINE Packet4ui pmax<Packet4ui>(const Packet4ui& a, const Packet4ui& b) { return vmaxq_u32(a,b); }
1273 template<> EIGEN_STRONG_INLINE Packet2l pmax<Packet2l>(const Packet2l& a, const Packet2l& b) {
1274  return vcombine_s64(
1275  vdup_n_s64((std::max)(vgetq_lane_s64(a, 0), vgetq_lane_s64(b, 0))),
1276  vdup_n_s64((std::max)(vgetq_lane_s64(a, 1), vgetq_lane_s64(b, 1))));
1277 }
1278 template<> EIGEN_STRONG_INLINE Packet2ul pmax<Packet2ul>(const Packet2ul& a, const Packet2ul& b) {
1279  return vcombine_u64(
1280  vdup_n_u64((std::max)(vgetq_lane_u64(a, 0), vgetq_lane_u64(b, 0))),
1281  vdup_n_u64((std::max)(vgetq_lane_u64(a, 1), vgetq_lane_u64(b, 1))));
1282 }
1283 
1284 template<> EIGEN_STRONG_INLINE Packet2f pcmp_le<Packet2f>(const Packet2f& a, const Packet2f& b)
1285 { return vreinterpret_f32_u32(vcle_f32(a,b)); }
1286 template<> EIGEN_STRONG_INLINE Packet4f pcmp_le<Packet4f>(const Packet4f& a, const Packet4f& b)
1287 { return vreinterpretq_f32_u32(vcleq_f32(a,b)); }
1288 template<> EIGEN_STRONG_INLINE Packet4c pcmp_le<Packet4c>(const Packet4c& a, const Packet4c& b)
1289 {
1290  return vget_lane_s32(vreinterpret_s32_u8(vcle_s8(
1291  vreinterpret_s8_s32(vdup_n_s32(a)),
1292  vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
1293 }
1294 template<> EIGEN_STRONG_INLINE Packet8c pcmp_le<Packet8c>(const Packet8c& a, const Packet8c& b)
1295 { return vreinterpret_s8_u8(vcle_s8(a,b)); }
1296 template<> EIGEN_STRONG_INLINE Packet16c pcmp_le<Packet16c>(const Packet16c& a, const Packet16c& b)
1297 { return vreinterpretq_s8_u8(vcleq_s8(a,b)); }
1298 template<> EIGEN_STRONG_INLINE Packet4uc pcmp_le<Packet4uc>(const Packet4uc& a, const Packet4uc& b)
1299 {
1300  return vget_lane_u32(vreinterpret_u32_u8(vcle_u8(
1301  vreinterpret_u8_u32(vdup_n_u32(a)),
1302  vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
1303 }
1304 template<> EIGEN_STRONG_INLINE Packet8uc pcmp_le<Packet8uc>(const Packet8uc& a, const Packet8uc& b)
1305 { return vcle_u8(a,b); }
1306 template<> EIGEN_STRONG_INLINE Packet16uc pcmp_le<Packet16uc>(const Packet16uc& a, const Packet16uc& b)
1307 { return vcleq_u8(a,b); }
1308 template<> EIGEN_STRONG_INLINE Packet4s pcmp_le<Packet4s>(const Packet4s& a, const Packet4s& b)
1309 { return vreinterpret_s16_u16(vcle_s16(a,b)); }
1310 template<> EIGEN_STRONG_INLINE Packet8s pcmp_le<Packet8s>(const Packet8s& a, const Packet8s& b)
1311 { return vreinterpretq_s16_u16(vcleq_s16(a,b)); }
1312 template<> EIGEN_STRONG_INLINE Packet4us pcmp_le<Packet4us>(const Packet4us& a, const Packet4us& b)
1313 { return vcle_u16(a,b); }
1314 template<> EIGEN_STRONG_INLINE Packet8us pcmp_le<Packet8us>(const Packet8us& a, const Packet8us& b)
1315 { return vcleq_u16(a,b); }
1316 template<> EIGEN_STRONG_INLINE Packet2i pcmp_le<Packet2i>(const Packet2i& a, const Packet2i& b)
1317 { return vreinterpret_s32_u32(vcle_s32(a,b)); }
1318 template<> EIGEN_STRONG_INLINE Packet4i pcmp_le<Packet4i>(const Packet4i& a, const Packet4i& b)
1319 { return vreinterpretq_s32_u32(vcleq_s32(a,b)); }
1320 template<> EIGEN_STRONG_INLINE Packet2ui pcmp_le<Packet2ui>(const Packet2ui& a, const Packet2ui& b)
1321 { return vcle_u32(a,b); }
1322 template<> EIGEN_STRONG_INLINE Packet4ui pcmp_le<Packet4ui>(const Packet4ui& a, const Packet4ui& b)
1323 { return vcleq_u32(a,b); }
1324 template<> EIGEN_STRONG_INLINE Packet2l pcmp_le<Packet2l>(const Packet2l& a, const Packet2l& b)
1325 {
1326 #if EIGEN_ARCH_ARM64
1327  return vreinterpretq_s64_u64(vcleq_s64(a,b));
1328 #else
1329  return vcombine_s64(
1330  vdup_n_s64(vgetq_lane_s64(a, 0) <= vgetq_lane_s64(b, 0) ? numext::int64_t(-1) : 0),
1331  vdup_n_s64(vgetq_lane_s64(a, 1) <= vgetq_lane_s64(b, 1) ? numext::int64_t(-1) : 0));
1332 #endif
1333 }
1334 template<> EIGEN_STRONG_INLINE Packet2ul pcmp_le<Packet2ul>(const Packet2ul& a, const Packet2ul& b)
1335 {
1336 #if EIGEN_ARCH_ARM64
1337  return vcleq_u64(a,b);
1338 #else
1339  return vcombine_u64(
1340  vdup_n_u64(vgetq_lane_u64(a, 0) <= vgetq_lane_u64(b, 0) ? numext::uint64_t(-1) : 0),
1341  vdup_n_u64(vgetq_lane_u64(a, 1) <= vgetq_lane_u64(b, 1) ? numext::uint64_t(-1) : 0));
1342 #endif
1343 }
1344 
1345 template<> EIGEN_STRONG_INLINE Packet2f pcmp_lt<Packet2f>(const Packet2f& a, const Packet2f& b)
1346 { return vreinterpret_f32_u32(vclt_f32(a,b)); }
1347 template<> EIGEN_STRONG_INLINE Packet4f pcmp_lt<Packet4f>(const Packet4f& a, const Packet4f& b)
1348 { return vreinterpretq_f32_u32(vcltq_f32(a,b)); }
1349 template<> EIGEN_STRONG_INLINE Packet4c pcmp_lt<Packet4c>(const Packet4c& a, const Packet4c& b)
1350 {
1351  return vget_lane_s32(vreinterpret_s32_u8(vclt_s8(
1352  vreinterpret_s8_s32(vdup_n_s32(a)),
1353  vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
1354 }
1355 template<> EIGEN_STRONG_INLINE Packet8c pcmp_lt<Packet8c>(const Packet8c& a, const Packet8c& b)
1356 { return vreinterpret_s8_u8(vclt_s8(a,b)); }
1357 template<> EIGEN_STRONG_INLINE Packet16c pcmp_lt<Packet16c>(const Packet16c& a, const Packet16c& b)
1358 { return vreinterpretq_s8_u8(vcltq_s8(a,b)); }
1359 template<> EIGEN_STRONG_INLINE Packet4uc pcmp_lt<Packet4uc>(const Packet4uc& a, const Packet4uc& b)
1360 {
1361  return vget_lane_u32(vreinterpret_u32_u8(vclt_u8(
1362  vreinterpret_u8_u32(vdup_n_u32(a)),
1363  vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
1364 }
1365 template<> EIGEN_STRONG_INLINE Packet8uc pcmp_lt<Packet8uc>(const Packet8uc& a, const Packet8uc& b)
1366 { return vclt_u8(a,b); }
1367 template<> EIGEN_STRONG_INLINE Packet16uc pcmp_lt<Packet16uc>(const Packet16uc& a, const Packet16uc& b)
1368 { return vcltq_u8(a,b); }
1369 template<> EIGEN_STRONG_INLINE Packet4s pcmp_lt<Packet4s>(const Packet4s& a, const Packet4s& b)
1370 { return vreinterpret_s16_u16(vclt_s16(a,b)); }
1371 template<> EIGEN_STRONG_INLINE Packet8s pcmp_lt<Packet8s>(const Packet8s& a, const Packet8s& b)
1372 { return vreinterpretq_s16_u16(vcltq_s16(a,b)); }
1373 template<> EIGEN_STRONG_INLINE Packet4us pcmp_lt<Packet4us>(const Packet4us& a, const Packet4us& b)
1374 { return vclt_u16(a,b); }
1375 template<> EIGEN_STRONG_INLINE Packet8us pcmp_lt<Packet8us>(const Packet8us& a, const Packet8us& b)
1376 { return vcltq_u16(a,b); }
1377 template<> EIGEN_STRONG_INLINE Packet2i pcmp_lt<Packet2i>(const Packet2i& a, const Packet2i& b)
1378 { return vreinterpret_s32_u32(vclt_s32(a,b)); }
1379 template<> EIGEN_STRONG_INLINE Packet4i pcmp_lt<Packet4i>(const Packet4i& a, const Packet4i& b)
1380 { return vreinterpretq_s32_u32(vcltq_s32(a,b)); }
1381 template<> EIGEN_STRONG_INLINE Packet2ui pcmp_lt<Packet2ui>(const Packet2ui& a, const Packet2ui& b)
1382 { return vclt_u32(a,b); }
1383 template<> EIGEN_STRONG_INLINE Packet4ui pcmp_lt<Packet4ui>(const Packet4ui& a, const Packet4ui& b)
1384 { return vcltq_u32(a,b); }
1385 template<> EIGEN_STRONG_INLINE Packet2l pcmp_lt<Packet2l>(const Packet2l& a, const Packet2l& b)
1386 {
1387 #if EIGEN_ARCH_ARM64
1388  return vreinterpretq_s64_u64(vcltq_s64(a,b));
1389 #else
1390  return vcombine_s64(
1391  vdup_n_s64(vgetq_lane_s64(a, 0) < vgetq_lane_s64(b, 0) ? numext::int64_t(-1) : 0),
1392  vdup_n_s64(vgetq_lane_s64(a, 1) < vgetq_lane_s64(b, 1) ? numext::int64_t(-1) : 0));
1393 #endif
1394 }
1395 template<> EIGEN_STRONG_INLINE Packet2ul pcmp_lt<Packet2ul>(const Packet2ul& a, const Packet2ul& b)
1396 {
1397 #if EIGEN_ARCH_ARM64
1398  return vcltq_u64(a,b);
1399 #else
1400  return vcombine_u64(
1401  vdup_n_u64(vgetq_lane_u64(a, 0) < vgetq_lane_u64(b, 0) ? numext::uint64_t(-1) : 0),
1402  vdup_n_u64(vgetq_lane_u64(a, 1) < vgetq_lane_u64(b, 1) ? numext::uint64_t(-1) : 0));
1403 #endif
1404 }
1405 
1406 template<> EIGEN_STRONG_INLINE Packet2f pcmp_eq<Packet2f>(const Packet2f& a, const Packet2f& b)
1407 { return vreinterpret_f32_u32(vceq_f32(a,b)); }
1408 template<> EIGEN_STRONG_INLINE Packet4f pcmp_eq<Packet4f>(const Packet4f& a, const Packet4f& b)
1409 { return vreinterpretq_f32_u32(vceqq_f32(a,b)); }
1410 template<> EIGEN_STRONG_INLINE Packet4c pcmp_eq<Packet4c>(const Packet4c& a, const Packet4c& b)
1411 {
1412  return vget_lane_s32(vreinterpret_s32_u8(vceq_s8(
1413  vreinterpret_s8_s32(vdup_n_s32(a)),
1414  vreinterpret_s8_s32(vdup_n_s32(b)))), 0);
1415 }
1416 template<> EIGEN_STRONG_INLINE Packet8c pcmp_eq<Packet8c>(const Packet8c& a, const Packet8c& b)
1417 { return vreinterpret_s8_u8(vceq_s8(a,b)); }
1418 template<> EIGEN_STRONG_INLINE Packet16c pcmp_eq<Packet16c>(const Packet16c& a, const Packet16c& b)
1419 { return vreinterpretq_s8_u8(vceqq_s8(a,b)); }
1420 template<> EIGEN_STRONG_INLINE Packet4uc pcmp_eq<Packet4uc>(const Packet4uc& a, const Packet4uc& b)
1421 {
1422  return vget_lane_u32(vreinterpret_u32_u8(vceq_u8(
1423  vreinterpret_u8_u32(vdup_n_u32(a)),
1424  vreinterpret_u8_u32(vdup_n_u32(b)))), 0);
1425 }
1426 template<> EIGEN_STRONG_INLINE Packet8uc pcmp_eq<Packet8uc>(const Packet8uc& a, const Packet8uc& b)
1427 { return vceq_u8(a,b); }
1428 template<> EIGEN_STRONG_INLINE Packet16uc pcmp_eq<Packet16uc>(const Packet16uc& a, const Packet16uc& b)
1429 { return vceqq_u8(a,b); }
1430 template<> EIGEN_STRONG_INLINE Packet4s pcmp_eq<Packet4s>(const Packet4s& a, const Packet4s& b)
1431 { return vreinterpret_s16_u16(vceq_s16(a,b)); }
1432 template<> EIGEN_STRONG_INLINE Packet8s pcmp_eq<Packet8s>(const Packet8s& a, const Packet8s& b)
1433 { return vreinterpretq_s16_u16(vceqq_s16(a,b)); }
1434 template<> EIGEN_STRONG_INLINE Packet4us pcmp_eq<Packet4us>(const Packet4us& a, const Packet4us& b)
1435 { return vceq_u16(a,b); }
1436 template<> EIGEN_STRONG_INLINE Packet8us pcmp_eq<Packet8us>(const Packet8us& a, const Packet8us& b)
1437 { return vceqq_u16(a,b); }
1438 template<> EIGEN_STRONG_INLINE Packet2i pcmp_eq<Packet2i>(const Packet2i& a, const Packet2i& b)
1439 { return vreinterpret_s32_u32(vceq_s32(a,b)); }
1440 template<> EIGEN_STRONG_INLINE Packet4i pcmp_eq<Packet4i>(const Packet4i& a, const Packet4i& b)
1441 { return vreinterpretq_s32_u32(vceqq_s32(a,b)); }
1442 template<> EIGEN_STRONG_INLINE Packet2ui pcmp_eq<Packet2ui>(const Packet2ui& a, const Packet2ui& b)
1443 { return vceq_u32(a,b); }
1444 template<> EIGEN_STRONG_INLINE Packet4ui pcmp_eq<Packet4ui>(const Packet4ui& a, const Packet4ui& b)
1445 { return vceqq_u32(a,b); }
1446 template<> EIGEN_STRONG_INLINE Packet2l pcmp_eq<Packet2l>(const Packet2l& a, const Packet2l& b)
1447 {
1448 #if EIGEN_ARCH_ARM64
1449  return vreinterpretq_s64_u64(vceqq_s64(a,b));
1450 #else
1451  return vcombine_s64(
1452  vdup_n_s64(vgetq_lane_s64(a, 0) == vgetq_lane_s64(b, 0) ? numext::int64_t(-1) : 0),
1453  vdup_n_s64(vgetq_lane_s64(a, 1) == vgetq_lane_s64(b, 1) ? numext::int64_t(-1) : 0));
1454 #endif
1455 }
1456 template<> EIGEN_STRONG_INLINE Packet2ul pcmp_eq<Packet2ul>(const Packet2ul& a, const Packet2ul& b)
1457 {
1458 #if EIGEN_ARCH_ARM64
1459  return vceqq_u64(a,b);
1460 #else
1461  return vcombine_u64(
1462  vdup_n_u64(vgetq_lane_u64(a, 0) == vgetq_lane_u64(b, 0) ? numext::uint64_t(-1) : 0),
1463  vdup_n_u64(vgetq_lane_u64(a, 1) == vgetq_lane_u64(b, 1) ? numext::uint64_t(-1) : 0));
1464 #endif
1465 }
1466 
1467 template<> EIGEN_STRONG_INLINE Packet2f pcmp_lt_or_nan<Packet2f>(const Packet2f& a, const Packet2f& b)
1468 { return vreinterpret_f32_u32(vmvn_u32(vcge_f32(a,b))); }
1469 template<> EIGEN_STRONG_INLINE Packet4f pcmp_lt_or_nan<Packet4f>(const Packet4f& a, const Packet4f& b)
1470 { return vreinterpretq_f32_u32(vmvnq_u32(vcgeq_f32(a,b))); }
1471 
1472 // Logical Operations are not supported for float, so we have to reinterpret casts using NEON intrinsics
1473 template<> EIGEN_STRONG_INLINE Packet2f pand<Packet2f>(const Packet2f& a, const Packet2f& b)
1474 { return vreinterpret_f32_u32(vand_u32(vreinterpret_u32_f32(a),vreinterpret_u32_f32(b))); }
1475 template<> EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(const Packet4f& a, const Packet4f& b)
1476 { return vreinterpretq_f32_u32(vandq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b))); }
1477 template<> EIGEN_STRONG_INLINE Packet4c pand<Packet4c>(const Packet4c& a, const Packet4c& b)
1478 { return a & b; }
1479 template<> EIGEN_STRONG_INLINE Packet8c pand<Packet8c>(const Packet8c& a, const Packet8c& b)
1480 { return vand_s8(a,b); }
1481 template<> EIGEN_STRONG_INLINE Packet16c pand<Packet16c>(const Packet16c& a, const Packet16c& b)
1482 { return vandq_s8(a,b); }
1483 template<> EIGEN_STRONG_INLINE Packet4uc pand<Packet4uc>(const Packet4uc& a, const Packet4uc& b)
1484 { return a & b; }
1485 template<> EIGEN_STRONG_INLINE Packet8uc pand<Packet8uc>(const Packet8uc& a, const Packet8uc& b)
1486 { return vand_u8(a,b); }
1487 template<> EIGEN_STRONG_INLINE Packet16uc pand<Packet16uc>(const Packet16uc& a, const Packet16uc& b)
1488 { return vandq_u8(a,b); }
1489 template<> EIGEN_STRONG_INLINE Packet4s pand<Packet4s>(const Packet4s& a, const Packet4s& b) { return vand_s16(a,b); }
1490 template<> EIGEN_STRONG_INLINE Packet8s pand<Packet8s>(const Packet8s& a, const Packet8s& b) { return vandq_s16(a,b); }
1491 template<> EIGEN_STRONG_INLINE Packet4us pand<Packet4us>(const Packet4us& a, const Packet4us& b)
1492 { return vand_u16(a,b); }
1493 template<> EIGEN_STRONG_INLINE Packet8us pand<Packet8us>(const Packet8us& a, const Packet8us& b)
1494 { return vandq_u16(a,b); }
1495 template<> EIGEN_STRONG_INLINE Packet2i pand<Packet2i>(const Packet2i& a, const Packet2i& b) { return vand_s32(a,b); }
1496 template<> EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(const Packet4i& a, const Packet4i& b) { return vandq_s32(a,b); }
1497 template<> EIGEN_STRONG_INLINE Packet2ui pand<Packet2ui>(const Packet2ui& a, const Packet2ui& b)
1498 { return vand_u32(a,b); }
1499 template<> EIGEN_STRONG_INLINE Packet4ui pand<Packet4ui>(const Packet4ui& a, const Packet4ui& b)
1500 { return vandq_u32(a,b); }
1501 template<> EIGEN_STRONG_INLINE Packet2l pand<Packet2l>(const Packet2l& a, const Packet2l& b) { return vandq_s64(a,b); }
1502 template<> EIGEN_STRONG_INLINE Packet2ul pand<Packet2ul>(const Packet2ul& a, const Packet2ul& b)
1503 { return vandq_u64(a,b); }
1504 
1505 template<> EIGEN_STRONG_INLINE Packet2f por<Packet2f>(const Packet2f& a, const Packet2f& b)
1506 { return vreinterpret_f32_u32(vorr_u32(vreinterpret_u32_f32(a),vreinterpret_u32_f32(b))); }
1507 template<> EIGEN_STRONG_INLINE Packet4f por<Packet4f>(const Packet4f& a, const Packet4f& b)
1508 { return vreinterpretq_f32_u32(vorrq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b))); }
1509 template<> EIGEN_STRONG_INLINE Packet4c por<Packet4c>(const Packet4c& a, const Packet4c& b)
1510 { return a | b; }
1511 template<> EIGEN_STRONG_INLINE Packet8c por<Packet8c>(const Packet8c& a, const Packet8c& b) { return vorr_s8(a,b); }
1512 template<> EIGEN_STRONG_INLINE Packet16c por<Packet16c>(const Packet16c& a, const Packet16c& b)
1513 { return vorrq_s8(a,b); }
1514 template<> EIGEN_STRONG_INLINE Packet4uc por<Packet4uc>(const Packet4uc& a, const Packet4uc& b)
1515 { return a | b; }
1516 template<> EIGEN_STRONG_INLINE Packet8uc por<Packet8uc>(const Packet8uc& a, const Packet8uc& b)
1517 { return vorr_u8(a,b); }
1518 template<> EIGEN_STRONG_INLINE Packet16uc por<Packet16uc>(const Packet16uc& a, const Packet16uc& b)
1519 { return vorrq_u8(a,b); }
1520 template<> EIGEN_STRONG_INLINE Packet4s por<Packet4s>(const Packet4s& a, const Packet4s& b)
1521 { return vorr_s16(a,b); }
1522 template<> EIGEN_STRONG_INLINE Packet8s por<Packet8s>(const Packet8s& a, const Packet8s& b)
1523 { return vorrq_s16(a,b); }
1524 template<> EIGEN_STRONG_INLINE Packet4us por<Packet4us>(const Packet4us& a, const Packet4us& b)
1525 { return vorr_u16(a,b); }
1526 template<> EIGEN_STRONG_INLINE Packet8us por<Packet8us>(const Packet8us& a, const Packet8us& b)
1527 { return vorrq_u16(a,b); }
1528 template<> EIGEN_STRONG_INLINE Packet2i por<Packet2i>(const Packet2i& a, const Packet2i& b) { return vorr_s32(a,b); }
1529 template<> EIGEN_STRONG_INLINE Packet4i por<Packet4i>(const Packet4i& a, const Packet4i& b) { return vorrq_s32(a,b); }
1530 template<> EIGEN_STRONG_INLINE Packet2ui por<Packet2ui>(const Packet2ui& a, const Packet2ui& b)
1531 { return vorr_u32(a,b); }
1532 template<> EIGEN_STRONG_INLINE Packet4ui por<Packet4ui>(const Packet4ui& a, const Packet4ui& b)
1533 { return vorrq_u32(a,b); }
1534 template<> EIGEN_STRONG_INLINE Packet2l por<Packet2l>(const Packet2l& a, const Packet2l& b)
1535 { return vorrq_s64(a,b); }
1536 template<> EIGEN_STRONG_INLINE Packet2ul por<Packet2ul>(const Packet2ul& a, const Packet2ul& b)
1537 { return vorrq_u64(a,b); }
1538 
1539 template<> EIGEN_STRONG_INLINE Packet2f pxor<Packet2f>(const Packet2f& a, const Packet2f& b)
1540 { return vreinterpret_f32_u32(veor_u32(vreinterpret_u32_f32(a),vreinterpret_u32_f32(b))); }
1541 template<> EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(const Packet4f& a, const Packet4f& b)
1542 { return vreinterpretq_f32_u32(veorq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b))); }
1543 template<> EIGEN_STRONG_INLINE Packet4c pxor<Packet4c>(const Packet4c& a, const Packet4c& b)
1544 { return a ^ b; }
1545 template<> EIGEN_STRONG_INLINE Packet8c pxor<Packet8c>(const Packet8c& a, const Packet8c& b)
1546 { return veor_s8(a,b); }
1547 template<> EIGEN_STRONG_INLINE Packet16c pxor<Packet16c>(const Packet16c& a, const Packet16c& b)
1548 { return veorq_s8(a,b); }
1549 template<> EIGEN_STRONG_INLINE Packet4uc pxor<Packet4uc>(const Packet4uc& a, const Packet4uc& b)
1550 { return a ^ b; }
1551 template<> EIGEN_STRONG_INLINE Packet8uc pxor<Packet8uc>(const Packet8uc& a, const Packet8uc& b)
1552 { return veor_u8(a,b); }
1553 template<> EIGEN_STRONG_INLINE Packet16uc pxor<Packet16uc>(const Packet16uc& a, const Packet16uc& b)
1554 { return veorq_u8(a,b); }
1555 template<> EIGEN_STRONG_INLINE Packet4s pxor<Packet4s>(const Packet4s& a, const Packet4s& b) { return veor_s16(a,b); }
1556 template<> EIGEN_STRONG_INLINE Packet8s pxor<Packet8s>(const Packet8s& a, const Packet8s& b) { return veorq_s16(a,b); }
1557 template<> EIGEN_STRONG_INLINE Packet4us pxor<Packet4us>(const Packet4us& a, const Packet4us& b)
1558 { return veor_u16(a,b); }
1559 template<> EIGEN_STRONG_INLINE Packet8us pxor<Packet8us>(const Packet8us& a, const Packet8us& b)
1560 { return veorq_u16(a,b); }
1561 template<> EIGEN_STRONG_INLINE Packet2i pxor<Packet2i>(const Packet2i& a, const Packet2i& b) { return veor_s32(a,b); }
1562 template<> EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(const Packet4i& a, const Packet4i& b) { return veorq_s32(a,b); }
1563 template<> EIGEN_STRONG_INLINE Packet2ui pxor<Packet2ui>(const Packet2ui& a, const Packet2ui& b)
1564 { return veor_u32(a,b); }
1565 template<> EIGEN_STRONG_INLINE Packet4ui pxor<Packet4ui>(const Packet4ui& a, const Packet4ui& b)
1566 { return veorq_u32(a,b); }
1567 template<> EIGEN_STRONG_INLINE Packet2l pxor<Packet2l>(const Packet2l& a, const Packet2l& b)
1568 { return veorq_s64(a,b); }
1569 template<> EIGEN_STRONG_INLINE Packet2ul pxor<Packet2ul>(const Packet2ul& a, const Packet2ul& b)
1570 { return veorq_u64(a,b); }
1571 
1572 template<> EIGEN_STRONG_INLINE Packet2f pandnot<Packet2f>(const Packet2f& a, const Packet2f& b)
1573 { return vreinterpret_f32_u32(vbic_u32(vreinterpret_u32_f32(a),vreinterpret_u32_f32(b))); }
1574 template<> EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(const Packet4f& a, const Packet4f& b)
1575 { return vreinterpretq_f32_u32(vbicq_u32(vreinterpretq_u32_f32(a),vreinterpretq_u32_f32(b))); }
1576 template<> EIGEN_STRONG_INLINE Packet4c pandnot<Packet4c>(const Packet4c& a, const Packet4c& b)
1577 { return a & ~b; }
1578 template<> EIGEN_STRONG_INLINE Packet8c pandnot<Packet8c>(const Packet8c& a, const Packet8c& b) { return vbic_s8(a,b); }
1579 template<> EIGEN_STRONG_INLINE Packet16c pandnot<Packet16c>(const Packet16c& a, const Packet16c& b) { return vbicq_s8(a,b); }
1580 template<> EIGEN_STRONG_INLINE Packet4uc pandnot<Packet4uc>(const Packet4uc& a, const Packet4uc& b)
1581 { return a & ~b; }
1582 template<> EIGEN_STRONG_INLINE Packet8uc pandnot<Packet8uc>(const Packet8uc& a, const Packet8uc& b)
1583 { return vbic_u8(a,b); }
1584 template<> EIGEN_STRONG_INLINE Packet16uc pandnot<Packet16uc>(const Packet16uc& a, const Packet16uc& b)
1585 { return vbicq_u8(a,b); }
1586 template<> EIGEN_STRONG_INLINE Packet4s pandnot<Packet4s>(const Packet4s& a, const Packet4s& b)
1587 { return vbic_s16(a,b); }
1588 template<> EIGEN_STRONG_INLINE Packet8s pandnot<Packet8s>(const Packet8s& a, const Packet8s& b)
1589 { return vbicq_s16(a,b); }
1590 template<> EIGEN_STRONG_INLINE Packet4us pandnot<Packet4us>(const Packet4us& a, const Packet4us& b)
1591 { return vbic_u16(a,b); }
1592 template<> EIGEN_STRONG_INLINE Packet8us pandnot<Packet8us>(const Packet8us& a, const Packet8us& b)
1593 { return vbicq_u16(a,b); }
1594 template<> EIGEN_STRONG_INLINE Packet2i pandnot<Packet2i>(const Packet2i& a, const Packet2i& b)
1595 { return vbic_s32(a,b); }
1596 template<> EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(const Packet4i& a, const Packet4i& b)
1597 { return vbicq_s32(a,b); }
1598 template<> EIGEN_STRONG_INLINE Packet2ui pandnot<Packet2ui>(const Packet2ui& a, const Packet2ui& b)
1599 { return vbic_u32(a,b); }
1600 template<> EIGEN_STRONG_INLINE Packet4ui pandnot<Packet4ui>(const Packet4ui& a, const Packet4ui& b)
1601 { return vbicq_u32(a,b); }
1602 template<> EIGEN_STRONG_INLINE Packet2l pandnot<Packet2l>(const Packet2l& a, const Packet2l& b)
1603 { return vbicq_s64(a,b); }
1604 template<> EIGEN_STRONG_INLINE Packet2ul pandnot<Packet2ul>(const Packet2ul& a, const Packet2ul& b)
1605 { return vbicq_u64(a,b); }
1606 
1607 
1608 template<int N> EIGEN_STRONG_INLINE Packet4c parithmetic_shift_right(Packet4c& a)
1609 { return vget_lane_s32(vreinterpret_s32_s8(vshr_n_s8(vreinterpret_s8_s32(vdup_n_s32(a)), N)), 0); }
1610 template<int N> EIGEN_STRONG_INLINE Packet8c parithmetic_shift_right(Packet8c a) { return vshr_n_s8(a,N); }
1611 template<int N> EIGEN_STRONG_INLINE Packet16c parithmetic_shift_right(Packet16c a) { return vshrq_n_s8(a,N); }
1612 template<int N> EIGEN_STRONG_INLINE Packet4uc parithmetic_shift_right(Packet4uc& a)
1613 { return vget_lane_u32(vreinterpret_u32_u8(vshr_n_u8(vreinterpret_u8_u32(vdup_n_u32(a)), N)), 0); }
1614 template<int N> EIGEN_STRONG_INLINE Packet8uc parithmetic_shift_right(Packet8uc a) { return vshr_n_u8(a,N); }
1615 template<int N> EIGEN_STRONG_INLINE Packet16uc parithmetic_shift_right(Packet16uc a) { return vshrq_n_u8(a,N); }
1616 template<int N> EIGEN_STRONG_INLINE Packet4s parithmetic_shift_right(Packet4s a) { return vshr_n_s16(a,N); }
1617 template<int N> EIGEN_STRONG_INLINE Packet8s parithmetic_shift_right(Packet8s a) { return vshrq_n_s16(a,N); }
1618 template<int N> EIGEN_STRONG_INLINE Packet4us parithmetic_shift_right(Packet4us a) { return vshr_n_u16(a,N); }
1619 template<int N> EIGEN_STRONG_INLINE Packet8us parithmetic_shift_right(Packet8us a) { return vshrq_n_u16(a,N); }
1620 template<int N> EIGEN_STRONG_INLINE Packet2i parithmetic_shift_right(Packet2i a) { return vshr_n_s32(a,N); }
1621 template<int N> EIGEN_STRONG_INLINE Packet4i parithmetic_shift_right(Packet4i a) { return vshrq_n_s32(a,N); }
1622 template<int N> EIGEN_STRONG_INLINE Packet2ui parithmetic_shift_right(Packet2ui a) { return vshr_n_u32(a,N); }
1623 template<int N> EIGEN_STRONG_INLINE Packet4ui parithmetic_shift_right(Packet4ui a) { return vshrq_n_u32(a,N); }
1624 template<int N> EIGEN_STRONG_INLINE Packet2l parithmetic_shift_right(Packet2l a) { return vshrq_n_s64(a,N); }
1625 template<int N> EIGEN_STRONG_INLINE Packet2ul parithmetic_shift_right(Packet2ul a) { return vshrq_n_u64(a,N); }
1626 
1627 template<int N> EIGEN_STRONG_INLINE Packet4c plogical_shift_right(Packet4c& a)
1628 { return vget_lane_s32(vreinterpret_s32_u8(vshr_n_u8(vreinterpret_u8_s32(vdup_n_s32(a)), N)), 0); }
1629 template<int N> EIGEN_STRONG_INLINE Packet8c plogical_shift_right(Packet8c a)
1630 { return vreinterpret_s8_u8(vshr_n_u8(vreinterpret_u8_s8(a),N)); }
1631 template<int N> EIGEN_STRONG_INLINE Packet16c plogical_shift_right(Packet16c a)
1632 { return vreinterpretq_s8_u8(vshrq_n_u8(vreinterpretq_u8_s8(a),N)); }
1633 template<int N> EIGEN_STRONG_INLINE Packet4uc plogical_shift_right(Packet4uc& a)
1634 { return vget_lane_u32(vreinterpret_u32_s8(vshr_n_s8(vreinterpret_s8_u32(vdup_n_u32(a)), N)), 0); }
1635 template<int N> EIGEN_STRONG_INLINE Packet8uc plogical_shift_right(Packet8uc a) { return vshr_n_u8(a,N); }
1636 template<int N> EIGEN_STRONG_INLINE Packet16uc plogical_shift_right(Packet16uc a) { return vshrq_n_u8(a,N); }
1637 template<int N> EIGEN_STRONG_INLINE Packet4s plogical_shift_right(Packet4s a)
1638 { return vreinterpret_s16_u16(vshr_n_u16(vreinterpret_u16_s16(a),N)); }
1639 template<int N> EIGEN_STRONG_INLINE Packet8s plogical_shift_right(Packet8s a)
1640 { return vreinterpretq_s16_u16(vshrq_n_u16(vreinterpretq_u16_s16(a),N)); }
1641 template<int N> EIGEN_STRONG_INLINE Packet4us plogical_shift_right(Packet4us a) { return vshr_n_u16(a,N); }
1642 template<int N> EIGEN_STRONG_INLINE Packet8us plogical_shift_right(Packet8us a) { return vshrq_n_u16(a,N); }
1643 template<int N> EIGEN_STRONG_INLINE Packet2i plogical_shift_right(Packet2i a)
1644 { return vreinterpret_s32_u32(vshr_n_u32(vreinterpret_u32_s32(a),N)); }
1645 template<int N> EIGEN_STRONG_INLINE Packet4i plogical_shift_right(Packet4i a)
1646 { return vreinterpretq_s32_u32(vshrq_n_u32(vreinterpretq_u32_s32(a),N)); }
1647 template<int N> EIGEN_STRONG_INLINE Packet2ui plogical_shift_right(Packet2ui a) { return vshr_n_u32(a,N); }
1648 template<int N> EIGEN_STRONG_INLINE Packet4ui plogical_shift_right(Packet4ui a) { return vshrq_n_u32(a,N); }
1649 template<int N> EIGEN_STRONG_INLINE Packet2l plogical_shift_right(Packet2l a)
1650 { return vreinterpretq_s64_u64(vshrq_n_u64(vreinterpretq_u64_s64(a),N)); }
1651 template<int N> EIGEN_STRONG_INLINE Packet2ul plogical_shift_right(Packet2ul a) { return vshrq_n_u64(a,N); }
1652 
1653 template<int N> EIGEN_STRONG_INLINE Packet4c plogical_shift_left(Packet4c& a)
1654 { return vget_lane_s32(vreinterpret_s32_s8(vshl_n_s8(vreinterpret_s8_s32(vdup_n_s32(a)), N)), 0); }
1655 template<int N> EIGEN_STRONG_INLINE Packet8c plogical_shift_left(Packet8c a) { return vshl_n_s8(a,N); }
1656 template<int N> EIGEN_STRONG_INLINE Packet16c plogical_shift_left(Packet16c a) { return vshlq_n_s8(a,N); }
1657 template<int N> EIGEN_STRONG_INLINE Packet4uc plogical_shift_left(Packet4uc& a)
1658 { return vget_lane_u32(vreinterpret_u32_u8(vshl_n_u8(vreinterpret_u8_u32(vdup_n_u32(a)), N)), 0); }
1659 template<int N> EIGEN_STRONG_INLINE Packet8uc plogical_shift_left(Packet8uc a) { return vshl_n_u8(a,N); }
1660 template<int N> EIGEN_STRONG_INLINE Packet16uc plogical_shift_left(Packet16uc a) { return vshlq_n_u8(a,N); }
1661 template<int N> EIGEN_STRONG_INLINE Packet4s plogical_shift_left(Packet4s a) { return vshl_n_s16(a,N); }
1662 template<int N> EIGEN_STRONG_INLINE Packet8s plogical_shift_left(Packet8s a) { return vshlq_n_s16(a,N); }
1663 template<int N> EIGEN_STRONG_INLINE Packet4us plogical_shift_left(Packet4us a) { return vshl_n_u16(a,N); }
1664 template<int N> EIGEN_STRONG_INLINE Packet8us plogical_shift_left(Packet8us a) { return vshlq_n_u16(a,N); }
1665 template<int N> EIGEN_STRONG_INLINE Packet2i plogical_shift_left(Packet2i a) { return vshl_n_s32(a,N); }
1666 template<int N> EIGEN_STRONG_INLINE Packet4i plogical_shift_left(Packet4i a) { return vshlq_n_s32(a,N); }
1667 template<int N> EIGEN_STRONG_INLINE Packet2ui plogical_shift_left(Packet2ui a) { return vshl_n_u32(a,N); }
1668 template<int N> EIGEN_STRONG_INLINE Packet4ui plogical_shift_left(Packet4ui a) { return vshlq_n_u32(a,N); }
1669 template<int N> EIGEN_STRONG_INLINE Packet2l plogical_shift_left(Packet2l a) { return vshlq_n_s64(a,N); }
1670 template<int N> EIGEN_STRONG_INLINE Packet2ul plogical_shift_left(Packet2ul a) { return vshlq_n_u64(a,N); }
1671 
1672 template<> EIGEN_STRONG_INLINE Packet2f pload<Packet2f>(const float* from)
1673 { EIGEN_DEBUG_ALIGNED_LOAD return vld1_f32(from); }
1674 template<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(const float* from)
1675 { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_f32(from); }
1676 template<> EIGEN_STRONG_INLINE Packet4c pload<Packet4c>(const int8_t* from)
1677 {
1678  Packet4c res;
1679  memcpy(&res, from, sizeof(Packet4c));
1680  return res;
1681 }
1682 template<> EIGEN_STRONG_INLINE Packet8c pload<Packet8c>(const int8_t* from)
1683 { EIGEN_DEBUG_ALIGNED_LOAD return vld1_s8(from); }
1684 template<> EIGEN_STRONG_INLINE Packet16c pload<Packet16c>(const int8_t* from)
1685 { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_s8(from); }
1686 template<> EIGEN_STRONG_INLINE Packet4uc pload<Packet4uc>(const uint8_t* from)
1687 {
1688  Packet4uc res;
1689  memcpy(&res, from, sizeof(Packet4uc));
1690  return res;
1691 }
1692 template<> EIGEN_STRONG_INLINE Packet8uc pload<Packet8uc>(const uint8_t* from)
1693 { EIGEN_DEBUG_ALIGNED_LOAD return vld1_u8(from); }
1694 template<> EIGEN_STRONG_INLINE Packet16uc pload<Packet16uc>(const uint8_t* from)
1695 { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_u8(from); }
1696 template<> EIGEN_STRONG_INLINE Packet4s pload<Packet4s>(const int16_t* from)
1697 { EIGEN_DEBUG_ALIGNED_LOAD return vld1_s16(from); }
1698 template<> EIGEN_STRONG_INLINE Packet8s pload<Packet8s>(const int16_t* from)
1699 { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_s16(from); }
1700 template<> EIGEN_STRONG_INLINE Packet4us pload<Packet4us>(const uint16_t* from)
1701 { EIGEN_DEBUG_ALIGNED_LOAD return vld1_u16(from); }
1702 template<> EIGEN_STRONG_INLINE Packet8us pload<Packet8us>(const uint16_t* from)
1703 { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_u16(from); }
1704 template<> EIGEN_STRONG_INLINE Packet2i pload<Packet2i>(const int32_t* from)
1705 { EIGEN_DEBUG_ALIGNED_LOAD return vld1_s32(from); }
1706 template<> EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(const int32_t* from)
1707 { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_s32(from); }
1708 template<> EIGEN_STRONG_INLINE Packet2ui pload<Packet2ui>(const uint32_t* from)
1709 { EIGEN_DEBUG_ALIGNED_LOAD return vld1_u32(from); }
1710 template<> EIGEN_STRONG_INLINE Packet4ui pload<Packet4ui>(const uint32_t* from)
1711 { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_u32(from); }
1712 template<> EIGEN_STRONG_INLINE Packet2l pload<Packet2l>(const int64_t* from)
1713 { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_s64(from); }
1714 template<> EIGEN_STRONG_INLINE Packet2ul pload<Packet2ul>(const uint64_t* from)
1715 { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_u64(from); }
1716 
1717 template<> EIGEN_STRONG_INLINE Packet2f ploadu<Packet2f>(const float* from)
1718 { EIGEN_DEBUG_UNALIGNED_LOAD return vld1_f32(from); }
1719 template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(const float* from)
1720 { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_f32(from); }
1721 template<> EIGEN_STRONG_INLINE Packet4c ploadu<Packet4c>(const int8_t* from)
1722 {
1723  Packet4c res;
1724  memcpy(&res, from, sizeof(Packet4c));
1725  return res;
1726 }
1727 template<> EIGEN_STRONG_INLINE Packet8c ploadu<Packet8c>(const int8_t* from)
1728 { EIGEN_DEBUG_UNALIGNED_LOAD return vld1_s8(from); }
1729 template<> EIGEN_STRONG_INLINE Packet16c ploadu<Packet16c>(const int8_t* from)
1730 { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_s8(from); }
1731 template<> EIGEN_STRONG_INLINE Packet4uc ploadu<Packet4uc>(const uint8_t* from)
1732 {
1733  Packet4uc res;
1734  memcpy(&res, from, sizeof(Packet4uc));
1735  return res;
1736 }
1737 template<> EIGEN_STRONG_INLINE Packet8uc ploadu<Packet8uc>(const uint8_t* from)
1738 { EIGEN_DEBUG_UNALIGNED_LOAD return vld1_u8(from); }
1739 template<> EIGEN_STRONG_INLINE Packet16uc ploadu<Packet16uc>(const uint8_t* from)
1740 { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_u8(from); }
1741 template<> EIGEN_STRONG_INLINE Packet4s ploadu<Packet4s>(const int16_t* from)
1742 { EIGEN_DEBUG_UNALIGNED_LOAD return vld1_s16(from); }
1743 template<> EIGEN_STRONG_INLINE Packet8s ploadu<Packet8s>(const int16_t* from)
1744 { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_s16(from); }
1745 template<> EIGEN_STRONG_INLINE Packet4us ploadu<Packet4us>(const uint16_t* from)
1746 { EIGEN_DEBUG_UNALIGNED_LOAD return vld1_u16(from); }
1747 template<> EIGEN_STRONG_INLINE Packet8us ploadu<Packet8us>(const uint16_t* from)
1748 { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_u16(from); }
1749 template<> EIGEN_STRONG_INLINE Packet2i ploadu<Packet2i>(const int32_t* from)
1750 { EIGEN_DEBUG_UNALIGNED_LOAD return vld1_s32(from); }
1751 template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(const int32_t* from)
1752 { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_s32(from); }
1753 template<> EIGEN_STRONG_INLINE Packet2ui ploadu<Packet2ui>(const uint32_t* from)
1754 { EIGEN_DEBUG_UNALIGNED_LOAD return vld1_u32(from); }
1755 template<> EIGEN_STRONG_INLINE Packet4ui ploadu<Packet4ui>(const uint32_t* from)
1756 { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_u32(from); }
1757 template<> EIGEN_STRONG_INLINE Packet2l ploadu<Packet2l>(const int64_t* from)
1758 { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_s64(from); }
1759 template<> EIGEN_STRONG_INLINE Packet2ul ploadu<Packet2ul>(const uint64_t* from)
1760 { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_u64(from); }
1761 
1762 template<> EIGEN_STRONG_INLINE Packet2f ploaddup<Packet2f>(const float* from)
1763 { return vld1_dup_f32(from); }
1764 template<> EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(const float* from)
1765 { return vcombine_f32(vld1_dup_f32(from), vld1_dup_f32(from+1)); }
1766 template<> EIGEN_STRONG_INLINE Packet4c ploaddup<Packet4c>(const int8_t* from)
1767 {
1768  const int8x8_t a = vreinterpret_s8_s32(vdup_n_s32(pload<Packet4c>(from)));
1769  return vget_lane_s32(vreinterpret_s32_s8(vzip_s8(a,a).val[0]), 0);
1770 }
1771 template<> EIGEN_STRONG_INLINE Packet8c ploaddup<Packet8c>(const int8_t* from)
1772 {
1773  const int8x8_t a = vld1_s8(from);
1774  return vzip_s8(a,a).val[0];
1775 }
1776 template<> EIGEN_STRONG_INLINE Packet16c ploaddup<Packet16c>(const int8_t* from)
1777 {
1778  const int8x8_t a = vld1_s8(from);
1779  const int8x8x2_t b = vzip_s8(a,a);
1780  return vcombine_s8(b.val[0], b.val[1]);
1781 }
1782 template<> EIGEN_STRONG_INLINE Packet4uc ploaddup<Packet4uc>(const uint8_t* from)
1783 {
1784  const uint8x8_t a = vreinterpret_u8_u32(vdup_n_u32(pload<Packet4uc>(from)));
1785  return vget_lane_u32(vreinterpret_u32_u8(vzip_u8(a,a).val[0]), 0);
1786 }
1787 template<> EIGEN_STRONG_INLINE Packet8uc ploaddup<Packet8uc>(const uint8_t* from)
1788 {
1789  const uint8x8_t a = vld1_u8(from);
1790  return vzip_u8(a,a).val[0];
1791 }
1792 template<> EIGEN_STRONG_INLINE Packet16uc ploaddup<Packet16uc>(const uint8_t* from)
1793 {
1794  const uint8x8_t a = vld1_u8(from);
1795  const uint8x8x2_t b = vzip_u8(a,a);
1796  return vcombine_u8(b.val[0], b.val[1]);
1797 }
1798 template<> EIGEN_STRONG_INLINE Packet4s ploaddup<Packet4s>(const int16_t* from)
1799 {
1800  return vreinterpret_s16_u32(vzip_u32(vreinterpret_u32_s16(vld1_dup_s16(from)),
1801  vreinterpret_u32_s16(vld1_dup_s16(from+1))).val[0]);
1802 }
1803 template<> EIGEN_STRONG_INLINE Packet8s ploaddup<Packet8s>(const int16_t* from)
1804 {
1805  const int16x4_t a = vld1_s16(from);
1806  const int16x4x2_t b = vzip_s16(a,a);
1807  return vcombine_s16(b.val[0], b.val[1]);
1808 }
1809 template<> EIGEN_STRONG_INLINE Packet4us ploaddup<Packet4us>(const uint16_t* from)
1810 {
1811  return vreinterpret_u16_u32(vzip_u32(vreinterpret_u32_u16(vld1_dup_u16(from)),
1812  vreinterpret_u32_u16(vld1_dup_u16(from+1))).val[0]);
1813 }
1814 template<> EIGEN_STRONG_INLINE Packet8us ploaddup<Packet8us>(const uint16_t* from)
1815 {
1816  const uint16x4_t a = vld1_u16(from);
1817  const uint16x4x2_t b = vzip_u16(a,a);
1818  return vcombine_u16(b.val[0], b.val[1]);
1819 }
1820 template<> EIGEN_STRONG_INLINE Packet2i ploaddup<Packet2i>(const int32_t* from)
1821 { return vld1_dup_s32(from); }
1822 template<> EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(const int32_t* from)
1823 { return vcombine_s32(vld1_dup_s32(from), vld1_dup_s32(from+1)); }
1824 template<> EIGEN_STRONG_INLINE Packet2ui ploaddup<Packet2ui>(const uint32_t* from)
1825 { return vld1_dup_u32(from); }
1826 template<> EIGEN_STRONG_INLINE Packet4ui ploaddup<Packet4ui>(const uint32_t* from)
1827 { return vcombine_u32(vld1_dup_u32(from), vld1_dup_u32(from+1)); }
1828 template<> EIGEN_STRONG_INLINE Packet2l ploaddup<Packet2l>(const int64_t* from)
1829 { return vld1q_dup_s64(from); }
1830 template<> EIGEN_STRONG_INLINE Packet2ul ploaddup<Packet2ul>(const uint64_t* from)
1831 { return vld1q_dup_u64(from); }
1832 
1833 template<> EIGEN_STRONG_INLINE Packet4f ploadquad<Packet4f>(const float* from) { return vld1q_dup_f32(from); }
1834 template<> EIGEN_STRONG_INLINE Packet4c ploadquad<Packet4c>(const int8_t* from)
1835 { return vget_lane_s32(vreinterpret_s32_s8(vld1_dup_s8(from)), 0); }
1836 template<> EIGEN_STRONG_INLINE Packet8c ploadquad<Packet8c>(const int8_t* from)
1837 {
1838  return vreinterpret_s8_u32(vzip_u32(
1839  vreinterpret_u32_s8(vld1_dup_s8(from)),
1840  vreinterpret_u32_s8(vld1_dup_s8(from+1))).val[0]);
1841 }
1842 template<> EIGEN_STRONG_INLINE Packet16c ploadquad<Packet16c>(const int8_t* from)
1843 {
1844  const int8x8_t a = vreinterpret_s8_u32(vzip_u32(
1845  vreinterpret_u32_s8(vld1_dup_s8(from)),
1846  vreinterpret_u32_s8(vld1_dup_s8(from+1))).val[0]);
1847  const int8x8_t b = vreinterpret_s8_u32(vzip_u32(
1848  vreinterpret_u32_s8(vld1_dup_s8(from+2)),
1849  vreinterpret_u32_s8(vld1_dup_s8(from+3))).val[0]);
1850  return vcombine_s8(a,b);
1851 }
1852 template<> EIGEN_STRONG_INLINE Packet4uc ploadquad<Packet4uc>(const uint8_t* from)
1853 { return vget_lane_u32(vreinterpret_u32_u8(vld1_dup_u8(from)), 0); }
1854 template<> EIGEN_STRONG_INLINE Packet8uc ploadquad<Packet8uc>(const uint8_t* from)
1855 {
1856  return vreinterpret_u8_u32(vzip_u32(
1857  vreinterpret_u32_u8(vld1_dup_u8(from)),
1858  vreinterpret_u32_u8(vld1_dup_u8(from+1))).val[0]);
1859 }
1860 template<> EIGEN_STRONG_INLINE Packet16uc ploadquad<Packet16uc>(const uint8_t* from)
1861 {
1862  const uint8x8_t a = vreinterpret_u8_u32(vzip_u32(
1863  vreinterpret_u32_u8(vld1_dup_u8(from)),
1864  vreinterpret_u32_u8(vld1_dup_u8(from+1))).val[0]);
1865  const uint8x8_t b = vreinterpret_u8_u32(vzip_u32(
1866  vreinterpret_u32_u8(vld1_dup_u8(from+2)),
1867  vreinterpret_u32_u8(vld1_dup_u8(from+3))).val[0]);
1868  return vcombine_u8(a,b);
1869 }
1870 template<> EIGEN_STRONG_INLINE Packet8s ploadquad<Packet8s>(const int16_t* from)
1871 { return vcombine_s16(vld1_dup_s16(from), vld1_dup_s16(from+1)); }
1872 template<> EIGEN_STRONG_INLINE Packet8us ploadquad<Packet8us>(const uint16_t* from)
1873 { return vcombine_u16(vld1_dup_u16(from), vld1_dup_u16(from+1)); }
1874 template<> EIGEN_STRONG_INLINE Packet4i ploadquad<Packet4i>(const int32_t* from) { return vld1q_dup_s32(from); }
1875 template<> EIGEN_STRONG_INLINE Packet4ui ploadquad<Packet4ui>(const uint32_t* from) { return vld1q_dup_u32(from); }
1876 
1877 template<> EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet2f& from)
1878 { EIGEN_DEBUG_ALIGNED_STORE vst1_f32(to,from); }
1879 template<> EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet4f& from)
1880 { EIGEN_DEBUG_ALIGNED_STORE vst1q_f32(to,from); }
1881 template<> EIGEN_STRONG_INLINE void pstore<int8_t>(int8_t* to, const Packet4c& from)
1882 { memcpy(to, &from, sizeof(from)); }
1883 template<> EIGEN_STRONG_INLINE void pstore<int8_t>(int8_t* to, const Packet8c& from)
1884 { EIGEN_DEBUG_ALIGNED_STORE vst1_s8(to,from); }
1885 template<> EIGEN_STRONG_INLINE void pstore<int8_t>(int8_t* to, const Packet16c& from)
1886 { EIGEN_DEBUG_ALIGNED_STORE vst1q_s8(to,from); }
1887 template<> EIGEN_STRONG_INLINE void pstore<uint8_t>(uint8_t* to, const Packet4uc& from)
1888 { memcpy(to, &from, sizeof(from)); }
1889 template<> EIGEN_STRONG_INLINE void pstore<uint8_t>(uint8_t* to, const Packet8uc& from)
1890 { EIGEN_DEBUG_ALIGNED_STORE vst1_u8(to,from); }
1891 template<> EIGEN_STRONG_INLINE void pstore<uint8_t>(uint8_t* to, const Packet16uc& from)
1892 { EIGEN_DEBUG_ALIGNED_STORE vst1q_u8(to,from); }
1893 template<> EIGEN_STRONG_INLINE void pstore<int16_t>(int16_t* to, const Packet4s& from)
1894 { EIGEN_DEBUG_ALIGNED_STORE vst1_s16(to,from); }
1895 template<> EIGEN_STRONG_INLINE void pstore<int16_t>(int16_t* to, const Packet8s& from)
1896 { EIGEN_DEBUG_ALIGNED_STORE vst1q_s16(to,from); }
1897 template<> EIGEN_STRONG_INLINE void pstore<uint16_t>(uint16_t* to, const Packet4us& from)
1898 { EIGEN_DEBUG_ALIGNED_STORE vst1_u16(to,from); }
1899 template<> EIGEN_STRONG_INLINE void pstore<uint16_t>(uint16_t* to, const Packet8us& from)
1900 { EIGEN_DEBUG_ALIGNED_STORE vst1q_u16(to,from); }
1901 template<> EIGEN_STRONG_INLINE void pstore<int32_t>(int32_t* to, const Packet2i& from)
1902 { EIGEN_DEBUG_ALIGNED_STORE vst1_s32(to,from); }
1903 template<> EIGEN_STRONG_INLINE void pstore<int32_t>(int32_t* to, const Packet4i& from)
1904 { EIGEN_DEBUG_ALIGNED_STORE vst1q_s32(to,from); }
1905 template<> EIGEN_STRONG_INLINE void pstore<uint32_t>(uint32_t* to, const Packet2ui& from)
1906 { EIGEN_DEBUG_ALIGNED_STORE vst1_u32(to,from); }
1907 template<> EIGEN_STRONG_INLINE void pstore<uint32_t>(uint32_t* to, const Packet4ui& from)
1908 { EIGEN_DEBUG_ALIGNED_STORE vst1q_u32(to,from); }
1909 template<> EIGEN_STRONG_INLINE void pstore<int64_t>(int64_t* to, const Packet2l& from)
1910 { EIGEN_DEBUG_ALIGNED_STORE vst1q_s64(to,from); }
1911 template<> EIGEN_STRONG_INLINE void pstore<uint64_t>(uint64_t* to, const Packet2ul& from)
1912 { EIGEN_DEBUG_ALIGNED_STORE vst1q_u64(to,from); }
1913 
1914 template<> EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet2f& from)
1915 { EIGEN_DEBUG_UNALIGNED_STORE vst1_f32(to,from); }
1916 template<> EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet4f& from)
1917 { EIGEN_DEBUG_UNALIGNED_STORE vst1q_f32(to,from); }
1918 template<> EIGEN_STRONG_INLINE void pstoreu<int8_t>(int8_t* to, const Packet4c& from)
1919 { memcpy(to, &from, sizeof(from)); }
1920 template<> EIGEN_STRONG_INLINE void pstoreu<int8_t>(int8_t* to, const Packet8c& from)
1921 { EIGEN_DEBUG_UNALIGNED_STORE vst1_s8(to,from); }
1922 template<> EIGEN_STRONG_INLINE void pstoreu<int8_t>(int8_t* to, const Packet16c& from)
1923 { EIGEN_DEBUG_UNALIGNED_STORE vst1q_s8(to,from); }
1924 template<> EIGEN_STRONG_INLINE void pstoreu<uint8_t>(uint8_t* to, const Packet4uc& from)
1925 { memcpy(to, &from, sizeof(from)); }
1926 template<> EIGEN_STRONG_INLINE void pstoreu<uint8_t>(uint8_t* to, const Packet8uc& from)
1927 { EIGEN_DEBUG_UNALIGNED_STORE vst1_u8(to,from); }
1928 template<> EIGEN_STRONG_INLINE void pstoreu<uint8_t>(uint8_t* to, const Packet16uc& from)
1929 { EIGEN_DEBUG_UNALIGNED_STORE vst1q_u8(to,from); }
1930 template<> EIGEN_STRONG_INLINE void pstoreu<int16_t>(int16_t* to, const Packet4s& from)
1931 { EIGEN_DEBUG_UNALIGNED_STORE vst1_s16(to,from); }
1932 template<> EIGEN_STRONG_INLINE void pstoreu<int16_t>(int16_t* to, const Packet8s& from)
1933 { EIGEN_DEBUG_UNALIGNED_STORE vst1q_s16(to,from); }
1934 template<> EIGEN_STRONG_INLINE void pstoreu<uint16_t>(uint16_t* to, const Packet4us& from)
1935 { EIGEN_DEBUG_UNALIGNED_STORE vst1_u16(to,from); }
1936 template<> EIGEN_STRONG_INLINE void pstoreu<uint16_t>(uint16_t* to, const Packet8us& from)
1937 { EIGEN_DEBUG_UNALIGNED_STORE vst1q_u16(to,from); }
1938 template<> EIGEN_STRONG_INLINE void pstoreu<int32_t>(int32_t* to, const Packet2i& from)
1939 { EIGEN_DEBUG_UNALIGNED_STORE vst1_s32(to,from); }
1940 template<> EIGEN_STRONG_INLINE void pstoreu<int32_t>(int32_t* to, const Packet4i& from)
1941 { EIGEN_DEBUG_UNALIGNED_STORE vst1q_s32(to,from); }
1942 template<> EIGEN_STRONG_INLINE void pstoreu<uint32_t>(uint32_t* to, const Packet2ui& from)
1943 { EIGEN_DEBUG_UNALIGNED_STORE vst1_u32(to,from); }
1944 template<> EIGEN_STRONG_INLINE void pstoreu<uint32_t>(uint32_t* to, const Packet4ui& from)
1945 { EIGEN_DEBUG_UNALIGNED_STORE vst1q_u32(to,from); }
1946 template<> EIGEN_STRONG_INLINE void pstoreu<int64_t>(int64_t* to, const Packet2l& from)
1947 { EIGEN_DEBUG_UNALIGNED_STORE vst1q_s64(to,from); }
1948 template<> EIGEN_STRONG_INLINE void pstoreu<uint64_t>(uint64_t* to, const Packet2ul& from)
1949 { EIGEN_DEBUG_UNALIGNED_STORE vst1q_u64(to,from); }
1950 
1951 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2f pgather<float, Packet2f>(const float* from, Index stride)
1952 {
1953  Packet2f res = vld1_dup_f32(from);
1954  res = vld1_lane_f32(from + 1*stride, res, 1);
1955  return res;
1956 }
1957 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4f pgather<float, Packet4f>(const float* from, Index stride)
1958 {
1959  Packet4f res = vld1q_dup_f32(from);
1960  res = vld1q_lane_f32(from + 1*stride, res, 1);
1961  res = vld1q_lane_f32(from + 2*stride, res, 2);
1962  res = vld1q_lane_f32(from + 3*stride, res, 3);
1963  return res;
1964 }
1965 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4c pgather<int8_t, Packet4c>(const int8_t* from, Index stride)
1966 {
1967  Packet4c res;
1968  for (int i = 0; i != 4; i++)
1969  reinterpret_cast<int8_t*>(&res)[i] = *(from + i * stride);
1970  return res;
1971 }
1972 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8c pgather<int8_t, Packet8c>(const int8_t* from, Index stride)
1973 {
1974  Packet8c res = vld1_dup_s8(from);
1975  res = vld1_lane_s8(from + 1*stride, res, 1);
1976  res = vld1_lane_s8(from + 2*stride, res, 2);
1977  res = vld1_lane_s8(from + 3*stride, res, 3);
1978  res = vld1_lane_s8(from + 4*stride, res, 4);
1979  res = vld1_lane_s8(from + 5*stride, res, 5);
1980  res = vld1_lane_s8(from + 6*stride, res, 6);
1981  res = vld1_lane_s8(from + 7*stride, res, 7);
1982  return res;
1983 }
1984 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16c pgather<int8_t, Packet16c>(const int8_t* from, Index stride)
1985 {
1986  Packet16c res = vld1q_dup_s8(from);
1987  res = vld1q_lane_s8(from + 1*stride, res, 1);
1988  res = vld1q_lane_s8(from + 2*stride, res, 2);
1989  res = vld1q_lane_s8(from + 3*stride, res, 3);
1990  res = vld1q_lane_s8(from + 4*stride, res, 4);
1991  res = vld1q_lane_s8(from + 5*stride, res, 5);
1992  res = vld1q_lane_s8(from + 6*stride, res, 6);
1993  res = vld1q_lane_s8(from + 7*stride, res, 7);
1994  res = vld1q_lane_s8(from + 8*stride, res, 8);
1995  res = vld1q_lane_s8(from + 9*stride, res, 9);
1996  res = vld1q_lane_s8(from + 10*stride, res, 10);
1997  res = vld1q_lane_s8(from + 11*stride, res, 11);
1998  res = vld1q_lane_s8(from + 12*stride, res, 12);
1999  res = vld1q_lane_s8(from + 13*stride, res, 13);
2000  res = vld1q_lane_s8(from + 14*stride, res, 14);
2001  res = vld1q_lane_s8(from + 15*stride, res, 15);
2002  return res;
2003 }
2004 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4uc pgather<uint8_t, Packet4uc>(const uint8_t* from, Index stride)
2005 {
2006  Packet4uc res;
2007  for (int i = 0; i != 4; i++)
2008  reinterpret_cast<uint8_t*>(&res)[i] = *(from + i * stride);
2009  return res;
2010 }
2011 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8uc pgather<uint8_t, Packet8uc>(const uint8_t* from, Index stride)
2012 {
2013  Packet8uc res = vld1_dup_u8(from);
2014  res = vld1_lane_u8(from + 1*stride, res, 1);
2015  res = vld1_lane_u8(from + 2*stride, res, 2);
2016  res = vld1_lane_u8(from + 3*stride, res, 3);
2017  res = vld1_lane_u8(from + 4*stride, res, 4);
2018  res = vld1_lane_u8(from + 5*stride, res, 5);
2019  res = vld1_lane_u8(from + 6*stride, res, 6);
2020  res = vld1_lane_u8(from + 7*stride, res, 7);
2021  return res;
2022 }
2023 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc pgather<uint8_t, Packet16uc>(const uint8_t* from, Index stride)
2024 {
2025  Packet16uc res = vld1q_dup_u8(from);
2026  res = vld1q_lane_u8(from + 1*stride, res, 1);
2027  res = vld1q_lane_u8(from + 2*stride, res, 2);
2028  res = vld1q_lane_u8(from + 3*stride, res, 3);
2029  res = vld1q_lane_u8(from + 4*stride, res, 4);
2030  res = vld1q_lane_u8(from + 5*stride, res, 5);
2031  res = vld1q_lane_u8(from + 6*stride, res, 6);
2032  res = vld1q_lane_u8(from + 7*stride, res, 7);
2033  res = vld1q_lane_u8(from + 8*stride, res, 8);
2034  res = vld1q_lane_u8(from + 9*stride, res, 9);
2035  res = vld1q_lane_u8(from + 10*stride, res, 10);
2036  res = vld1q_lane_u8(from + 11*stride, res, 11);
2037  res = vld1q_lane_u8(from + 12*stride, res, 12);
2038  res = vld1q_lane_u8(from + 13*stride, res, 13);
2039  res = vld1q_lane_u8(from + 14*stride, res, 14);
2040  res = vld1q_lane_u8(from + 15*stride, res, 15);
2041  return res;
2042 }
2043 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4s pgather<int16_t, Packet4s>(const int16_t* from, Index stride)
2044 {
2045  Packet4s res = vld1_dup_s16(from);
2046  res = vld1_lane_s16(from + 1*stride, res, 1);
2047  res = vld1_lane_s16(from + 2*stride, res, 2);
2048  res = vld1_lane_s16(from + 3*stride, res, 3);
2049  return res;
2050 }
2051 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s pgather<int16_t, Packet8s>(const int16_t* from, Index stride)
2052 {
2053  Packet8s res = vld1q_dup_s16(from);
2054  res = vld1q_lane_s16(from + 1*stride, res, 1);
2055  res = vld1q_lane_s16(from + 2*stride, res, 2);
2056  res = vld1q_lane_s16(from + 3*stride, res, 3);
2057  res = vld1q_lane_s16(from + 4*stride, res, 4);
2058  res = vld1q_lane_s16(from + 5*stride, res, 5);
2059  res = vld1q_lane_s16(from + 6*stride, res, 6);
2060  res = vld1q_lane_s16(from + 7*stride, res, 7);
2061  return res;
2062 }
2063 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us pgather<uint16_t, Packet4us>(const uint16_t* from, Index stride)
2064 {
2065  Packet4us res = vld1_dup_u16(from);
2066  res = vld1_lane_u16(from + 1*stride, res, 1);
2067  res = vld1_lane_u16(from + 2*stride, res, 2);
2068  res = vld1_lane_u16(from + 3*stride, res, 3);
2069  return res;
2070 }
2071 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us pgather<uint16_t, Packet8us>(const uint16_t* from, Index stride)
2072 {
2073  Packet8us res = vld1q_dup_u16(from);
2074  res = vld1q_lane_u16(from + 1*stride, res, 1);
2075  res = vld1q_lane_u16(from + 2*stride, res, 2);
2076  res = vld1q_lane_u16(from + 3*stride, res, 3);
2077  res = vld1q_lane_u16(from + 4*stride, res, 4);
2078  res = vld1q_lane_u16(from + 5*stride, res, 5);
2079  res = vld1q_lane_u16(from + 6*stride, res, 6);
2080  res = vld1q_lane_u16(from + 7*stride, res, 7);
2081  return res;
2082 }
2083 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2i pgather<int32_t, Packet2i>(const int32_t* from, Index stride)
2084 {
2085  Packet2i res = vld1_dup_s32(from);
2086  res = vld1_lane_s32(from + 1*stride, res, 1);
2087  return res;
2088 }
2089 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4i pgather<int32_t, Packet4i>(const int32_t* from, Index stride)
2090 {
2091  Packet4i res = vld1q_dup_s32(from);
2092  res = vld1q_lane_s32(from + 1*stride, res, 1);
2093  res = vld1q_lane_s32(from + 2*stride, res, 2);
2094  res = vld1q_lane_s32(from + 3*stride, res, 3);
2095  return res;
2096 }
2097 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ui pgather<uint32_t, Packet2ui>(const uint32_t* from, Index stride)
2098 {
2099  Packet2ui res = vld1_dup_u32(from);
2100  res = vld1_lane_u32(from + 1*stride, res, 1);
2101  return res;
2102 }
2103 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4ui pgather<uint32_t, Packet4ui>(const uint32_t* from, Index stride)
2104 {
2105  Packet4ui res = vld1q_dup_u32(from);
2106  res = vld1q_lane_u32(from + 1*stride, res, 1);
2107  res = vld1q_lane_u32(from + 2*stride, res, 2);
2108  res = vld1q_lane_u32(from + 3*stride, res, 3);
2109  return res;
2110 }
2111 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2l pgather<int64_t, Packet2l>(const int64_t* from, Index stride)
2112 {
2113  Packet2l res = vld1q_dup_s64(from);
2114  res = vld1q_lane_s64(from + 1*stride, res, 1);
2115  return res;
2116 }
2117 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ul pgather<uint64_t, Packet2ul>(const uint64_t* from, Index stride)
2118 {
2119  Packet2ul res = vld1q_dup_u64(from);
2120  res = vld1q_lane_u64(from + 1*stride, res, 1);
2121  return res;
2122 }
2123 
2124 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<float, Packet2f>(float* to, const Packet2f& from, Index stride)
2125 {
2126  vst1_lane_f32(to + stride*0, from, 0);
2127  vst1_lane_f32(to + stride*1, from, 1);
2128 }
2129 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<float, Packet4f>(float* to, const Packet4f& from, Index stride)
2130 {
2131  vst1q_lane_f32(to + stride*0, from, 0);
2132  vst1q_lane_f32(to + stride*1, from, 1);
2133  vst1q_lane_f32(to + stride*2, from, 2);
2134  vst1q_lane_f32(to + stride*3, from, 3);
2135 }
2136 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int8_t, Packet4c>(int8_t* to, const Packet4c& from, Index stride)
2137 {
2138  for (int i = 0; i != 4; i++)
2139  *(to + i * stride) = reinterpret_cast<const int8_t*>(&from)[i];
2140 }
2141 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int8_t, Packet8c>(int8_t* to, const Packet8c& from, Index stride)
2142 {
2143  vst1_lane_s8(to + stride*0, from, 0);
2144  vst1_lane_s8(to + stride*1, from, 1);
2145  vst1_lane_s8(to + stride*2, from, 2);
2146  vst1_lane_s8(to + stride*3, from, 3);
2147  vst1_lane_s8(to + stride*4, from, 4);
2148  vst1_lane_s8(to + stride*5, from, 5);
2149  vst1_lane_s8(to + stride*6, from, 6);
2150  vst1_lane_s8(to + stride*7, from, 7);
2151 }
2152 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int8_t, Packet16c>(int8_t* to, const Packet16c& from, Index stride)
2153 {
2154  vst1q_lane_s8(to + stride*0, from, 0);
2155  vst1q_lane_s8(to + stride*1, from, 1);
2156  vst1q_lane_s8(to + stride*2, from, 2);
2157  vst1q_lane_s8(to + stride*3, from, 3);
2158  vst1q_lane_s8(to + stride*4, from, 4);
2159  vst1q_lane_s8(to + stride*5, from, 5);
2160  vst1q_lane_s8(to + stride*6, from, 6);
2161  vst1q_lane_s8(to + stride*7, from, 7);
2162  vst1q_lane_s8(to + stride*8, from, 8);
2163  vst1q_lane_s8(to + stride*9, from, 9);
2164  vst1q_lane_s8(to + stride*10, from, 10);
2165  vst1q_lane_s8(to + stride*11, from, 11);
2166  vst1q_lane_s8(to + stride*12, from, 12);
2167  vst1q_lane_s8(to + stride*13, from, 13);
2168  vst1q_lane_s8(to + stride*14, from, 14);
2169  vst1q_lane_s8(to + stride*15, from, 15);
2170 }
2171 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint8_t, Packet4uc>(uint8_t* to, const Packet4uc& from, Index stride)
2172 {
2173  for (int i = 0; i != 4; i++)
2174  *(to + i * stride) = reinterpret_cast<const uint8_t*>(&from)[i];
2175 }
2176 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint8_t, Packet8uc>(uint8_t* to, const Packet8uc& from, Index stride)
2177 {
2178  vst1_lane_u8(to + stride*0, from, 0);
2179  vst1_lane_u8(to + stride*1, from, 1);
2180  vst1_lane_u8(to + stride*2, from, 2);
2181  vst1_lane_u8(to + stride*3, from, 3);
2182  vst1_lane_u8(to + stride*4, from, 4);
2183  vst1_lane_u8(to + stride*5, from, 5);
2184  vst1_lane_u8(to + stride*6, from, 6);
2185  vst1_lane_u8(to + stride*7, from, 7);
2186 }
2187 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint8_t, Packet16uc>(uint8_t* to, const Packet16uc& from, Index stride)
2188 {
2189  vst1q_lane_u8(to + stride*0, from, 0);
2190  vst1q_lane_u8(to + stride*1, from, 1);
2191  vst1q_lane_u8(to + stride*2, from, 2);
2192  vst1q_lane_u8(to + stride*3, from, 3);
2193  vst1q_lane_u8(to + stride*4, from, 4);
2194  vst1q_lane_u8(to + stride*5, from, 5);
2195  vst1q_lane_u8(to + stride*6, from, 6);
2196  vst1q_lane_u8(to + stride*7, from, 7);
2197  vst1q_lane_u8(to + stride*8, from, 8);
2198  vst1q_lane_u8(to + stride*9, from, 9);
2199  vst1q_lane_u8(to + stride*10, from, 10);
2200  vst1q_lane_u8(to + stride*11, from, 11);
2201  vst1q_lane_u8(to + stride*12, from, 12);
2202  vst1q_lane_u8(to + stride*13, from, 13);
2203  vst1q_lane_u8(to + stride*14, from, 14);
2204  vst1q_lane_u8(to + stride*15, from, 15);
2205 }
2206 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int16_t, Packet4s>(int16_t* to, const Packet4s& from, Index stride)
2207 {
2208  vst1_lane_s16(to + stride*0, from, 0);
2209  vst1_lane_s16(to + stride*1, from, 1);
2210  vst1_lane_s16(to + stride*2, from, 2);
2211  vst1_lane_s16(to + stride*3, from, 3);
2212 }
2213 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int16_t, Packet8s>(int16_t* to, const Packet8s& from, Index stride)
2214 {
2215  vst1q_lane_s16(to + stride*0, from, 0);
2216  vst1q_lane_s16(to + stride*1, from, 1);
2217  vst1q_lane_s16(to + stride*2, from, 2);
2218  vst1q_lane_s16(to + stride*3, from, 3);
2219  vst1q_lane_s16(to + stride*4, from, 4);
2220  vst1q_lane_s16(to + stride*5, from, 5);
2221  vst1q_lane_s16(to + stride*6, from, 6);
2222  vst1q_lane_s16(to + stride*7, from, 7);
2223 }
2224 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint16_t, Packet4us>(uint16_t* to, const Packet4us& from, Index stride)
2225 {
2226  vst1_lane_u16(to + stride*0, from, 0);
2227  vst1_lane_u16(to + stride*1, from, 1);
2228  vst1_lane_u16(to + stride*2, from, 2);
2229  vst1_lane_u16(to + stride*3, from, 3);
2230 }
2231 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint16_t, Packet8us>(uint16_t* to, const Packet8us& from, Index stride)
2232 {
2233  vst1q_lane_u16(to + stride*0, from, 0);
2234  vst1q_lane_u16(to + stride*1, from, 1);
2235  vst1q_lane_u16(to + stride*2, from, 2);
2236  vst1q_lane_u16(to + stride*3, from, 3);
2237  vst1q_lane_u16(to + stride*4, from, 4);
2238  vst1q_lane_u16(to + stride*5, from, 5);
2239  vst1q_lane_u16(to + stride*6, from, 6);
2240  vst1q_lane_u16(to + stride*7, from, 7);
2241 }
2242 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int32_t, Packet2i>(int32_t* to, const Packet2i& from, Index stride)
2243 {
2244  vst1_lane_s32(to + stride*0, from, 0);
2245  vst1_lane_s32(to + stride*1, from, 1);
2246 }
2247 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int32_t, Packet4i>(int32_t* to, const Packet4i& from, Index stride)
2248 {
2249  vst1q_lane_s32(to + stride*0, from, 0);
2250  vst1q_lane_s32(to + stride*1, from, 1);
2251  vst1q_lane_s32(to + stride*2, from, 2);
2252  vst1q_lane_s32(to + stride*3, from, 3);
2253 }
2254 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint32_t, Packet2ui>(uint32_t* to, const Packet2ui& from, Index stride)
2255 {
2256  vst1_lane_u32(to + stride*0, from, 0);
2257  vst1_lane_u32(to + stride*1, from, 1);
2258 }
2259 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint32_t, Packet4ui>(uint32_t* to, const Packet4ui& from, Index stride)
2260 {
2261  vst1q_lane_u32(to + stride*0, from, 0);
2262  vst1q_lane_u32(to + stride*1, from, 1);
2263  vst1q_lane_u32(to + stride*2, from, 2);
2264  vst1q_lane_u32(to + stride*3, from, 3);
2265 }
2266 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<int64_t, Packet2l>(int64_t* to, const Packet2l& from, Index stride)
2267 {
2268  vst1q_lane_s64(to + stride*0, from, 0);
2269  vst1q_lane_s64(to + stride*1, from, 1);
2270 }
2271 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<uint64_t, Packet2ul>(uint64_t* to, const Packet2ul& from, Index stride)
2272 {
2273  vst1q_lane_u64(to + stride*0, from, 0);
2274  vst1q_lane_u64(to + stride*1, from, 1);
2275 }
2276 
2277 template<> EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { EIGEN_ARM_PREFETCH(addr); }
2278 template<> EIGEN_STRONG_INLINE void prefetch<int8_t>(const int8_t* addr) { EIGEN_ARM_PREFETCH(addr); }
2279 template<> EIGEN_STRONG_INLINE void prefetch<uint8_t>(const uint8_t* addr) { EIGEN_ARM_PREFETCH(addr); }
2280 template<> EIGEN_STRONG_INLINE void prefetch<int16_t>(const int16_t* addr) { EIGEN_ARM_PREFETCH(addr); }
2281 template<> EIGEN_STRONG_INLINE void prefetch<uint16_t>(const uint16_t* addr) { EIGEN_ARM_PREFETCH(addr); }
2282 template<> EIGEN_STRONG_INLINE void prefetch<int32_t>(const int32_t* addr) { EIGEN_ARM_PREFETCH(addr); }
2283 template<> EIGEN_STRONG_INLINE void prefetch<uint32_t>(const uint32_t* addr) { EIGEN_ARM_PREFETCH(addr); }
2284 template<> EIGEN_STRONG_INLINE void prefetch<int64_t>(const int64_t* addr) { EIGEN_ARM_PREFETCH(addr); }
2285 template<> EIGEN_STRONG_INLINE void prefetch<uint64_t>(const uint64_t* addr) { EIGEN_ARM_PREFETCH(addr); }
2286 
2287 template<> EIGEN_STRONG_INLINE float pfirst<Packet2f>(const Packet2f& a) { return vget_lane_f32(a,0); }
2288 template<> EIGEN_STRONG_INLINE float pfirst<Packet4f>(const Packet4f& a) { return vgetq_lane_f32(a,0); }
2289 template<> EIGEN_STRONG_INLINE int8_t pfirst<Packet4c>(const Packet4c& a) { return static_cast<int8_t>(a & 0xff); }
2290 template<> EIGEN_STRONG_INLINE int8_t pfirst<Packet8c>(const Packet8c& a) { return vget_lane_s8(a,0); }
2291 template<> EIGEN_STRONG_INLINE int8_t pfirst<Packet16c>(const Packet16c& a) { return vgetq_lane_s8(a,0); }
2292 template<> EIGEN_STRONG_INLINE uint8_t pfirst<Packet4uc>(const Packet4uc& a) { return static_cast<uint8_t>(a & 0xff); }
2293 template<> EIGEN_STRONG_INLINE uint8_t pfirst<Packet8uc>(const Packet8uc& a) { return vget_lane_u8(a,0); }
2294 template<> EIGEN_STRONG_INLINE uint8_t pfirst<Packet16uc>(const Packet16uc& a) { return vgetq_lane_u8(a,0); }
2295 template<> EIGEN_STRONG_INLINE int16_t pfirst<Packet4s>(const Packet4s& a) { return vget_lane_s16(a,0); }
2296 template<> EIGEN_STRONG_INLINE int16_t pfirst<Packet8s>(const Packet8s& a) { return vgetq_lane_s16(a,0); }
2297 template<> EIGEN_STRONG_INLINE uint16_t pfirst<Packet4us>(const Packet4us& a) { return vget_lane_u16(a,0); }
2298 template<> EIGEN_STRONG_INLINE uint16_t pfirst<Packet8us>(const Packet8us& a) { return vgetq_lane_u16(a,0); }
2299 template<> EIGEN_STRONG_INLINE int32_t pfirst<Packet2i>(const Packet2i& a) { return vget_lane_s32(a,0); }
2300 template<> EIGEN_STRONG_INLINE int32_t pfirst<Packet4i>(const Packet4i& a) { return vgetq_lane_s32(a,0); }
2301 template<> EIGEN_STRONG_INLINE uint32_t pfirst<Packet2ui>(const Packet2ui& a) { return vget_lane_u32(a,0); }
2302 template<> EIGEN_STRONG_INLINE uint32_t pfirst<Packet4ui>(const Packet4ui& a) { return vgetq_lane_u32(a,0); }
2303 template<> EIGEN_STRONG_INLINE int64_t pfirst<Packet2l>(const Packet2l& a) { return vgetq_lane_s64(a,0); }
2304 template<> EIGEN_STRONG_INLINE uint64_t pfirst<Packet2ul>(const Packet2ul& a) { return vgetq_lane_u64(a,0); }
2305 
2306 template<> EIGEN_STRONG_INLINE Packet2f preverse(const Packet2f& a) { return vrev64_f32(a); }
2307 template<> EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a)
2308 {
2309  const float32x4_t a_r64 = vrev64q_f32(a);
2310  return vcombine_f32(vget_high_f32(a_r64), vget_low_f32(a_r64));
2311 }
2312 template<> EIGEN_STRONG_INLINE Packet4c preverse(const Packet4c& a)
2313 { return vget_lane_s32(vreinterpret_s32_s8(vrev64_s8(vreinterpret_s8_s32(vdup_n_s32(a)))), 0); }
2314 template<> EIGEN_STRONG_INLINE Packet8c preverse(const Packet8c& a) { return vrev64_s8(a); }
2315 template<> EIGEN_STRONG_INLINE Packet16c preverse(const Packet16c& a)
2316 {
2317  const int8x16_t a_r64 = vrev64q_s8(a);
2318  return vcombine_s8(vget_high_s8(a_r64), vget_low_s8(a_r64));
2319 }
2320 template<> EIGEN_STRONG_INLINE Packet4uc preverse(const Packet4uc& a)
2321 { return vget_lane_u32(vreinterpret_u32_u8(vrev64_u8(vreinterpret_u8_u32(vdup_n_u32(a)))), 0); }
2322 template<> EIGEN_STRONG_INLINE Packet8uc preverse(const Packet8uc& a) { return vrev64_u8(a); }
2323 template<> EIGEN_STRONG_INLINE Packet16uc preverse(const Packet16uc& a)
2324 {
2325  const uint8x16_t a_r64 = vrev64q_u8(a);
2326  return vcombine_u8(vget_high_u8(a_r64), vget_low_u8(a_r64));
2327 }
2328 template<> EIGEN_STRONG_INLINE Packet4s preverse(const Packet4s& a) { return vrev64_s16(a); }
2329 template<> EIGEN_STRONG_INLINE Packet8s preverse(const Packet8s& a)
2330 {
2331  const int16x8_t a_r64 = vrev64q_s16(a);
2332  return vcombine_s16(vget_high_s16(a_r64), vget_low_s16(a_r64));
2333 }
2334 template<> EIGEN_STRONG_INLINE Packet4us preverse(const Packet4us& a) { return vrev64_u16(a); }
2335 template<> EIGEN_STRONG_INLINE Packet8us preverse(const Packet8us& a)
2336 {
2337  const uint16x8_t a_r64 = vrev64q_u16(a);
2338  return vcombine_u16(vget_high_u16(a_r64), vget_low_u16(a_r64));
2339 }
2340 template<> EIGEN_STRONG_INLINE Packet2i preverse(const Packet2i& a) { return vrev64_s32(a); }
2341 template<> EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a)
2342 {
2343  const int32x4_t a_r64 = vrev64q_s32(a);
2344  return vcombine_s32(vget_high_s32(a_r64), vget_low_s32(a_r64));
2345 }
2346 template<> EIGEN_STRONG_INLINE Packet2ui preverse(const Packet2ui& a) { return vrev64_u32(a); }
2347 template<> EIGEN_STRONG_INLINE Packet4ui preverse(const Packet4ui& a)
2348 {
2349  const uint32x4_t a_r64 = vrev64q_u32(a);
2350  return vcombine_u32(vget_high_u32(a_r64), vget_low_u32(a_r64));
2351 }
2352 template<> EIGEN_STRONG_INLINE Packet2l preverse(const Packet2l& a)
2353 { return vcombine_s64(vget_high_s64(a), vget_low_s64(a)); }
2354 template<> EIGEN_STRONG_INLINE Packet2ul preverse(const Packet2ul& a)
2355 { return vcombine_u64(vget_high_u64(a), vget_low_u64(a)); }
2356 
2357 template<> EIGEN_STRONG_INLINE Packet2f pabs(const Packet2f& a) { return vabs_f32(a); }
2358 template<> EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a) { return vabsq_f32(a); }
2359 template<> EIGEN_STRONG_INLINE Packet4c pabs<Packet4c>(const Packet4c& a)
2360 { return vget_lane_s32(vreinterpret_s32_s8(vabs_s8(vreinterpret_s8_s32(vdup_n_s32(a)))), 0); }
2361 template<> EIGEN_STRONG_INLINE Packet8c pabs(const Packet8c& a) { return vabs_s8(a); }
2362 template<> EIGEN_STRONG_INLINE Packet16c pabs(const Packet16c& a) { return vabsq_s8(a); }
2363 template<> EIGEN_STRONG_INLINE Packet4uc pabs(const Packet4uc& a) { return a; }
2364 template<> EIGEN_STRONG_INLINE Packet8uc pabs(const Packet8uc& a) { return a; }
2365 template<> EIGEN_STRONG_INLINE Packet16uc pabs(const Packet16uc& a) { return a; }
2366 template<> EIGEN_STRONG_INLINE Packet4s pabs(const Packet4s& a) { return vabs_s16(a); }
2367 template<> EIGEN_STRONG_INLINE Packet8s pabs(const Packet8s& a) { return vabsq_s16(a); }
2368 template<> EIGEN_STRONG_INLINE Packet4us pabs(const Packet4us& a) { return a; }
2369 template<> EIGEN_STRONG_INLINE Packet8us pabs(const Packet8us& a) { return a; }
2370 template<> EIGEN_STRONG_INLINE Packet2i pabs(const Packet2i& a) { return vabs_s32(a); }
2371 template<> EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a) { return vabsq_s32(a); }
2372 template<> EIGEN_STRONG_INLINE Packet2ui pabs(const Packet2ui& a) { return a; }
2373 template<> EIGEN_STRONG_INLINE Packet4ui pabs(const Packet4ui& a) { return a; }
2374 template<> EIGEN_STRONG_INLINE Packet2l pabs(const Packet2l& a) {
2375 #if EIGEN_ARCH_ARM64
2376  return vabsq_s64(a);
2377 #else
2378  return vcombine_s64(
2379  vdup_n_s64((std::abs)(vgetq_lane_s64(a, 0))),
2380  vdup_n_s64((std::abs)(vgetq_lane_s64(a, 1))));
2381 #endif
2382 }
2383 template<> EIGEN_STRONG_INLINE Packet2ul pabs(const Packet2ul& a) { return a; }
2384 
2385 template <>
2386 EIGEN_STRONG_INLINE Packet2f psignbit(const Packet2f& a) {
2387  return vreinterpret_f32_s32(vshr_n_s32(vreinterpret_s32_f32(a), 31));
2388 }
2389 template <>
2390 EIGEN_STRONG_INLINE Packet4f psignbit(const Packet4f& a) {
2391  return vreinterpretq_f32_s32(vshrq_n_s32(vreinterpretq_s32_f32(a), 31));
2392 }
2393 
2394 template<> EIGEN_STRONG_INLINE Packet2f pfrexp<Packet2f>(const Packet2f& a, Packet2f& exponent)
2395 { return pfrexp_generic(a,exponent); }
2396 template<> EIGEN_STRONG_INLINE Packet4f pfrexp<Packet4f>(const Packet4f& a, Packet4f& exponent)
2397 { return pfrexp_generic(a,exponent); }
2398 
2399 template<> EIGEN_STRONG_INLINE Packet2f pldexp<Packet2f>(const Packet2f& a, const Packet2f& exponent)
2400 { return pldexp_generic(a,exponent); }
2401 template<> EIGEN_STRONG_INLINE Packet4f pldexp<Packet4f>(const Packet4f& a, const Packet4f& exponent)
2402 { return pldexp_generic(a,exponent); }
2403 
2404 #if EIGEN_ARCH_ARM64
2405 template<> EIGEN_STRONG_INLINE float predux<Packet2f>(const Packet2f& a) { return vaddv_f32(a); }
2406 template<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a) { return vaddvq_f32(a); }
2407 #else
2408 template<> EIGEN_STRONG_INLINE float predux<Packet2f>(const Packet2f& a) { return vget_lane_f32(vpadd_f32(a,a), 0); }
2409 template<> EIGEN_STRONG_INLINE float predux<Packet4f>(const Packet4f& a)
2410 {
2411  const float32x2_t sum = vadd_f32(vget_low_f32(a), vget_high_f32(a));
2412  return vget_lane_f32(vpadd_f32(sum, sum), 0);
2413 }
2414 #endif
2415 template<> EIGEN_STRONG_INLINE int8_t predux<Packet4c>(const Packet4c& a)
2416 {
2417  const int8x8_t a_dup = vreinterpret_s8_s32(vdup_n_s32(a));
2418  int8x8_t sum = vpadd_s8(a_dup, a_dup);
2419  sum = vpadd_s8(sum, sum);
2420  return vget_lane_s8(sum, 0);
2421 }
2422 #if EIGEN_ARCH_ARM64
2423 template<> EIGEN_STRONG_INLINE int8_t predux<Packet8c>(const Packet8c& a) { return vaddv_s8(a); }
2424 template<> EIGEN_STRONG_INLINE int8_t predux<Packet16c>(const Packet16c& a) { return vaddvq_s8(a); }
2425 #else
2426 template<> EIGEN_STRONG_INLINE int8_t predux<Packet8c>(const Packet8c& a)
2427 {
2428  int8x8_t sum = vpadd_s8(a,a);
2429  sum = vpadd_s8(sum, sum);
2430  sum = vpadd_s8(sum, sum);
2431  return vget_lane_s8(sum, 0);
2432 }
2433 template<> EIGEN_STRONG_INLINE int8_t predux<Packet16c>(const Packet16c& a)
2434 {
2435  int8x8_t sum = vadd_s8(vget_low_s8(a), vget_high_s8(a));
2436  sum = vpadd_s8(sum, sum);
2437  sum = vpadd_s8(sum, sum);
2438  sum = vpadd_s8(sum, sum);
2439  return vget_lane_s8(sum, 0);
2440 }
2441 #endif
2442 template<> EIGEN_STRONG_INLINE uint8_t predux<Packet4uc>(const Packet4uc& a)
2443 {
2444  const uint8x8_t a_dup = vreinterpret_u8_u32(vdup_n_u32(a));
2445  uint8x8_t sum = vpadd_u8(a_dup, a_dup);
2446  sum = vpadd_u8(sum, sum);
2447  return vget_lane_u8(sum, 0);
2448 }
2449 #if EIGEN_ARCH_ARM64
2450 template<> EIGEN_STRONG_INLINE uint8_t predux<Packet8uc>(const Packet8uc& a) { return vaddv_u8(a); }
2451 template<> EIGEN_STRONG_INLINE uint8_t predux<Packet16uc>(const Packet16uc& a) { return vaddvq_u8(a); }
2452 template<> EIGEN_STRONG_INLINE int16_t predux<Packet4s>(const Packet4s& a) { return vaddv_s16(a); }
2453 template<> EIGEN_STRONG_INLINE int16_t predux<Packet8s>(const Packet8s& a) { return vaddvq_s16(a); }
2454 template<> EIGEN_STRONG_INLINE uint16_t predux<Packet4us>(const Packet4us& a) { return vaddv_u16(a); }
2455 template<> EIGEN_STRONG_INLINE uint16_t predux<Packet8us>(const Packet8us& a) { return vaddvq_u16(a); }
2456 template<> EIGEN_STRONG_INLINE int32_t predux<Packet2i>(const Packet2i& a) { return vaddv_s32(a); }
2457 template<> EIGEN_STRONG_INLINE int32_t predux<Packet4i>(const Packet4i& a) { return vaddvq_s32(a); }
2458 template<> EIGEN_STRONG_INLINE uint32_t predux<Packet2ui>(const Packet2ui& a) { return vaddv_u32(a); }
2459 template<> EIGEN_STRONG_INLINE uint32_t predux<Packet4ui>(const Packet4ui& a) { return vaddvq_u32(a); }
2460 template<> EIGEN_STRONG_INLINE int64_t predux<Packet2l>(const Packet2l& a) { return vaddvq_s64(a); }
2461 template<> EIGEN_STRONG_INLINE uint64_t predux<Packet2ul>(const Packet2ul& a) { return vaddvq_u64(a); }
2462 #else
2463 template<> EIGEN_STRONG_INLINE uint8_t predux<Packet8uc>(const Packet8uc& a)
2464 {
2465  uint8x8_t sum = vpadd_u8(a,a);
2466  sum = vpadd_u8(sum, sum);
2467  sum = vpadd_u8(sum, sum);
2468  return vget_lane_u8(sum, 0);
2469 }
2470 template<> EIGEN_STRONG_INLINE uint8_t predux<Packet16uc>(const Packet16uc& a)
2471 {
2472  uint8x8_t sum = vadd_u8(vget_low_u8(a), vget_high_u8(a));
2473  sum = vpadd_u8(sum, sum);
2474  sum = vpadd_u8(sum, sum);
2475  sum = vpadd_u8(sum, sum);
2476  return vget_lane_u8(sum, 0);
2477 }
2478 template<> EIGEN_STRONG_INLINE int16_t predux<Packet4s>(const Packet4s& a)
2479 {
2480  const int16x4_t sum = vpadd_s16(a,a);
2481  return vget_lane_s16(vpadd_s16(sum, sum), 0);
2482 }
2483 template<> EIGEN_STRONG_INLINE int16_t predux<Packet8s>(const Packet8s& a)
2484 {
2485  int16x4_t sum = vadd_s16(vget_low_s16(a), vget_high_s16(a));
2486  sum = vpadd_s16(sum, sum);
2487  sum = vpadd_s16(sum, sum);
2488  return vget_lane_s16(sum, 0);
2489 }
2490 template<> EIGEN_STRONG_INLINE uint16_t predux<Packet4us>(const Packet4us& a)
2491 {
2492  const uint16x4_t sum = vpadd_u16(a,a);
2493  return vget_lane_u16(vpadd_u16(sum, sum), 0);
2494 }
2495 template<> EIGEN_STRONG_INLINE uint16_t predux<Packet8us>(const Packet8us& a)
2496 {
2497  uint16x4_t sum = vadd_u16(vget_low_u16(a), vget_high_u16(a));
2498  sum = vpadd_u16(sum, sum);
2499  sum = vpadd_u16(sum, sum);
2500  return vget_lane_u16(sum, 0);
2501 }
2502 template<> EIGEN_STRONG_INLINE int32_t predux<Packet2i>(const Packet2i& a) { return vget_lane_s32(vpadd_s32(a,a), 0); }
2503 template<> EIGEN_STRONG_INLINE int32_t predux<Packet4i>(const Packet4i& a)
2504 {
2505  const int32x2_t sum = vadd_s32(vget_low_s32(a), vget_high_s32(a));
2506  return vget_lane_s32(vpadd_s32(sum, sum), 0);
2507 }
2508 template<> EIGEN_STRONG_INLINE uint32_t predux<Packet2ui>(const Packet2ui& a) { return vget_lane_u32(vpadd_u32(a,a), 0); }
2509 template<> EIGEN_STRONG_INLINE uint32_t predux<Packet4ui>(const Packet4ui& a)
2510 {
2511  const uint32x2_t sum = vadd_u32(vget_low_u32(a), vget_high_u32(a));
2512  return vget_lane_u32(vpadd_u32(sum, sum), 0);
2513 }
2514 template<> EIGEN_STRONG_INLINE int64_t predux<Packet2l>(const Packet2l& a)
2515 { return vgetq_lane_s64(a, 0) + vgetq_lane_s64(a, 1); }
2516 template<> EIGEN_STRONG_INLINE uint64_t predux<Packet2ul>(const Packet2ul& a)
2517 { return vgetq_lane_u64(a, 0) + vgetq_lane_u64(a, 1); }
2518 #endif
2519 
2520 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4c predux_half_dowto4(const Packet8c& a)
2521 {
2522  return vget_lane_s32(vreinterpret_s32_s8(vadd_s8(a,
2523  vreinterpret_s8_s32(vrev64_s32(vreinterpret_s32_s8(a))))), 0);
2524 }
2525 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8c predux_half_dowto4(const Packet16c& a)
2526 { return vadd_s8(vget_high_s8(a), vget_low_s8(a)); }
2527 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4uc predux_half_dowto4(const Packet8uc& a)
2528 {
2529  return vget_lane_u32(vreinterpret_u32_u8(vadd_u8(a,
2530  vreinterpret_u8_u32(vrev64_u32(vreinterpret_u32_u8(a))))), 0);
2531 }
2532 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8uc predux_half_dowto4(const Packet16uc& a)
2533 { return vadd_u8(vget_high_u8(a), vget_low_u8(a)); }
2534 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4s predux_half_dowto4(const Packet8s& a)
2535 { return vadd_s16(vget_high_s16(a), vget_low_s16(a)); }
2536 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us predux_half_dowto4(const Packet8us& a)
2537 { return vadd_u16(vget_high_u16(a), vget_low_u16(a)); }
2538 
2539 // Other reduction functions:
2540 // mul
2541 template<> EIGEN_STRONG_INLINE float predux_mul<Packet2f>(const Packet2f& a)
2542 { return vget_lane_f32(a, 0) * vget_lane_f32(a, 1); }
2543 template<> EIGEN_STRONG_INLINE float predux_mul<Packet4f>(const Packet4f& a)
2544 { return predux_mul<Packet2f>(vmul_f32(vget_low_f32(a), vget_high_f32(a))); }
2545 template<> EIGEN_STRONG_INLINE int8_t predux_mul<Packet4c>(const Packet4c& a)
2546 {
2547  int8x8_t prod = vreinterpret_s8_s32(vdup_n_s32(a));
2548  prod = vmul_s8(prod, vrev16_s8(prod));
2549  return vget_lane_s8(prod, 0) * vget_lane_s8(prod, 2);
2550 }
2551 template<> EIGEN_STRONG_INLINE int8_t predux_mul<Packet8c>(const Packet8c& a)
2552 {
2553  int8x8_t prod = vmul_s8(a, vrev16_s8(a));
2554  prod = vmul_s8(prod, vrev32_s8(prod));
2555  return vget_lane_s8(prod, 0) * vget_lane_s8(prod, 4);
2556 }
2557 template<> EIGEN_STRONG_INLINE int8_t predux_mul<Packet16c>(const Packet16c& a)
2558 { return predux_mul<Packet8c>(vmul_s8(vget_low_s8(a), vget_high_s8(a))); }
2559 template<> EIGEN_STRONG_INLINE uint8_t predux_mul<Packet4uc>(const Packet4uc& a)
2560 {
2561  uint8x8_t prod = vreinterpret_u8_u32(vdup_n_u32(a));
2562  prod = vmul_u8(prod, vrev16_u8(prod));
2563  return vget_lane_u8(prod, 0) * vget_lane_u8(prod, 2);
2564 }
2565 template<> EIGEN_STRONG_INLINE uint8_t predux_mul<Packet8uc>(const Packet8uc& a)
2566 {
2567  uint8x8_t prod = vmul_u8(a, vrev16_u8(a));
2568  prod = vmul_u8(prod, vrev32_u8(prod));
2569  return vget_lane_u8(prod, 0) * vget_lane_u8(prod, 4);
2570 }
2571 template<> EIGEN_STRONG_INLINE uint8_t predux_mul<Packet16uc>(const Packet16uc& a)
2572 { return predux_mul<Packet8uc>(vmul_u8(vget_low_u8(a), vget_high_u8(a))); }
2573 template<> EIGEN_STRONG_INLINE int16_t predux_mul<Packet4s>(const Packet4s& a)
2574 {
2575  const int16x4_t prod = vmul_s16(a, vrev32_s16(a));
2576  return vget_lane_s16(prod, 0) * vget_lane_s16(prod, 2);
2577 }
2578 template<> EIGEN_STRONG_INLINE int16_t predux_mul<Packet8s>(const Packet8s& a)
2579 {
2580  int16x4_t prod;
2581 
2582  // Get the product of a_lo * a_hi -> |a1*a5|a2*a6|a3*a7|a4*a8|
2583  prod = vmul_s16(vget_low_s16(a), vget_high_s16(a));
2584  // Swap and multiply |a1*a5*a2*a6|a3*a7*a4*a8|
2585  prod = vmul_s16(prod, vrev32_s16(prod));
2586  // Multiply |a1*a5*a2*a6*a3*a7*a4*a8|
2587  return vget_lane_s16(prod, 0) * vget_lane_s16(prod, 2);
2588 }
2589 template<> EIGEN_STRONG_INLINE uint16_t predux_mul<Packet4us>(const Packet4us& a)
2590 {
2591  const uint16x4_t prod = vmul_u16(a, vrev32_u16(a));
2592  return vget_lane_u16(prod, 0) * vget_lane_u16(prod, 2);
2593 }
2594 template<> EIGEN_STRONG_INLINE uint16_t predux_mul<Packet8us>(const Packet8us& a)
2595 {
2596  uint16x4_t prod;
2597 
2598  // Get the product of a_lo * a_hi -> |a1*a5|a2*a6|a3*a7|a4*a8|
2599  prod = vmul_u16(vget_low_u16(a), vget_high_u16(a));
2600  // Swap and multiply |a1*a5*a2*a6|a3*a7*a4*a8|
2601  prod = vmul_u16(prod, vrev32_u16(prod));
2602  // Multiply |a1*a5*a2*a6*a3*a7*a4*a8|
2603  return vget_lane_u16(prod, 0) * vget_lane_u16(prod, 2);
2604 }
2605 template<> EIGEN_STRONG_INLINE int32_t predux_mul<Packet2i>(const Packet2i& a)
2606 { return vget_lane_s32(a, 0) * vget_lane_s32(a, 1); }
2607 template<> EIGEN_STRONG_INLINE int32_t predux_mul<Packet4i>(const Packet4i& a)
2608 { return predux_mul<Packet2i>(vmul_s32(vget_low_s32(a), vget_high_s32(a))); }
2609 template<> EIGEN_STRONG_INLINE uint32_t predux_mul<Packet2ui>(const Packet2ui& a)
2610 { return vget_lane_u32(a, 0) * vget_lane_u32(a, 1); }
2611 template<> EIGEN_STRONG_INLINE uint32_t predux_mul<Packet4ui>(const Packet4ui& a)
2612 { return predux_mul<Packet2ui>(vmul_u32(vget_low_u32(a), vget_high_u32(a))); }
2613 template<> EIGEN_STRONG_INLINE int64_t predux_mul<Packet2l>(const Packet2l& a)
2614 { return vgetq_lane_s64(a, 0) * vgetq_lane_s64(a, 1); }
2615 template<> EIGEN_STRONG_INLINE uint64_t predux_mul<Packet2ul>(const Packet2ul& a)
2616 { return vgetq_lane_u64(a, 0) * vgetq_lane_u64(a, 1); }
2617 
2618 // min
2619 #if EIGEN_ARCH_ARM64
2620 template<> EIGEN_STRONG_INLINE float predux_min<Packet2f>(const Packet2f& a) { return vminv_f32(a); }
2621 template<> EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a) { return vminvq_f32(a); }
2622 #else
2623 template<> EIGEN_STRONG_INLINE float predux_min<Packet2f>(const Packet2f& a)
2624 { return vget_lane_f32(vpmin_f32(a,a), 0); }
2625 template<> EIGEN_STRONG_INLINE float predux_min<Packet4f>(const Packet4f& a)
2626 {
2627  const float32x2_t min = vmin_f32(vget_low_f32(a), vget_high_f32(a));
2628  return vget_lane_f32(vpmin_f32(min, min), 0);
2629 }
2630 #endif
2631 template<> EIGEN_STRONG_INLINE int8_t predux_min<Packet4c>(const Packet4c& a)
2632 {
2633  const int8x8_t a_dup = vreinterpret_s8_s32(vdup_n_s32(a));
2634  int8x8_t min = vpmin_s8(a_dup, a_dup);
2635  min = vpmin_s8(min, min);
2636  return vget_lane_s8(min, 0);
2637 }
2638 #if EIGEN_ARCH_ARM64
2639 template<> EIGEN_STRONG_INLINE int8_t predux_min<Packet8c>(const Packet8c& a) { return vminv_s8(a); }
2640 template<> EIGEN_STRONG_INLINE int8_t predux_min<Packet16c>(const Packet16c& a) { return vminvq_s8(a); }
2641 #else
2642 template<> EIGEN_STRONG_INLINE int8_t predux_min<Packet8c>(const Packet8c& a)
2643 {
2644  int8x8_t min = vpmin_s8(a,a);
2645  min = vpmin_s8(min, min);
2646  min = vpmin_s8(min, min);
2647  return vget_lane_s8(min, 0);
2648 }
2649 template<> EIGEN_STRONG_INLINE int8_t predux_min<Packet16c>(const Packet16c& a)
2650 {
2651  int8x8_t min = vmin_s8(vget_low_s8(a), vget_high_s8(a));
2652  min = vpmin_s8(min, min);
2653  min = vpmin_s8(min, min);
2654  min = vpmin_s8(min, min);
2655  return vget_lane_s8(min, 0);
2656 }
2657 #endif
2658 template<> EIGEN_STRONG_INLINE uint8_t predux_min<Packet4uc>(const Packet4uc& a)
2659 {
2660  const uint8x8_t a_dup = vreinterpret_u8_u32(vdup_n_u32(a));
2661  uint8x8_t min = vpmin_u8(a_dup, a_dup);
2662  min = vpmin_u8(min, min);
2663  return vget_lane_u8(min, 0);
2664 }
2665 #if EIGEN_ARCH_ARM64
2666 template<> EIGEN_STRONG_INLINE uint8_t predux_min<Packet8uc>(const Packet8uc& a) { return vminv_u8(a); }
2667 template<> EIGEN_STRONG_INLINE uint8_t predux_min<Packet16uc>(const Packet16uc& a) { return vminvq_u8(a); }
2668 template<> EIGEN_STRONG_INLINE int16_t predux_min<Packet4s>(const Packet4s& a) { return vminv_s16(a); }
2669 template<> EIGEN_STRONG_INLINE int16_t predux_min<Packet8s>(const Packet8s& a) { return vminvq_s16(a); }
2670 template<> EIGEN_STRONG_INLINE uint16_t predux_min<Packet4us>(const Packet4us& a) { return vminv_u16(a); }
2671 template<> EIGEN_STRONG_INLINE uint16_t predux_min<Packet8us>(const Packet8us& a) { return vminvq_u16(a); }
2672 template<> EIGEN_STRONG_INLINE int32_t predux_min<Packet2i>(const Packet2i& a) { return vminv_s32(a); }
2673 template<> EIGEN_STRONG_INLINE int32_t predux_min<Packet4i>(const Packet4i& a) { return vminvq_s32(a); }
2674 template<> EIGEN_STRONG_INLINE uint32_t predux_min<Packet2ui>(const Packet2ui& a) { return vminv_u32(a); }
2675 template<> EIGEN_STRONG_INLINE uint32_t predux_min<Packet4ui>(const Packet4ui& a) { return vminvq_u32(a); }
2676 #else
2677 template<> EIGEN_STRONG_INLINE uint8_t predux_min<Packet8uc>(const Packet8uc& a)
2678 {
2679  uint8x8_t min = vpmin_u8(a,a);
2680  min = vpmin_u8(min, min);
2681  min = vpmin_u8(min, min);
2682  return vget_lane_u8(min, 0);
2683 }
2684 template<> EIGEN_STRONG_INLINE uint8_t predux_min<Packet16uc>(const Packet16uc& a)
2685 {
2686  uint8x8_t min = vmin_u8(vget_low_u8(a), vget_high_u8(a));
2687  min = vpmin_u8(min, min);
2688  min = vpmin_u8(min, min);
2689  min = vpmin_u8(min, min);
2690  return vget_lane_u8(min, 0);
2691 }
2692 template<> EIGEN_STRONG_INLINE int16_t predux_min<Packet4s>(const Packet4s& a)
2693 {
2694  const int16x4_t min = vpmin_s16(a,a);
2695  return vget_lane_s16(vpmin_s16(min, min), 0);
2696 }
2697 template<> EIGEN_STRONG_INLINE int16_t predux_min<Packet8s>(const Packet8s& a)
2698 {
2699  int16x4_t min = vmin_s16(vget_low_s16(a), vget_high_s16(a));
2700  min = vpmin_s16(min, min);
2701  min = vpmin_s16(min, min);
2702  return vget_lane_s16(min, 0);
2703 }
2704 template<> EIGEN_STRONG_INLINE uint16_t predux_min<Packet4us>(const Packet4us& a)
2705 {
2706  const uint16x4_t min = vpmin_u16(a,a);
2707  return vget_lane_u16(vpmin_u16(min, min), 0);
2708 }
2709 template<> EIGEN_STRONG_INLINE uint16_t predux_min<Packet8us>(const Packet8us& a)
2710 {
2711  uint16x4_t min = vmin_u16(vget_low_u16(a), vget_high_u16(a));
2712  min = vpmin_u16(min, min);
2713  min = vpmin_u16(min, min);
2714  return vget_lane_u16(min, 0);
2715 }
2716 template<> EIGEN_STRONG_INLINE int32_t predux_min<Packet2i>(const Packet2i& a)
2717 { return vget_lane_s32(vpmin_s32(a,a), 0); }
2718 template<> EIGEN_STRONG_INLINE int32_t predux_min<Packet4i>(const Packet4i& a)
2719 {
2720  const int32x2_t min = vmin_s32(vget_low_s32(a), vget_high_s32(a));
2721  return vget_lane_s32(vpmin_s32(min, min), 0);
2722 }
2723 template<> EIGEN_STRONG_INLINE uint32_t predux_min<Packet2ui>(const Packet2ui& a)
2724 { return vget_lane_u32(vpmin_u32(a,a), 0); }
2725 template<> EIGEN_STRONG_INLINE uint32_t predux_min<Packet4ui>(const Packet4ui& a)
2726 {
2727  const uint32x2_t min = vmin_u32(vget_low_u32(a), vget_high_u32(a));
2728  return vget_lane_u32(vpmin_u32(min, min), 0);
2729 }
2730 #endif
2731 template<> EIGEN_STRONG_INLINE int64_t predux_min<Packet2l>(const Packet2l& a)
2732 { return (std::min)(vgetq_lane_s64(a, 0), vgetq_lane_s64(a, 1)); }
2733 template<> EIGEN_STRONG_INLINE uint64_t predux_min<Packet2ul>(const Packet2ul& a)
2734 { return (std::min)(vgetq_lane_u64(a, 0), vgetq_lane_u64(a, 1)); }
2735 
2736 // max
2737 #if EIGEN_ARCH_ARM64
2738 template<> EIGEN_STRONG_INLINE float predux_max<Packet2f>(const Packet2f& a) { return vmaxv_f32(a); }
2739 template<> EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a) { return vmaxvq_f32(a); }
2740 #else
2741 template<> EIGEN_STRONG_INLINE float predux_max<Packet2f>(const Packet2f& a)
2742 { return vget_lane_f32(vpmax_f32(a,a), 0); }
2743 template<> EIGEN_STRONG_INLINE float predux_max<Packet4f>(const Packet4f& a)
2744 {
2745  const float32x2_t max = vmax_f32(vget_low_f32(a), vget_high_f32(a));
2746  return vget_lane_f32(vpmax_f32(max, max), 0);
2747 }
2748 #endif
2749 template<> EIGEN_STRONG_INLINE int8_t predux_max<Packet4c>(const Packet4c& a)
2750 {
2751  const int8x8_t a_dup = vreinterpret_s8_s32(vdup_n_s32(a));
2752  int8x8_t max = vpmax_s8(a_dup, a_dup);
2753  max = vpmax_s8(max, max);
2754  return vget_lane_s8(max, 0);
2755 }
2756 #if EIGEN_ARCH_ARM64
2757 template<> EIGEN_STRONG_INLINE int8_t predux_max<Packet8c>(const Packet8c& a) { return vmaxv_s8(a); }
2758 template<> EIGEN_STRONG_INLINE int8_t predux_max<Packet16c>(const Packet16c& a) { return vmaxvq_s8(a); }
2759 #else
2760 template<> EIGEN_STRONG_INLINE int8_t predux_max<Packet8c>(const Packet8c& a)
2761 {
2762  int8x8_t max = vpmax_s8(a,a);
2763  max = vpmax_s8(max, max);
2764  max = vpmax_s8(max, max);
2765  return vget_lane_s8(max, 0);
2766 }
2767 template<> EIGEN_STRONG_INLINE int8_t predux_max<Packet16c>(const Packet16c& a)
2768 {
2769  int8x8_t max = vmax_s8(vget_low_s8(a), vget_high_s8(a));
2770  max = vpmax_s8(max, max);
2771  max = vpmax_s8(max, max);
2772  max = vpmax_s8(max, max);
2773  return vget_lane_s8(max, 0);
2774 }
2775 #endif
2776 template<> EIGEN_STRONG_INLINE uint8_t predux_max<Packet4uc>(const Packet4uc& a)
2777 {
2778  const uint8x8_t a_dup = vreinterpret_u8_u32(vdup_n_u32(a));
2779  uint8x8_t max = vpmax_u8(a_dup, a_dup);
2780  max = vpmax_u8(max, max);
2781  return vget_lane_u8(max, 0);
2782 }
2783 #if EIGEN_ARCH_ARM64
2784 template<> EIGEN_STRONG_INLINE uint8_t predux_max<Packet8uc>(const Packet8uc& a) { return vmaxv_u8(a); }
2785 template<> EIGEN_STRONG_INLINE uint8_t predux_max<Packet16uc>(const Packet16uc& a) { return vmaxvq_u8(a); }
2786 template<> EIGEN_STRONG_INLINE int16_t predux_max<Packet4s>(const Packet4s& a) { return vmaxv_s16(a); }
2787 template<> EIGEN_STRONG_INLINE int16_t predux_max<Packet8s>(const Packet8s& a) { return vmaxvq_s16(a); }
2788 template<> EIGEN_STRONG_INLINE uint16_t predux_max<Packet4us>(const Packet4us& a) { return vmaxv_u16(a); }
2789 template<> EIGEN_STRONG_INLINE uint16_t predux_max<Packet8us>(const Packet8us& a) { return vmaxvq_u16(a); }
2790 template<> EIGEN_STRONG_INLINE int32_t predux_max<Packet2i>(const Packet2i& a) { return vmaxv_s32(a); }
2791 template<> EIGEN_STRONG_INLINE int32_t predux_max<Packet4i>(const Packet4i& a) { return vmaxvq_s32(a); }
2792 template<> EIGEN_STRONG_INLINE uint32_t predux_max<Packet2ui>(const Packet2ui& a) { return vmaxv_u32(a); }
2793 template<> EIGEN_STRONG_INLINE uint32_t predux_max<Packet4ui>(const Packet4ui& a) { return vmaxvq_u32(a); }
2794 #else
2795 template<> EIGEN_STRONG_INLINE uint8_t predux_max<Packet8uc>(const Packet8uc& a)
2796 {
2797  uint8x8_t max = vpmax_u8(a,a);
2798  max = vpmax_u8(max, max);
2799  max = vpmax_u8(max, max);
2800  return vget_lane_u8(max, 0);
2801 }
2802 template<> EIGEN_STRONG_INLINE uint8_t predux_max<Packet16uc>(const Packet16uc& a)
2803 {
2804  uint8x8_t max = vmax_u8(vget_low_u8(a), vget_high_u8(a));
2805  max = vpmax_u8(max, max);
2806  max = vpmax_u8(max, max);
2807  max = vpmax_u8(max, max);
2808  return vget_lane_u8(max, 0);
2809 }
2810 template<> EIGEN_STRONG_INLINE int16_t predux_max<Packet4s>(const Packet4s& a)
2811 {
2812  const int16x4_t max = vpmax_s16(a,a);
2813  return vget_lane_s16(vpmax_s16(max, max), 0);
2814 }
2815 template<> EIGEN_STRONG_INLINE int16_t predux_max<Packet8s>(const Packet8s& a)
2816 {
2817  int16x4_t max = vmax_s16(vget_low_s16(a), vget_high_s16(a));
2818  max = vpmax_s16(max, max);
2819  max = vpmax_s16(max, max);
2820  return vget_lane_s16(max, 0);
2821 }
2822 template<> EIGEN_STRONG_INLINE uint16_t predux_max<Packet4us>(const Packet4us& a)
2823 {
2824  const uint16x4_t max = vpmax_u16(a,a);
2825  return vget_lane_u16(vpmax_u16(max, max), 0);
2826 }
2827 template<> EIGEN_STRONG_INLINE uint16_t predux_max<Packet8us>(const Packet8us& a)
2828 {
2829  uint16x4_t max = vmax_u16(vget_low_u16(a), vget_high_u16(a));
2830  max = vpmax_u16(max, max);
2831  max = vpmax_u16(max, max);
2832  return vget_lane_u16(max, 0);
2833 }
2834 template<> EIGEN_STRONG_INLINE int32_t predux_max<Packet2i>(const Packet2i& a)
2835 { return vget_lane_s32(vpmax_s32(a,a), 0); }
2836 template<> EIGEN_STRONG_INLINE int32_t predux_max<Packet4i>(const Packet4i& a)
2837 {
2838  const int32x2_t max = vmax_s32(vget_low_s32(a), vget_high_s32(a));
2839  return vget_lane_s32(vpmax_s32(max, max), 0);
2840 }
2841 template<> EIGEN_STRONG_INLINE uint32_t predux_max<Packet2ui>(const Packet2ui& a)
2842 { return vget_lane_u32(vpmax_u32(a,a), 0); }
2843 template<> EIGEN_STRONG_INLINE uint32_t predux_max<Packet4ui>(const Packet4ui& a)
2844 {
2845  const uint32x2_t max = vmax_u32(vget_low_u32(a), vget_high_u32(a));
2846  return vget_lane_u32(vpmax_u32(max, max), 0);
2847 }
2848 #endif
2849 template<> EIGEN_STRONG_INLINE int64_t predux_max<Packet2l>(const Packet2l& a)
2850 { return (std::max)(vgetq_lane_s64(a, 0), vgetq_lane_s64(a, 1)); }
2851 template<> EIGEN_STRONG_INLINE uint64_t predux_max<Packet2ul>(const Packet2ul& a)
2852 { return (std::max)(vgetq_lane_u64(a, 0), vgetq_lane_u64(a, 1)); }
2853 
2854 template<> EIGEN_STRONG_INLINE bool predux_any(const Packet4f& x)
2855 {
2856  uint32x2_t tmp = vorr_u32(vget_low_u32( vreinterpretq_u32_f32(x)),
2857  vget_high_u32(vreinterpretq_u32_f32(x)));
2858  return vget_lane_u32(vpmax_u32(tmp, tmp), 0);
2859 }
2860 
2861 // Helpers for ptranspose.
2862 namespace detail {
2863 
2864 template<typename Packet>
2865 void zip_in_place(Packet& p1, Packet& p2);
2866 
2867 template<>
2868 EIGEN_ALWAYS_INLINE void zip_in_place<Packet2f>(Packet2f& p1, Packet2f& p2) {
2869  const float32x2x2_t tmp = vzip_f32(p1, p2);
2870  p1 = tmp.val[0];
2871  p2 = tmp.val[1];
2872 }
2873 
2874 template<>
2875 EIGEN_ALWAYS_INLINE void zip_in_place<Packet4f>(Packet4f& p1, Packet4f& p2) {
2876  const float32x4x2_t tmp = vzipq_f32(p1, p2);
2877  p1 = tmp.val[0];
2878  p2 = tmp.val[1];
2879 }
2880 
2881 template<>
2882 EIGEN_ALWAYS_INLINE void zip_in_place<Packet8c>(Packet8c& p1, Packet8c& p2) {
2883  const int8x8x2_t tmp = vzip_s8(p1, p2);
2884  p1 = tmp.val[0];
2885  p2 = tmp.val[1];
2886 }
2887 
2888 template<>
2889 EIGEN_ALWAYS_INLINE void zip_in_place<Packet16c>(Packet16c& p1, Packet16c& p2) {
2890  const int8x16x2_t tmp = vzipq_s8(p1, p2);
2891  p1 = tmp.val[0];
2892  p2 = tmp.val[1];
2893 }
2894 
2895 template<>
2896 EIGEN_ALWAYS_INLINE void zip_in_place<Packet8uc>(Packet8uc& p1, Packet8uc& p2) {
2897  const uint8x8x2_t tmp = vzip_u8(p1, p2);
2898  p1 = tmp.val[0];
2899  p2 = tmp.val[1];
2900 }
2901 
2902 template<>
2903 EIGEN_ALWAYS_INLINE void zip_in_place<Packet16uc>(Packet16uc& p1, Packet16uc& p2) {
2904  const uint8x16x2_t tmp = vzipq_u8(p1, p2);
2905  p1 = tmp.val[0];
2906  p2 = tmp.val[1];
2907 }
2908 
2909 template<>
2910 EIGEN_ALWAYS_INLINE void zip_in_place<Packet2i>(Packet2i& p1, Packet2i& p2) {
2911  const int32x2x2_t tmp = vzip_s32(p1, p2);
2912  p1 = tmp.val[0];
2913  p2 = tmp.val[1];
2914 }
2915 
2916 template<>
2917 EIGEN_ALWAYS_INLINE void zip_in_place<Packet4i>(Packet4i& p1, Packet4i& p2) {
2918  const int32x4x2_t tmp = vzipq_s32(p1, p2);
2919  p1 = tmp.val[0];
2920  p2 = tmp.val[1];
2921 }
2922 
2923 template<>
2924 EIGEN_ALWAYS_INLINE void zip_in_place<Packet2ui>(Packet2ui& p1, Packet2ui& p2) {
2925  const uint32x2x2_t tmp = vzip_u32(p1, p2);
2926  p1 = tmp.val[0];
2927  p2 = tmp.val[1];
2928 }
2929 
2930 template<>
2931 EIGEN_ALWAYS_INLINE void zip_in_place<Packet4ui>(Packet4ui& p1, Packet4ui& p2) {
2932  const uint32x4x2_t tmp = vzipq_u32(p1, p2);
2933  p1 = tmp.val[0];
2934  p2 = tmp.val[1];
2935 }
2936 
2937 template<>
2938 EIGEN_ALWAYS_INLINE void zip_in_place<Packet4s>(Packet4s& p1, Packet4s& p2) {
2939  const int16x4x2_t tmp = vzip_s16(p1, p2);
2940  p1 = tmp.val[0];
2941  p2 = tmp.val[1];
2942 }
2943 
2944 template<>
2945 EIGEN_ALWAYS_INLINE void zip_in_place<Packet8s>(Packet8s& p1, Packet8s& p2) {
2946  const int16x8x2_t tmp = vzipq_s16(p1, p2);
2947  p1 = tmp.val[0];
2948  p2 = tmp.val[1];
2949 }
2950 
2951 template<>
2952 EIGEN_ALWAYS_INLINE void zip_in_place<Packet4us>(Packet4us& p1, Packet4us& p2) {
2953  const uint16x4x2_t tmp = vzip_u16(p1, p2);
2954  p1 = tmp.val[0];
2955  p2 = tmp.val[1];
2956 }
2957 
2958 template<>
2959 EIGEN_ALWAYS_INLINE void zip_in_place<Packet8us>(Packet8us& p1, Packet8us& p2) {
2960  const uint16x8x2_t tmp = vzipq_u16(p1, p2);
2961  p1 = tmp.val[0];
2962  p2 = tmp.val[1];
2963 }
2964 
2965 template<typename Packet>
2966 EIGEN_ALWAYS_INLINE void ptranspose_impl(PacketBlock<Packet, 2>& kernel) {
2967  zip_in_place(kernel.packet[0], kernel.packet[1]);
2968 }
2969 
2970 template<typename Packet>
2971 EIGEN_ALWAYS_INLINE void ptranspose_impl(PacketBlock<Packet, 4>& kernel) {
2972  zip_in_place(kernel.packet[0], kernel.packet[2]);
2973  zip_in_place(kernel.packet[1], kernel.packet[3]);
2974  zip_in_place(kernel.packet[0], kernel.packet[1]);
2975  zip_in_place(kernel.packet[2], kernel.packet[3]);
2976 }
2977 
2978 template<typename Packet>
2979 EIGEN_ALWAYS_INLINE void ptranspose_impl(PacketBlock<Packet, 8>& kernel) {
2980  zip_in_place(kernel.packet[0], kernel.packet[4]);
2981  zip_in_place(kernel.packet[1], kernel.packet[5]);
2982  zip_in_place(kernel.packet[2], kernel.packet[6]);
2983  zip_in_place(kernel.packet[3], kernel.packet[7]);
2984 
2985  zip_in_place(kernel.packet[0], kernel.packet[2]);
2986  zip_in_place(kernel.packet[1], kernel.packet[3]);
2987  zip_in_place(kernel.packet[4], kernel.packet[6]);
2988  zip_in_place(kernel.packet[5], kernel.packet[7]);
2989 
2990  zip_in_place(kernel.packet[0], kernel.packet[1]);
2991  zip_in_place(kernel.packet[2], kernel.packet[3]);
2992  zip_in_place(kernel.packet[4], kernel.packet[5]);
2993  zip_in_place(kernel.packet[6], kernel.packet[7]);
2994 }
2995 
2996 template<typename Packet>
2997 EIGEN_ALWAYS_INLINE void ptranspose_impl(PacketBlock<Packet, 16>& kernel) {
2998  EIGEN_UNROLL_LOOP
2999  for (int i=0; i<4; ++i) {
3000  const int m = (1 << i);
3001  EIGEN_UNROLL_LOOP
3002  for (int j=0; j<m; ++j) {
3003  const int n = (1 << (3-i));
3004  EIGEN_UNROLL_LOOP
3005  for (int k=0; k<n; ++k) {
3006  const int idx = 2*j*n+k;
3007  zip_in_place(kernel.packet[idx], kernel.packet[idx + n]);
3008  }
3009  }
3010  }
3011 }
3012 
3013 } // namespace detail
3014 
3015 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet2f, 2>& kernel) {
3016  detail::ptranspose_impl(kernel);
3017 }
3018 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4f, 4>& kernel) {
3019  detail::ptranspose_impl(kernel);
3020 }
3021 
3022 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4c, 4>& kernel)
3023 {
3024  const int8x8_t a = vreinterpret_s8_s32(vset_lane_s32(kernel.packet[2], vdup_n_s32(kernel.packet[0]), 1));
3025  const int8x8_t b = vreinterpret_s8_s32(vset_lane_s32(kernel.packet[3], vdup_n_s32(kernel.packet[1]), 1));
3026 
3027  const int8x8x2_t zip8 = vzip_s8(a,b);
3028  const int16x4x2_t zip16 = vzip_s16(vreinterpret_s16_s8(zip8.val[0]), vreinterpret_s16_s8(zip8.val[1]));
3029 
3030  kernel.packet[0] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[0]), 0);
3031  kernel.packet[1] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[0]), 1);
3032  kernel.packet[2] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[1]), 0);
3033  kernel.packet[3] = vget_lane_s32(vreinterpret_s32_s16(zip16.val[1]), 1);
3034 }
3035 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8c, 8>& kernel) {
3036  detail::ptranspose_impl(kernel);
3037 }
3038 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8c, 4>& kernel) {
3039  detail::ptranspose_impl(kernel);
3040 }
3041 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16c, 16>& kernel) {
3042  detail::ptranspose_impl(kernel);
3043 }
3044 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16c, 8>& kernel) {
3045  detail::ptranspose_impl(kernel);
3046 }
3047 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16c, 4>& kernel) {
3048  detail::ptranspose_impl(kernel);
3049 }
3050 
3051 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4uc, 4>& kernel)
3052 {
3053  const uint8x8_t a = vreinterpret_u8_u32(vset_lane_u32(kernel.packet[2], vdup_n_u32(kernel.packet[0]), 1));
3054  const uint8x8_t b = vreinterpret_u8_u32(vset_lane_u32(kernel.packet[3], vdup_n_u32(kernel.packet[1]), 1));
3055 
3056  const uint8x8x2_t zip8 = vzip_u8(a,b);
3057  const uint16x4x2_t zip16 = vzip_u16(vreinterpret_u16_u8(zip8.val[0]), vreinterpret_u16_u8(zip8.val[1]));
3058 
3059  kernel.packet[0] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[0]), 0);
3060  kernel.packet[1] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[0]), 1);
3061  kernel.packet[2] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[1]), 0);
3062  kernel.packet[3] = vget_lane_u32(vreinterpret_u32_u16(zip16.val[1]), 1);
3063 }
3064 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8uc, 8>& kernel) {
3065  detail::ptranspose_impl(kernel);
3066 }
3067 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8uc, 4>& kernel) {
3068  detail::ptranspose_impl(kernel);
3069 }
3070 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16uc, 16>& kernel) {
3071  detail::ptranspose_impl(kernel);
3072 }
3073 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16uc, 8>& kernel) {
3074  detail::ptranspose_impl(kernel);
3075 }
3076 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16uc, 4>& kernel) {
3077  detail::ptranspose_impl(kernel);
3078 }
3079 
3080 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4s, 4>& kernel) {
3081  detail::ptranspose_impl(kernel);
3082 }
3083 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8s, 8>& kernel) {
3084  detail::ptranspose_impl(kernel);
3085 }
3086 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8s, 4>& kernel) {
3087  detail::ptranspose_impl(kernel);
3088 }
3089 
3090 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4us, 4>& kernel) {
3091  detail::ptranspose_impl(kernel);
3092 }
3093 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8us, 8>& kernel) {
3094  detail::ptranspose_impl(kernel);
3095 }
3096 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8us, 4>& kernel) {
3097  detail::ptranspose_impl(kernel);
3098 }
3099 
3100 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet2i, 2>& kernel) {
3101  detail::ptranspose_impl(kernel);
3102 }
3103 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4i, 4>& kernel) {
3104  detail::ptranspose_impl(kernel);
3105 }
3106 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet2ui, 2>& kernel) {
3107  detail::zip_in_place(kernel.packet[0], kernel.packet[1]);
3108 }
3109 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4ui, 4>& kernel) {
3110  detail::ptranspose_impl(kernel);
3111 }
3112 
3113 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void
3114 ptranspose(PacketBlock<Packet2l, 2>& kernel)
3115 {
3116 #if EIGEN_ARCH_ARM64
3117  const int64x2_t tmp1 = vzip1q_s64(kernel.packet[0], kernel.packet[1]);
3118  kernel.packet[1] = vzip2q_s64(kernel.packet[0], kernel.packet[1]);
3119  kernel.packet[0] = tmp1;
3120 #else
3121  const int64x1_t tmp[2][2] = {
3122  { vget_low_s64(kernel.packet[0]), vget_high_s64(kernel.packet[0]) },
3123  { vget_low_s64(kernel.packet[1]), vget_high_s64(kernel.packet[1]) }
3124  };
3125 
3126  kernel.packet[0] = vcombine_s64(tmp[0][0], tmp[1][0]);
3127  kernel.packet[1] = vcombine_s64(tmp[0][1], tmp[1][1]);
3128 #endif
3129 }
3130 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void
3131 ptranspose(PacketBlock<Packet2ul, 2>& kernel)
3132 {
3133 #if EIGEN_ARCH_ARM64
3134  const uint64x2_t tmp1 = vzip1q_u64(kernel.packet[0], kernel.packet[1]);
3135  kernel.packet[1] = vzip2q_u64(kernel.packet[0], kernel.packet[1]);
3136  kernel.packet[0] = tmp1;
3137 #else
3138  const uint64x1_t tmp[2][2] = {
3139  { vget_low_u64(kernel.packet[0]), vget_high_u64(kernel.packet[0]) },
3140  { vget_low_u64(kernel.packet[1]), vget_high_u64(kernel.packet[1]) }
3141  };
3142 
3143  kernel.packet[0] = vcombine_u64(tmp[0][0], tmp[1][0]);
3144  kernel.packet[1] = vcombine_u64(tmp[0][1], tmp[1][1]);
3145 #endif
3146 }
3147 
3148 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2f pselect( const Packet2f& mask, const Packet2f& a, const Packet2f& b)
3149 { return vbsl_f32(vreinterpret_u32_f32(mask), a, b); }
3150 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4f pselect(const Packet4f& mask, const Packet4f& a, const Packet4f& b)
3151 { return vbslq_f32(vreinterpretq_u32_f32(mask), a, b); }
3152 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8c pselect(const Packet8c& mask, const Packet8c& a, const Packet8c& b)
3153 { return vbsl_s8(vreinterpret_u8_s8(mask), a, b); }
3154 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16c pselect(const Packet16c& mask, const Packet16c& a, const Packet16c& b)
3155 { return vbslq_s8(vreinterpretq_u8_s8(mask), a, b); }
3156 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8uc pselect(const Packet8uc& mask, const Packet8uc& a, const Packet8uc& b)
3157 { return vbsl_u8(mask, a, b); }
3158 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc pselect(const Packet16uc& mask, const Packet16uc& a, const Packet16uc& b)
3159 { return vbslq_u8(mask, a, b); }
3160 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4s pselect(const Packet4s& mask, const Packet4s& a, const Packet4s& b)
3161 { return vbsl_s16(vreinterpret_u16_s16(mask), a, b); }
3162 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s pselect(const Packet8s& mask, const Packet8s& a, const Packet8s& b)
3163 { return vbslq_s16(vreinterpretq_u16_s16(mask), a, b); }
3164 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4us pselect(const Packet4us& mask, const Packet4us& a, const Packet4us& b)
3165 { return vbsl_u16(mask, a, b); }
3166 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us pselect(const Packet8us& mask, const Packet8us& a, const Packet8us& b)
3167 { return vbslq_u16(mask, a, b); }
3168 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2i pselect(const Packet2i& mask, const Packet2i& a, const Packet2i& b)
3169 { return vbsl_s32(vreinterpret_u32_s32(mask), a, b); }
3170 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4i pselect(const Packet4i& mask, const Packet4i& a, const Packet4i& b)
3171 { return vbslq_s32(vreinterpretq_u32_s32(mask), a, b); }
3172 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ui pselect(const Packet2ui& mask, const Packet2ui& a, const Packet2ui& b)
3173 { return vbsl_u32(mask, a, b); }
3174 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4ui pselect(const Packet4ui& mask, const Packet4ui& a, const Packet4ui& b)
3175 { return vbslq_u32(mask, a, b); }
3176 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2l pselect(const Packet2l& mask, const Packet2l& a, const Packet2l& b)
3177 { return vbslq_s64(vreinterpretq_u64_s64(mask), a, b); }
3178 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ul pselect(const Packet2ul& mask, const Packet2ul& a, const Packet2ul& b)
3179 { return vbslq_u64(mask, a, b); }
3180 
3181 // Use armv8 rounding intinsics if available.
3182 #if EIGEN_ARCH_ARMV8
3183 template<> EIGEN_STRONG_INLINE Packet2f print<Packet2f>(const Packet2f& a)
3184 { return vrndn_f32(a); }
3185 
3186 template<> EIGEN_STRONG_INLINE Packet4f print<Packet4f>(const Packet4f& a)
3187 { return vrndnq_f32(a); }
3188 
3189 template<> EIGEN_STRONG_INLINE Packet2f pfloor<Packet2f>(const Packet2f& a)
3190 { return vrndm_f32(a); }
3191 
3192 template<> EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(const Packet4f& a)
3193 { return vrndmq_f32(a); }
3194 
3195 template<> EIGEN_STRONG_INLINE Packet2f pceil<Packet2f>(const Packet2f& a)
3196 { return vrndp_f32(a); }
3197 
3198 template<> EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(const Packet4f& a)
3199 { return vrndpq_f32(a); }
3200 
3201 #else
3202 
3203 template<> EIGEN_STRONG_INLINE Packet4f print(const Packet4f& a) {
3204  // Adds and subtracts signum(a) * 2^23 to force rounding.
3205  const Packet4f limit = pset1<Packet4f>(static_cast<float>(1<<23));
3206  const Packet4f abs_a = pabs(a);
3207  Packet4f r = padd(abs_a, limit);
3208  // Don't compile-away addition and subtraction.
3209  EIGEN_OPTIMIZATION_BARRIER(r);
3210  r = psub(r, limit);
3211  // If greater than limit, simply return a. Otherwise, account for sign.
3212  r = pselect(pcmp_lt(abs_a, limit),
3213  pselect(pcmp_lt(a, pzero(a)), pnegate(r), r), a);
3214  return r;
3215 }
3216 
3217 template<> EIGEN_STRONG_INLINE Packet2f print(const Packet2f& a) {
3218  // Adds and subtracts signum(a) * 2^23 to force rounding.
3219  const Packet2f limit = pset1<Packet2f>(static_cast<float>(1<<23));
3220  const Packet2f abs_a = pabs(a);
3221  Packet2f r = padd(abs_a, limit);
3222  // Don't compile-away addition and subtraction.
3223  EIGEN_OPTIMIZATION_BARRIER(r);
3224  r = psub(r, limit);
3225  // If greater than limit, simply return a. Otherwise, account for sign.
3226  r = pselect(pcmp_lt(abs_a, limit),
3227  pselect(pcmp_lt(a, pzero(a)), pnegate(r), r), a);
3228  return r;
3229 }
3230 
3231 template<> EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(const Packet4f& a)
3232 {
3233  const Packet4f cst_1 = pset1<Packet4f>(1.0f);
3234  Packet4f tmp = print<Packet4f>(a);
3235  // If greater, subtract one.
3236  Packet4f mask = pcmp_lt(a, tmp);
3237  mask = pand(mask, cst_1);
3238  return psub(tmp, mask);
3239 }
3240 
3241 template<> EIGEN_STRONG_INLINE Packet2f pfloor<Packet2f>(const Packet2f& a)
3242 {
3243  const Packet2f cst_1 = pset1<Packet2f>(1.0f);
3244  Packet2f tmp = print<Packet2f>(a);
3245  // If greater, subtract one.
3246  Packet2f mask = pcmp_lt(a, tmp);
3247  mask = pand(mask, cst_1);
3248  return psub(tmp, mask);
3249 }
3250 
3251 template<> EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(const Packet4f& a)
3252 {
3253  const Packet4f cst_1 = pset1<Packet4f>(1.0f);
3254  Packet4f tmp = print<Packet4f>(a);
3255  // If smaller, add one.
3256  Packet4f mask = pcmp_lt(tmp, a);
3257  mask = pand(mask, cst_1);
3258  return padd(tmp, mask);
3259 }
3260 
3261 template<> EIGEN_STRONG_INLINE Packet2f pceil<Packet2f>(const Packet2f& a)
3262 {
3263  const Packet2f cst_1 = pset1<Packet2f>(1.0);
3264  Packet2f tmp = print<Packet2f>(a);
3265  // If smaller, add one.
3266  Packet2f mask = pcmp_lt(tmp, a);
3267  mask = pand(mask, cst_1);
3268  return padd(tmp, mask);
3269 }
3270 
3271 #endif
3272 
3279 template<> EIGEN_STRONG_INLINE Packet4uc psqrt(const Packet4uc& a) {
3280  uint8x8_t x = vreinterpret_u8_u32(vdup_n_u32(a));
3281  uint8x8_t res = vdup_n_u8(0);
3282  uint8x8_t add = vdup_n_u8(0x8);
3283  for (int i = 0; i < 4; i++)
3284  {
3285  const uint8x8_t temp = vorr_u8(res, add);
3286  res = vbsl_u8(vcge_u8(x, vmul_u8(temp, temp)), temp, res);
3287  add = vshr_n_u8(add, 1);
3288  }
3289  return vget_lane_u32(vreinterpret_u32_u8(res), 0);
3290 }
3292 template<> EIGEN_STRONG_INLINE Packet8uc psqrt(const Packet8uc& a) {
3293  uint8x8_t res = vdup_n_u8(0);
3294  uint8x8_t add = vdup_n_u8(0x8);
3295  for (int i = 0; i < 4; i++)
3296  {
3297  const uint8x8_t temp = vorr_u8(res, add);
3298  res = vbsl_u8(vcge_u8(a, vmul_u8(temp, temp)), temp, res);
3299  add = vshr_n_u8(add, 1);
3300  }
3301  return res;
3302 }
3304 template<> EIGEN_STRONG_INLINE Packet16uc psqrt(const Packet16uc& a) {
3305  uint8x16_t res = vdupq_n_u8(0);
3306  uint8x16_t add = vdupq_n_u8(0x8);
3307  for (int i = 0; i < 4; i++)
3308  {
3309  const uint8x16_t temp = vorrq_u8(res, add);
3310  res = vbslq_u8(vcgeq_u8(a, vmulq_u8(temp, temp)), temp, res);
3311  add = vshrq_n_u8(add, 1);
3312  }
3313  return res;
3314 }
3316 template<> EIGEN_STRONG_INLINE Packet4us psqrt(const Packet4us& a) {
3317  uint16x4_t res = vdup_n_u16(0);
3318  uint16x4_t add = vdup_n_u16(0x80);
3319  for (int i = 0; i < 8; i++)
3320  {
3321  const uint16x4_t temp = vorr_u16(res, add);
3322  res = vbsl_u16(vcge_u16(a, vmul_u16(temp, temp)), temp, res);
3323  add = vshr_n_u16(add, 1);
3324  }
3325  return res;
3326 }
3328 template<> EIGEN_STRONG_INLINE Packet8us psqrt(const Packet8us& a) {
3329  uint16x8_t res = vdupq_n_u16(0);
3330  uint16x8_t add = vdupq_n_u16(0x80);
3331  for (int i = 0; i < 8; i++)
3332  {
3333  const uint16x8_t temp = vorrq_u16(res, add);
3334  res = vbslq_u16(vcgeq_u16(a, vmulq_u16(temp, temp)), temp, res);
3335  add = vshrq_n_u16(add, 1);
3336  }
3337  return res;
3338 }
3340 template<> EIGEN_STRONG_INLINE Packet2ui psqrt(const Packet2ui& a) {
3341  uint32x2_t res = vdup_n_u32(0);
3342  uint32x2_t add = vdup_n_u32(0x8000);
3343  for (int i = 0; i < 16; i++)
3344  {
3345  const uint32x2_t temp = vorr_u32(res, add);
3346  res = vbsl_u32(vcge_u32(a, vmul_u32(temp, temp)), temp, res);
3347  add = vshr_n_u32(add, 1);
3348  }
3349  return res;
3350 }
3352 template<> EIGEN_STRONG_INLINE Packet4ui psqrt(const Packet4ui& a) {
3353  uint32x4_t res = vdupq_n_u32(0);
3354  uint32x4_t add = vdupq_n_u32(0x8000);
3355  for (int i = 0; i < 16; i++)
3356  {
3357  const uint32x4_t temp = vorrq_u32(res, add);
3358  res = vbslq_u32(vcgeq_u32(a, vmulq_u32(temp, temp)), temp, res);
3359  add = vshrq_n_u32(add, 1);
3360  }
3361  return res;
3362 }
3363 
3364 template<> EIGEN_STRONG_INLINE Packet4f prsqrt(const Packet4f& a) {
3365  // Do Newton iterations for 1/sqrt(x).
3366  return generic_rsqrt_newton_step<Packet4f, /*Steps=*/2>::run(a, vrsqrteq_f32(a));
3367 }
3368 
3369 template<> EIGEN_STRONG_INLINE Packet2f prsqrt(const Packet2f& a) {
3370  // Compute approximate reciprocal sqrt.
3371  return generic_rsqrt_newton_step<Packet2f, /*Steps=*/2>::run(a, vrsqrte_f32(a));
3372 }
3373 
3374 // Unfortunately vsqrt_f32 is only available for A64.
3375 #if EIGEN_ARCH_ARM64
3376 template<> EIGEN_STRONG_INLINE Packet4f psqrt(const Packet4f& _x){return vsqrtq_f32(_x);}
3377 template<> EIGEN_STRONG_INLINE Packet2f psqrt(const Packet2f& _x){return vsqrt_f32(_x); }
3378 #else
3379 template<> EIGEN_STRONG_INLINE Packet4f psqrt(const Packet4f& a) {
3380  return generic_sqrt_newton_step<Packet4f>::run(a, prsqrt(a));
3381 }
3382 template<> EIGEN_STRONG_INLINE Packet2f psqrt(const Packet2f& a) {
3383  return generic_sqrt_newton_step<Packet2f>::run(a, prsqrt(a));
3384 }
3385 #endif
3386 
3387 //---------- bfloat16 ----------
3388 // TODO: Add support for native armv8.6-a bfloat16_t
3389 
3390 // TODO: Guard if we have native bfloat16 support
3391 typedef eigen_packet_wrapper<uint16x4_t, 19> Packet4bf;
3392 
3393 template<> struct is_arithmetic<Packet4bf> { enum { value = true }; };
3394 
3395 template<> struct packet_traits<bfloat16> : default_packet_traits
3396 {
3397  typedef Packet4bf type;
3398  typedef Packet4bf half;
3399  enum
3400  {
3401  Vectorizable = 1,
3402  AlignedOnScalar = 1,
3403  size = 4,
3404 
3405  HasCmp = 1,
3406  HasAdd = 1,
3407  HasSub = 1,
3408  HasShift = 1,
3409  HasMul = 1,
3410  HasNegate = 1,
3411  HasAbs = 1,
3412  HasArg = 0,
3413  HasAbs2 = 1,
3414  HasAbsDiff = 1,
3415  HasMin = 1,
3416  HasMax = 1,
3417  HasConj = 1,
3418  HasSetLinear = 1,
3419  HasBlend = 0,
3420  HasDiv = 1,
3421  HasFloor = 1,
3422  HasCeil = 1,
3423  HasRint = 1,
3424 
3425  HasSin = EIGEN_FAST_MATH,
3426  HasCos = EIGEN_FAST_MATH,
3427  HasLog = 1,
3428  HasExp = 1,
3429  HasSqrt = 0,
3430  HasTanh = EIGEN_FAST_MATH,
3431  HasErf = EIGEN_FAST_MATH,
3432  HasBessel = 0, // Issues with accuracy.
3433  HasNdtri = 0
3434  };
3435 };
3436 
3437 template<> struct unpacket_traits<Packet4bf>
3438 {
3439  typedef bfloat16 type;
3440  typedef Packet4bf half;
3441  enum
3442  {
3443  size = 4,
3444  alignment = Aligned16,
3445  vectorizable = true,
3446  masked_load_available = false,
3447  masked_store_available = false
3448  };
3449 };
3450 
3451 namespace detail {
3452 template<>
3453 EIGEN_ALWAYS_INLINE void zip_in_place<Packet4bf>(Packet4bf& p1, Packet4bf& p2) {
3454  const uint16x4x2_t tmp = vzip_u16(p1, p2);
3455  p1 = tmp.val[0];
3456  p2 = tmp.val[1];
3457 }
3458 } // namespace detail
3459 
3460 EIGEN_STRONG_INLINE Packet4bf F32ToBf16(const Packet4f& p)
3461 {
3462  // See the scalar implementation in BFloat16.h for a comprehensible explanation
3463  // of this fast rounding algorithm
3464  Packet4ui input = Packet4ui(vreinterpretq_u32_f32(p));
3465 
3466  // lsb = (input >> 16) & 1
3467  Packet4ui lsb = vandq_u32(vshrq_n_u32(input, 16), vdupq_n_u32(1));
3468 
3469  // rounding_bias = 0x7fff + lsb
3470  Packet4ui rounding_bias = vaddq_u32(lsb, vdupq_n_u32(0x7fff));
3471 
3472  // input += rounding_bias
3473  input = vaddq_u32(input, rounding_bias);
3474 
3475  // input = input >> 16
3476  input = vshrq_n_u32(input, 16);
3477 
3478  // Replace float-nans by bfloat16-nans, that is 0x7fc0
3479  const Packet4ui bf16_nan = vdupq_n_u32(0x7fc0);
3480  const Packet4ui mask = vceqq_f32(p, p);
3481  input = vbslq_u32(mask, input, bf16_nan);
3482 
3483  // output = static_cast<uint16_t>(input)
3484  return vmovn_u32(input);
3485 }
3486 
3487 EIGEN_STRONG_INLINE Packet4f Bf16ToF32(const Packet4bf& p)
3488 {
3489  return Packet4f(vreinterpretq_f32_u32(vshlq_n_u32(vmovl_u16(p), 16)));
3490 }
3491 
3492 EIGEN_STRONG_INLINE Packet4bf F32MaskToBf16Mask(const Packet4f& p) {
3493  return vmovn_u32(vreinterpretq_u32_f32(p));
3494 }
3495 
3496 template<> EIGEN_STRONG_INLINE Packet4bf pset1<Packet4bf>(const bfloat16& from) {
3497  return Packet4bf(pset1<Packet4us>(from.value));
3498 }
3499 
3500 template<> EIGEN_STRONG_INLINE bfloat16 pfirst<Packet4bf>(const Packet4bf& from) {
3501  return bfloat16_impl::raw_uint16_to_bfloat16(static_cast<uint16_t>(pfirst<Packet4us>(Packet4us(from))));
3502 }
3503 
3504 template<> EIGEN_STRONG_INLINE Packet4bf pload<Packet4bf>(const bfloat16* from)
3505 {
3506  return Packet4bf(pload<Packet4us>(reinterpret_cast<const uint16_t*>(from)));
3507 }
3508 
3509 template<> EIGEN_STRONG_INLINE Packet4bf ploadu<Packet4bf>(const bfloat16* from)
3510 {
3511  return Packet4bf(ploadu<Packet4us>(reinterpret_cast<const uint16_t*>(from)));
3512 }
3513 
3514 template<> EIGEN_STRONG_INLINE void pstore<bfloat16>(bfloat16* to, const Packet4bf& from)
3515 {
3516  EIGEN_DEBUG_ALIGNED_STORE vst1_u16(reinterpret_cast<uint16_t*>(to), from);
3517 }
3518 
3519 template<> EIGEN_STRONG_INLINE void pstoreu<bfloat16>(bfloat16* to, const Packet4bf& from)
3520 {
3521  EIGEN_DEBUG_UNALIGNED_STORE vst1_u16(reinterpret_cast<uint16_t*>(to), from);
3522 }
3523 
3524 template<> EIGEN_STRONG_INLINE Packet4bf ploaddup<Packet4bf>(const bfloat16* from)
3525 {
3526  return Packet4bf(ploaddup<Packet4us>(reinterpret_cast<const uint16_t*>(from)));
3527 }
3528 
3529 template <> EIGEN_STRONG_INLINE Packet4bf pabs(const Packet4bf& a) {
3530  return F32ToBf16(pabs<Packet4f>(Bf16ToF32(a)));
3531 }
3532 
3533 template <> EIGEN_STRONG_INLINE Packet4bf pmin<PropagateNumbers, Packet4bf>(const Packet4bf &a,
3534  const Packet4bf &b)
3535 {
3536  return F32ToBf16(pmin<PropagateNumbers, Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3537 }
3538 template <> EIGEN_STRONG_INLINE Packet4bf pmin<PropagateNaN, Packet4bf>(const Packet4bf &a,
3539  const Packet4bf &b)
3540 {
3541  return F32ToBf16(pmin<PropagateNaN, Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3542 }
3543 
3544 template <> EIGEN_STRONG_INLINE Packet4bf pmin<Packet4bf>(const Packet4bf &a,
3545  const Packet4bf &b)
3546 {
3547  return F32ToBf16(pmin<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3548 }
3549 
3550 template <> EIGEN_STRONG_INLINE Packet4bf pmax<PropagateNumbers, Packet4bf>(const Packet4bf &a,
3551  const Packet4bf &b)
3552 {
3553  return F32ToBf16(pmax<PropagateNumbers, Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3554 }
3555 template <> EIGEN_STRONG_INLINE Packet4bf pmax<PropagateNaN, Packet4bf>(const Packet4bf &a,
3556  const Packet4bf &b)
3557 {
3558  return F32ToBf16(pmax<PropagateNaN, Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3559 }
3560 
3561 template <> EIGEN_STRONG_INLINE Packet4bf pmax<Packet4bf>(const Packet4bf &a,
3562  const Packet4bf &b)
3563 {
3564  return F32ToBf16(pmax<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3565 }
3566 
3567 template<> EIGEN_STRONG_INLINE Packet4bf plset<Packet4bf>(const bfloat16& a)
3568 {
3569  return F32ToBf16(plset<Packet4f>(static_cast<float>(a)));
3570 }
3571 
3572 template<> EIGEN_STRONG_INLINE Packet4bf por(const Packet4bf& a,const Packet4bf& b) {
3573  return Packet4bf(por<Packet4us>(Packet4us(a), Packet4us(b)));
3574 }
3575 
3576 template<> EIGEN_STRONG_INLINE Packet4bf pxor(const Packet4bf& a,const Packet4bf& b) {
3577  return Packet4bf(pxor<Packet4us>(Packet4us(a), Packet4us(b)));
3578 }
3579 
3580 template<> EIGEN_STRONG_INLINE Packet4bf pand(const Packet4bf& a,const Packet4bf& b) {
3581  return Packet4bf(pand<Packet4us>(Packet4us(a), Packet4us(b)));
3582 }
3583 
3584 template<> EIGEN_STRONG_INLINE Packet4bf pandnot(const Packet4bf& a,const Packet4bf& b) {
3585  return Packet4bf(pandnot<Packet4us>(Packet4us(a), Packet4us(b)));
3586 }
3587 
3588 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4bf pselect(const Packet4bf& mask, const Packet4bf& a,
3589  const Packet4bf& b)
3590 {
3591  return Packet4bf(pselect<Packet4us>(Packet4us(mask), Packet4us(a), Packet4us(b)));
3592 }
3593 
3594 template<> EIGEN_STRONG_INLINE Packet4bf print<Packet4bf>(const Packet4bf& a)
3595 {
3596  return F32ToBf16(print<Packet4f>(Bf16ToF32(a)));
3597 }
3598 
3599 template<> EIGEN_STRONG_INLINE Packet4bf pfloor<Packet4bf>(const Packet4bf& a)
3600 {
3601  return F32ToBf16(pfloor<Packet4f>(Bf16ToF32(a)));
3602 }
3603 
3604 template<> EIGEN_STRONG_INLINE Packet4bf pceil<Packet4bf>(const Packet4bf& a)
3605 {
3606  return F32ToBf16(pceil<Packet4f>(Bf16ToF32(a)));
3607 }
3608 
3609 template<> EIGEN_STRONG_INLINE Packet4bf pconj(const Packet4bf& a) { return a; }
3610 
3611 template<> EIGEN_STRONG_INLINE Packet4bf padd<Packet4bf>(const Packet4bf& a, const Packet4bf& b) {
3612  return F32ToBf16(padd<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3613 }
3614 
3615 template<> EIGEN_STRONG_INLINE Packet4bf psub<Packet4bf>(const Packet4bf& a, const Packet4bf& b) {
3616  return F32ToBf16(psub<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3617 }
3618 
3619 template<> EIGEN_STRONG_INLINE Packet4bf pmul<Packet4bf>(const Packet4bf& a, const Packet4bf& b) {
3620  return F32ToBf16(pmul<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3621 }
3622 
3623 template<> EIGEN_STRONG_INLINE Packet4bf pdiv<Packet4bf>(const Packet4bf& a, const Packet4bf& b) {
3624  return F32ToBf16(pdiv<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3625 }
3626 
3627 template<>
3628 EIGEN_STRONG_INLINE Packet4bf pgather<bfloat16, Packet4bf>(const bfloat16* from, Index stride)
3629 {
3630  return Packet4bf(pgather<uint16_t, Packet4us>(reinterpret_cast<const uint16_t*>(from), stride));
3631 }
3632 
3633 template<>
3634 EIGEN_STRONG_INLINE void pscatter<bfloat16, Packet4bf>(bfloat16* to, const Packet4bf& from, Index stride)
3635 {
3636  pscatter<uint16_t, Packet4us>(reinterpret_cast<uint16_t*>(to), Packet4us(from), stride);
3637 }
3638 
3639 template<> EIGEN_STRONG_INLINE bfloat16 predux<Packet4bf>(const Packet4bf& a)
3640 {
3641  return static_cast<bfloat16>(predux<Packet4f>(Bf16ToF32(a)));
3642 }
3643 
3644 template<> EIGEN_STRONG_INLINE bfloat16 predux_max<Packet4bf>(const Packet4bf& a)
3645 {
3646  return static_cast<bfloat16>(predux_max<Packet4f>(Bf16ToF32(a)));
3647 }
3648 
3649 template<> EIGEN_STRONG_INLINE bfloat16 predux_min<Packet4bf>(const Packet4bf& a)
3650 {
3651  return static_cast<bfloat16>(predux_min<Packet4f>(Bf16ToF32(a)));
3652 }
3653 
3654 template<> EIGEN_STRONG_INLINE bfloat16 predux_mul<Packet4bf>(const Packet4bf& a)
3655 {
3656  return static_cast<bfloat16>(predux_mul<Packet4f>(Bf16ToF32(a)));
3657 }
3658 
3659 template<> EIGEN_STRONG_INLINE Packet4bf preverse<Packet4bf>(const Packet4bf& a)
3660 {
3661  return Packet4bf(preverse<Packet4us>(Packet4us(a)));
3662 }
3663 
3664 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4bf, 4>& kernel)
3665 {
3666  detail::ptranspose_impl(kernel);
3667 }
3668 
3669 template<> EIGEN_STRONG_INLINE Packet4bf pabsdiff<Packet4bf>(const Packet4bf& a, const Packet4bf& b)
3670 {
3671  return F32ToBf16(pabsdiff<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3672 }
3673 
3674 template<> EIGEN_STRONG_INLINE Packet4bf pcmp_eq<Packet4bf>(const Packet4bf& a, const Packet4bf& b)
3675 {
3676  return F32MaskToBf16Mask(pcmp_eq<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3677 }
3678 
3679 template<> EIGEN_STRONG_INLINE Packet4bf pcmp_lt<Packet4bf>(const Packet4bf& a, const Packet4bf& b)
3680 {
3681  return F32MaskToBf16Mask(pcmp_lt<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3682 }
3683 
3684 template<> EIGEN_STRONG_INLINE Packet4bf pcmp_lt_or_nan<Packet4bf>(const Packet4bf& a, const Packet4bf& b)
3685 {
3686  return F32MaskToBf16Mask(pcmp_lt_or_nan<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3687 }
3688 
3689 template<> EIGEN_STRONG_INLINE Packet4bf pcmp_le<Packet4bf>(const Packet4bf& a, const Packet4bf& b)
3690 {
3691  return F32MaskToBf16Mask(pcmp_le<Packet4f>(Bf16ToF32(a), Bf16ToF32(b)));
3692 }
3693 
3694 template<> EIGEN_STRONG_INLINE Packet4bf pnegate<Packet4bf>(const Packet4bf& a)
3695 {
3696  return Packet4bf(pxor<Packet4us>(Packet4us(a), pset1<Packet4us>(static_cast<uint16_t>(0x8000))));
3697 }
3698 
3699 //---------- double ----------
3700 
3701 // Clang 3.5 in the iOS toolchain has an ICE triggered by NEON intrisics for double.
3702 // Confirmed at least with __apple_build_version__ = 6000054.
3703 #if EIGEN_COMP_CLANGAPPLE
3704 // Let's hope that by the time __apple_build_version__ hits the 601* range, the bug will be fixed.
3705 // https://gist.github.com/yamaya/2924292 suggests that the 3 first digits are only updated with
3706 // major toolchain updates.
3707 #define EIGEN_APPLE_DOUBLE_NEON_BUG (EIGEN_COMP_CLANGAPPLE < 6010000)
3708 #else
3709 #define EIGEN_APPLE_DOUBLE_NEON_BUG 0
3710 #endif
3711 
3712 #if EIGEN_ARCH_ARM64 && !EIGEN_APPLE_DOUBLE_NEON_BUG
3713 
3714 #if EIGEN_COMP_GNUC
3715 // Bug 907: workaround missing declarations of the following two functions in the ADK
3716 // Defining these functions as templates ensures that if these intrinsics are
3717 // already defined in arm_neon.h, then our workaround doesn't cause a conflict
3718 // and has lower priority in overload resolution.
3719 // This doesn't work with MSVC though, since the function names are macros.
3720 template <typename T> uint64x2_t vreinterpretq_u64_f64(T a) { return (uint64x2_t) a; }
3721 
3722 template <typename T> float64x2_t vreinterpretq_f64_u64(T a) { return (float64x2_t) a; }
3723 #endif
3724 
3725 #if EIGEN_COMP_MSVC_STRICT
3726 typedef eigen_packet_wrapper<float64x2_t, 18> Packet2d;
3727 typedef eigen_packet_wrapper<float64x1_t, 19> Packet1d;
3728 
3729 EIGEN_ALWAYS_INLINE Packet2d make_packet2d(double a, double b) {
3730  double from[2] = {a, b};
3731  return vld1q_f64(from);
3732 }
3733 
3734 #else
3735 typedef float64x2_t Packet2d;
3736 typedef float64x1_t Packet1d;
3737 
3738 EIGEN_ALWAYS_INLINE Packet2d make_packet2d(double a, double b) { return Packet2d{a, b}; }
3739 #endif
3740 
3741 
3742 // fuctionally equivalent to _mm_shuffle_pd in SSE (i.e. shuffle(m, n, mask) equals _mm_shuffle_pd(m,n,mask))
3743 // Currently used in LU/arch/InverseSize4.h to enable a shared implementation
3744 // for fast inversion of matrices of size 4.
3745 EIGEN_STRONG_INLINE Packet2d shuffle(const Packet2d& m, const Packet2d& n, int mask)
3746 {
3747  const double* a = reinterpret_cast<const double*>(&m);
3748  const double* b = reinterpret_cast<const double*>(&n);
3749  Packet2d res = make_packet2d(*(a + (mask & 1)), *(b + ((mask >> 1) & 1)));
3750  return res;
3751 }
3752 
3753 EIGEN_STRONG_INLINE Packet2d vec2d_swizzle2(const Packet2d& a, const Packet2d& b, int mask)
3754 {
3755  return shuffle(a, b, mask);
3756 }
3757 EIGEN_STRONG_INLINE Packet2d vec2d_unpacklo(const Packet2d& a,const Packet2d& b)
3758 {
3759  return shuffle(a, b, 0);
3760 }
3761 EIGEN_STRONG_INLINE Packet2d vec2d_unpackhi(const Packet2d& a,const Packet2d& b)
3762 {
3763  return shuffle(a, b, 3);
3764 }
3765 #define vec2d_duplane(a, p) \
3766  Packet2d(vdupq_laneq_f64(a, p))
3767 
3768 template<> struct packet_traits<double> : default_packet_traits
3769 {
3770  typedef Packet2d type;
3771  typedef Packet2d half;
3772  enum
3773  {
3774  Vectorizable = 1,
3775  AlignedOnScalar = 1,
3776  size = 2,
3777 
3778  HasCmp = 1,
3779  HasAdd = 1,
3780  HasSub = 1,
3781  HasShift = 1,
3782  HasMul = 1,
3783  HasNegate = 1,
3784  HasAbs = 1,
3785  HasArg = 0,
3786  HasAbs2 = 1,
3787  HasAbsDiff = 1,
3788  HasMin = 1,
3789  HasMax = 1,
3790  HasConj = 1,
3791  HasSetLinear = 1,
3792  HasBlend = 0,
3793 
3794  HasDiv = 1,
3795  HasFloor = 1,
3796  HasCeil = 1,
3797  HasRint = 1,
3798 
3799 #if EIGEN_ARCH_ARM64 && !EIGEN_APPLE_DOUBLE_NEON_BUG
3800  HasExp = 1,
3801  HasLog = 1,
3802  HasATan = 1,
3803 #endif
3804  HasSin = 0,
3805  HasCos = 0,
3806  HasSqrt = 1,
3807  HasRsqrt = 1,
3808  HasTanh = 0,
3809  HasErf = 0
3810  };
3811 };
3812 
3813 template<> struct unpacket_traits<Packet2d>
3814 {
3815  typedef double type;
3816  typedef Packet2d half;
3817  typedef Packet2l integer_packet;
3818  enum
3819  {
3820  size = 2,
3821  alignment = Aligned16,
3822  vectorizable = true,
3823  masked_load_available = false,
3824  masked_store_available = false
3825  };
3826 };
3827 
3828 template<> EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(const double& from) { return vdupq_n_f64(from); }
3829 
3830 template<> EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(const double& a)
3831 {
3832  const double c[] = {0.0,1.0};
3833  return vaddq_f64(pset1<Packet2d>(a), vld1q_f64(c));
3834 }
3835 
3836 template<> EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(const Packet2d& a, const Packet2d& b) { return vaddq_f64(a,b); }
3837 
3838 template<> EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(const Packet2d& a, const Packet2d& b) { return vsubq_f64(a,b); }
3839 
3840 template<> EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d& , const Packet2d& );
3841 template<> EIGEN_STRONG_INLINE Packet2d paddsub<Packet2d>(const Packet2d& a, const Packet2d& b){
3842  const Packet2d mask = make_packet2d(numext::bit_cast<double>(0x8000000000000000ull), 0.0);
3843  return padd(a, pxor(mask, b));
3844 }
3845 
3846 template<> EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a) { return vnegq_f64(a); }
3847 
3848 template<> EIGEN_STRONG_INLINE Packet2d pconj(const Packet2d& a) { return a; }
3849 
3850 template<> EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(const Packet2d& a, const Packet2d& b) { return vmulq_f64(a,b); }
3851 
3852 template<> EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(const Packet2d& a, const Packet2d& b) { return vdivq_f64(a,b); }
3853 
3854 #ifdef __ARM_FEATURE_FMA
3855 // See bug 936. See above comment about FMA for float.
3856 template<> EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c)
3857 { return vfmaq_f64(c,a,b); }
3858 #else
3859 template<> EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c)
3860 { return vmlaq_f64(c,a,b); }
3861 #endif
3862 
3863 template<> EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(const Packet2d& a, const Packet2d& b) { return vminq_f64(a,b); }
3864 
3865 #ifdef __ARM_FEATURE_NUMERIC_MAXMIN
3866 // numeric max and min are only available if ARM_FEATURE_NUMERIC_MAXMIN is defined (which can only be the case for Armv8 systems).
3867 template<> EIGEN_STRONG_INLINE Packet2d pmin<PropagateNumbers, Packet2d>(const Packet2d& a, const Packet2d& b) { return vminnmq_f64(a, b); }
3868 template<> EIGEN_STRONG_INLINE Packet2d pmax<PropagateNumbers, Packet2d>(const Packet2d& a, const Packet2d& b) { return vmaxnmq_f64(a, b); }
3869 
3870 #endif
3871 
3872 template<> EIGEN_STRONG_INLINE Packet2d pmin<PropagateNaN, Packet2d>(const Packet2d& a, const Packet2d& b) { return pmin<Packet2d>(a, b); }
3873 
3874 template<> EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(const Packet2d& a, const Packet2d& b) { return vmaxq_f64(a,b); }
3875 
3876 
3877 template<> EIGEN_STRONG_INLINE Packet2d pmax<PropagateNaN, Packet2d>(const Packet2d& a, const Packet2d& b) { return pmax<Packet2d>(a, b); }
3878 
3879 // Logical Operations are not supported for float, so we have to reinterpret casts using NEON intrinsics
3880 template<> EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(const Packet2d& a, const Packet2d& b)
3881 { return vreinterpretq_f64_u64(vandq_u64(vreinterpretq_u64_f64(a),vreinterpretq_u64_f64(b))); }
3882 
3883 template<> EIGEN_STRONG_INLINE Packet2d por<Packet2d>(const Packet2d& a, const Packet2d& b)
3884 { return vreinterpretq_f64_u64(vorrq_u64(vreinterpretq_u64_f64(a),vreinterpretq_u64_f64(b))); }
3885 
3886 template<> EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(const Packet2d& a, const Packet2d& b)
3887 { return vreinterpretq_f64_u64(veorq_u64(vreinterpretq_u64_f64(a),vreinterpretq_u64_f64(b))); }
3888 
3889 template<> EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(const Packet2d& a, const Packet2d& b)
3890 { return vreinterpretq_f64_u64(vbicq_u64(vreinterpretq_u64_f64(a),vreinterpretq_u64_f64(b))); }
3891 
3892 template<> EIGEN_STRONG_INLINE Packet2d pcmp_le(const Packet2d& a, const Packet2d& b)
3893 { return vreinterpretq_f64_u64(vcleq_f64(a,b)); }
3894 
3895 template<> EIGEN_STRONG_INLINE Packet2d pcmp_lt(const Packet2d& a, const Packet2d& b)
3896 { return vreinterpretq_f64_u64(vcltq_f64(a,b)); }
3897 
3898 template<> EIGEN_STRONG_INLINE Packet2d pcmp_lt_or_nan(const Packet2d& a, const Packet2d& b)
3899 { return vreinterpretq_f64_u32(vmvnq_u32(vreinterpretq_u32_u64(vcgeq_f64(a,b)))); }
3900 
3901 template<> EIGEN_STRONG_INLINE Packet2d pcmp_eq(const Packet2d& a, const Packet2d& b)
3902 { return vreinterpretq_f64_u64(vceqq_f64(a,b)); }
3903 
3904 template<> EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(const double* from)
3905 { EIGEN_DEBUG_ALIGNED_LOAD return vld1q_f64(from); }
3906 
3907 template<> EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(const double* from)
3908 { EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_f64(from); }
3909 
3910 template<> EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(const double* from) { return vld1q_dup_f64(from); }
3911 template<> EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet2d& from)
3912 { EIGEN_DEBUG_ALIGNED_STORE vst1q_f64(to,from); }
3913 
3914 template<> EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet2d& from)
3915 { EIGEN_DEBUG_UNALIGNED_STORE vst1q_f64(to,from); }
3916 
3917 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2d pgather<double, Packet2d>(const double* from, Index stride)
3918 {
3919  Packet2d res = pset1<Packet2d>(0.0);
3920  res = vld1q_lane_f64(from + 0*stride, res, 0);
3921  res = vld1q_lane_f64(from + 1*stride, res, 1);
3922  return res;
3923 }
3924 
3925 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<double, Packet2d>(double* to, const Packet2d& from, Index stride)
3926 {
3927  vst1q_lane_f64(to + stride*0, from, 0);
3928  vst1q_lane_f64(to + stride*1, from, 1);
3929 }
3930 
3931 template<> EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) { EIGEN_ARM_PREFETCH(addr); }
3932 
3933 // FIXME only store the 2 first elements ?
3934 template<> EIGEN_STRONG_INLINE double pfirst<Packet2d>(const Packet2d& a) { return vgetq_lane_f64(a,0); }
3935 
3936 template<> EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a)
3937 { return vcombine_f64(vget_high_f64(a), vget_low_f64(a)); }
3938 
3939 template<> EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a) { return vabsq_f64(a); }
3940 
3941 template <>
3942 EIGEN_STRONG_INLINE Packet2d psignbit(const Packet2d& a) {
3943  return vreinterpretq_f64_s64(vshrq_n_s64(vreinterpretq_s64_f64(a), 63));
3944 }
3945 
3946 template<> EIGEN_STRONG_INLINE double predux<Packet2d>(const Packet2d& a)
3947 { return vaddvq_f64(a); }
3948 
3949 // Other reduction functions:
3950 // mul
3951 #if EIGEN_COMP_CLANGAPPLE
3952 template<> EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a)
3953 { return (vget_low_f64(a) * vget_high_f64(a))[0]; }
3954 #else
3955 template<> EIGEN_STRONG_INLINE double predux_mul<Packet2d>(const Packet2d& a)
3956 { return vget_lane_f64(vmul_f64(vget_low_f64(a), vget_high_f64(a)), 0); }
3957 #endif
3958 
3959 // min
3960 template<> EIGEN_STRONG_INLINE double predux_min<Packet2d>(const Packet2d& a)
3961 { return vminvq_f64(a); }
3962 
3963 // max
3964 template<> EIGEN_STRONG_INLINE double predux_max<Packet2d>(const Packet2d& a)
3965 { return vmaxvq_f64(a); }
3966 
3967 
3968 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void
3969 ptranspose(PacketBlock<Packet2d, 2>& kernel)
3970 {
3971  const float64x2_t tmp1 = vzip1q_f64(kernel.packet[0], kernel.packet[1]);
3972  const float64x2_t tmp2 = vzip2q_f64(kernel.packet[0], kernel.packet[1]);
3973 
3974  kernel.packet[0] = tmp1;
3975  kernel.packet[1] = tmp2;
3976 }
3977 
3978 template<> EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2d pselect( const Packet2d& mask, const Packet2d& a, const Packet2d& b)
3979 { return vbslq_f64(vreinterpretq_u64_f64(mask), a, b); }
3980 
3981 template<> EIGEN_STRONG_INLINE Packet2d print<Packet2d>(const Packet2d& a)
3982 { return vrndnq_f64(a); }
3983 
3984 template<> EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(const Packet2d& a)
3985 { return vrndmq_f64(a); }
3986 
3987 template<> EIGEN_STRONG_INLINE Packet2d pceil<Packet2d>(const Packet2d& a)
3988 { return vrndpq_f64(a); }
3989 
3990 template<> EIGEN_STRONG_INLINE Packet2d pldexp<Packet2d>(const Packet2d& a, const Packet2d& exponent)
3991 { return pldexp_generic(a, exponent); }
3992 
3993 template<> EIGEN_STRONG_INLINE Packet2d pfrexp<Packet2d>(const Packet2d& a, Packet2d& exponent)
3994 { return pfrexp_generic(a,exponent); }
3995 
3996 template<> EIGEN_STRONG_INLINE Packet2d pset1frombits<Packet2d>(uint64_t from)
3997 { return vreinterpretq_f64_u64(vdupq_n_u64(from)); }
3998 
3999 template<> EIGEN_STRONG_INLINE Packet2d prsqrt(const Packet2d& a) {
4000  // Do Newton iterations for 1/sqrt(x).
4001  return generic_rsqrt_newton_step<Packet2d, /*Steps=*/3>::run(a, vrsqrteq_f64(a));
4002 }
4003 
4004 template<> EIGEN_STRONG_INLINE Packet2d psqrt(const Packet2d& _x){ return vsqrtq_f64(_x); }
4005 
4006 #endif // EIGEN_ARCH_ARM64 && !EIGEN_APPLE_DOUBLE_NEON_BUG
4007 
4008 // Do we have an fp16 types and supporting Neon intrinsics?
4009 #if EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC
4010 typedef float16x4_t Packet4hf;
4011 typedef float16x8_t Packet8hf;
4012 
4013 template <>
4014 struct packet_traits<Eigen::half> : default_packet_traits {
4015  typedef Packet8hf type;
4016  typedef Packet4hf half;
4017  enum {
4018  Vectorizable = 1,
4019  AlignedOnScalar = 1,
4020  size = 8,
4021 
4022  HasCmp = 1,
4023  HasCast = 1,
4024  HasAdd = 1,
4025  HasSub = 1,
4026  HasShift = 1,
4027  HasMul = 1,
4028  HasNegate = 1,
4029  HasAbs = 1,
4030  HasArg = 0,
4031  HasAbs2 = 1,
4032  HasAbsDiff = 0,
4033  HasMin = 1,
4034  HasMax = 1,
4035  HasConj = 1,
4036  HasSetLinear = 1,
4037  HasBlend = 0,
4038  HasInsert = 1,
4039  HasReduxp = 1,
4040  HasDiv = 1,
4041  HasFloor = 1,
4042  HasCeil = 1,
4043  HasRint = 1,
4044  HasSin = 0,
4045  HasCos = 0,
4046  HasLog = 0,
4047  HasExp = 0,
4048  HasTanh = packet_traits<float>::HasTanh, // tanh<half> calls tanh<float>
4049  HasSqrt = 1,
4050  HasRsqrt = 1,
4051  HasErf = EIGEN_FAST_MATH,
4052  HasBessel = 0, // Issues with accuracy.
4053  HasNdtri = 0
4054  };
4055 };
4056 
4057 template <>
4058 struct unpacket_traits<Packet4hf> {
4059  typedef Eigen::half type;
4060  typedef Packet4hf half;
4061  enum {
4062  size = 4,
4063  alignment = Aligned16,
4064  vectorizable = true,
4065  masked_load_available = false,
4066  masked_store_available = false
4067  };
4068 };
4069 
4070 template <>
4071 struct unpacket_traits<Packet8hf> {
4072  typedef Eigen::half type;
4073  typedef Packet4hf half;
4074  enum {
4075  size = 8,
4076  alignment = Aligned16,
4077  vectorizable = true,
4078  masked_load_available = false,
4079  masked_store_available = false
4080  };
4081 };
4082 
4083 template<>
4084 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf predux_half_dowto4<Packet8hf>(const Packet8hf& a) {
4085  return vadd_f16(vget_low_f16(a), vget_high_f16(a));
4086 }
4087 
4088 template <>
4089 EIGEN_STRONG_INLINE Packet8hf pset1<Packet8hf>(const Eigen::half& from) {
4090  return vdupq_n_f16(from.x);
4091 }
4092 
4093 template <>
4094 EIGEN_STRONG_INLINE Packet4hf pset1<Packet4hf>(const Eigen::half& from) {
4095  return vdup_n_f16(from.x);
4096 }
4097 
4098 template <>
4099 EIGEN_STRONG_INLINE Packet8hf plset<Packet8hf>(const Eigen::half& a) {
4100  const float16_t f[] = {0, 1, 2, 3, 4, 5, 6, 7};
4101  Packet8hf countdown = vld1q_f16(f);
4102  return vaddq_f16(pset1<Packet8hf>(a), countdown);
4103 }
4104 
4105 template <>
4106 EIGEN_STRONG_INLINE Packet4hf plset<Packet4hf>(const Eigen::half& a) {
4107  const float16_t f[] = {0, 1, 2, 3};
4108  Packet4hf countdown = vld1_f16(f);
4109  return vadd_f16(pset1<Packet4hf>(a), countdown);
4110 }
4111 
4112 template <>
4113 EIGEN_STRONG_INLINE Packet8hf padd<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
4114  return vaddq_f16(a, b);
4115 }
4116 
4117 template <>
4118 EIGEN_STRONG_INLINE Packet4hf padd<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
4119  return vadd_f16(a, b);
4120 }
4121 
4122 template <>
4123 EIGEN_STRONG_INLINE Packet8hf psub<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
4124  return vsubq_f16(a, b);
4125 }
4126 
4127 template <>
4128 EIGEN_STRONG_INLINE Packet4hf psub<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
4129  return vsub_f16(a, b);
4130 }
4131 
4132 template <>
4133 EIGEN_STRONG_INLINE Packet8hf pnegate(const Packet8hf& a) {
4134  return vnegq_f16(a);
4135 }
4136 
4137 template <>
4138 EIGEN_STRONG_INLINE Packet4hf pnegate(const Packet4hf& a) {
4139  return vneg_f16(a);
4140 }
4141 
4142 template <>
4143 EIGEN_STRONG_INLINE Packet8hf pconj(const Packet8hf& a) {
4144  return a;
4145 }
4146 
4147 template <>
4148 EIGEN_STRONG_INLINE Packet4hf pconj(const Packet4hf& a) {
4149  return a;
4150 }
4151 
4152 template <>
4153 EIGEN_STRONG_INLINE Packet8hf pmul<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
4154  return vmulq_f16(a, b);
4155 }
4156 
4157 template <>
4158 EIGEN_STRONG_INLINE Packet4hf pmul<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
4159  return vmul_f16(a, b);
4160 }
4161 
4162 template <>
4163 EIGEN_STRONG_INLINE Packet8hf pdiv<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
4164  return vdivq_f16(a, b);
4165 }
4166 
4167 template <>
4168 EIGEN_STRONG_INLINE Packet4hf pdiv<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
4169  return vdiv_f16(a, b);
4170 }
4171 
4172 template <>
4173 EIGEN_STRONG_INLINE Packet8hf pmadd(const Packet8hf& a, const Packet8hf& b, const Packet8hf& c) {
4174  return vfmaq_f16(c, a, b);
4175 }
4176 
4177 template <>
4178 EIGEN_STRONG_INLINE Packet4hf pmadd(const Packet4hf& a, const Packet4hf& b, const Packet4hf& c) {
4179  return vfma_f16(c, a, b);
4180 }
4181 
4182 template <>
4183 EIGEN_STRONG_INLINE Packet8hf pmin<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
4184  return vminq_f16(a, b);
4185 }
4186 
4187 template <>
4188 EIGEN_STRONG_INLINE Packet4hf pmin<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
4189  return vmin_f16(a, b);
4190 }
4191 
4192 #ifdef __ARM_FEATURE_NUMERIC_MAXMIN
4193 // numeric max and min are only available if ARM_FEATURE_NUMERIC_MAXMIN is defined (which can only be the case for Armv8 systems).
4194 template<> EIGEN_STRONG_INLINE Packet4hf pmin<PropagateNumbers, Packet4hf>(const Packet4hf& a, const Packet4hf& b) { return vminnm_f16(a, b); }
4195 template<> EIGEN_STRONG_INLINE Packet8hf pmin<PropagateNumbers, Packet8hf>(const Packet8hf& a, const Packet8hf& b) { return vminnmq_f16(a, b); }
4196 #endif
4197 
4198 template<> EIGEN_STRONG_INLINE Packet4hf pmin<PropagateNaN, Packet4hf>(const Packet4hf& a, const Packet4hf& b) { return pmin<Packet4hf>(a, b); }
4199 
4200 template<> EIGEN_STRONG_INLINE Packet8hf pmin<PropagateNaN, Packet8hf>(const Packet8hf& a, const Packet8hf& b) { return pmin<Packet8hf>(a, b); }
4201 
4202 template <>
4203 EIGEN_STRONG_INLINE Packet8hf pmax<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
4204  return vmaxq_f16(a, b);
4205 }
4206 
4207 template <>
4208 EIGEN_STRONG_INLINE Packet4hf pmax<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
4209  return vmax_f16(a, b);
4210 }
4211 
4212 #ifdef __ARM_FEATURE_NUMERIC_MAXMIN
4213 // numeric max and min are only available if ARM_FEATURE_NUMERIC_MAXMIN is defined (which can only be the case for Armv8 systems).
4214 template<> EIGEN_STRONG_INLINE Packet4hf pmax<PropagateNumbers, Packet4hf>(const Packet4hf& a, const Packet4hf& b) { return vmaxnm_f16(a, b); }
4215 template<> EIGEN_STRONG_INLINE Packet8hf pmax<PropagateNumbers, Packet8hf>(const Packet8hf& a, const Packet8hf& b) { return vmaxnmq_f16(a, b); }
4216 #endif
4217 
4218 template<> EIGEN_STRONG_INLINE Packet4hf pmax<PropagateNaN, Packet4hf>(const Packet4hf& a, const Packet4hf& b) { return pmax<Packet4hf>(a, b); }
4219 
4220 template<> EIGEN_STRONG_INLINE Packet8hf pmax<PropagateNaN, Packet8hf>(const Packet8hf& a, const Packet8hf& b) { return pmax<Packet8hf>(a, b); }
4221 
4222 #define EIGEN_MAKE_ARM_FP16_CMP_8(name) \
4223  template <> \
4224  EIGEN_STRONG_INLINE Packet8hf pcmp_##name(const Packet8hf& a, const Packet8hf& b) { \
4225  return vreinterpretq_f16_u16(vc##name##q_f16(a, b)); \
4226  }
4227 
4228 #define EIGEN_MAKE_ARM_FP16_CMP_4(name) \
4229  template <> \
4230  EIGEN_STRONG_INLINE Packet4hf pcmp_##name(const Packet4hf& a, const Packet4hf& b) { \
4231  return vreinterpret_f16_u16(vc##name##_f16(a, b)); \
4232  }
4233 
4234 EIGEN_MAKE_ARM_FP16_CMP_8(eq)
4235 EIGEN_MAKE_ARM_FP16_CMP_8(lt)
4236 EIGEN_MAKE_ARM_FP16_CMP_8(le)
4237 
4238 EIGEN_MAKE_ARM_FP16_CMP_4(eq)
4239 EIGEN_MAKE_ARM_FP16_CMP_4(lt)
4240 EIGEN_MAKE_ARM_FP16_CMP_4(le)
4241 
4242 #undef EIGEN_MAKE_ARM_FP16_CMP_8
4243 #undef EIGEN_MAKE_ARM_FP16_CMP_4
4244 
4245 template <>
4246 EIGEN_STRONG_INLINE Packet8hf pcmp_lt_or_nan<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
4247  return vreinterpretq_f16_u16(vmvnq_u16(vcgeq_f16(a, b)));
4248 }
4249 
4250 template <>
4251 EIGEN_STRONG_INLINE Packet4hf pcmp_lt_or_nan<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
4252  return vreinterpret_f16_u16(vmvn_u16(vcge_f16(a, b)));
4253 }
4254 
4255 template <>
4256 EIGEN_STRONG_INLINE Packet8hf print<Packet8hf>(const Packet8hf& a)
4257 { return vrndnq_f16(a); }
4258 
4259 template <>
4260 EIGEN_STRONG_INLINE Packet4hf print<Packet4hf>(const Packet4hf& a)
4261 { return vrndn_f16(a); }
4262 
4263 template <>
4264 EIGEN_STRONG_INLINE Packet8hf pfloor<Packet8hf>(const Packet8hf& a)
4265 { return vrndmq_f16(a); }
4266 
4267 template <>
4268 EIGEN_STRONG_INLINE Packet4hf pfloor<Packet4hf>(const Packet4hf& a)
4269 { return vrndm_f16(a); }
4270 
4271 template <>
4272 EIGEN_STRONG_INLINE Packet8hf pceil<Packet8hf>(const Packet8hf& a)
4273 { return vrndpq_f16(a); }
4274 
4275 template <>
4276 EIGEN_STRONG_INLINE Packet4hf pceil<Packet4hf>(const Packet4hf& a)
4277 { return vrndp_f16(a); }
4278 
4279 template <>
4280 EIGEN_STRONG_INLINE Packet8hf psqrt<Packet8hf>(const Packet8hf& a) {
4281  return vsqrtq_f16(a);
4282 }
4283 
4284 template <>
4285 EIGEN_STRONG_INLINE Packet4hf psqrt<Packet4hf>(const Packet4hf& a) {
4286  return vsqrt_f16(a);
4287 }
4288 
4289 template <>
4290 EIGEN_STRONG_INLINE Packet8hf pand<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
4291  return vreinterpretq_f16_u16(vandq_u16(vreinterpretq_u16_f16(a), vreinterpretq_u16_f16(b)));
4292 }
4293 
4294 template <>
4295 EIGEN_STRONG_INLINE Packet4hf pand<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
4296  return vreinterpret_f16_u16(vand_u16(vreinterpret_u16_f16(a), vreinterpret_u16_f16(b)));
4297 }
4298 
4299 template <>
4300 EIGEN_STRONG_INLINE Packet8hf por<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
4301  return vreinterpretq_f16_u16(vorrq_u16(vreinterpretq_u16_f16(a), vreinterpretq_u16_f16(b)));
4302 }
4303 
4304 template <>
4305 EIGEN_STRONG_INLINE Packet4hf por<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
4306  return vreinterpret_f16_u16(vorr_u16(vreinterpret_u16_f16(a), vreinterpret_u16_f16(b)));
4307 }
4308 
4309 template <>
4310 EIGEN_STRONG_INLINE Packet8hf pxor<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
4311  return vreinterpretq_f16_u16(veorq_u16(vreinterpretq_u16_f16(a), vreinterpretq_u16_f16(b)));
4312 }
4313 
4314 template <>
4315 EIGEN_STRONG_INLINE Packet4hf pxor<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
4316  return vreinterpret_f16_u16(veor_u16(vreinterpret_u16_f16(a), vreinterpret_u16_f16(b)));
4317 }
4318 
4319 template <>
4320 EIGEN_STRONG_INLINE Packet8hf pandnot<Packet8hf>(const Packet8hf& a, const Packet8hf& b) {
4321  return vreinterpretq_f16_u16(vbicq_u16(vreinterpretq_u16_f16(a), vreinterpretq_u16_f16(b)));
4322 }
4323 
4324 template <>
4325 EIGEN_STRONG_INLINE Packet4hf pandnot<Packet4hf>(const Packet4hf& a, const Packet4hf& b) {
4326  return vreinterpret_f16_u16(vbic_u16(vreinterpret_u16_f16(a), vreinterpret_u16_f16(b)));
4327 }
4328 
4329 template <>
4330 EIGEN_STRONG_INLINE Packet8hf pload<Packet8hf>(const Eigen::half* from) {
4331  EIGEN_DEBUG_ALIGNED_LOAD return vld1q_f16(reinterpret_cast<const float16_t*>(from));
4332 }
4333 
4334 template <>
4335 EIGEN_STRONG_INLINE Packet4hf pload<Packet4hf>(const Eigen::half* from) {
4336  EIGEN_DEBUG_ALIGNED_LOAD return vld1_f16(reinterpret_cast<const float16_t*>(from));
4337 }
4338 
4339 template <>
4340 EIGEN_STRONG_INLINE Packet8hf ploadu<Packet8hf>(const Eigen::half* from) {
4341  EIGEN_DEBUG_UNALIGNED_LOAD return vld1q_f16(reinterpret_cast<const float16_t*>(from));
4342 }
4343 
4344 template <>
4345 EIGEN_STRONG_INLINE Packet4hf ploadu<Packet4hf>(const Eigen::half* from) {
4346  EIGEN_DEBUG_UNALIGNED_LOAD return vld1_f16(reinterpret_cast<const float16_t*>(from));
4347 }
4348 
4349 template <>
4350 EIGEN_STRONG_INLINE Packet8hf ploaddup<Packet8hf>(const Eigen::half* from) {
4351  Packet8hf packet;
4352  packet[0] = from[0].x;
4353  packet[1] = from[0].x;
4354  packet[2] = from[1].x;
4355  packet[3] = from[1].x;
4356  packet[4] = from[2].x;
4357  packet[5] = from[2].x;
4358  packet[6] = from[3].x;
4359  packet[7] = from[3].x;
4360  return packet;
4361 }
4362 
4363 template <>
4364 EIGEN_STRONG_INLINE Packet4hf ploaddup<Packet4hf>(const Eigen::half* from) {
4365  float16x4_t packet;
4366  float16_t* tmp;
4367  tmp = (float16_t*)&packet;
4368  tmp[0] = from[0].x;
4369  tmp[1] = from[0].x;
4370  tmp[2] = from[1].x;
4371  tmp[3] = from[1].x;
4372  return packet;
4373 }
4374 
4375 template <>
4376 EIGEN_STRONG_INLINE Packet8hf ploadquad<Packet8hf>(const Eigen::half* from) {
4377  Packet4hf lo, hi;
4378  lo = vld1_dup_f16(reinterpret_cast<const float16_t*>(from));
4379  hi = vld1_dup_f16(reinterpret_cast<const float16_t*>(from+1));
4380  return vcombine_f16(lo, hi);
4381 }
4382 
4383 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pinsertfirst(const Packet8hf& a, Eigen::half b) { return vsetq_lane_f16(b.x, a, 0); }
4384 
4385 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pinsertfirst(const Packet4hf& a, Eigen::half b) { return vset_lane_f16(b.x, a, 0); }
4386 
4387 template <>
4388 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pselect(const Packet8hf& mask, const Packet8hf& a, const Packet8hf& b) {
4389  return vbslq_f16(vreinterpretq_u16_f16(mask), a, b);
4390 }
4391 
4392 template <>
4393 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pselect(const Packet4hf& mask, const Packet4hf& a, const Packet4hf& b) {
4394  return vbsl_f16(vreinterpret_u16_f16(mask), a, b);
4395 }
4396 
4397 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pinsertlast(const Packet8hf& a, Eigen::half b) { return vsetq_lane_f16(b.x, a, 7); }
4398 
4399 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pinsertlast(const Packet4hf& a, Eigen::half b) { return vset_lane_f16(b.x, a, 3); }
4400 
4401 template <>
4402 EIGEN_STRONG_INLINE void pstore<Eigen::half>(Eigen::half* to, const Packet8hf& from) {
4403  EIGEN_DEBUG_ALIGNED_STORE vst1q_f16(reinterpret_cast<float16_t*>(to), from);
4404 }
4405 
4406 template <>
4407 EIGEN_STRONG_INLINE void pstore<Eigen::half>(Eigen::half* to, const Packet4hf& from) {
4408  EIGEN_DEBUG_ALIGNED_STORE vst1_f16(reinterpret_cast<float16_t*>(to), from);
4409 }
4410 
4411 template <>
4412 EIGEN_STRONG_INLINE void pstoreu<Eigen::half>(Eigen::half* to, const Packet8hf& from) {
4413  EIGEN_DEBUG_UNALIGNED_STORE vst1q_f16(reinterpret_cast<float16_t*>(to), from);
4414 }
4415 
4416 template <>
4417 EIGEN_STRONG_INLINE void pstoreu<Eigen::half>(Eigen::half* to, const Packet4hf& from) {
4418  EIGEN_DEBUG_UNALIGNED_STORE vst1_f16(reinterpret_cast<float16_t*>(to), from);
4419 }
4420 
4421 template <>
4422 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf pgather<Eigen::half, Packet8hf>(const Eigen::half* from, Index stride) {
4423  Packet8hf res = pset1<Packet8hf>(Eigen::half(0.f));
4424  res = vsetq_lane_f16(from[0 * stride].x, res, 0);
4425  res = vsetq_lane_f16(from[1 * stride].x, res, 1);
4426  res = vsetq_lane_f16(from[2 * stride].x, res, 2);
4427  res = vsetq_lane_f16(from[3 * stride].x, res, 3);
4428  res = vsetq_lane_f16(from[4 * stride].x, res, 4);
4429  res = vsetq_lane_f16(from[5 * stride].x, res, 5);
4430  res = vsetq_lane_f16(from[6 * stride].x, res, 6);
4431  res = vsetq_lane_f16(from[7 * stride].x, res, 7);
4432  return res;
4433 }
4434 
4435 template <>
4436 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf pgather<Eigen::half, Packet4hf>(const Eigen::half* from, Index stride) {
4437  Packet4hf res = pset1<Packet4hf>(Eigen::half(0.f));
4438  res = vset_lane_f16(from[0 * stride].x, res, 0);
4439  res = vset_lane_f16(from[1 * stride].x, res, 1);
4440  res = vset_lane_f16(from[2 * stride].x, res, 2);
4441  res = vset_lane_f16(from[3 * stride].x, res, 3);
4442  return res;
4443 }
4444 
4445 template <>
4446 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<Eigen::half, Packet8hf>(Eigen::half* to, const Packet8hf& from, Index stride) {
4447  to[stride * 0].x = vgetq_lane_f16(from, 0);
4448  to[stride * 1].x = vgetq_lane_f16(from, 1);
4449  to[stride * 2].x = vgetq_lane_f16(from, 2);
4450  to[stride * 3].x = vgetq_lane_f16(from, 3);
4451  to[stride * 4].x = vgetq_lane_f16(from, 4);
4452  to[stride * 5].x = vgetq_lane_f16(from, 5);
4453  to[stride * 6].x = vgetq_lane_f16(from, 6);
4454  to[stride * 7].x = vgetq_lane_f16(from, 7);
4455 }
4456 
4457 template <>
4458 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter<Eigen::half, Packet4hf>(Eigen::half* to, const Packet4hf& from, Index stride) {
4459  to[stride * 0].x = vget_lane_f16(from, 0);
4460  to[stride * 1].x = vget_lane_f16(from, 1);
4461  to[stride * 2].x = vget_lane_f16(from, 2);
4462  to[stride * 3].x = vget_lane_f16(from, 3);
4463 }
4464 
4465 template <>
4466 EIGEN_STRONG_INLINE void prefetch<Eigen::half>(const Eigen::half* addr) {
4467  EIGEN_ARM_PREFETCH(addr);
4468 }
4469 
4470 template <>
4471 EIGEN_STRONG_INLINE Eigen::half pfirst<Packet8hf>(const Packet8hf& a) {
4472  float16_t x[8];
4473  vst1q_f16(x, a);
4474  Eigen::half h;
4475  h.x = x[0];
4476  return h;
4477 }
4478 
4479 template <>
4480 EIGEN_STRONG_INLINE Eigen::half pfirst<Packet4hf>(const Packet4hf& a) {
4481  float16_t x[4];
4482  vst1_f16(x, a);
4483  Eigen::half h;
4484  h.x = x[0];
4485  return h;
4486 }
4487 
4488 template<> EIGEN_STRONG_INLINE Packet8hf preverse(const Packet8hf& a) {
4489  float16x4_t a_lo, a_hi;
4490  Packet8hf a_r64;
4491 
4492  a_r64 = vrev64q_f16(a);
4493  a_lo = vget_low_f16(a_r64);
4494  a_hi = vget_high_f16(a_r64);
4495  return vcombine_f16(a_hi, a_lo);
4496 }
4497 
4498 template <>
4499 EIGEN_STRONG_INLINE Packet4hf preverse<Packet4hf>(const Packet4hf& a) {
4500  return vrev64_f16(a);
4501 }
4502 
4503 template <>
4504 EIGEN_STRONG_INLINE Packet8hf pabs<Packet8hf>(const Packet8hf& a) {
4505  return vabsq_f16(a);
4506 }
4507 
4508 template<>
4509 EIGEN_STRONG_INLINE Packet8hf psignbit(const Packet8hf& a) {
4510  return vreinterpretq_f16_s16(vshrq_n_s16(vreinterpretq_s16_f16(a), 15));
4511 }
4512 
4513 template <>
4514 EIGEN_STRONG_INLINE Packet4hf pabs<Packet4hf>(const Packet4hf& a) {
4515  return vabs_f16(a);
4516 }
4517 
4518 template <>
4519 EIGEN_STRONG_INLINE Packet4hf psignbit(const Packet4hf& a) {
4520  return vreinterpret_f16_s16( vshr_n_s16( vreinterpret_s16_f16(a), 15));
4521 }
4522 
4523 template <>
4524 EIGEN_STRONG_INLINE Eigen::half predux<Packet8hf>(const Packet8hf& a) {
4525  float16x4_t a_lo, a_hi, sum;
4526 
4527  a_lo = vget_low_f16(a);
4528  a_hi = vget_high_f16(a);
4529  sum = vpadd_f16(a_lo, a_hi);
4530  sum = vpadd_f16(sum, sum);
4531  sum = vpadd_f16(sum, sum);
4532 
4533  Eigen::half h;
4534  h.x = vget_lane_f16(sum, 0);
4535  return h;
4536 }
4537 
4538 template <>
4539 EIGEN_STRONG_INLINE Eigen::half predux<Packet4hf>(const Packet4hf& a) {
4540  float16x4_t sum;
4541 
4542  sum = vpadd_f16(a, a);
4543  sum = vpadd_f16(sum, sum);
4544  Eigen::half h;
4545  h.x = vget_lane_f16(sum, 0);
4546  return h;
4547 }
4548 
4549 template <>
4550 EIGEN_STRONG_INLINE Eigen::half predux_mul<Packet8hf>(const Packet8hf& a) {
4551  float16x4_t a_lo, a_hi, prod;
4552 
4553  a_lo = vget_low_f16(a);
4554  a_hi = vget_high_f16(a);
4555  prod = vmul_f16(a_lo, a_hi);
4556  prod = vmul_f16(prod, vrev64_f16(prod));
4557 
4558  Eigen::half h;
4559  h.x = vmulh_f16(vget_lane_f16(prod, 0), vget_lane_f16(prod, 1));
4560  return h;
4561 }
4562 
4563 template <>
4564 EIGEN_STRONG_INLINE Eigen::half predux_mul<Packet4hf>(const Packet4hf& a) {
4565  float16x4_t prod;
4566  prod = vmul_f16(a, vrev64_f16(a));
4567  Eigen::half h;
4568  h.x = vmulh_f16(vget_lane_f16(prod, 0), vget_lane_f16(prod, 1));
4569  return h;
4570 }
4571 
4572 template <>
4573 EIGEN_STRONG_INLINE Eigen::half predux_min<Packet8hf>(const Packet8hf& a) {
4574  Eigen::half h;
4575  h.x = vminvq_f16(a);
4576  return h;
4577 }
4578 
4579 template <>
4580 EIGEN_STRONG_INLINE Eigen::half predux_min<Packet4hf>(const Packet4hf& a) {
4581  Eigen::half h;
4582  h.x = vminv_f16(a);
4583  return h;
4584 }
4585 
4586 template <>
4587 EIGEN_STRONG_INLINE Eigen::half predux_max<Packet8hf>(const Packet8hf& a) {
4588  Eigen::half h;
4589  h.x = vmaxvq_f16(a);
4590  return h;
4591 }
4592 
4593 template <>
4594 EIGEN_STRONG_INLINE Eigen::half predux_max<Packet4hf>(const Packet4hf& a) {
4595  Eigen::half h;
4596  h.x = vmaxv_f16(a);
4597  return h;
4598 }
4599 
4600 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8hf, 4>& kernel)
4601 {
4602  const float16x8x2_t zip16_1 = vzipq_f16(kernel.packet[0], kernel.packet[1]);
4603  const float16x8x2_t zip16_2 = vzipq_f16(kernel.packet[2], kernel.packet[3]);
4604 
4605  const float32x4x2_t zip32_1 = vzipq_f32(vreinterpretq_f32_f16(zip16_1.val[0]), vreinterpretq_f32_f16(zip16_2.val[0]));
4606  const float32x4x2_t zip32_2 = vzipq_f32(vreinterpretq_f32_f16(zip16_1.val[1]), vreinterpretq_f32_f16(zip16_2.val[1]));
4607 
4608  kernel.packet[0] = vreinterpretq_f16_f32(zip32_1.val[0]);
4609  kernel.packet[1] = vreinterpretq_f16_f32(zip32_1.val[1]);
4610  kernel.packet[2] = vreinterpretq_f16_f32(zip32_2.val[0]);
4611  kernel.packet[3] = vreinterpretq_f16_f32(zip32_2.val[1]);
4612 }
4613 
4614 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet4hf, 4>& kernel) {
4615  EIGEN_ALIGN16 float16x4x4_t tmp_x4;
4616  float16_t* tmp = (float16_t*)&kernel;
4617  tmp_x4 = vld4_f16(tmp);
4618 
4619  kernel.packet[0] = tmp_x4.val[0];
4620  kernel.packet[1] = tmp_x4.val[1];
4621  kernel.packet[2] = tmp_x4.val[2];
4622  kernel.packet[3] = tmp_x4.val[3];
4623 }
4624 
4625 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet8hf, 8>& kernel) {
4626  float16x8x2_t T_1[4];
4627 
4628  T_1[0] = vuzpq_f16(kernel.packet[0], kernel.packet[1]);
4629  T_1[1] = vuzpq_f16(kernel.packet[2], kernel.packet[3]);
4630  T_1[2] = vuzpq_f16(kernel.packet[4], kernel.packet[5]);
4631  T_1[3] = vuzpq_f16(kernel.packet[6], kernel.packet[7]);
4632 
4633  float16x8x2_t T_2[4];
4634  T_2[0] = vuzpq_f16(T_1[0].val[0], T_1[1].val[0]);
4635  T_2[1] = vuzpq_f16(T_1[0].val[1], T_1[1].val[1]);
4636  T_2[2] = vuzpq_f16(T_1[2].val[0], T_1[3].val[0]);
4637  T_2[3] = vuzpq_f16(T_1[2].val[1], T_1[3].val[1]);
4638 
4639  float16x8x2_t T_3[4];
4640  T_3[0] = vuzpq_f16(T_2[0].val[0], T_2[2].val[0]);
4641  T_3[1] = vuzpq_f16(T_2[0].val[1], T_2[2].val[1]);
4642  T_3[2] = vuzpq_f16(T_2[1].val[0], T_2[3].val[0]);
4643  T_3[3] = vuzpq_f16(T_2[1].val[1], T_2[3].val[1]);
4644 
4645  kernel.packet[0] = T_3[0].val[0];
4646  kernel.packet[1] = T_3[2].val[0];
4647  kernel.packet[2] = T_3[1].val[0];
4648  kernel.packet[3] = T_3[3].val[0];
4649  kernel.packet[4] = T_3[0].val[1];
4650  kernel.packet[5] = T_3[2].val[1];
4651  kernel.packet[6] = T_3[1].val[1];
4652  kernel.packet[7] = T_3[3].val[1];
4653 }
4654 #endif // end EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC
4655 
4656 } // end namespace internal
4657 
4658 } // end namespace Eigen
4659 
4660 #endif // EIGEN_PACKET_MATH_NEON_H
m
Matrix3f m
Definition: AngleAxis_mimic_euler.cpp:1
abs
const AbsReturnType abs() const
Definition: ArrayCwiseUnaryOps.h:52
a
ArrayXXi a
Definition: Array_initializer_list_23_cxx11.cpp:1
b
Array< int, 3, 1 > b
Definition: Array_variadic_ctor_cxx11.cpp:2
n
int n
Definition: BiCGSTAB_simple.cpp:1
x
x
Definition: BiCGSTAB_simple.cpp:7
i
int i
Definition: BiCGSTAB_step_by_step.cpp:9
EIGEN_ALIGN16
#define EIGEN_ALIGN16
Definition: ConfigureVectorization.h:133
c
Array33i c
Definition: Cwise_product.cpp:2
EIGEN_DEBUG_ALIGNED_STORE
#define EIGEN_DEBUG_ALIGNED_STORE
Definition: GenericPacketMath.h:37
EIGEN_DEBUG_ALIGNED_LOAD
#define EIGEN_DEBUG_ALIGNED_LOAD
Definition: GenericPacketMath.h:29
EIGEN_DEBUG_UNALIGNED_STORE
#define EIGEN_DEBUG_UNALIGNED_STORE
Definition: GenericPacketMath.h:41
EIGEN_DEBUG_UNALIGNED_LOAD
#define EIGEN_DEBUG_UNALIGNED_LOAD
Definition: GenericPacketMath.h:33
EIGEN_ALWAYS_INLINE
#define EIGEN_ALWAYS_INLINE
Definition: Macros.h:836
EIGEN_UNROLL_LOOP
#define EIGEN_UNROLL_LOOP
Definition: Macros.h:1290
EIGEN_DEVICE_FUNC
#define EIGEN_DEVICE_FUNC
Definition: Macros.h:883
eigen_assert
#define eigen_assert(x)
Definition: Macros.h:902
EIGEN_FAST_MATH
#define EIGEN_FAST_MATH
Definition: Macros.h:50
EIGEN_OPTIMIZATION_BARRIER
#define EIGEN_OPTIMIZATION_BARRIER(X)
Definition: Macros.h:1039
p1
Vector3f p1
Definition: MatrixBase_all.cpp:2
EIGEN_ARM_PREFETCH
#define EIGEN_ARM_PREFETCH(ADDR)
Definition: NEON/PacketMath.h:177
res
cout<< "Here is the matrix m:"<< endl<< m<< endl;Matrix< ptrdiff_t, 3, 1 > res
Definition: PartialRedux_count.cpp:3
vec2d_swizzle2
#define vec2d_swizzle2(a, b, mask)
Definition: SSE/PacketMath.h:109
size
const int size
Definition: Tutorial_AdvancedInitialization_ThreeWays.cpp:1
p
float * p
Definition: Tutorial_Map_using.cpp:9
Eigen::Unaligned
@ Unaligned
Definition: Constants.h:235
Eigen::Aligned16
@ Aligned16
Definition: Constants.h:237
Eigen::bfloat16_impl::max
bfloat16() max(const bfloat16 &a, const bfloat16 &b)
Definition: BFloat16.h:690
Eigen::bfloat16_impl::raw_uint16_to_bfloat16
EIGEN_CONSTEXPR __bfloat16_raw raw_uint16_to_bfloat16(unsigned short value)
Eigen::bfloat16_impl::min
bfloat16() min(const bfloat16 &a, const bfloat16 &b)
Definition: BFloat16.h:684
Eigen::internal::pdiv< Packet2ui >
Packet2ui pdiv< Packet2ui >(const Packet2ui &, const Packet2ui &)
Definition: NEON/PacketMath.h:1069
Eigen::internal::pset1< Packet4uc >
Packet4uc pset1< Packet4uc >(const uint8_t &from)
Definition: NEON/PacketMath.h:708
Eigen::internal::pcmp_lt< Packet4f >
Packet4f pcmp_lt< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition: NEON/PacketMath.h:1347
Eigen::internal::predux_max< Packet4s >
int16_t predux_max< Packet4s >(const Packet4s &a)
Definition: NEON/PacketMath.h:2810
Eigen::internal::pfirst< Packet16c >
signed char pfirst< Packet16c >(const Packet16c &a)
Definition: AltiVec/PacketMath.h:1532
Eigen::internal::pand< Packet8us >
Packet8us pand< Packet8us >(const Packet8us &a, const Packet8us &b)
Definition: AltiVec/PacketMath.h:1157
Eigen::internal::pand< Packet4c >
Packet4c pand< Packet4c >(const Packet4c &a, const Packet4c &b)
Definition: NEON/PacketMath.h:1477
Eigen::internal::pdiv< Packet2d >
Packet2d pdiv< Packet2d >(const Packet2d &a, const Packet2d &b)
Definition: MSA/PacketMath.h:922
Eigen::internal::pstoreu< int8_t >
void pstoreu< int8_t >(int8_t *to, const Packet4c &from)
Definition: NEON/PacketMath.h:1918
Eigen::internal::pxor< Packet8uc >
Packet8uc pxor< Packet8uc >(const Packet8uc &a, const Packet8uc &b)
Definition: NEON/PacketMath.h:1551
Eigen::internal::pcmp_eq< Packet4s >
Packet4s pcmp_eq< Packet4s >(const Packet4s &a, const Packet4s &b)
Definition: NEON/PacketMath.h:1430
Eigen::internal::prefetch< int8_t >
void prefetch< int8_t >(const int8_t *addr)
Definition: NEON/PacketMath.h:2278
Eigen::internal::pabsdiff< Packet16c >
Packet16c pabsdiff< Packet16c >(const Packet16c &a, const Packet16c &b)
Definition: NEON/PacketMath.h:1159
Eigen::internal::psub< Packet16uc >
Packet16uc psub< Packet16uc >(const Packet16uc &a, const Packet16uc &b)
Definition: AltiVec/PacketMath.h:994
Eigen::internal::pandnot< Packet8us >
Packet8us pandnot< Packet8us >(const Packet8us &a, const Packet8us &b)
Definition: NEON/PacketMath.h:1592
Eigen::internal::padd
Packet padd(const Packet &a, const Packet &b)
Definition: GenericPacketMath.h:308
Eigen::internal::ploadu< Packet16uc >
Packet16uc ploadu< Packet16uc >(const unsigned char *from)
Definition: AltiVec/PacketMath.h:1258
Eigen::internal::pmin< PropagateNumbers, Packet4f >
Packet4f pmin< PropagateNumbers, Packet4f >(const Packet4f &a, const Packet4f &b)
Definition: SSE/PacketMath.h:698
Eigen::internal::pcmp_lt< Packet2ul >
Packet2ul pcmp_lt< Packet2ul >(const Packet2ul &a, const Packet2ul &b)
Definition: NEON/PacketMath.h:1395
Eigen::internal::pfirst< Packet2ui >
uint32_t pfirst< Packet2ui >(const Packet2ui &a)
Definition: NEON/PacketMath.h:2301
Eigen::internal::pload< Packet2i >
Packet2i pload< Packet2i >(const int32_t *from)
Definition: NEON/PacketMath.h:1704
Eigen::internal::pmax< Packet8s >
Packet8s pmax< Packet8s >(const Packet8s &a, const Packet8s &b)
Definition: AltiVec/PacketMath.h:1112
Eigen::internal::pfirst< Packet2ul >
uint64_t pfirst< Packet2ul >(const Packet2ul &a)
Definition: NEON/PacketMath.h:2304
Eigen::internal::pand< Packet2l >
Packet2l pand< Packet2l >(const Packet2l &a, const Packet2l &b)
Definition: NEON/PacketMath.h:1501
Eigen::internal::pscatter< uint16_t, Packet8us >
void pscatter< uint16_t, Packet8us >(uint16_t *to, const Packet8us &from, Index stride)
Definition: NEON/PacketMath.h:2231
Eigen::internal::predux_max< Packet2d >
double predux_max< Packet2d >(const Packet2d &a)
Definition: MSA/PacketMath.h:1115
Eigen::internal::predux_min< Packet4uc >
uint8_t predux_min< Packet4uc >(const Packet4uc &a)
Definition: NEON/PacketMath.h:2658
Eigen::internal::plset< Packet8uc >
Packet8uc plset< Packet8uc >(const uint8_t &a)
Definition: NEON/PacketMath.h:752
Eigen::internal::pzero
Packet8f pzero(const Packet8f &)
Definition: AVX/PacketMath.h:677
Eigen::internal::por< Packet2l >
Packet2l por< Packet2l >(const Packet2l &a, const Packet2l &b)
Definition: NEON/PacketMath.h:1534
Eigen::internal::pstore< float >
void pstore< float >(float *to, const Packet4f &from)
Definition: AltiVec/PacketMath.h:611
Eigen::internal::pxor< Packet4ui >
Packet4ui pxor< Packet4ui >(const Packet4ui &a, const Packet4ui &b)
Definition: NEON/PacketMath.h:1565
Eigen::internal::pandnot< Packet16uc >
Packet16uc pandnot< Packet16uc >(const Packet16uc &a, const Packet16uc &b)
Definition: NEON/PacketMath.h:1584
Eigen::internal::pgather< uint16_t, Packet8us >
Packet8us pgather< uint16_t, Packet8us >(const uint16_t *from, Index stride)
Definition: NEON/PacketMath.h:2071
Eigen::internal::pscatter< int64_t, Packet2l >
void pscatter< int64_t, Packet2l >(int64_t *to, const Packet2l &from, Index stride)
Definition: NEON/PacketMath.h:2266
Eigen::internal::pfirst< Packet4ui >
uint32_t pfirst< Packet4ui >(const Packet4ui &a)
Definition: NEON/PacketMath.h:2302
Eigen::internal::pmul< Packet2i >
Packet2i pmul< Packet2i >(const Packet2i &a, const Packet2i &b)
Definition: NEON/PacketMath.h:943
Eigen::internal::pcmp_le< Packet4us >
Packet4us pcmp_le< Packet4us >(const Packet4us &a, const Packet4us &b)
Definition: NEON/PacketMath.h:1312
Eigen::internal::ploaddup< Packet4s >
Packet4s ploaddup< Packet4s >(const int16_t *from)
Definition: NEON/PacketMath.h:1798
Eigen::internal::psub< Packet8c >
Packet8c psub< Packet8c >(const Packet8c &a, const Packet8c &b)
Definition: NEON/PacketMath.h:850
Eigen::internal::pand< Packet16uc >
Packet16uc pand< Packet16uc >(const Packet16uc &a, const Packet16uc &b)
Definition: NEON/PacketMath.h:1487
Eigen::internal::pgather< int8_t, Packet8c >
Packet8c pgather< int8_t, Packet8c >(const int8_t *from, Index stride)
Definition: NEON/PacketMath.h:1972
Eigen::internal::ploaddup< Packet2l >
Packet2l ploaddup< Packet2l >(const int64_t *from)
Definition: NEON/PacketMath.h:1828
Eigen::internal::pset1frombits< Packet2d >
Packet2d pset1frombits< Packet2d >(uint64_t from)
Definition: SSE/PacketMath.h:302
Eigen::internal::pset1< Packet4c >
Packet4c pset1< Packet4c >(const int8_t &from)
Definition: NEON/PacketMath.h:704
Eigen::internal::predux_max< Packet4f >
float predux_max< Packet4f >(const Packet4f &a)
Definition: AltiVec/PacketMath.h:2275
Eigen::internal::predux_min< Packet2d >
double predux_min< Packet2d >(const Packet2d &a)
Definition: MSA/PacketMath.h:1100
Eigen::internal::pstoreu< int32_t >
void pstoreu< int32_t >(int32_t *to, const Packet4i &from)
Definition: MSA/PacketMath.h:450
Eigen::internal::plset< Packet2d >
Packet2d plset< Packet2d >(const double &a)
Definition: MSA/PacketMath.h:886
Eigen::internal::ploadquad< Packet4c >
Packet4c ploadquad< Packet4c >(const int8_t *from)
Definition: NEON/PacketMath.h:1834
Eigen::internal::Packet2ui
uint32x2_t Packet2ui
Definition: NEON/PacketMath.h:88
Eigen::internal::psub< Packet4s >
Packet4s psub< Packet4s >(const Packet4s &a, const Packet4s &b)
Definition: NEON/PacketMath.h:860
Eigen::internal::plset< Packet2l >
Packet2l plset< Packet2l >(const int64_t &a)
Definition: NEON/PacketMath.h:802
Eigen::internal::plset< Packet2ul >
Packet2ul plset< Packet2ul >(const uint64_t &a)
Definition: NEON/PacketMath.h:807
Eigen::internal::pabsdiff< Packet4bf >
Packet4bf pabsdiff< Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition: NEON/PacketMath.h:3669
Eigen::internal::pand< Packet2ul >
Packet2ul pand< Packet2ul >(const Packet2ul &a, const Packet2ul &b)
Definition: NEON/PacketMath.h:1502
Eigen::internal::Packet4i
__vector int Packet4i
Definition: AltiVec/PacketMath.h:33
Eigen::internal::pfirst< Packet4us >
uint16_t pfirst< Packet4us >(const Packet4us &a)
Definition: NEON/PacketMath.h:2297
Eigen::internal::predux_mul< Packet2i >
int32_t predux_mul< Packet2i >(const Packet2i &a)
Definition: NEON/PacketMath.h:2605
Eigen::internal::pmin< PropagateNaN, Packet4f >
Packet4f pmin< PropagateNaN, Packet4f >(const Packet4f &a, const Packet4f &b)
Definition: NEON/PacketMath.h:1197
Eigen::internal::pcmp_le< Packet4bf >
Packet4bf pcmp_le< Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition: NEON/PacketMath.h:3689
Eigen::internal::pcmp_le< Packet4ui >
Packet4ui pcmp_le< Packet4ui >(const Packet4ui &a, const Packet4ui &b)
Definition: NEON/PacketMath.h:1322
Eigen::internal::pand< Packet4s >
Packet4s pand< Packet4s >(const Packet4s &a, const Packet4s &b)
Definition: NEON/PacketMath.h:1489
Eigen::internal::plset< Packet8c >
Packet8c plset< Packet8c >(const int8_t &a)
Definition: NEON/PacketMath.h:740
Eigen::internal::pxor< Packet4s >
Packet4s pxor< Packet4s >(const Packet4s &a, const Packet4s &b)
Definition: NEON/PacketMath.h:1555
Eigen::internal::pmin< Packet4c >
Packet4c pmin< Packet4c >(const Packet4c &a, const Packet4c &b)
Definition: NEON/PacketMath.h:1201
Eigen::internal::pmul< Packet4c >
Packet4c pmul< Packet4c >(const Packet4c &a, const Packet4c &b)
Definition: NEON/PacketMath.h:923
Eigen::internal::pgather< uint8_t, Packet16uc >
Packet16uc pgather< uint8_t, Packet16uc >(const uint8_t *from, Index stride)
Definition: NEON/PacketMath.h:2023
Eigen::internal::pmin< Packet2d >
Packet2d pmin< Packet2d >(const Packet2d &a, const Packet2d &b)
Definition: MSA/PacketMath.h:973
Eigen::internal::pmul< Packet2d >
Packet2d pmul< Packet2d >(const Packet2d &a, const Packet2d &b)
Definition: MSA/PacketMath.h:915
Eigen::internal::eigen_neon_shuffle_mask
static int eigen_neon_shuffle_mask(int p, int q, int r, int s)
Definition: NEON/PacketMath.h:126
Eigen::internal::por< Packet4uc >
Packet4uc por< Packet4uc >(const Packet4uc &a, const Packet4uc &b)
Definition: NEON/PacketMath.h:1514
Eigen::internal::pmax< Packet2f >
Packet2f pmax< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition: NEON/PacketMath.h:1236
Eigen::internal::pload< Packet16c >
Packet16c pload< Packet16c >(const signed char *from)
Definition: AltiVec/PacketMath.h:481
Eigen::internal::pand< Packet8c >
Packet8c pand< Packet8c >(const Packet8c &a, const Packet8c &b)
Definition: NEON/PacketMath.h:1479
Eigen::internal::pcmp_lt_or_nan
Packet4f pcmp_lt_or_nan(const Packet4f &a, const Packet4f &b)
Definition: AltiVec/PacketMath.h:1123
Eigen::internal::pscatter< float, Packet2f >
void pscatter< float, Packet2f >(float *to, const Packet2f &from, Index stride)
Definition: NEON/PacketMath.h:2124
Eigen::internal::pabsdiff< Packet4ui >
Packet4ui pabsdiff< Packet4ui >(const Packet4ui &a, const Packet4ui &b)
Definition: NEON/PacketMath.h:1185
Eigen::internal::pgather< uint8_t, Packet8uc >
Packet8uc pgather< uint8_t, Packet8uc >(const uint8_t *from, Index stride)
Definition: NEON/PacketMath.h:2011
Eigen::internal::pgather< uint8_t, Packet4uc >
Packet4uc pgather< uint8_t, Packet4uc >(const uint8_t *from, Index stride)
Definition: NEON/PacketMath.h:2004
Eigen::internal::vec4f_unpackhi
Packet4f vec4f_unpackhi(const Packet4f &a, const Packet4f &b)
Definition: NEON/PacketMath.h:148
Eigen::internal::pgather< uint64_t, Packet2ul >
Packet2ul pgather< uint64_t, Packet2ul >(const uint64_t *from, Index stride)
Definition: NEON/PacketMath.h:2117
Eigen::internal::predux< Packet4c >
int8_t predux< Packet4c >(const Packet4c &a)
Definition: NEON/PacketMath.h:2415
Eigen::internal::predux_mul< Packet4s >
int16_t predux_mul< Packet4s >(const Packet4s &a)
Definition: NEON/PacketMath.h:2573
Eigen::internal::predux< Packet2ul >
uint64_t predux< Packet2ul >(const Packet2ul &a)
Definition: NEON/PacketMath.h:2516
Eigen::internal::pmin< Packet2ul >
Packet2ul pmin< Packet2ul >(const Packet2ul &a, const Packet2ul &b)
Definition: NEON/PacketMath.h:1230
Eigen::internal::pmax< Packet8c >
Packet8c pmax< Packet8c >(const Packet8c &a, const Packet8c &b)
Definition: NEON/PacketMath.h:1255
Eigen::internal::prefetch< int64_t >
void prefetch< int64_t >(const int64_t *addr)
Definition: NEON/PacketMath.h:2284
Eigen::internal::pmax< Packet4uc >
Packet4uc pmax< Packet4uc >(const Packet4uc &a, const Packet4uc &b)
Definition: NEON/PacketMath.h:1257
Eigen::internal::pmax< Packet4us >
Packet4us pmax< Packet4us >(const Packet4us &a, const Packet4us &b)
Definition: NEON/PacketMath.h:1267
Eigen::internal::pxor< Packet4f >
Packet4f pxor< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition: AltiVec/PacketMath.h:1171
Eigen::internal::pfrexp< Packet2d >
Packet2d pfrexp< Packet2d >(const Packet2d &a, Packet2d &exponent)
Definition: SSE/PacketMath.h:1108
Eigen::internal::Packet16uc
__vector unsigned char Packet16uc
Definition: AltiVec/PacketMath.h:40
Eigen::internal::F32ToBf16
Packet8bf F32ToBf16(Packet4f p4f)
Definition: AltiVec/PacketMath.h:1675
Eigen::internal::pmax< Packet4c >
Packet4c pmax< Packet4c >(const Packet4c &a, const Packet4c &b)
Definition: NEON/PacketMath.h:1249
Eigen::internal::pcmp_le< Packet8uc >
Packet8uc pcmp_le< Packet8uc >(const Packet8uc &a, const Packet8uc &b)
Definition: NEON/PacketMath.h:1304
Eigen::internal::ploaddup< Packet4us >
Packet4us ploaddup< Packet4us >(const uint16_t *from)
Definition: NEON/PacketMath.h:1809
Eigen::internal::vec4f_unpacklo
Packet4f vec4f_unpacklo(const Packet4f &a, const Packet4f &b)
Definition: NEON/PacketMath.h:144
Eigen::internal::pldexp< Packet2f >
Packet2f pldexp< Packet2f >(const Packet2f &a, const Packet2f &exponent)
Definition: NEON/PacketMath.h:2399
Eigen::internal::plset< Packet16uc >
Packet16uc plset< Packet16uc >(const unsigned char &a)
Definition: AltiVec/PacketMath.h:979
Eigen::internal::F32MaskToBf16Mask
Packet4bf F32MaskToBf16Mask(const Packet4f &p)
Definition: NEON/PacketMath.h:3492
Eigen::internal::ploaddup< Packet2ul >
Packet2ul ploaddup< Packet2ul >(const uint64_t *from)
Definition: NEON/PacketMath.h:1830
Eigen::internal::predux_mul< Packet2ul >
uint64_t predux_mul< Packet2ul >(const Packet2ul &a)
Definition: NEON/PacketMath.h:2615
Eigen::internal::shuffle2< true >
Packet4f shuffle2< true >(const Packet4f &m, const Packet4f &n, int mask)
Definition: NEON/PacketMath.h:118
Eigen::internal::pabsdiff< Packet16uc >
Packet16uc pabsdiff< Packet16uc >(const Packet16uc &a, const Packet16uc &b)
Definition: NEON/PacketMath.h:1169
Eigen::internal::pscatter< uint16_t, Packet4us >
void pscatter< uint16_t, Packet4us >(uint16_t *to, const Packet4us &from, Index stride)
Definition: NEON/PacketMath.h:2224
Eigen::internal::ploaddup< Packet8us >
Packet8us ploaddup< Packet8us >(const unsigned short int *from)
Definition: AltiVec/PacketMath.h:1353
Eigen::internal::pcmp_lt< Packet16c >
Packet16c pcmp_lt< Packet16c >(const Packet16c &a, const Packet16c &b)
Definition: NEON/PacketMath.h:1357
Eigen::internal::Packet4bf
eigen_packet_wrapper< uint16x4_t, 19 > Packet4bf
Definition: NEON/PacketMath.h:3391
Eigen::internal::predux_min< Packet2ui >
uint32_t predux_min< Packet2ui >(const Packet2ui &a)
Definition: NEON/PacketMath.h:2723
Eigen::internal::psub< Packet4c >
Packet4c psub< Packet4c >(const Packet4c &a, const Packet4c &b)
Definition: NEON/PacketMath.h:844
Eigen::internal::pand< Packet2i >
Packet2i pand< Packet2i >(const Packet2i &a, const Packet2i &b)
Definition: NEON/PacketMath.h:1495
Eigen::internal::pdiv< Packet2ul >
Packet2ul pdiv< Packet2ul >(const Packet2ul &, const Packet2ul &)
Definition: NEON/PacketMath.h:1084
Eigen::internal::pdiv< Packet8c >
Packet8c pdiv< Packet8c >(const Packet8c &, const Packet8c &)
Definition: NEON/PacketMath.h:1014
Eigen::internal::pdiv< Packet4bf >
Packet4bf pdiv< Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition: NEON/PacketMath.h:3623
Eigen::internal::pcmp_eq< Packet16uc >
Packet16uc pcmp_eq< Packet16uc >(const Packet16uc &a, const Packet16uc &b)
Definition: NEON/PacketMath.h:1428
Eigen::internal::ploadu< Packet2ul >
Packet2ul ploadu< Packet2ul >(const uint64_t *from)
Definition: NEON/PacketMath.h:1759
Eigen::internal::plset< Packet2f >
Packet2f plset< Packet2f >(const float &a)
Definition: NEON/PacketMath.h:728
Eigen::internal::pload< Packet8uc >
Packet8uc pload< Packet8uc >(const uint8_t *from)
Definition: NEON/PacketMath.h:1692
Eigen::internal::predux_mul< Packet8s >
short int predux_mul< Packet8s >(const Packet8s &a)
Definition: AltiVec/PacketMath.h:2129
Eigen::internal::pmax< PropagateNaN, Packet2d >
Packet2d pmax< PropagateNaN, Packet2d >(const Packet2d &a, const Packet2d &b)
Definition: SSE/PacketMath.h:726
Eigen::internal::padd< Packet8s >
Packet8s padd< Packet8s >(const Packet8s &a, const Packet8s &b)
Definition: AltiVec/PacketMath.h:984
Eigen::internal::por< Packet8us >
Packet8us por< Packet8us >(const Packet8us &a, const Packet8us &b)
Definition: AltiVec/PacketMath.h:1166
Eigen::internal::pcmp_eq< Packet2ul >
Packet2ul pcmp_eq< Packet2ul >(const Packet2ul &a, const Packet2ul &b)
Definition: NEON/PacketMath.h:1456
Eigen::internal::predux_max< Packet2l >
int64_t predux_max< Packet2l >(const Packet2l &a)
Definition: NEON/PacketMath.h:2849
Eigen::internal::ploaddup< Packet8s >
Packet8s ploaddup< Packet8s >(const short int *from)
Definition: AltiVec/PacketMath.h:1345
Eigen::internal::pset1< Packet16uc >
Packet16uc pset1< Packet16uc >(const unsigned char &from)
Definition: AltiVec/PacketMath.h:759
Eigen::internal::plset< Packet4ui >
Packet4ui plset< Packet4ui >(const uint32_t &a)
Definition: NEON/PacketMath.h:797
Eigen::internal::predux< Packet4i >
int predux< Packet4i >(const Packet4i &a)
Definition: AltiVec/PacketMath.h:2036
Eigen::internal::pmin< Packet4f >
Packet4f pmin< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition: AltiVec/PacketMath.h:1082
Eigen::internal::predux_max< Packet2f >
float predux_max< Packet2f >(const Packet2f &a)
Definition: NEON/PacketMath.h:2741
Eigen::internal::ploadu< Packet8s >
Packet8s ploadu< Packet8s >(const short int *from)
Definition: AltiVec/PacketMath.h:1242
Eigen::internal::predux_max< Packet2i >
int32_t predux_max< Packet2i >(const Packet2i &a)
Definition: NEON/PacketMath.h:2834
Eigen::internal::pabsdiff< Packet4s >
Packet4s pabsdiff< Packet4s >(const Packet4s &a, const Packet4s &b)
Definition: NEON/PacketMath.h:1171
Eigen::internal::predux_min< Packet4f >
float predux_min< Packet4f >(const Packet4f &a)
Definition: AltiVec/PacketMath.h:2195
Eigen::internal::pset1frombits< Packet4f >
Packet4f pset1frombits< Packet4f >(unsigned int from)
Definition: AltiVec/PacketMath.h:763
Eigen::internal::pceil< Packet2d >
Packet2d pceil< Packet2d >(const Packet2d &a)
Definition: MSA/PacketMath.h:1181
Eigen::internal::pdiv< Packet8s >
Packet8s pdiv< Packet8s >(const Packet8s &, const Packet8s &)
Definition: NEON/PacketMath.h:1044
Eigen::internal::plset< Packet4s >
Packet4s plset< Packet4s >(const int16_t &a)
Definition: NEON/PacketMath.h:762
Eigen::internal::pscatter< int8_t, Packet4c >
void pscatter< int8_t, Packet4c >(int8_t *to, const Packet4c &from, Index stride)
Definition: NEON/PacketMath.h:2136
Eigen::internal::pscatter< uint32_t, Packet4ui >
void pscatter< uint32_t, Packet4ui >(uint32_t *to, const Packet4ui &from, Index stride)
Definition: NEON/PacketMath.h:2259
Eigen::internal::pmax< Packet4ui >
Packet4ui pmax< Packet4ui >(const Packet4ui &a, const Packet4ui &b)
Definition: NEON/PacketMath.h:1272
Eigen::internal::pabsdiff< Packet2ui >
Packet2ui pabsdiff< Packet2ui >(const Packet2ui &a, const Packet2ui &b)
Definition: NEON/PacketMath.h:1183
Eigen::internal::pand< Packet16c >
Packet16c pand< Packet16c >(const Packet16c &a, const Packet16c &b)
Definition: NEON/PacketMath.h:1481
Eigen::internal::pmin< Packet2l >
Packet2l pmin< Packet2l >(const Packet2l &a, const Packet2l &b)
Definition: NEON/PacketMath.h:1225
Eigen::internal::shuffle2
Packet4f shuffle2(const Packet4f &m, const Packet4f &n, int mask)
Definition: NEON/PacketMath.h:109
Eigen::internal::pmin< PropagateNaN, Packet2f >
Packet2f pmin< PropagateNaN, Packet2f >(const Packet2f &a, const Packet2f &b)
Definition: NEON/PacketMath.h:1199
Eigen::internal::pload< Packet4us >
Packet4us pload< Packet4us >(const uint16_t *from)
Definition: NEON/PacketMath.h:1700
Eigen::internal::por< Packet4f >
Packet4f por< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition: AltiVec/PacketMath.h:1163
Eigen::internal::pandnot< Packet2i >
Packet2i pandnot< Packet2i >(const Packet2i &a, const Packet2i &b)
Definition: NEON/PacketMath.h:1594
Eigen::internal::pabsdiff< Packet2f >
Packet2f pabsdiff< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition: NEON/PacketMath.h:1147
Eigen::internal::pandnot< Packet8c >
Packet8c pandnot< Packet8c >(const Packet8c &a, const Packet8c &b)
Definition: NEON/PacketMath.h:1578
Eigen::internal::pabsdiff< Packet8us >
Packet8us pabsdiff< Packet8us >(const Packet8us &a, const Packet8us &b)
Definition: NEON/PacketMath.h:1177
Eigen::internal::pcmp_lt< Packet4uc >
Packet4uc pcmp_lt< Packet4uc >(const Packet4uc &a, const Packet4uc &b)
Definition: NEON/PacketMath.h:1359
Eigen::internal::pstoreu< uint32_t >
void pstoreu< uint32_t >(uint32_t *to, const Packet8ui &from)
Definition: AVX/PacketMath.h:1295
Eigen::internal::pmul< Packet2l >
Packet2l pmul< Packet2l >(const Packet2l &a, const Packet2l &b)
Definition: NEON/PacketMath.h:947
Eigen::internal::pceil< Packet4bf >
Packet4bf pceil< Packet4bf >(const Packet4bf &a)
Definition: NEON/PacketMath.h:3604
Eigen::internal::predux< Packet8c >
int8_t predux< Packet8c >(const Packet8c &a)
Definition: NEON/PacketMath.h:2426
Eigen::internal::pload< Packet2ui >
Packet2ui pload< Packet2ui >(const uint32_t *from)
Definition: NEON/PacketMath.h:1708
Eigen::internal::ploadu< Packet4bf >
Packet4bf ploadu< Packet4bf >(const bfloat16 *from)
Definition: NEON/PacketMath.h:3509
Eigen::internal::predux< Packet2ui >
uint32_t predux< Packet2ui >(const Packet2ui &a)
Definition: NEON/PacketMath.h:2508
Eigen::internal::Packet4c
eigen_packet_wrapper< int32_t,2 > Packet4c
Definition: NEON/PacketMath.h:76
Eigen::internal::pload< Packet8c >
Packet8c pload< Packet8c >(const int8_t *from)
Definition: NEON/PacketMath.h:1682
Eigen::internal::psub< Packet4us >
Packet4us psub< Packet4us >(const Packet4us &a, const Packet4us &b)
Definition: NEON/PacketMath.h:862
Eigen::internal::pdiv< Packet16c >
Packet16c pdiv< Packet16c >(const Packet16c &, const Packet16c &)
Definition: NEON/PacketMath.h:1019
Eigen::internal::pfirst< Packet4bf >
bfloat16 pfirst< Packet4bf >(const Packet4bf &from)
Definition: NEON/PacketMath.h:3500
Eigen::internal::pandnot< Packet4f >
Packet4f pandnot< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition: AltiVec/PacketMath.h:1178
Eigen::internal::pandnot
Packet8h pandnot(const Packet8h &a, const Packet8h &b)
Definition: AVX/PacketMath.h:1886
Eigen::internal::pldexp< Packet4f >
Packet4f pldexp< Packet4f >(const Packet4f &a, const Packet4f &exponent)
Definition: AltiVec/PacketMath.h:1934
Eigen::internal::padd< Packet8uc >
Packet8uc padd< Packet8uc >(const Packet8uc &a, const Packet8uc &b)
Definition: NEON/PacketMath.h:829
Eigen::internal::pabs
Packet4f pabs(const Packet4f &a)
Definition: AltiVec/PacketMath.h:1579
Eigen::internal::pmax< PropagateNumbers, Packet4bf >
Packet4bf pmax< PropagateNumbers, Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition: NEON/PacketMath.h:3550
Eigen::internal::Packet8us
__vector unsigned short int Packet8us
Definition: AltiVec/PacketMath.h:37
Eigen::internal::pcmp_lt< Packet4c >
Packet4c pcmp_lt< Packet4c >(const Packet4c &a, const Packet4c &b)
Definition: NEON/PacketMath.h:1349
Eigen::internal::pxor< Packet4c >
Packet4c pxor< Packet4c >(const Packet4c &a, const Packet4c &b)
Definition: NEON/PacketMath.h:1543
Eigen::internal::plset< Packet4bf >
Packet4bf plset< Packet4bf >(const bfloat16 &a)
Definition: NEON/PacketMath.h:3567
Eigen::internal::pset1frombits< Packet2f >
Packet2f pset1frombits< Packet2f >(uint32_t from)
Definition: NEON/PacketMath.h:723
Eigen::internal::Bf16ToF32
Packet8f Bf16ToF32(const Packet8bf &a)
Definition: AVX/PacketMath.h:2098
Eigen::internal::psub< Packet8s >
Packet8s psub< Packet8s >(const Packet8s &a, const Packet8s &b)
Definition: AltiVec/PacketMath.h:991
Eigen::internal::pmax< Packet4s >
Packet4s pmax< Packet4s >(const Packet4s &a, const Packet4s &b)
Definition: NEON/PacketMath.h:1265
Eigen::internal::pcmp_eq< Packet2l >
Packet2l pcmp_eq< Packet2l >(const Packet2l &a, const Packet2l &b)
Definition: NEON/PacketMath.h:1446
Eigen::internal::predux< Packet4f >
float predux< Packet4f >(const Packet4f &a)
Definition: AltiVec/PacketMath.h:2026
Eigen::internal::pabsdiff< Packet4c >
Packet4c pabsdiff< Packet4c >(const Packet4c &a, const Packet4c &b)
Definition: NEON/PacketMath.h:1151
Eigen::internal::pmax< Packet2ul >
Packet2ul pmax< Packet2ul >(const Packet2ul &a, const Packet2ul &b)
Definition: NEON/PacketMath.h:1278
Eigen::internal::pfirst< Packet4s >
int16_t pfirst< Packet4s >(const Packet4s &a)
Definition: NEON/PacketMath.h:2295
Eigen::internal::ploadu< Packet4s >
Packet4s ploadu< Packet4s >(const int16_t *from)
Definition: NEON/PacketMath.h:1741
Eigen::internal::plset< Packet4f >
Packet4f plset< Packet4f >(const float &a)
Definition: AltiVec/PacketMath.h:974
Eigen::internal::ploadquad< Packet8s >
Packet8s ploadquad< Packet8s >(const short int *from)
Definition: AltiVec/PacketMath.h:1361
Eigen::internal::predux< Packet8us >
unsigned short int predux< Packet8us >(const Packet8us &a)
Definition: AltiVec/PacketMath.h:2076
Eigen::internal::predux< Packet4uc >
uint8_t predux< Packet4uc >(const Packet4uc &a)
Definition: NEON/PacketMath.h:2442
Eigen::internal::pload< Packet4ui >
Packet4ui pload< Packet4ui >(const uint32_t *from)
Definition: NEON/PacketMath.h:1710
Eigen::internal::pnegate< Packet4bf >
Packet4bf pnegate< Packet4bf >(const Packet4bf &a)
Definition: NEON/PacketMath.h:3694
Eigen::internal::pnegate
Packet2cf pnegate(const Packet2cf &a)
Definition: AltiVec/Complex.h:214
Eigen::internal::predux< Packet2d >
double predux< Packet2d >(const Packet2d &a)
Definition: MSA/PacketMath.h:1081
Eigen::internal::predux_max< Packet2ui >
uint32_t predux_max< Packet2ui >(const Packet2ui &a)
Definition: NEON/PacketMath.h:2841
Eigen::internal::predux< Packet4bf >
bfloat16 predux< Packet4bf >(const Packet4bf &a)
Definition: NEON/PacketMath.h:3639
Eigen::internal::paddsub< Packet2f >
Packet2f paddsub< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition: NEON/PacketMath.h:872
Eigen::internal::pdiv< Packet2i >
Packet2i pdiv< Packet2i >(const Packet2i &, const Packet2i &)
Definition: NEON/PacketMath.h:1059
Eigen::internal::preverse< Packet4bf >
Packet4bf preverse< Packet4bf >(const Packet4bf &a)
Definition: NEON/PacketMath.h:3659
Eigen::internal::ploaddup< Packet2ui >
Packet2ui ploaddup< Packet2ui >(const uint32_t *from)
Definition: NEON/PacketMath.h:1824
Eigen::internal::pcmp_lt< Packet2l >
Packet2l pcmp_lt< Packet2l >(const Packet2l &a, const Packet2l &b)
Definition: NEON/PacketMath.h:1385
Eigen::internal::pcmp_lt< Packet2i >
Packet2i pcmp_lt< Packet2i >(const Packet2i &a, const Packet2i &b)
Definition: NEON/PacketMath.h:1377
Eigen::internal::pand< Packet4f >
Packet4f pand< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition: AltiVec/PacketMath.h:1154
Eigen::internal::pgather< uint16_t, Packet4us >
Packet4us pgather< uint16_t, Packet4us >(const uint16_t *from, Index stride)
Definition: NEON/PacketMath.h:2063
Eigen::internal::predux_mul< Packet4bf >
bfloat16 predux_mul< Packet4bf >(const Packet4bf &a)
Definition: NEON/PacketMath.h:3654
Eigen::internal::pabsdiff< Packet2i >
Packet2i pabsdiff< Packet2i >(const Packet2i &a, const Packet2i &b)
Definition: NEON/PacketMath.h:1179
Eigen::internal::pandnot< Packet8uc >
Packet8uc pandnot< Packet8uc >(const Packet8uc &a, const Packet8uc &b)
Definition: NEON/PacketMath.h:1582
Eigen::internal::pmax< PropagateNaN, Packet4bf >
Packet4bf pmax< PropagateNaN, Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition: NEON/PacketMath.h:3555
Eigen::internal::pcmp_le< Packet2ul >
Packet2ul pcmp_le< Packet2ul >(const Packet2ul &a, const Packet2ul &b)
Definition: NEON/PacketMath.h:1334
Eigen::internal::pand< Packet2ui >
Packet2ui pand< Packet2ui >(const Packet2ui &a, const Packet2ui &b)
Definition: NEON/PacketMath.h:1497
Eigen::internal::predux_mul< Packet2l >
int64_t predux_mul< Packet2l >(const Packet2l &a)
Definition: NEON/PacketMath.h:2613
Eigen::internal::pand< Packet8s >
Packet8s pand< Packet8s >(const Packet8s &a, const Packet8s &b)
Definition: NEON/PacketMath.h:1490
Eigen::internal::predux_min< Packet8us >
unsigned short int predux_min< Packet8us >(const Packet8us &a)
Definition: AltiVec/PacketMath.h:2228
Eigen::internal::pset1< Packet8s >
Packet8s pset1< Packet8s >(const short int &from)
Definition: AltiVec/PacketMath.h:747
Eigen::internal::plset< Packet16c >
Packet16c plset< Packet16c >(const signed char &a)
Definition: AltiVec/PacketMath.h:978
Eigen::internal::pcmp_eq< Packet8c >
Packet8c pcmp_eq< Packet8c >(const Packet8c &a, const Packet8c &b)
Definition: NEON/PacketMath.h:1416
Eigen::internal::pcmp_lt< Packet4bf >
Packet4bf pcmp_lt< Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition: NEON/PacketMath.h:3679
Eigen::internal::pmin< Packet2ui >
Packet2ui pmin< Packet2ui >(const Packet2ui &a, const Packet2ui &b)
Definition: NEON/PacketMath.h:1223
Eigen::internal::pscatter< uint32_t, Packet2ui >
void pscatter< uint32_t, Packet2ui >(uint32_t *to, const Packet2ui &from, Index stride)
Definition: NEON/PacketMath.h:2254
Eigen::internal::pmax< Packet16uc >
Packet16uc pmax< Packet16uc >(const Packet16uc &a, const Packet16uc &b)
Definition: AltiVec/PacketMath.h:1115
Eigen::internal::predux_max< Packet4i >
int predux_max< Packet4i >(const Packet4i &a)
Definition: AltiVec/PacketMath.h:2280
Eigen::internal::pcmp_le< Packet8c >
Packet8c pcmp_le< Packet8c >(const Packet8c &a, const Packet8c &b)
Definition: NEON/PacketMath.h:1294
Eigen::internal::pset1< Packet4bf >
Packet4bf pset1< Packet4bf >(const bfloat16 &from)
Definition: NEON/PacketMath.h:3496
Eigen::internal::pabsdiff< Packet8c >
Packet8c pabsdiff< Packet8c >(const Packet8c &a, const Packet8c &b)
Definition: NEON/PacketMath.h:1157
Eigen::internal::ploadu< Packet4c >
Packet4c ploadu< Packet4c >(const int8_t *from)
Definition: NEON/PacketMath.h:1721
Eigen::internal::pcmp_eq< Packet4f >
Packet4f pcmp_eq< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition: NEON/PacketMath.h:1408
Eigen::internal::predux_mul< Packet4c >
int8_t predux_mul< Packet4c >(const Packet4c &a)
Definition: NEON/PacketMath.h:2545
Eigen::internal::predux_max< Packet4bf >
bfloat16 predux_max< Packet4bf >(const Packet4bf &a)
Definition: NEON/PacketMath.h:3644
Eigen::internal::pmax< Packet2i >
Packet2i pmax< Packet2i >(const Packet2i &a, const Packet2i &b)
Definition: NEON/PacketMath.h:1269
Eigen::internal::plogical_shift_right
Packet4i plogical_shift_right(const Packet4i &a)
Definition: AltiVec/PacketMath.h:1595
Eigen::internal::predux< Packet16c >
signed char predux< Packet16c >(const Packet16c &a)
Definition: AltiVec/PacketMath.h:2103
Eigen::internal::ploadquad< Packet8us >
Packet8us ploadquad< Packet8us >(const unsigned short int *from)
Definition: AltiVec/PacketMath.h:1369
Eigen::internal::ploadu< Packet4ui >
Packet4ui ploadu< Packet4ui >(const uint32_t *from)
Definition: NEON/PacketMath.h:1755
Eigen::internal::pcmp_le< Packet2f >
Packet2f pcmp_le< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition: NEON/PacketMath.h:1284
Eigen::internal::predux_mul< Packet16c >
signed char predux_mul< Packet16c >(const Packet16c &a)
Definition: AltiVec/PacketMath.h:2160
Eigen::internal::pcmp_le< Packet4f >
Packet4f pcmp_le< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition: NEON/PacketMath.h:1286
Eigen::internal::pand< Packet2d >
Packet2d pand< Packet2d >(const Packet2d &a, const Packet2d &b)
Definition: MSA/PacketMath.h:938
Eigen::internal::ploadquad< Packet16c >
Packet16c ploadquad< Packet16c >(const int8_t *from)
Definition: NEON/PacketMath.h:1842
Eigen::internal::ploadquad< Packet8uc >
Packet8uc ploadquad< Packet8uc >(const uint8_t *from)
Definition: NEON/PacketMath.h:1854
Eigen::internal::pxor< Packet4us >
Packet4us pxor< Packet4us >(const Packet4us &a, const Packet4us &b)
Definition: NEON/PacketMath.h:1557
Eigen::internal::pcmp_le< Packet4s >
Packet4s pcmp_le< Packet4s >(const Packet4s &a, const Packet4s &b)
Definition: NEON/PacketMath.h:1308
Eigen::internal::pmin< Packet4ui >
Packet4ui pmin< Packet4ui >(const Packet4ui &a, const Packet4ui &b)
Definition: NEON/PacketMath.h:1224
Eigen::internal::pscatter< int16_t, Packet4s >
void pscatter< int16_t, Packet4s >(int16_t *to, const Packet4s &from, Index stride)
Definition: NEON/PacketMath.h:2206
Eigen::internal::predux_mul< Packet2d >
double predux_mul< Packet2d >(const Packet2d &a)
Definition: MSA/PacketMath.h:1091
Eigen::internal::pdiv< Packet4f >
Packet4f pdiv< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition: AltiVec/PacketMath.h:1040
Eigen::internal::pload< Packet4bf >
Packet4bf pload< Packet4bf >(const bfloat16 *from)
Definition: NEON/PacketMath.h:3504
Eigen::internal::predux_max< Packet4ui >
uint32_t predux_max< Packet4ui >(const Packet4ui &a)
Definition: NEON/PacketMath.h:2843
Eigen::internal::psub< Packet4bf >
Packet4bf psub< Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition: NEON/PacketMath.h:3615
Eigen::internal::ploaddup< Packet2i >
Packet2i ploaddup< Packet2i >(const int32_t *from)
Definition: NEON/PacketMath.h:1820
Eigen::internal::pcmp_le< Packet16c >
Packet16c pcmp_le< Packet16c >(const Packet16c &a, const Packet16c &b)
Definition: NEON/PacketMath.h:1296
Eigen::internal::pandnot< Packet2ul >
Packet2ul pandnot< Packet2ul >(const Packet2ul &a, const Packet2ul &b)
Definition: NEON/PacketMath.h:1604
Eigen::internal::pandnot< Packet2ui >
Packet2ui pandnot< Packet2ui >(const Packet2ui &a, const Packet2ui &b)
Definition: NEON/PacketMath.h:1598
Eigen::internal::padd< Packet2ui >
Packet2ui padd< Packet2ui >(const Packet2ui &a, const Packet2ui &b)
Definition: NEON/PacketMath.h:837
Eigen::internal::pmin< Packet16uc >
Packet16uc pmin< Packet16uc >(const Packet16uc &a, const Packet16uc &b)
Definition: AltiVec/PacketMath.h:1097
Eigen::internal::pandnot< Packet2l >
Packet2l pandnot< Packet2l >(const Packet2l &a, const Packet2l &b)
Definition: NEON/PacketMath.h:1602
Eigen::internal::pdiv< Packet8uc >
Packet8uc pdiv< Packet8uc >(const Packet8uc &, const Packet8uc &)
Definition: NEON/PacketMath.h:1029
Eigen::internal::predux< Packet2i >
int32_t predux< Packet2i >(const Packet2i &a)
Definition: NEON/PacketMath.h:2502
Eigen::internal::pstore< uint16_t >
void pstore< uint16_t >(uint16_t *to, const Packet4us &from)
Definition: NEON/PacketMath.h:1897
Eigen::internal::pmadd
Packet4f pmadd(const Packet4f &a, const Packet4f &b, const Packet4f &c)
Definition: AltiVec/PacketMath.h:1071
Eigen::internal::pstoreu< uint64_t >
void pstoreu< uint64_t >(uint64_t *to, const Packet2ul &from)
Definition: NEON/PacketMath.h:1948
Eigen::internal::pfirst< Packet2l >
int64_t pfirst< Packet2l >(const Packet2l &a)
Definition: NEON/PacketMath.h:2303
Eigen::internal::pmul< Packet8c >
Packet8c pmul< Packet8c >(const Packet8c &a, const Packet8c &b)
Definition: NEON/PacketMath.h:929
Eigen::internal::pxor< Packet2l >
Packet2l pxor< Packet2l >(const Packet2l &a, const Packet2l &b)
Definition: NEON/PacketMath.h:1567
Eigen::internal::pmul< Packet16c >
Packet16c pmul< Packet16c >(const Packet16c &a, const Packet16c &b)
Definition: AltiVec/PacketMath.h:1036
Eigen::internal::pmax< Packet2ui >
Packet2ui pmax< Packet2ui >(const Packet2ui &a, const Packet2ui &b)
Definition: NEON/PacketMath.h:1271
Eigen::internal::ploadu< Packet2l >
Packet2l ploadu< Packet2l >(const int64_t *from)
Definition: NEON/PacketMath.h:1757
Eigen::internal::pcmp_eq< Packet4ui >
Packet4ui pcmp_eq< Packet4ui >(const Packet4ui &a, const Packet4ui &b)
Definition: NEON/PacketMath.h:1444
Eigen::internal::psub< Packet2ul >
Packet2ul psub< Packet2ul >(const Packet2ul &a, const Packet2ul &b)
Definition: NEON/PacketMath.h:869
Eigen::internal::ploaddup< Packet2d >
Packet2d ploaddup< Packet2d >(const double *from)
Definition: MSA/PacketMath.h:1010
Eigen::internal::predux_min< Packet4ui >
uint32_t predux_min< Packet4ui >(const Packet4ui &a)
Definition: NEON/PacketMath.h:2725
Eigen::internal::pgather< int32_t, Packet2i >
Packet2i pgather< int32_t, Packet2i >(const int32_t *from, Index stride)
Definition: NEON/PacketMath.h:2083
Eigen::internal::pmin< Packet4s >
Packet4s pmin< Packet4s >(const Packet4s &a, const Packet4s &b)
Definition: NEON/PacketMath.h:1217
Eigen::internal::pmul< Packet4i >
Packet4i pmul< Packet4i >(const Packet4i &a, const Packet4i &b)
Definition: AltiVec/PacketMath.h:1033
Eigen::internal::Packet16c
__vector signed char Packet16c
Definition: AltiVec/PacketMath.h:39
Eigen::internal::pandnot< Packet2f >
Packet2f pandnot< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition: NEON/PacketMath.h:1572
Eigen::internal::padd< Packet4ui >
Packet4ui padd< Packet4ui >(const Packet4ui &a, const Packet4ui &b)
Definition: AltiVec/PacketMath.h:983
Eigen::internal::pmin< Packet2i >
Packet2i pmin< Packet2i >(const Packet2i &a, const Packet2i &b)
Definition: NEON/PacketMath.h:1221
Eigen::internal::pcmp_eq
Packet2cf pcmp_eq(const Packet2cf &a, const Packet2cf &b)
Definition: AltiVec/Complex.h:281
Eigen::internal::paddsub< Packet4f >
Packet4f paddsub< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition: NEON/PacketMath.h:877
Eigen::internal::pset1< Packet2i >
Packet2i pset1< Packet2i >(const int32_t &from)
Definition: NEON/PacketMath.h:716
Eigen::internal::pscatter< bfloat16, Packet4bf >
void pscatter< bfloat16, Packet4bf >(bfloat16 *to, const Packet4bf &from, Index stride)
Definition: NEON/PacketMath.h:3634
Eigen::internal::predux_max< Packet2ul >
uint64_t predux_max< Packet2ul >(const Packet2ul &a)
Definition: NEON/PacketMath.h:2851
Eigen::internal::pmul< Packet4ui >
Packet4ui pmul< Packet4ui >(const Packet4ui &a, const Packet4ui &b)
Definition: NEON/PacketMath.h:946
Eigen::internal::pscatter< int8_t, Packet8c >
void pscatter< int8_t, Packet8c >(int8_t *to, const Packet8c &from, Index stride)
Definition: NEON/PacketMath.h:2141
Eigen::internal::pscatter< int32_t, Packet2i >
void pscatter< int32_t, Packet2i >(int32_t *to, const Packet2i &from, Index stride)
Definition: NEON/PacketMath.h:2242
Eigen::internal::pmin< Packet4bf >
Packet4bf pmin< Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition: NEON/PacketMath.h:3544
Eigen::internal::por< Packet4c >
Packet4c por< Packet4c >(const Packet4c &a, const Packet4c &b)
Definition: NEON/PacketMath.h:1509
Eigen::internal::padd< Packet16c >
Packet16c padd< Packet16c >(const Packet16c &a, const Packet16c &b)
Definition: AltiVec/PacketMath.h:986
Eigen::internal::pgather< double, Packet2d >
Packet2d pgather< double, Packet2d >(const double *from, Index stride)
Definition: MSA/PacketMath.h:1032
Eigen::internal::pmul< Packet4uc >
Packet4uc pmul< Packet4uc >(const Packet4uc &a, const Packet4uc &b)
Definition: NEON/PacketMath.h:931
Eigen::internal::padd< Packet8us >
Packet8us padd< Packet8us >(const Packet8us &a, const Packet8us &b)
Definition: AltiVec/PacketMath.h:985
Eigen::internal::ploadu< Packet2ui >
Packet2ui ploadu< Packet2ui >(const uint32_t *from)
Definition: NEON/PacketMath.h:1753
Eigen::internal::pmin< Packet8s >
Packet8s pmin< Packet8s >(const Packet8s &a, const Packet8s &b)
Definition: AltiVec/PacketMath.h:1094
Eigen::internal::predux_mul< Packet2f >
float predux_mul< Packet2f >(const Packet2f &a)
Definition: NEON/PacketMath.h:2541
Eigen::internal::pand< Packet4us >
Packet4us pand< Packet4us >(const Packet4us &a, const Packet4us &b)
Definition: NEON/PacketMath.h:1491
Eigen::internal::pdiv< Packet8us >
Packet8us pdiv< Packet8us >(const Packet8us &, const Packet8us &)
Definition: NEON/PacketMath.h:1054
Eigen::internal::pstoreu< double >
void pstoreu< double >(double *to, const Packet4d &from)
Definition: AVX/PacketMath.h:1293
Eigen::internal::pload< Packet2ul >
Packet2ul pload< Packet2ul >(const uint64_t *from)
Definition: NEON/PacketMath.h:1714
Eigen::internal::pselect
Packet4f pselect(const Packet4f &mask, const Packet4f &a, const Packet4f &b)
Definition: AltiVec/PacketMath.h:1181
Eigen::internal::pscatter< int16_t, Packet8s >
void pscatter< int16_t, Packet8s >(int16_t *to, const Packet8s &from, Index stride)
Definition: NEON/PacketMath.h:2213
Eigen::internal::pmin< Packet8uc >
Packet8uc pmin< Packet8uc >(const Packet8uc &a, const Packet8uc &b)
Definition: NEON/PacketMath.h:1215
Eigen::internal::prefetch< uint16_t >
void prefetch< uint16_t >(const uint16_t *addr)
Definition: NEON/PacketMath.h:2281
Eigen::internal::ploaddup< Packet16c >
Packet16c ploaddup< Packet16c >(const signed char *from)
Definition: AltiVec/PacketMath.h:1382
Eigen::internal::pdiv< Packet16uc >
Packet16uc pdiv< Packet16uc >(const Packet16uc &, const Packet16uc &)
Definition: NEON/PacketMath.h:1034
Eigen::internal::predux_min< Packet4c >
int8_t predux_min< Packet4c >(const Packet4c &a)
Definition: NEON/PacketMath.h:2631
Eigen::internal::Packet4ui
__vector unsigned int Packet4ui
Definition: AltiVec/PacketMath.h:34
Eigen::internal::ploaddup< Packet16uc >
Packet16uc ploaddup< Packet16uc >(const unsigned char *from)
Definition: AltiVec/PacketMath.h:1390
Eigen::internal::pandnot< Packet8s >
Packet8s pandnot< Packet8s >(const Packet8s &a, const Packet8s &b)
Definition: NEON/PacketMath.h:1588
Eigen::internal::ploadu< Packet2i >
Packet2i ploadu< Packet2i >(const int32_t *from)
Definition: NEON/PacketMath.h:1749
Eigen::internal::pcmp_le< Packet4c >
Packet4c pcmp_le< Packet4c >(const Packet4c &a, const Packet4c &b)
Definition: NEON/PacketMath.h:1288
Eigen::internal::ploadquad< Packet4uc >
Packet4uc ploadquad< Packet4uc >(const uint8_t *from)
Definition: NEON/PacketMath.h:1852
Eigen::internal::predux_min< Packet2i >
int32_t predux_min< Packet2i >(const Packet2i &a)
Definition: NEON/PacketMath.h:2716
Eigen::internal::ptranspose
void ptranspose(PacketBlock< Packet2cf, 2 > &kernel)
Definition: AltiVec/Complex.h:269
Eigen::internal::predux_min< Packet2ul >
uint64_t predux_min< Packet2ul >(const Packet2ul &a)
Definition: NEON/PacketMath.h:2733
Eigen::internal::pcmp_le< Packet8us >
Packet8us pcmp_le< Packet8us >(const Packet8us &a, const Packet8us &b)
Definition: NEON/PacketMath.h:1314
Eigen::internal::pmin< Packet4uc >
Packet4uc pmin< Packet4uc >(const Packet4uc &a, const Packet4uc &b)
Definition: NEON/PacketMath.h:1209
Eigen::internal::pload< Packet2l >
Packet2l pload< Packet2l >(const int64_t *from)
Definition: NEON/PacketMath.h:1712
Eigen::internal::pfirst< Packet4uc >
uint8_t pfirst< Packet4uc >(const Packet4uc &a)
Definition: NEON/PacketMath.h:2292
Eigen::internal::pscatter< uint64_t, Packet2ul >
void pscatter< uint64_t, Packet2ul >(uint64_t *to, const Packet2ul &from, Index stride)
Definition: NEON/PacketMath.h:2271
Eigen::internal::pdiv< Packet4uc >
Packet4uc pdiv< Packet4uc >(const Packet4uc &, const Packet4uc &)
Definition: NEON/PacketMath.h:1024
Eigen::internal::pcmp_lt< Packet4ui >
Packet4ui pcmp_lt< Packet4ui >(const Packet4ui &a, const Packet4ui &b)
Definition: NEON/PacketMath.h:1383
Eigen::internal::ploadu< Packet4i >
Packet4i ploadu< Packet4i >(const int *from)
Definition: AltiVec/PacketMath.h:1238
Eigen::internal::predux_max< Packet16c >
signed char predux_max< Packet16c >(const Packet16c &a)
Definition: AltiVec/PacketMath.h:2323
Eigen::internal::padd< Packet2f >
Packet2f padd< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition: NEON/PacketMath.h:813
Eigen::internal::ploadu< Packet8c >
Packet8c ploadu< Packet8c >(const int8_t *from)
Definition: NEON/PacketMath.h:1727
Eigen::internal::ploadu< Packet8uc >
Packet8uc ploadu< Packet8uc >(const uint8_t *from)
Definition: NEON/PacketMath.h:1737
Eigen::internal::predux_max< Packet8uc >
uint8_t predux_max< Packet8uc >(const Packet8uc &a)
Definition: NEON/PacketMath.h:2795
Eigen::internal::pfirst< Packet4f >
float pfirst< Packet4f >(const Packet4f &a)
Definition: AltiVec/PacketMath.h:1515
Eigen::internal::predux_min< Packet8c >
int8_t predux_min< Packet8c >(const Packet8c &a)
Definition: NEON/PacketMath.h:2642
Eigen::internal::pmax< PropagateNumbers, Packet2d >
Packet2d pmax< PropagateNumbers, Packet2d >(const Packet2d &a, const Packet2d &b)
Definition: SSE/PacketMath.h:710
Eigen::internal::pabs< Packet4c >
Packet4c pabs< Packet4c >(const Packet4c &a)
Definition: NEON/PacketMath.h:2359
Eigen::internal::pfrexp_generic
Packet pfrexp_generic(const Packet &a, Packet &exponent)
Definition: GenericPacketMathFunctions.h:42
Eigen::internal::predux< Packet16uc >
unsigned char predux< Packet16uc >(const Packet16uc &a)
Definition: AltiVec/PacketMath.h:2108
Eigen::internal::pcmp_eq< Packet8uc >
Packet8uc pcmp_eq< Packet8uc >(const Packet8uc &a, const Packet8uc &b)
Definition: NEON/PacketMath.h:1426
Eigen::internal::ploadquad< Packet4i >
Packet4i ploadquad< Packet4i >(const int32_t *from)
Definition: NEON/PacketMath.h:1874
Eigen::internal::pfirst< Packet4c >
int8_t pfirst< Packet4c >(const Packet4c &a)
Definition: NEON/PacketMath.h:2289
Eigen::internal::predux_min< Packet4us >
uint16_t predux_min< Packet4us >(const Packet4us &a)
Definition: NEON/PacketMath.h:2704
Eigen::internal::pfirst< Packet2d >
double pfirst< Packet2d >(const Packet2d &a)
Definition: MSA/PacketMath.h:1060
Eigen::internal::pldexp_generic
Packet pldexp_generic(const Packet &a, const Packet &exponent)
Definition: GenericPacketMathFunctions.h:86
Eigen::internal::pgather< int64_t, Packet2l >
Packet2l pgather< int64_t, Packet2l >(const int64_t *from, Index stride)
Definition: NEON/PacketMath.h:2111
Eigen::internal::pfloor< Packet4bf >
Packet4bf pfloor< Packet4bf >(const Packet4bf &a)
Definition: NEON/PacketMath.h:3599
Eigen::internal::pabsdiff< Packet8uc >
Packet8uc pabsdiff< Packet8uc >(const Packet8uc &a, const Packet8uc &b)
Definition: NEON/PacketMath.h:1167
Eigen::internal::ploadquad< Packet8c >
Packet8c ploadquad< Packet8c >(const int8_t *from)
Definition: NEON/PacketMath.h:1836
Eigen::internal::pcmp_eq< Packet8us >
Packet8us pcmp_eq< Packet8us >(const Packet8us &a, const Packet8us &b)
Definition: NEON/PacketMath.h:1436
Eigen::internal::pxor< Packet4i >
Packet4i pxor< Packet4i >(const Packet4i &a, const Packet4i &b)
Definition: AltiVec/PacketMath.h:1172
Eigen::internal::pxor< Packet8s >
Packet8s pxor< Packet8s >(const Packet8s &a, const Packet8s &b)
Definition: NEON/PacketMath.h:1556
Eigen::internal::pmul< Packet2ui >
Packet2ui pmul< Packet2ui >(const Packet2ui &a, const Packet2ui &b)
Definition: NEON/PacketMath.h:945
Eigen::internal::pstore< int16_t >
void pstore< int16_t >(int16_t *to, const Packet4s &from)
Definition: NEON/PacketMath.h:1893
Eigen::internal::pstore< uint32_t >
void pstore< uint32_t >(uint32_t *to, const Packet8ui &from)
Definition: AVX/PacketMath.h:1290
Eigen::internal::predux_min< Packet4s >
int16_t predux_min< Packet4s >(const Packet4s &a)
Definition: NEON/PacketMath.h:2692
Eigen::internal::por< Packet2f >
Packet2f por< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition: NEON/PacketMath.h:1505
Eigen::internal::pfirst< Packet2f >
float pfirst< Packet2f >(const Packet2f &a)
Definition: NEON/PacketMath.h:2287
Eigen::internal::pstoreu< bfloat16 >
void pstoreu< bfloat16 >(bfloat16 *to, const Packet8bf &from)
Definition: AltiVec/PacketMath.h:1436
Eigen::internal::ploaddup< Packet4c >
Packet4c ploaddup< Packet4c >(const int8_t *from)
Definition: NEON/PacketMath.h:1766
Eigen::internal::pmul< Packet8uc >
Packet8uc pmul< Packet8uc >(const Packet8uc &a, const Packet8uc &b)
Definition: NEON/PacketMath.h:937
Eigen::internal::predux_max< Packet4c >
int8_t predux_max< Packet4c >(const Packet4c &a)
Definition: NEON/PacketMath.h:2749
Eigen::internal::pmin< Packet16c >
Packet16c pmin< Packet16c >(const Packet16c &a, const Packet16c &b)
Definition: AltiVec/PacketMath.h:1096
Eigen::internal::pandnot< Packet4i >
Packet4i pandnot< Packet4i >(const Packet4i &a, const Packet4i &b)
Definition: AltiVec/PacketMath.h:1179
Eigen::internal::pstoreu< uint8_t >
void pstoreu< uint8_t >(uint8_t *to, const Packet4uc &from)
Definition: NEON/PacketMath.h:1924
Eigen::internal::pmin< PropagateNaN, Packet4bf >
Packet4bf pmin< PropagateNaN, Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition: NEON/PacketMath.h:3538
Eigen::internal::pcmp_lt_or_nan< Packet2f >
Packet2f pcmp_lt_or_nan< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition: NEON/PacketMath.h:1467
Eigen::internal::pset1< Packet4i >
Packet4i pset1< Packet4i >(const int &from)
Definition: AltiVec/PacketMath.h:743
Eigen::internal::pset1< Packet2l >
Packet2l pset1< Packet2l >(const int64_t &from)
Definition: NEON/PacketMath.h:720
Eigen::internal::pmax< Packet4bf >
Packet4bf pmax< Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition: NEON/PacketMath.h:3561
Eigen::internal::prefetch< uint32_t >
void prefetch< uint32_t >(const uint32_t *addr)
Definition: AVX/PacketMath.h:1398
Eigen::internal::pfirst< Packet8c >
int8_t pfirst< Packet8c >(const Packet8c &a)
Definition: NEON/PacketMath.h:2290
Eigen::internal::predux_mul< Packet4uc >
uint8_t predux_mul< Packet4uc >(const Packet4uc &a)
Definition: NEON/PacketMath.h:2559
Eigen::internal::predux_mul< Packet2ui >
uint32_t predux_mul< Packet2ui >(const Packet2ui &a)
Definition: NEON/PacketMath.h:2609
Eigen::internal::pcmp_lt< Packet16uc >
Packet16uc pcmp_lt< Packet16uc >(const Packet16uc &a, const Packet16uc &b)
Definition: NEON/PacketMath.h:1367
Eigen::internal::pload< Packet8s >
Packet8s pload< Packet8s >(const short int *from)
Definition: AltiVec/PacketMath.h:471
Eigen::internal::ploadu< Packet4uc >
Packet4uc ploadu< Packet4uc >(const uint8_t *from)
Definition: NEON/PacketMath.h:1731
Eigen::internal::pgather< int16_t, Packet4s >
Packet4s pgather< int16_t, Packet4s >(const int16_t *from, Index stride)
Definition: NEON/PacketMath.h:2043
Eigen::internal::pset1< Packet16c >
Packet16c pset1< Packet16c >(const signed char &from)
Definition: AltiVec/PacketMath.h:755
Eigen::internal::padd< Packet4f >
Packet4f padd< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition: AltiVec/PacketMath.h:981
Eigen::internal::pcmp_eq< Packet4i >
Packet4i pcmp_eq< Packet4i >(const Packet4i &a, const Packet4i &b)
Definition: NEON/PacketMath.h:1440
Eigen::internal::pcmp_eq< Packet2ui >
Packet2ui pcmp_eq< Packet2ui >(const Packet2ui &a, const Packet2ui &b)
Definition: NEON/PacketMath.h:1442
Eigen::internal::plset< Packet2ui >
Packet2ui plset< Packet2ui >(const uint32_t &a)
Definition: NEON/PacketMath.h:792
Eigen::internal::pxor< Packet2ul >
Packet2ul pxor< Packet2ul >(const Packet2ul &a, const Packet2ul &b)
Definition: NEON/PacketMath.h:1569
Eigen::internal::pcmp_eq< Packet4uc >
Packet4uc pcmp_eq< Packet4uc >(const Packet4uc &a, const Packet4uc &b)
Definition: NEON/PacketMath.h:1420
Eigen::internal::pload< Packet4i >
Packet4i pload< Packet4i >(const int *from)
Definition: AltiVec/PacketMath.h:466
Eigen::internal::ploadu< Packet4f >
Packet4f ploadu< Packet4f >(const float *from)
Definition: AltiVec/PacketMath.h:1234
Eigen::internal::por< Packet4us >
Packet4us por< Packet4us >(const Packet4us &a, const Packet4us &b)
Definition: NEON/PacketMath.h:1524
Eigen::internal::predux_mul< Packet4f >
float predux_mul< Packet4f >(const Packet4f &a)
Definition: AltiVec/PacketMath.h:2115
Eigen::internal::padd< Packet2d >
Packet2d padd< Packet2d >(const Packet2d &a, const Packet2d &b)
Definition: MSA/PacketMath.h:879
Eigen::internal::por< Packet2ul >
Packet2ul por< Packet2ul >(const Packet2ul &a, const Packet2ul &b)
Definition: NEON/PacketMath.h:1536
Eigen::internal::ploadquad< Packet4f >
Packet4f ploadquad< Packet4f >(const float *from)
Definition: NEON/PacketMath.h:1833
Eigen::internal::pfirst< Packet8s >
short int pfirst< Packet8s >(const Packet8s &a)
Definition: AltiVec/PacketMath.h:1524
Eigen::internal::psqrt
Packet4f psqrt(const Packet4f &a)
Definition: MSA/PacketMath.h:723
Eigen::internal::predux_min< Packet16uc >
unsigned char predux_min< Packet16uc >(const Packet16uc &a)
Definition: AltiVec/PacketMath.h:2255
Eigen::internal::padd< Packet8c >
Packet8c padd< Packet8c >(const Packet8c &a, const Packet8c &b)
Definition: NEON/PacketMath.h:821
Eigen::internal::por< Packet2ui >
Packet2ui por< Packet2ui >(const Packet2ui &a, const Packet2ui &b)
Definition: NEON/PacketMath.h:1530
Eigen::internal::pandnot< Packet2d >
Packet2d pandnot< Packet2d >(const Packet2d &a, const Packet2d &b)
Definition: MSA/PacketMath.h:959
Eigen::internal::pscatter< float, Packet4f >
EIGEN_ALWAYS_INLINE void pscatter< float, Packet4f >(float *to, const Packet4f &from, Index stride)
Definition: AltiVec/PacketMath.h:904
Eigen::internal::pmax< PropagateNaN, Packet2f >
Packet2f pmax< PropagateNaN, Packet2f >(const Packet2f &a, const Packet2f &b)
Definition: NEON/PacketMath.h:1247
Eigen::internal::plset< Packet8us >
Packet8us plset< Packet8us >(const unsigned short int &a)
Definition: AltiVec/PacketMath.h:977
Eigen::internal::pdiv< Packet4ui >
Packet4ui pdiv< Packet4ui >(const Packet4ui &, const Packet4ui &)
Definition: NEON/PacketMath.h:1074
Eigen::internal::pload< Packet4s >
Packet4s pload< Packet4s >(const int16_t *from)
Definition: NEON/PacketMath.h:1696
Eigen::internal::predux_mul< Packet4us >
uint16_t predux_mul< Packet4us >(const Packet4us &a)
Definition: NEON/PacketMath.h:2589
Eigen::internal::Packet8s
__vector short int Packet8s
Definition: AltiVec/PacketMath.h:36
Eigen::internal::vec2d_unpackhi
Packet2d vec2d_unpackhi(const Packet2d &a, const Packet2d &b)
Definition: SSE/PacketMath.h:116
Eigen::internal::print
Packet4f print(const Packet4f &a)
Definition: NEON/PacketMath.h:3203
Eigen::internal::pand< Packet2f >
Packet2f pand< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition: NEON/PacketMath.h:1473
Eigen::internal::pcmp_le< Packet2i >
Packet2i pcmp_le< Packet2i >(const Packet2i &a, const Packet2i &b)
Definition: NEON/PacketMath.h:1316
Eigen::internal::psub
Packet psub(const Packet &a, const Packet &b)
Definition: GenericPacketMath.h:324
Eigen::internal::padd< Packet4uc >
Packet4uc padd< Packet4uc >(const Packet4uc &a, const Packet4uc &b)
Definition: NEON/PacketMath.h:823
Eigen::internal::pstoreu< int64_t >
void pstoreu< int64_t >(int64_t *to, const Packet2l &from)
Definition: NEON/PacketMath.h:1946
Eigen::internal::pscatter< int8_t, Packet16c >
void pscatter< int8_t, Packet16c >(int8_t *to, const Packet16c &from, Index stride)
Definition: NEON/PacketMath.h:2152
Eigen::internal::pgather< int32_t, Packet4i >
Packet4i pgather< int32_t, Packet4i >(const int32_t *from, Index stride)
Definition: MSA/PacketMath.h:469
Eigen::internal::ploaddup< Packet4i >
Packet4i ploaddup< Packet4i >(const int *from)
Definition: AltiVec/PacketMath.h:1340
Eigen::internal::pandnot< Packet4s >
Packet4s pandnot< Packet4s >(const Packet4s &a, const Packet4s &b)
Definition: NEON/PacketMath.h:1586
Eigen::internal::prefetch< float >
void prefetch< float >(const float *addr)
Definition: AltiVec/PacketMath.h:1512
Eigen::internal::vec4f_swizzle2
Packet4f vec4f_swizzle2(const Packet4f &a, const Packet4f &b, int p, int q, int r, int s)
Definition: NEON/PacketMath.h:132
Eigen::internal::plset< Packet4c >
Packet4c plset< Packet4c >(const int8_t &a)
Definition: NEON/PacketMath.h:738
Eigen::internal::prefetch< double >
void prefetch< double >(const double *addr)
Definition: AVX/PacketMath.h:1396
Eigen::internal::pset1< Packet8us >
Packet8us pset1< Packet8us >(const unsigned short int &from)
Definition: AltiVec/PacketMath.h:751
Eigen::internal::Packet4us
uint16x4_t Packet4us
Definition: NEON/PacketMath.h:84
Eigen::internal::psub< Packet2f >
Packet2f psub< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition: NEON/PacketMath.h:842
Eigen::internal::pmul< Packet4f >
Packet4f pmul< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition: AltiVec/PacketMath.h:1032
Eigen::internal::por< Packet8uc >
Packet8uc por< Packet8uc >(const Packet8uc &a, const Packet8uc &b)
Definition: NEON/PacketMath.h:1516
Eigen::internal::predux< Packet8uc >
uint8_t predux< Packet8uc >(const Packet8uc &a)
Definition: NEON/PacketMath.h:2463
Eigen::internal::pset1< Packet2d >
Packet2d pset1< Packet2d >(const double &from)
Definition: MSA/PacketMath.h:871
Eigen::internal::psub< Packet2l >
Packet2l psub< Packet2l >(const Packet2l &a, const Packet2l &b)
Definition: NEON/PacketMath.h:868
Eigen::internal::pdiv< Packet4i >
Packet4i pdiv< Packet4i >(const Packet4i &a, const Packet4i &b)
Definition: AltiVec/PacketMath.h:1058
Eigen::internal::shuffle1
Packet4f shuffle1(const Packet4f &m, int mask)
Definition: NEON/PacketMath.h:98
Eigen::internal::paddsub< Packet2d >
Packet2d paddsub< Packet2d >(const Packet2d &a, const Packet2d &b)
Definition: SSE/PacketMath.h:359
Eigen::internal::pgather< uint32_t, Packet2ui >
Packet2ui pgather< uint32_t, Packet2ui >(const uint32_t *from, Index stride)
Definition: NEON/PacketMath.h:2097
Eigen::internal::padd< Packet4s >
Packet4s padd< Packet4s >(const Packet4s &a, const Packet4s &b)
Definition: NEON/PacketMath.h:831
Eigen::internal::pmul< Packet4s >
Packet4s pmul< Packet4s >(const Packet4s &a, const Packet4s &b)
Definition: NEON/PacketMath.h:939
Eigen::internal::pand
Packet8h pand(const Packet8h &a, const Packet8h &b)
Definition: AVX/PacketMath.h:1883
Eigen::internal::predux_max< Packet4us >
uint16_t predux_max< Packet4us >(const Packet4us &a)
Definition: NEON/PacketMath.h:2822
Eigen::internal::pfrexp< Packet4f >
Packet4f pfrexp< Packet4f >(const Packet4f &a, Packet4f &exponent)
Definition: AltiVec/PacketMath.h:1941
Eigen::internal::predux< Packet4ui >
uint32_t predux< Packet4ui >(const Packet4ui &a)
Definition: NEON/PacketMath.h:2509
Eigen::internal::ploadu< Packet2d >
Packet2d ploadu< Packet2d >(const double *from)
Definition: MSA/PacketMath.h:1003
Eigen::internal::pstore< int64_t >
void pstore< int64_t >(int64_t *to, const Packet2l &from)
Definition: NEON/PacketMath.h:1909
Eigen::internal::pmul< Packet8us >
Packet8us pmul< Packet8us >(const Packet8us &a, const Packet8us &b)
Definition: AltiVec/PacketMath.h:1035
Eigen::internal::predux_min< Packet8s >
short int predux_min< Packet8s >(const Packet8s &a)
Definition: AltiVec/PacketMath.h:2213
Eigen::internal::Packet8uc
uint8x8_t Packet8uc
Definition: NEON/PacketMath.h:80
Eigen::internal::pload< Packet4uc >
Packet4uc pload< Packet4uc >(const uint8_t *from)
Definition: NEON/PacketMath.h:1686
Eigen::internal::plset< Packet4i >
Packet4i plset< Packet4i >(const int &a)
Definition: AltiVec/PacketMath.h:975
Eigen::internal::pload< Packet16uc >
Packet16uc pload< Packet16uc >(const unsigned char *from)
Definition: AltiVec/PacketMath.h:486
Eigen::internal::pand< Packet4uc >
Packet4uc pand< Packet4uc >(const Packet4uc &a, const Packet4uc &b)
Definition: NEON/PacketMath.h:1483
Eigen::internal::pxor< Packet2f >
Packet2f pxor< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition: NEON/PacketMath.h:1539
Eigen::internal::ploadu< Packet16c >
Packet16c ploadu< Packet16c >(const signed char *from)
Definition: AltiVec/PacketMath.h:1254
Eigen::internal::por< Packet8c >
Packet8c por< Packet8c >(const Packet8c &a, const Packet8c &b)
Definition: NEON/PacketMath.h:1511
Eigen::internal::pand< Packet8uc >
Packet8uc pand< Packet8uc >(const Packet8uc &a, const Packet8uc &b)
Definition: NEON/PacketMath.h:1485
Eigen::internal::pstoreu< float >
void pstoreu< float >(float *to, const Packet4f &from)
Definition: AltiVec/PacketMath.h:1420
Eigen::internal::pxor< Packet2i >
Packet2i pxor< Packet2i >(const Packet2i &a, const Packet2i &b)
Definition: NEON/PacketMath.h:1561
Eigen::internal::prefetch< int16_t >
void prefetch< int16_t >(const int16_t *addr)
Definition: NEON/PacketMath.h:2280
Eigen::internal::predux_max< Packet16uc >
unsigned char predux_max< Packet16uc >(const Packet16uc &a)
Definition: AltiVec/PacketMath.h:2335
Eigen::internal::pset1< Packet2f >
Packet2f pset1< Packet2f >(const float &from)
Definition: NEON/PacketMath.h:702
Eigen::internal::pmax< Packet16c >
Packet16c pmax< Packet16c >(const Packet16c &a, const Packet16c &b)
Definition: AltiVec/PacketMath.h:1114
Eigen::internal::pfrexp< Packet2f >
Packet2f pfrexp< Packet2f >(const Packet2f &a, Packet2f &exponent)
Definition: NEON/PacketMath.h:2394
Eigen::internal::predux< Packet4s >
int16_t predux< Packet4s >(const Packet4s &a)
Definition: NEON/PacketMath.h:2478
Eigen::internal::predux_mul< Packet8c >
int8_t predux_mul< Packet8c >(const Packet8c &a)
Definition: NEON/PacketMath.h:2551
Eigen::internal::make_packet2f
EIGEN_ALWAYS_INLINE Packet2f make_packet2f(float a, float b)
Definition: NEON/PacketMath.h:94
Eigen::internal::prefetch< uint64_t >
void prefetch< uint64_t >(const uint64_t *addr)
Definition: NEON/PacketMath.h:2285
Eigen::internal::prefetch< int32_t >
void prefetch< int32_t >(const int32_t *addr)
Definition: MSA/PacketMath.h:516
Eigen::internal::Packet2f
float32x2_t Packet2f
Definition: NEON/PacketMath.h:74
Eigen::internal::pdiv< Packet2l >
Packet2l pdiv< Packet2l >(const Packet2l &, const Packet2l &)
Definition: NEON/PacketMath.h:1079
Eigen::internal::pmul< Packet8s >
Packet8s pmul< Packet8s >(const Packet8s &a, const Packet8s &b)
Definition: AltiVec/PacketMath.h:1034
Eigen::internal::pset1< Packet4us >
Packet4us pset1< Packet4us >(const uint16_t &from)
Definition: NEON/PacketMath.h:714
Eigen::internal::pmax< PropagateNumbers, Packet4f >
Packet4f pmax< PropagateNumbers, Packet4f >(const Packet4f &a, const Packet4f &b)
Definition: SSE/PacketMath.h:706
Eigen::internal::pcmp_eq< Packet4c >
Packet4c pcmp_eq< Packet4c >(const Packet4c &a, const Packet4c &b)
Definition: NEON/PacketMath.h:1410
Eigen::internal::psub< Packet2d >
Packet2d psub< Packet2d >(const Packet2d &a, const Packet2d &b)
Definition: MSA/PacketMath.h:894
Eigen::internal::pfloor< Packet2d >
Packet2d pfloor< Packet2d >(const Packet2d &a)
Definition: MSA/PacketMath.h:1162
Eigen::internal::prefetch< uint8_t >
void prefetch< uint8_t >(const uint8_t *addr)
Definition: NEON/PacketMath.h:2279
Eigen::internal::plset< Packet8s >
Packet8s plset< Packet8s >(const short int &a)
Definition: AltiVec/PacketMath.h:976
Eigen::internal::pdiv< Packet4us >
Packet4us pdiv< Packet4us >(const Packet4us &, const Packet4us &)
Definition: NEON/PacketMath.h:1049
Eigen::internal::pstore< int32_t >
void pstore< int32_t >(int32_t *to, const Packet4i &from)
Definition: MSA/PacketMath.h:436
Eigen::internal::pgather< float, Packet4f >
EIGEN_ALWAYS_INLINE Packet4f pgather< float, Packet4f >(const float *from, Index stride)
Definition: AltiVec/PacketMath.h:815
Eigen::internal::pldexp< Packet2d >
Packet2d pldexp< Packet2d >(const Packet2d &a, const Packet2d &exponent)
Definition: SSE/PacketMath.h:1118
Eigen::internal::por< Packet16uc >
Packet16uc por< Packet16uc >(const Packet16uc &a, const Packet16uc &b)
Definition: NEON/PacketMath.h:1518
Eigen::internal::predux_max< Packet4uc >
uint8_t predux_max< Packet4uc >(const Packet4uc &a)
Definition: NEON/PacketMath.h:2776
Eigen::internal::predux_min< Packet2l >
int64_t predux_min< Packet2l >(const Packet2l &a)
Definition: NEON/PacketMath.h:2731
Eigen::internal::pxor
Packet8h pxor(const Packet8h &a, const Packet8h &b)
Definition: AVX/PacketMath.h:1880
Eigen::internal::pmin< PropagateNaN, Packet2d >
Packet2d pmin< PropagateNaN, Packet2d >(const Packet2d &a, const Packet2d &b)
Definition: SSE/PacketMath.h:718
Eigen::internal::psub< Packet8us >
Packet8us psub< Packet8us >(const Packet8us &a, const Packet8us &b)
Definition: AltiVec/PacketMath.h:992
Eigen::internal::pmul< Packet2ul >
Packet2ul pmul< Packet2ul >(const Packet2ul &a, const Packet2ul &b)
Definition: NEON/PacketMath.h:952
Eigen::internal::pscatter< uint8_t, Packet4uc >
void pscatter< uint8_t, Packet4uc >(uint8_t *to, const Packet4uc &from, Index stride)
Definition: NEON/PacketMath.h:2171
Eigen::internal::pcmp_lt< Packet4us >
Packet4us pcmp_lt< Packet4us >(const Packet4us &a, const Packet4us &b)
Definition: NEON/PacketMath.h:1373
Eigen::internal::padd< Packet4us >
Packet4us padd< Packet4us >(const Packet4us &a, const Packet4us &b)
Definition: NEON/PacketMath.h:833
Eigen::internal::ploaddup< Packet4uc >
Packet4uc ploaddup< Packet4uc >(const uint8_t *from)
Definition: NEON/PacketMath.h:1782
Eigen::internal::psub< Packet16c >
Packet16c psub< Packet16c >(const Packet16c &a, const Packet16c &b)
Definition: AltiVec/PacketMath.h:993
Eigen::internal::pmin< PropagateNumbers, Packet2d >
Packet2d pmin< PropagateNumbers, Packet2d >(const Packet2d &a, const Packet2d &b)
Definition: SSE/PacketMath.h:702
Eigen::internal::predux_min< Packet4i >
int predux_min< Packet4i >(const Packet4i &a)
Definition: AltiVec/PacketMath.h:2200
Eigen::internal::pmax< Packet8us >
Packet8us pmax< Packet8us >(const Packet8us &a, const Packet8us &b)
Definition: AltiVec/PacketMath.h:1113
Eigen::internal::pcmp_lt< Packet4i >
Packet4i pcmp_lt< Packet4i >(const Packet4i &a, const Packet4i &b)
Definition: NEON/PacketMath.h:1379
Eigen::internal::padd< Packet4i >
Packet4i padd< Packet4i >(const Packet4i &a, const Packet4i &b)
Definition: AltiVec/PacketMath.h:982
Eigen::internal::pandnot< Packet4us >
Packet4us pandnot< Packet4us >(const Packet4us &a, const Packet4us &b)
Definition: NEON/PacketMath.h:1590
Eigen::internal::por< Packet2i >
Packet2i por< Packet2i >(const Packet2i &a, const Packet2i &b)
Definition: NEON/PacketMath.h:1528
Eigen::internal::pabs< Packet4f >
Packet4f pabs< Packet4f >(const Packet4f &a)
Definition: ZVector/PacketMath.h:797
Eigen::internal::psub< Packet4i >
Packet4i psub< Packet4i >(const Packet4i &a, const Packet4i &b)
Definition: AltiVec/PacketMath.h:990
Eigen::internal::pcmp_lt< Packet8s >
Packet8s pcmp_lt< Packet8s >(const Packet8s &a, const Packet8s &b)
Definition: NEON/PacketMath.h:1371
Eigen::internal::padd< Packet4c >
Packet4c padd< Packet4c >(const Packet4c &a, const Packet4c &b)
Definition: NEON/PacketMath.h:815
Eigen::internal::psub< Packet4ui >
Packet4ui psub< Packet4ui >(const Packet4ui &a, const Packet4ui &b)
Definition: NEON/PacketMath.h:867
Eigen::internal::pxor< Packet16c >
Packet16c pxor< Packet16c >(const Packet16c &a, const Packet16c &b)
Definition: NEON/PacketMath.h:1547
Eigen::internal::pfirst< Packet2i >
int32_t pfirst< Packet2i >(const Packet2i &a)
Definition: NEON/PacketMath.h:2299
Eigen::internal::pxor< Packet2d >
Packet2d pxor< Packet2d >(const Packet2d &a, const Packet2d &b)
Definition: MSA/PacketMath.h:952
Eigen::internal::pfirst< Packet8us >
unsigned short int pfirst< Packet8us >(const Packet8us &a)
Definition: AltiVec/PacketMath.h:1528
Eigen::internal::ploaddup< Packet4ui >
Packet4ui ploaddup< Packet4ui >(const uint32_t *from)
Definition: NEON/PacketMath.h:1826
Eigen::internal::psignbit
Packet8bf psignbit(const Packet8bf &a)
Definition: AltiVec/PacketMath.h:1590
Eigen::internal::pcmp_eq< Packet8s >
Packet8s pcmp_eq< Packet8s >(const Packet8s &a, const Packet8s &b)
Definition: NEON/PacketMath.h:1432
Eigen::internal::ploadquad< Packet4ui >
Packet4ui ploadquad< Packet4ui >(const uint32_t *from)
Definition: NEON/PacketMath.h:1875
Eigen::internal::predux_mul< Packet4i >
int predux_mul< Packet4i >(const Packet4i &a)
Definition: AltiVec/PacketMath.h:2122
Eigen::internal::predux_min< Packet4bf >
bfloat16 predux_min< Packet4bf >(const Packet4bf &a)
Definition: NEON/PacketMath.h:3649
Eigen::internal::pxor< Packet16uc >
Packet16uc pxor< Packet16uc >(const Packet16uc &a, const Packet16uc &b)
Definition: NEON/PacketMath.h:1553
Eigen::internal::pcmp_lt< Packet4s >
Packet4s pcmp_lt< Packet4s >(const Packet4s &a, const Packet4s &b)
Definition: NEON/PacketMath.h:1369
Eigen::internal::pmin< Packet4us >
Packet4us pmin< Packet4us >(const Packet4us &a, const Packet4us &b)
Definition: NEON/PacketMath.h:1219
Eigen::internal::pxor< Packet2ui >
Packet2ui pxor< Packet2ui >(const Packet2ui &a, const Packet2ui &b)
Definition: NEON/PacketMath.h:1563
Eigen::internal::pfirst< Packet4i >
int pfirst< Packet4i >(const Packet4i &a)
Definition: AltiVec/PacketMath.h:1516
Eigen::internal::pmax< Packet2l >
Packet2l pmax< Packet2l >(const Packet2l &a, const Packet2l &b)
Definition: NEON/PacketMath.h:1273
Eigen::internal::pmax< Packet8uc >
Packet8uc pmax< Packet8uc >(const Packet8uc &a, const Packet8uc &b)
Definition: NEON/PacketMath.h:1263
Eigen::internal::pmin< Packet2f >
Packet2f pmin< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition: NEON/PacketMath.h:1188
Eigen::internal::pset1< Packet4s >
Packet4s pset1< Packet4s >(const int16_t &from)
Definition: NEON/PacketMath.h:712
Eigen::internal::padd< Packet2l >
Packet2l padd< Packet2l >(const Packet2l &a, const Packet2l &b)
Definition: NEON/PacketMath.h:839
Eigen::internal::pgather< bfloat16, Packet4bf >
Packet4bf pgather< bfloat16, Packet4bf >(const bfloat16 *from, Index stride)
Definition: NEON/PacketMath.h:3628
Eigen::internal::pandnot< Packet4c >
Packet4c pandnot< Packet4c >(const Packet4c &a, const Packet4c &b)
Definition: NEON/PacketMath.h:1576
Eigen::internal::pstore< uint64_t >
void pstore< uint64_t >(uint64_t *to, const Packet2ul &from)
Definition: NEON/PacketMath.h:1911
Eigen::internal::pcmp_lt< Packet2ui >
Packet2ui pcmp_lt< Packet2ui >(const Packet2ui &a, const Packet2ui &b)
Definition: NEON/PacketMath.h:1381
Eigen::internal::pmax< Packet4i >
Packet4i pmax< Packet4i >(const Packet4i &a, const Packet4i &b)
Definition: AltiVec/PacketMath.h:1111
Eigen::internal::pfloor< Packet2f >
Packet2f pfloor< Packet2f >(const Packet2f &a)
Definition: NEON/PacketMath.h:3241
Eigen::internal::vec2d_unpacklo
Packet2d vec2d_unpacklo(const Packet2d &a, const Packet2d &b)
Definition: SSE/PacketMath.h:112
Eigen::internal::pandnot< Packet16c >
Packet16c pandnot< Packet16c >(const Packet16c &a, const Packet16c &b)
Definition: NEON/PacketMath.h:1579
Eigen::internal::print< Packet4bf >
Packet4bf print< Packet4bf >(const Packet4bf &a)
Definition: NEON/PacketMath.h:3594
Eigen::internal::pgather< uint32_t, Packet4ui >
Packet4ui pgather< uint32_t, Packet4ui >(const uint32_t *from, Index stride)
Definition: NEON/PacketMath.h:2103
Eigen::internal::por< Packet4i >
Packet4i por< Packet4i >(const Packet4i &a, const Packet4i &b)
Definition: AltiVec/PacketMath.h:1164
Eigen::internal::pconj
Packet2cf pconj(const Packet2cf &a)
Definition: AltiVec/Complex.h:215
Eigen::internal::por< Packet2d >
Packet2d por< Packet2d >(const Packet2d &a, const Packet2d &b)
Definition: MSA/PacketMath.h:945
Eigen::internal::ploadu< Packet4us >
Packet4us ploadu< Packet4us >(const uint16_t *from)
Definition: NEON/PacketMath.h:1745
Eigen::internal::predux< Packet8s >
short int predux< Packet8s >(const Packet8s &a)
Definition: AltiVec/PacketMath.h:2071
Eigen::internal::pfirst< Packet16uc >
unsigned char pfirst< Packet16uc >(const Packet16uc &a)
Definition: AltiVec/PacketMath.h:1537
Eigen::internal::psub< Packet2ui >
Packet2ui psub< Packet2ui >(const Packet2ui &a, const Packet2ui &b)
Definition: NEON/PacketMath.h:866
Eigen::internal::pcmp_eq< Packet2i >
Packet2i pcmp_eq< Packet2i >(const Packet2i &a, const Packet2i &b)
Definition: NEON/PacketMath.h:1438
Eigen::internal::padd< Packet4bf >
Packet4bf padd< Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition: NEON/PacketMath.h:3611
Eigen::internal::pmax< Packet4f >
Packet4f pmax< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition: AltiVec/PacketMath.h:1100
Eigen::internal::pcmp_le< Packet16uc >
Packet16uc pcmp_le< Packet16uc >(const Packet16uc &a, const Packet16uc &b)
Definition: NEON/PacketMath.h:1306
Eigen::internal::pmul< Packet4us >
Packet4us pmul< Packet4us >(const Packet4us &a, const Packet4us &b)
Definition: NEON/PacketMath.h:941
Eigen::internal::pgather< int16_t, Packet8s >
Packet8s pgather< int16_t, Packet8s >(const int16_t *from, Index stride)
Definition: NEON/PacketMath.h:2051
Eigen::internal::pmin< Packet4i >
Packet4i pmin< Packet4i >(const Packet4i &a, const Packet4i &b)
Definition: AltiVec/PacketMath.h:1093
Eigen::internal::pand< Packet4i >
Packet4i pand< Packet4i >(const Packet4i &a, const Packet4i &b)
Definition: AltiVec/PacketMath.h:1155
Eigen::internal::pceil< Packet2f >
Packet2f pceil< Packet2f >(const Packet2f &a)
Definition: NEON/PacketMath.h:3261
Eigen::internal::psub< Packet8uc >
Packet8uc psub< Packet8uc >(const Packet8uc &a, const Packet8uc &b)
Definition: NEON/PacketMath.h:858
Eigen::internal::ploadu< Packet2f >
Packet2f ploadu< Packet2f >(const float *from)
Definition: NEON/PacketMath.h:1717
Eigen::internal::pxor< Packet8us >
Packet8us pxor< Packet8us >(const Packet8us &a, const Packet8us &b)
Definition: AltiVec/PacketMath.h:1173
Eigen::internal::pload< Packet2d >
Packet2d pload< Packet2d >(const double *from)
Definition: MSA/PacketMath.h:966
Eigen::internal::psub< Packet2i >
Packet2i psub< Packet2i >(const Packet2i &a, const Packet2i &b)
Definition: NEON/PacketMath.h:864
Eigen::internal::plset< Packet2i >
Packet2i plset< Packet2i >(const int32_t &a)
Definition: NEON/PacketMath.h:782
Eigen::internal::psub< Packet4f >
Packet4f psub< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition: AltiVec/PacketMath.h:989
Eigen::internal::pcmp_eq< Packet4us >
Packet4us pcmp_eq< Packet4us >(const Packet4us &a, const Packet4us &b)
Definition: NEON/PacketMath.h:1434
Eigen::internal::plogical_shift_left
Packet4i plogical_shift_left(const Packet4i &a)
Definition: AltiVec/PacketMath.h:1597
Eigen::internal::pstore< int8_t >
void pstore< int8_t >(int8_t *to, const Packet4c &from)
Definition: NEON/PacketMath.h:1881
Eigen::internal::pscatter< int32_t, Packet4i >
void pscatter< int32_t, Packet4i >(int32_t *to, const Packet4i &from, Index stride)
Definition: MSA/PacketMath.h:495
Eigen::internal::preverse
Packet2cf preverse(const Packet2cf &a)
Definition: AltiVec/Complex.h:232
Eigen::internal::predux_max< Packet8c >
int8_t predux_max< Packet8c >(const Packet8c &a)
Definition: NEON/PacketMath.h:2760
Eigen::internal::parithmetic_shift_right
Packet4i parithmetic_shift_right(const Packet4i &a)
Definition: AltiVec/PacketMath.h:1593
Eigen::internal::pabsdiff< Packet4us >
Packet4us pabsdiff< Packet4us >(const Packet4us &a, const Packet4us &b)
Definition: NEON/PacketMath.h:1175
Eigen::internal::pstoreu< uint16_t >
void pstoreu< uint16_t >(uint16_t *to, const Packet4us &from)
Definition: NEON/PacketMath.h:1934
Eigen::internal::pcmp_le< Packet4uc >
Packet4uc pcmp_le< Packet4uc >(const Packet4uc &a, const Packet4uc &b)
Definition: NEON/PacketMath.h:1298
Eigen::internal::pstore< uint8_t >
void pstore< uint8_t >(uint8_t *to, const Packet4uc &from)
Definition: NEON/PacketMath.h:1887
Eigen::internal::pcmp_le< Packet8s >
Packet8s pcmp_le< Packet8s >(const Packet8s &a, const Packet8s &b)
Definition: NEON/PacketMath.h:1310
Eigen::internal::pload< Packet4f >
Packet4f pload< Packet4f >(const float *from)
Definition: AltiVec/PacketMath.h:461
Eigen::internal::pcmp_lt_or_nan< Packet4f >
Packet4f pcmp_lt_or_nan< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition: NEON/PacketMath.h:1469
Eigen::internal::predux_mul< Packet8uc >
uint8_t predux_mul< Packet8uc >(const Packet8uc &a)
Definition: NEON/PacketMath.h:2565
Eigen::internal::plset< Packet4uc >
Packet4uc plset< Packet4uc >(const uint8_t &a)
Definition: NEON/PacketMath.h:750
Eigen::internal::por
Packet8h por(const Packet8h &a, const Packet8h &b)
Definition: AVX/PacketMath.h:1875
Eigen::internal::pscatter< uint8_t, Packet8uc >
void pscatter< uint8_t, Packet8uc >(uint8_t *to, const Packet8uc &from, Index stride)
Definition: NEON/PacketMath.h:2176
Eigen::internal::pset1< Packet4f >
Packet4f pset1< Packet4f >(const float &from)
Definition: AltiVec/PacketMath.h:739
Eigen::internal::pcmp_lt
Packet4i pcmp_lt(const Packet4i &a, const Packet4i &b)
Definition: AltiVec/PacketMath.h:1131
Eigen::internal::pmul< Packet2f >
Packet2f pmul< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition: NEON/PacketMath.h:921
Eigen::internal::ploaddup< Packet4f >
Packet4f ploaddup< Packet4f >(const float *from)
Definition: AltiVec/PacketMath.h:1336
Eigen::internal::por< Packet16c >
Packet16c por< Packet16c >(const Packet16c &a, const Packet16c &b)
Definition: NEON/PacketMath.h:1512
Eigen::internal::pgather< int8_t, Packet4c >
Packet4c pgather< int8_t, Packet4c >(const int8_t *from, Index stride)
Definition: NEON/PacketMath.h:1965
Eigen::internal::pload< Packet2f >
Packet2f pload< Packet2f >(const float *from)
Definition: NEON/PacketMath.h:1672
Eigen::internal::ploaddup< Packet2f >
Packet2f ploaddup< Packet2f >(const float *from)
Definition: NEON/PacketMath.h:1762
Eigen::internal::pset1< Packet4ui >
Packet4ui pset1< Packet4ui >(const uint32_t &from)
Definition: NEON/PacketMath.h:719
Eigen::internal::pcmp_lt< Packet2f >
Packet2f pcmp_lt< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition: NEON/PacketMath.h:1345
Eigen::internal::pabsdiff< Packet8s >
Packet8s pabsdiff< Packet8s >(const Packet8s &a, const Packet8s &b)
Definition: NEON/PacketMath.h:1173
Eigen::internal::padd< Packet2i >
Packet2i padd< Packet2i >(const Packet2i &a, const Packet2i &b)
Definition: NEON/PacketMath.h:835
Eigen::internal::predux_mul< Packet16uc >
unsigned char predux_mul< Packet16uc >(const Packet16uc &a)
Definition: AltiVec/PacketMath.h:2172
Eigen::internal::padd< Packet2ul >
Packet2ul padd< Packet2ul >(const Packet2ul &a, const Packet2ul &b)
Definition: NEON/PacketMath.h:840
Eigen::internal::Packet4f
__vector float Packet4f
Definition: AltiVec/PacketMath.h:32
Eigen::internal::pdiv< Packet4c >
Packet4c pdiv< Packet4c >(const Packet4c &, const Packet4c &)
Definition: NEON/PacketMath.h:1009
Eigen::internal::pfirst< Packet8uc >
uint8_t pfirst< Packet8uc >(const Packet8uc &a)
Definition: NEON/PacketMath.h:2293
Eigen::internal::predux< Packet2f >
float predux< Packet2f >(const Packet2f &a)
Definition: NEON/PacketMath.h:2408
Eigen::internal::pmax< PropagateNaN, Packet4f >
Packet4f pmax< PropagateNaN, Packet4f >(const Packet4f &a, const Packet4f &b)
Definition: NEON/PacketMath.h:1245
Eigen::internal::pset1< Packet8uc >
Packet8uc pset1< Packet8uc >(const uint8_t &from)
Definition: NEON/PacketMath.h:710
Eigen::internal::predux_half_dowto4
Packet4c predux_half_dowto4(const Packet8c &a)
Definition: NEON/PacketMath.h:2520
Eigen::internal::pfloor< Packet4f >
Packet4f pfloor< Packet4f >(const Packet4f &a)
Definition: AltiVec/PacketMath.h:1203
Eigen::internal::predux_max< Packet8s >
short int predux_max< Packet8s >(const Packet8s &a)
Definition: AltiVec/PacketMath.h:2293
Eigen::internal::Packet2i
int32x2_t Packet2i
Definition: NEON/PacketMath.h:86
Eigen::internal::padd< Packet16uc >
Packet16uc padd< Packet16uc >(const Packet16uc &a, const Packet16uc &b)
Definition: AltiVec/PacketMath.h:987
Eigen::internal::pxor< Packet4uc >
Packet4uc pxor< Packet4uc >(const Packet4uc &a, const Packet4uc &b)
Definition: NEON/PacketMath.h:1549
Eigen::internal::predux_max< Packet8us >
unsigned short int predux_max< Packet8us >(const Packet8us &a)
Definition: AltiVec/PacketMath.h:2308
Eigen::internal::pgather< float, Packet2f >
Packet2f pgather< float, Packet2f >(const float *from, Index stride)
Definition: NEON/PacketMath.h:1951
Eigen::internal::pcmp_eq< Packet16c >
Packet16c pcmp_eq< Packet16c >(const Packet16c &a, const Packet16c &b)
Definition: NEON/PacketMath.h:1418
Eigen::internal::por< Packet8s >
Packet8s por< Packet8s >(const Packet8s &a, const Packet8s &b)
Definition: AltiVec/PacketMath.h:1165
Eigen::internal::pcmp_le< Packet2l >
Packet2l pcmp_le< Packet2l >(const Packet2l &a, const Packet2l &b)
Definition: NEON/PacketMath.h:1324
Eigen::internal::pstore< bfloat16 >
void pstore< bfloat16 >(bfloat16 *to, const Packet8bf &from)
Definition: AltiVec/PacketMath.h:631
Eigen::internal::pmul< Packet16uc >
Packet16uc pmul< Packet16uc >(const Packet16uc &a, const Packet16uc &b)
Definition: AltiVec/PacketMath.h:1037
Eigen::internal::pdiv< Packet2f >
Packet2f pdiv< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition: NEON/PacketMath.h:958
Eigen::internal::pmul< Packet4bf >
Packet4bf pmul< Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition: NEON/PacketMath.h:3619
Eigen::internal::pcmp_le< Packet4i >
Packet4i pcmp_le< Packet4i >(const Packet4i &a, const Packet4i &b)
Definition: NEON/PacketMath.h:1318
Eigen::internal::make_packet4f
EIGEN_ALWAYS_INLINE Packet4f make_packet4f(float a, float b, float c, float d)
Definition: NEON/PacketMath.h:93
Eigen::internal::ploadquad< Packet16uc >
Packet16uc ploadquad< Packet16uc >(const uint8_t *from)
Definition: NEON/PacketMath.h:1860
Eigen::internal::psub< Packet4uc >
Packet4uc psub< Packet4uc >(const Packet4uc &a, const Packet4uc &b)
Definition: NEON/PacketMath.h:852
Eigen::internal::Packet4uc
eigen_packet_wrapper< uint32_t,5 > Packet4uc
Definition: NEON/PacketMath.h:79
Eigen::internal::pabsdiff< Packet4i >
Packet4i pabsdiff< Packet4i >(const Packet4i &a, const Packet4i &b)
Definition: NEON/PacketMath.h:1181
Eigen::internal::pcmp_eq< Packet2f >
Packet2f pcmp_eq< Packet2f >(const Packet2f &a, const Packet2f &b)
Definition: NEON/PacketMath.h:1406
Eigen::internal::predux< Packet4us >
uint16_t predux< Packet4us >(const Packet4us &a)
Definition: NEON/PacketMath.h:2490
Eigen::internal::pdiv< Packet4s >
Packet4s pdiv< Packet4s >(const Packet4s &, const Packet4s &)
Definition: NEON/PacketMath.h:1039
Eigen::internal::pcmp_lt< Packet8c >
Packet8c pcmp_lt< Packet8c >(const Packet8c &a, const Packet8c &b)
Definition: NEON/PacketMath.h:1355
Eigen::internal::pmin< PropagateNumbers, Packet4bf >
Packet4bf pmin< PropagateNumbers, Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition: NEON/PacketMath.h:3533
Eigen::internal::pmin< Packet8c >
Packet8c pmin< Packet8c >(const Packet8c &a, const Packet8c &b)
Definition: NEON/PacketMath.h:1207
Eigen::internal::pand< Packet4ui >
Packet4ui pand< Packet4ui >(const Packet4ui &a, const Packet4ui &b)
Definition: AltiVec/PacketMath.h:1156
Eigen::internal::ploaddup< Packet8uc >
Packet8uc ploaddup< Packet8uc >(const uint8_t *from)
Definition: NEON/PacketMath.h:1787
Eigen::internal::pscatter< double, Packet2d >
void pscatter< double, Packet2d >(double *to, const Packet2d &from, Index stride)
Definition: MSA/PacketMath.h:1043
Eigen::internal::pcmp_lt< Packet8us >
Packet8us pcmp_lt< Packet8us >(const Packet8us &a, const Packet8us &b)
Definition: NEON/PacketMath.h:1375
Eigen::internal::predux_mul< Packet8us >
unsigned short int predux_mul< Packet8us >(const Packet8us &a)
Definition: AltiVec/PacketMath.h:2140
Eigen::internal::pscatter< uint8_t, Packet16uc >
void pscatter< uint8_t, Packet16uc >(uint8_t *to, const Packet16uc &from, Index stride)
Definition: NEON/PacketMath.h:2187
Eigen::internal::pabsdiff< Packet4f >
Packet4f pabsdiff< Packet4f >(const Packet4f &a, const Packet4f &b)
Definition: NEON/PacketMath.h:1149
Eigen::internal::ploadu< Packet8us >
Packet8us ploadu< Packet8us >(const unsigned short int *from)
Definition: AltiVec/PacketMath.h:1246
Eigen::internal::vec4f_movelh
Packet4f vec4f_movelh(const Packet4f &a, const Packet4f &b)
Definition: NEON/PacketMath.h:136
Eigen::internal::pload< Packet8us >
Packet8us pload< Packet8us >(const unsigned short int *from)
Definition: AltiVec/PacketMath.h:476
Eigen::internal::predux_min< Packet16c >
signed char predux_min< Packet16c >(const Packet16c &a)
Definition: AltiVec/PacketMath.h:2243
Eigen::internal::pcmp_le< Packet2ui >
Packet2ui pcmp_le< Packet2ui >(const Packet2ui &a, const Packet2ui &b)
Definition: NEON/PacketMath.h:1320
Eigen::internal::predux_any
bool predux_any(const Packet4f &x)
Definition: AltiVec/PacketMath.h:2347
Eigen::internal::pcmp_eq< Packet4bf >
Packet4bf pcmp_eq< Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition: NEON/PacketMath.h:3674
Eigen::internal::vec4f_movehl
Packet4f vec4f_movehl(const Packet4f &a, const Packet4f &b)
Definition: NEON/PacketMath.h:140
Eigen::internal::pgather< int8_t, Packet16c >
Packet16c pgather< int8_t, Packet16c >(const int8_t *from, Index stride)
Definition: NEON/PacketMath.h:1984
Eigen::internal::predux_min< Packet2f >
float predux_min< Packet2f >(const Packet2f &a)
Definition: NEON/PacketMath.h:2623
Eigen::internal::pload< Packet4c >
Packet4c pload< Packet4c >(const int8_t *from)
Definition: NEON/PacketMath.h:1676
Eigen::internal::pset1< Packet2ul >
Packet2ul pset1< Packet2ul >(const uint64_t &from)
Definition: NEON/PacketMath.h:721
Eigen::internal::por< Packet4ui >
Packet4ui por< Packet4ui >(const Packet4ui &a, const Packet4ui &b)
Definition: NEON/PacketMath.h:1532
Eigen::internal::pandnot< Packet4ui >
Packet4ui pandnot< Packet4ui >(const Packet4ui &a, const Packet4ui &b)
Definition: NEON/PacketMath.h:1600
Eigen::internal::pcmp_lt< Packet8uc >
Packet8uc pcmp_lt< Packet8uc >(const Packet8uc &a, const Packet8uc &b)
Definition: NEON/PacketMath.h:1365
Eigen::internal::pmin< Packet8us >
Packet8us pmin< Packet8us >(const Packet8us &a, const Packet8us &b)
Definition: AltiVec/PacketMath.h:1095
Eigen::internal::pstore< double >
void pstore< double >(double *to, const Packet4d &from)
Definition: AVX/PacketMath.h:1288
Eigen::internal::por< Packet4s >
Packet4s por< Packet4s >(const Packet4s &a, const Packet4s &b)
Definition: NEON/PacketMath.h:1520
Eigen::internal::pcmp_le
Packet4f pcmp_le(const Packet4f &a, const Packet4f &b)
Definition: AltiVec/PacketMath.h:1117
Eigen::internal::pceil< Packet4f >
Packet4f pceil< Packet4f >(const Packet4f &a)
Definition: AltiVec/PacketMath.h:1202
Eigen::internal::predux_mul< Packet4ui >
uint32_t predux_mul< Packet4ui >(const Packet4ui &a)
Definition: NEON/PacketMath.h:2611
Eigen::internal::ploaddup< Packet4bf >
Packet4bf ploaddup< Packet4bf >(const bfloat16 *from)
Definition: NEON/PacketMath.h:3524
Eigen::internal::Packet4s
int16x4_t Packet4s
Definition: NEON/PacketMath.h:82
Eigen::internal::vec4f_swizzle1
Packet4f vec4f_swizzle1(const Packet4f &a, int p, int q, int r, int s)
Definition: NEON/PacketMath.h:128
Eigen::internal::pset1< Packet8c >
Packet8c pset1< Packet8c >(const int8_t &from)
Definition: NEON/PacketMath.h:706
Eigen::internal::predux_min< Packet8uc >
uint8_t predux_min< Packet8uc >(const Packet8uc &a)
Definition: NEON/PacketMath.h:2677
Eigen::internal::plset< Packet4us >
Packet4us plset< Packet4us >(const uint16_t &a)
Definition: NEON/PacketMath.h:767
Eigen::internal::ploaddup< Packet8c >
Packet8c ploaddup< Packet8c >(const int8_t *from)
Definition: NEON/PacketMath.h:1771
Eigen::internal::pxor< Packet8c >
Packet8c pxor< Packet8c >(const Packet8c &a, const Packet8c &b)
Definition: NEON/PacketMath.h:1545
Eigen::internal::pandnot< Packet4uc >
Packet4uc pandnot< Packet4uc >(const Packet4uc &a, const Packet4uc &b)
Definition: NEON/PacketMath.h:1580
Eigen::internal::pabsdiff< Packet4uc >
Packet4uc pabsdiff< Packet4uc >(const Packet4uc &a, const Packet4uc &b)
Definition: NEON/PacketMath.h:1161
Eigen::internal::prsqrt
Packet4f prsqrt(const Packet4f &a)
Definition: MSA/PacketMath.h:730
Eigen::internal::pset1< Packet2ui >
Packet2ui pset1< Packet2ui >(const uint32_t &from)
Definition: NEON/PacketMath.h:718
Eigen::internal::pcmp_lt_or_nan< Packet4bf >
Packet4bf pcmp_lt_or_nan< Packet4bf >(const Packet4bf &a, const Packet4bf &b)
Definition: NEON/PacketMath.h:3684
Eigen::internal::pmax< Packet2d >
Packet2d pmax< Packet2d >(const Packet2d &a, const Packet2d &b)
Definition: MSA/PacketMath.h:988
Eigen::internal::pstoreu< int16_t >
void pstoreu< int16_t >(int16_t *to, const Packet4s &from)
Definition: NEON/PacketMath.h:1930
Eigen::internal::predux< Packet2l >
int64_t predux< Packet2l >(const Packet2l &a)
Definition: NEON/PacketMath.h:2514
Eigen::numext::int32_t
std::int32_t int32_t
Definition: Meta.h:40
Eigen::numext::int8_t
std::int8_t int8_t
Definition: Meta.h:36
Eigen::numext::uint8_t
std::uint8_t uint8_t
Definition: Meta.h:35
Eigen::numext::int16_t
std::int16_t int16_t
Definition: Meta.h:38
Eigen::numext::int64_t
std::int64_t int64_t
Definition: Meta.h:42
Eigen::numext::uint16_t
std::uint16_t uint16_t
Definition: Meta.h:37
Eigen::numext::uint32_t
std::uint32_t uint32_t
Definition: Meta.h:39
Eigen::numext::uint64_t
std::uint64_t uint64_t
Definition: Meta.h:41
Eigen
: InteropHeaders
Definition: Core:139
Eigen::Index
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition: Meta.h:82
Eigen::bfloat16_impl::__bfloat16_raw::value
unsigned short value
Definition: BFloat16.h:76
Eigen::half_impl::__half_raw::x
numext::uint16_t x
Definition: Half.h:104
j
std::ptrdiff_t j
Definition: tut_arithmetic_redux_minmax.cpp:2
Generated on Tue Jul 18 2023 15:28:17 for Eigen by Doxygen 1.9.1 and the doxyYoda theme. This is an unofficial themed variant maintained by Rohit Goswami, based on the official documentation.