AVX512/PacketMath.h
Go to the documentation of this file.
1 // This file is part of Eigen, a lightweight C++ template library
2 // for linear algebra.
3 //
4 // Copyright (C) 2016 Benoit Steiner (benoit.steiner.goog@gmail.com)
5 //
6 // This Source Code Form is subject to the terms of the Mozilla
7 // Public License v. 2.0. If a copy of the MPL was not distributed
8 // with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9 
10 #ifndef EIGEN_PACKET_MATH_AVX512_H
11 #define EIGEN_PACKET_MATH_AVX512_H
12 
13 #include "../../InternalHeaderCheck.h"
14 
15 namespace Eigen {
16 
17 namespace internal {
18 
19 #ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
20 #define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
21 #endif
22 
23 #ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
24 #define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32
25 #endif
26 
27 #ifdef EIGEN_VECTORIZE_FMA
28 #ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
29 #define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
30 #endif
31 #endif
32 
33 typedef __m512 Packet16f;
34 typedef __m512i Packet16i;
35 typedef __m512d Packet8d;
36 #ifndef EIGEN_VECTORIZE_AVX512FP16
37 typedef eigen_packet_wrapper<__m256i, 1> Packet16h;
38 #endif
39 typedef eigen_packet_wrapper<__m256i, 2> Packet16bf;
40 
41 template <>
42 struct is_arithmetic<__m512> {
43  enum { value = true };
44 };
45 template <>
46 struct is_arithmetic<__m512i> {
47  enum { value = true };
48 };
49 template <>
50 struct is_arithmetic<__m512d> {
51  enum { value = true };
52 };
53 
54 #ifndef EIGEN_VECTORIZE_AVX512FP16
55 template<> struct is_arithmetic<Packet16h> { enum { value = true }; };
56 
57 template <>
58 struct packet_traits<half> : default_packet_traits {
59  typedef Packet16h type;
60  // There is no half-size packet for Packet16h.
61  typedef Packet16h half;
62  enum {
63  Vectorizable = 1,
64  AlignedOnScalar = 1,
65  size = 16,
66 
67  HasCmp = 1,
68  HasAdd = 1,
69  HasSub = 1,
70  HasMul = 1,
71  HasDiv = 1,
72  HasNegate = 1,
73  HasAbs = 1,
74  HasAbs2 = 0,
75  HasMin = 1,
76  HasMax = 1,
77  HasConj = 1,
78  HasSetLinear = 0,
79  HasSqrt = 1,
80  HasRsqrt = 1,
81  HasLog = 1,
82  HasLog1p = 1,
83  HasExp = 1,
84  HasExpm1 = 1,
85  HasBessel = 1,
86  HasNdtri = 1,
87  HasSin = EIGEN_FAST_MATH,
88  HasCos = EIGEN_FAST_MATH,
89  HasTanh = EIGEN_FAST_MATH,
90  HasErf = EIGEN_FAST_MATH,
91  HasBlend = 0,
92  HasRound = 1,
93  HasFloor = 1,
94  HasCeil = 1,
95  HasRint = 1
96  };
97 };
98 #endif
99 
100 template<> struct packet_traits<float> : default_packet_traits
101 {
102  typedef Packet16f type;
103  typedef Packet8f half;
104  enum {
105  Vectorizable = 1,
106  AlignedOnScalar = 1,
107  size = 16,
108 
109  HasAbs = 1,
110  HasMin = 1,
111  HasMax = 1,
112  HasConj = 1,
113  HasBlend = 1,
114  HasSin = EIGEN_FAST_MATH,
115  HasCos = EIGEN_FAST_MATH,
116  HasACos = 1,
117  HasASin = 1,
118  HasATan = 1,
119  HasATanh = 1,
120  HasSqrt = 1,
121  HasRsqrt = 1,
122  HasLog = 1,
123  HasLog1p = 1,
124  HasExpm1 = 1,
125  HasNdtri = 1,
126  HasBessel = 1,
127  HasExp = 1,
128  HasReciprocal = EIGEN_FAST_MATH,
129  HasTanh = EIGEN_FAST_MATH,
130  HasErf = EIGEN_FAST_MATH,
131  HasCmp = 1,
132  HasDiv = 1,
133  HasRound = 1,
134  HasFloor = 1,
135  HasCeil = 1,
136  HasRint = 1
137  };
138  };
139 template<> struct packet_traits<double> : default_packet_traits
140 {
141  typedef Packet8d type;
142  typedef Packet4d half;
143  enum {
144  Vectorizable = 1,
145  AlignedOnScalar = 1,
146  size = 8,
147  HasBlend = 1,
148  HasSqrt = 1,
149  HasRsqrt = 1,
150  HasLog = 1,
151  HasExp = 1,
152  HasATan = 1,
153  HasCmp = 1,
154  HasDiv = 1,
155  HasRound = 1,
156  HasFloor = 1,
157  HasCeil = 1,
158  HasRint = 1
159  };
160 };
161 
162 template<> struct packet_traits<int> : default_packet_traits
163 {
164  typedef Packet16i type;
165  typedef Packet8i half;
166  enum {
167  Vectorizable = 1,
168  AlignedOnScalar = 1,
169  HasBlend = 0,
170  HasCmp = 1,
171  HasDiv = 1,
172  size=16
173  };
174 };
175 
176 template <>
177 struct unpacket_traits<Packet16f> {
178  typedef float type;
179  typedef Packet8f half;
180  typedef Packet16i integer_packet;
181  typedef uint16_t mask_t;
182  enum { size = 16, alignment=Aligned64, vectorizable=true, masked_load_available=true, masked_store_available=true, masked_fpops_available=true };
183 };
184 template <>
185 struct unpacket_traits<Packet8d> {
186  typedef double type;
187  typedef Packet4d half;
188  typedef uint8_t mask_t;
189  enum { size = 8, alignment=Aligned64, vectorizable=true, masked_load_available=true, masked_store_available=true, masked_fpops_available=true };
190 };
191 template <>
192 struct unpacket_traits<Packet16i> {
193  typedef int type;
194  typedef Packet8i half;
195  enum { size = 16, alignment=Aligned64, vectorizable=true, masked_load_available=false, masked_store_available=false };
196 };
197 
198 #ifndef EIGEN_VECTORIZE_AVX512FP16
199 template<>
200 struct unpacket_traits<Packet16h> {
201  typedef Eigen::half type;
202  typedef Packet8h half;
203  enum {size=16, alignment=Aligned32, vectorizable=true, masked_load_available=false, masked_store_available=false};
204 };
205 #endif
206 
207 template <>
208 EIGEN_STRONG_INLINE Packet16f pset1<Packet16f>(const float& from) {
209  return _mm512_set1_ps(from);
210 }
211 template <>
212 EIGEN_STRONG_INLINE Packet8d pset1<Packet8d>(const double& from) {
213  return _mm512_set1_pd(from);
214 }
215 template <>
216 EIGEN_STRONG_INLINE Packet16i pset1<Packet16i>(const int& from) {
217  return _mm512_set1_epi32(from);
218 }
219 
220 template <>
221 EIGEN_STRONG_INLINE Packet16f pset1frombits<Packet16f>(unsigned int from) {
222  return _mm512_castsi512_ps(_mm512_set1_epi32(from));
223 }
224 
225 template <>
226 EIGEN_STRONG_INLINE Packet8d pset1frombits<Packet8d>(const numext::uint64_t from) {
227  return _mm512_castsi512_pd(_mm512_set1_epi64(from));
228 }
229 
230 template<> EIGEN_STRONG_INLINE Packet16f pzero(const Packet16f& /*a*/) { return _mm512_setzero_ps(); }
231 template<> EIGEN_STRONG_INLINE Packet8d pzero(const Packet8d& /*a*/) { return _mm512_setzero_pd(); }
232 template<> EIGEN_STRONG_INLINE Packet16i pzero(const Packet16i& /*a*/) { return _mm512_setzero_si512(); }
233 
234 template<> EIGEN_STRONG_INLINE Packet16f peven_mask(const Packet16f& /*a*/) {
235  return _mm512_castsi512_ps(_mm512_set_epi32(0, -1, 0, -1, 0, -1, 0, -1,
236  0, -1, 0, -1, 0, -1, 0, -1));
237 }
238 template<> EIGEN_STRONG_INLINE Packet16i peven_mask(const Packet16i& /*a*/) {
239  return _mm512_set_epi32(0, -1, 0, -1, 0, -1, 0, -1,
240  0, -1, 0, -1, 0, -1, 0, -1);
241 }
242 template<> EIGEN_STRONG_INLINE Packet8d peven_mask(const Packet8d& /*a*/) {
243  return _mm512_castsi512_pd(_mm512_set_epi32(0, 0, -1, -1, 0, 0, -1, -1,
244  0, 0, -1, -1, 0, 0, -1, -1));
245 }
246 
247 template <>
248 EIGEN_STRONG_INLINE Packet16f pload1<Packet16f>(const float* from) {
249 #if (EIGEN_COMP_GNUC != 0) || (EIGEN_COMP_CLANG != 0)
250  // Inline asm here helps reduce some register spilling in TRSM kernels.
251  // See note in unrolls::gemm::microKernel in TrsmKernel.h
252  Packet16f ret;
253  __asm__ ("vbroadcastss %[mem], %[dst]" : [dst] "=v" (ret) : [mem] "m" (*from));
254  return ret;
255 #else
256  return _mm512_broadcastss_ps(_mm_load_ps1(from));
257 #endif
258 }
259 template <>
260 EIGEN_STRONG_INLINE Packet8d pload1<Packet8d>(const double* from) {
261 #if (EIGEN_COMP_GNUC != 0) || (EIGEN_COMP_CLANG != 0)
262  Packet8d ret;
263  __asm__ ("vbroadcastsd %[mem], %[dst]" : [dst] "=v" (ret) : [mem] "m" (*from));
264  return ret;
265 #else
266  return _mm512_set1_pd(*from);
267 #endif
268 }
269 
270 template <>
271 EIGEN_STRONG_INLINE Packet16f plset<Packet16f>(const float& a) {
272  return _mm512_add_ps(
273  _mm512_set1_ps(a),
274  _mm512_set_ps(15.0f, 14.0f, 13.0f, 12.0f, 11.0f, 10.0f, 9.0f, 8.0f, 7.0f, 6.0f, 5.0f,
275  4.0f, 3.0f, 2.0f, 1.0f, 0.0f));
276 }
277 template <>
278 EIGEN_STRONG_INLINE Packet8d plset<Packet8d>(const double& a) {
279  return _mm512_add_pd(_mm512_set1_pd(a),
280  _mm512_set_pd(7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.0));
281 }
282 template <>
283 EIGEN_STRONG_INLINE Packet16i plset<Packet16i>(const int& a) {
284  return _mm512_add_epi32(
285  _mm512_set1_epi32(a),
286  _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0));
287 }
288 
289 template <>
290 EIGEN_STRONG_INLINE Packet16f padd<Packet16f>(const Packet16f& a,
291  const Packet16f& b) {
292  return _mm512_add_ps(a, b);
293 }
294 template <>
295 EIGEN_STRONG_INLINE Packet8d padd<Packet8d>(const Packet8d& a,
296  const Packet8d& b) {
297  return _mm512_add_pd(a, b);
298 }
299 template <>
300 EIGEN_STRONG_INLINE Packet16i padd<Packet16i>(const Packet16i& a,
301  const Packet16i& b) {
302  return _mm512_add_epi32(a, b);
303 }
304 
305 template <>
306 EIGEN_STRONG_INLINE Packet16f padd<Packet16f>(const Packet16f& a,
307  const Packet16f& b,
308  uint16_t umask) {
309  __mmask16 mask = static_cast<__mmask16>(umask);
310  return _mm512_maskz_add_ps(mask, a, b);
311 }
312 template <>
313 EIGEN_STRONG_INLINE Packet8d padd<Packet8d>(const Packet8d& a,
314  const Packet8d& b,
315  uint8_t umask) {
316  __mmask8 mask = static_cast<__mmask8>(umask);
317  return _mm512_maskz_add_pd(mask, a, b);
318 }
319 
320 template <>
321 EIGEN_STRONG_INLINE Packet16f psub<Packet16f>(const Packet16f& a,
322  const Packet16f& b) {
323  return _mm512_sub_ps(a, b);
324 }
325 template <>
326 EIGEN_STRONG_INLINE Packet8d psub<Packet8d>(const Packet8d& a,
327  const Packet8d& b) {
328  return _mm512_sub_pd(a, b);
329 }
330 template <>
331 EIGEN_STRONG_INLINE Packet16i psub<Packet16i>(const Packet16i& a,
332  const Packet16i& b) {
333  return _mm512_sub_epi32(a, b);
334 }
335 
336 template <>
337 EIGEN_STRONG_INLINE Packet16f pnegate(const Packet16f& a) {
338  // NOTE: MSVC seems to struggle with _mm512_set1_epi32, leading to random results.
339  // The intel docs give it a relatively high latency as well, so we're probably
340  // better off with using _mm512_set_epi32 directly anyways.
341  const __m512i mask = _mm512_set_epi32(0x80000000,0x80000000,0x80000000,0x80000000,
342  0x80000000,0x80000000,0x80000000,0x80000000,
343  0x80000000,0x80000000,0x80000000,0x80000000,
344  0x80000000,0x80000000,0x80000000,0x80000000);
345  return _mm512_castsi512_ps(_mm512_xor_epi32(_mm512_castps_si512(a), mask));
346 }
347 template <>
348 EIGEN_STRONG_INLINE Packet8d pnegate(const Packet8d& a) {
349  const __m512i mask = _mm512_set_epi64(0x8000000000000000ULL, 0x8000000000000000ULL, 0x8000000000000000ULL, 0x8000000000000000ULL,
350  0x8000000000000000ULL, 0x8000000000000000ULL, 0x8000000000000000ULL, 0x8000000000000000ULL);
351  return _mm512_castsi512_pd(_mm512_xor_epi64(_mm512_castpd_si512(a), mask));
352 }
353 template <>
354 EIGEN_STRONG_INLINE Packet16i pnegate(const Packet16i& a) {
355  return _mm512_sub_epi32(_mm512_setzero_si512(), a);
356 }
357 
358 template <>
359 EIGEN_STRONG_INLINE Packet16f pconj(const Packet16f& a) {
360  return a;
361 }
362 template <>
363 EIGEN_STRONG_INLINE Packet8d pconj(const Packet8d& a) {
364  return a;
365 }
366 template <>
367 EIGEN_STRONG_INLINE Packet16i pconj(const Packet16i& a) {
368  return a;
369 }
370 
371 template <>
372 EIGEN_STRONG_INLINE Packet16f pmul<Packet16f>(const Packet16f& a,
373  const Packet16f& b) {
374  return _mm512_mul_ps(a, b);
375 }
376 template <>
377 EIGEN_STRONG_INLINE Packet8d pmul<Packet8d>(const Packet8d& a,
378  const Packet8d& b) {
379  return _mm512_mul_pd(a, b);
380 }
381 template <>
382 EIGEN_STRONG_INLINE Packet16i pmul<Packet16i>(const Packet16i& a,
383  const Packet16i& b) {
384  return _mm512_mullo_epi32(a, b);
385 }
386 
387 template <>
388 EIGEN_STRONG_INLINE Packet16f pdiv<Packet16f>(const Packet16f& a,
389  const Packet16f& b) {
390  return _mm512_div_ps(a, b);
391 }
392 
393 template <>
394 EIGEN_STRONG_INLINE Packet8d pdiv<Packet8d>(const Packet8d& a,
395  const Packet8d& b) {
396  return _mm512_div_pd(a, b);
397 }
398 
399 template <>
400 EIGEN_STRONG_INLINE Packet16i pdiv<Packet16i>(const Packet16i& a,
401  const Packet16i& b) {
402  Packet8i q_lo = pdiv<Packet8i>(_mm512_extracti64x4_epi64(a, 0), _mm512_extracti64x4_epi64(b,0));
403  Packet8i q_hi = pdiv<Packet8i>(_mm512_extracti64x4_epi64(a, 1), _mm512_extracti64x4_epi64(b, 1));
404  return _mm512_inserti64x4(_mm512_castsi256_si512(q_lo), q_hi, 1);
405 }
406 
407 #ifdef EIGEN_VECTORIZE_FMA
408 template <>
409 EIGEN_STRONG_INLINE Packet16f pmadd(const Packet16f& a, const Packet16f& b,
410  const Packet16f& c) {
411  return _mm512_fmadd_ps(a, b, c);
412 }
413 template <>
414 EIGEN_STRONG_INLINE Packet8d pmadd(const Packet8d& a, const Packet8d& b,
415  const Packet8d& c) {
416  return _mm512_fmadd_pd(a, b, c);
417 }
418 
419 template <>
420 EIGEN_STRONG_INLINE Packet16f pmsub(const Packet16f& a, const Packet16f& b,
421  const Packet16f& c) {
422  return _mm512_fmsub_ps(a, b, c);
423 }
424 template <>
425 EIGEN_STRONG_INLINE Packet8d pmsub(const Packet8d& a, const Packet8d& b,
426  const Packet8d& c) {
427  return _mm512_fmsub_pd(a, b, c);
428 }
429 
430 template <>
431 EIGEN_STRONG_INLINE Packet16f pnmadd(const Packet16f& a, const Packet16f& b,
432  const Packet16f& c) {
433  return _mm512_fnmadd_ps(a, b, c);
434 }
435 template <>
436 EIGEN_STRONG_INLINE Packet8d pnmadd(const Packet8d& a, const Packet8d& b,
437  const Packet8d& c) {
438  return _mm512_fnmadd_pd(a, b, c);
439 }
440 
441 template <>
442 EIGEN_STRONG_INLINE Packet16f pnmsub(const Packet16f& a, const Packet16f& b,
443  const Packet16f& c) {
444  return _mm512_fnmsub_ps(a, b, c);
445 }
446 template <>
447 EIGEN_STRONG_INLINE Packet8d pnmsub(const Packet8d& a, const Packet8d& b,
448  const Packet8d& c) {
449  return _mm512_fnmsub_pd(a, b, c);
450 }
451 #endif
452 
453 template <>
454 EIGEN_DEVICE_FUNC inline Packet16f pselect(const Packet16f& mask,
455  const Packet16f& a,
456  const Packet16f& b) {
457  __mmask16 mask16 = _mm512_cmpeq_epi32_mask(_mm512_castps_si512(mask), _mm512_setzero_epi32());
458  return _mm512_mask_blend_ps(mask16, a, b);
459 }
460 
461 template <>
462 EIGEN_DEVICE_FUNC inline Packet16i pselect(const Packet16i& mask,
463  const Packet16i& a,
464  const Packet16i& b) {
465  __mmask16 mask16 = _mm512_cmpeq_epi32_mask(mask, _mm512_setzero_epi32());
466  return _mm512_mask_blend_epi32(mask16, a, b);
467 }
468 
469 template <>
470 EIGEN_DEVICE_FUNC inline Packet8d pselect(const Packet8d& mask,
471  const Packet8d& a,
472  const Packet8d& b) {
473  __mmask8 mask8 = _mm512_cmp_epi64_mask(_mm512_castpd_si512(mask),
474  _mm512_setzero_epi32(), _MM_CMPINT_EQ);
475  return _mm512_mask_blend_pd(mask8, a, b);
476 }
477 
478 template <>
479 EIGEN_STRONG_INLINE Packet16f pmin<Packet16f>(const Packet16f& a,
480  const Packet16f& b) {
481  // Arguments are reversed to match NaN propagation behavior of std::min.
482  return _mm512_min_ps(b, a);
483 }
484 template <>
485 EIGEN_STRONG_INLINE Packet8d pmin<Packet8d>(const Packet8d& a,
486  const Packet8d& b) {
487  // Arguments are reversed to match NaN propagation behavior of std::min.
488  return _mm512_min_pd(b, a);
489 }
490 template <>
491 EIGEN_STRONG_INLINE Packet16i pmin<Packet16i>(const Packet16i& a,
492  const Packet16i& b) {
493  return _mm512_min_epi32(b, a);
494 }
495 
496 template <>
497 EIGEN_STRONG_INLINE Packet16f pmax<Packet16f>(const Packet16f& a,
498  const Packet16f& b) {
499  // Arguments are reversed to match NaN propagation behavior of std::max.
500  return _mm512_max_ps(b, a);
501 }
502 template <>
503 EIGEN_STRONG_INLINE Packet8d pmax<Packet8d>(const Packet8d& a,
504  const Packet8d& b) {
505  // Arguments are reversed to match NaN propagation behavior of std::max.
506  return _mm512_max_pd(b, a);
507 }
508 template <>
509 EIGEN_STRONG_INLINE Packet16i pmax<Packet16i>(const Packet16i& a,
510  const Packet16i& b) {
511  return _mm512_max_epi32(b, a);
512 }
513 
514 // Add specializations for min/max with prescribed NaN progation.
515 template<>
516 EIGEN_STRONG_INLINE Packet16f pmin<PropagateNumbers, Packet16f>(const Packet16f& a, const Packet16f& b) {
517  return pminmax_propagate_numbers(a, b, pmin<Packet16f>);
518 }
519 template<>
520 EIGEN_STRONG_INLINE Packet8d pmin<PropagateNumbers, Packet8d>(const Packet8d& a, const Packet8d& b) {
521  return pminmax_propagate_numbers(a, b, pmin<Packet8d>);
522 }
523 template<>
524 EIGEN_STRONG_INLINE Packet16f pmax<PropagateNumbers, Packet16f>(const Packet16f& a, const Packet16f& b) {
525  return pminmax_propagate_numbers(a, b, pmax<Packet16f>);
526 }
527 template<>
528 EIGEN_STRONG_INLINE Packet8d pmax<PropagateNumbers, Packet8d>(const Packet8d& a, const Packet8d& b) {
529  return pminmax_propagate_numbers(a, b, pmax<Packet8d>);
530 }
531 template<>
532 EIGEN_STRONG_INLINE Packet16f pmin<PropagateNaN, Packet16f>(const Packet16f& a, const Packet16f& b) {
533  return pminmax_propagate_nan(a, b, pmin<Packet16f>);
534 }
535 template<>
536 EIGEN_STRONG_INLINE Packet8d pmin<PropagateNaN, Packet8d>(const Packet8d& a, const Packet8d& b) {
537  return pminmax_propagate_nan(a, b, pmin<Packet8d>);
538 }
539 template<>
540 EIGEN_STRONG_INLINE Packet16f pmax<PropagateNaN, Packet16f>(const Packet16f& a, const Packet16f& b) {
541  return pminmax_propagate_nan(a, b, pmax<Packet16f>);
542 }
543 template<>
544 EIGEN_STRONG_INLINE Packet8d pmax<PropagateNaN, Packet8d>(const Packet8d& a, const Packet8d& b) {
545  return pminmax_propagate_nan(a, b, pmax<Packet8d>);
546 }
547 
548 
549 #ifdef EIGEN_VECTORIZE_AVX512DQ
550 template<int I_> EIGEN_STRONG_INLINE Packet8f extract256(Packet16f x) { return _mm512_extractf32x8_ps(x,I_); }
551 template<int I_> EIGEN_STRONG_INLINE Packet2d extract128(Packet8d x) { return _mm512_extractf64x2_pd(x,I_); }
552 EIGEN_STRONG_INLINE Packet16f cat256(Packet8f a, Packet8f b) { return _mm512_insertf32x8(_mm512_castps256_ps512(a),b,1); }
553 EIGEN_STRONG_INLINE Packet16i cat256i(Packet8i a, Packet8i b) { return _mm512_inserti32x8(_mm512_castsi256_si512(a), b, 1); }
554 #else
555 // AVX512F does not define _mm512_extractf32x8_ps to extract _m256 from _m512
556 template<int I_> EIGEN_STRONG_INLINE Packet8f extract256(Packet16f x) {
557  return _mm256_castsi256_ps(_mm512_extracti64x4_epi64( _mm512_castps_si512(x),I_));
558 }
559 
560 // AVX512F does not define _mm512_extractf64x2_pd to extract _m128 from _m512
561 template<int I_> EIGEN_STRONG_INLINE Packet2d extract128(Packet8d x) {
562  return _mm_castsi128_pd(_mm512_extracti32x4_epi32( _mm512_castpd_si512(x),I_));
563 }
564 
565 EIGEN_STRONG_INLINE Packet16f cat256(Packet8f a, Packet8f b) {
566  return _mm512_castsi512_ps(_mm512_inserti64x4(_mm512_castsi256_si512(_mm256_castps_si256(a)),
567  _mm256_castps_si256(b),1));
568 }
569 EIGEN_STRONG_INLINE Packet16i cat256i(Packet8i a, Packet8i b) {
570  return _mm512_inserti64x4(_mm512_castsi256_si512(a), b, 1);
571 }
572 #endif
573 
574 // Helper function for bit packing snippet of low precision comparison.
575 // It packs the flags from 32x16 to 16x16.
576 EIGEN_STRONG_INLINE __m256i Pack32To16(Packet16f rf) {
577  // Split data into small pieces and handle with AVX instructions
578  // to guarantee internal order of vector.
579  // Operation:
580  // dst[15:0] := Saturate16(rf[31:0])
581  // dst[31:16] := Saturate16(rf[63:32])
582  // ...
583  // dst[255:240] := Saturate16(rf[255:224])
584  __m256i lo = _mm256_castps_si256(extract256<0>(rf));
585  __m256i hi = _mm256_castps_si256(extract256<1>(rf));
586  __m128i result_lo = _mm_packs_epi32(_mm256_extractf128_si256(lo, 0),
587  _mm256_extractf128_si256(lo, 1));
588  __m128i result_hi = _mm_packs_epi32(_mm256_extractf128_si256(hi, 0),
589  _mm256_extractf128_si256(hi, 1));
590  return _mm256_insertf128_si256(_mm256_castsi128_si256(result_lo), result_hi, 1);
591 }
592 
593 template <>
594 EIGEN_STRONG_INLINE Packet16f pisnan(const Packet16f& a) {
595  __mmask16 mask = _mm512_cmp_ps_mask(a, a, _CMP_UNORD_Q);
596  return _mm512_castsi512_ps(_mm512_maskz_set1_epi32(mask, 0xffffffffu));
597 }
598 
599 template <>
600 EIGEN_STRONG_INLINE Packet16f pcmp_eq(const Packet16f& a, const Packet16f& b) {
601  __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_EQ_OQ);
602  return _mm512_castsi512_ps(
603  _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, 0xffffffffu));
604 }
605 template<> EIGEN_STRONG_INLINE Packet16f pcmp_le(const Packet16f& a, const Packet16f& b) {
606  __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_LE_OQ);
607  return _mm512_castsi512_ps(
608  _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, 0xffffffffu));
609 }
610 
611 template<> EIGEN_STRONG_INLINE Packet16f pcmp_lt(const Packet16f& a, const Packet16f& b) {
612  __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_LT_OQ);
613  return _mm512_castsi512_ps(
614  _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, 0xffffffffu));
615 }
616 
617 template<> EIGEN_STRONG_INLINE Packet16f pcmp_lt_or_nan(const Packet16f& a, const Packet16f& b) {
618  __mmask16 mask = _mm512_cmp_ps_mask(a, b, _CMP_NGE_UQ);
619  return _mm512_castsi512_ps(
620  _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, 0xffffffffu));
621 }
622 
623 template<> EIGEN_STRONG_INLINE Packet16i pcmp_eq(const Packet16i& a, const Packet16i& b) {
624  __mmask16 mask = _mm512_cmp_epi32_mask(a, b, _MM_CMPINT_EQ);
625  return _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, 0xffffffffu);
626 }
627 template<> EIGEN_STRONG_INLINE Packet16i pcmp_le(const Packet16i& a, const Packet16i& b) {
628  __mmask16 mask = _mm512_cmp_epi32_mask(a, b, _MM_CMPINT_LE);
629  return _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, 0xffffffffu);
630 }
631 template<> EIGEN_STRONG_INLINE Packet16i pcmp_lt(const Packet16i& a, const Packet16i& b) {
632  __mmask16 mask = _mm512_cmp_epi32_mask(a, b, _MM_CMPINT_LT);
633  return _mm512_mask_set1_epi32(_mm512_setzero_epi32(), mask, 0xffffffffu);
634 }
635 
636 template <>
637 EIGEN_STRONG_INLINE Packet8d pcmp_eq(const Packet8d& a, const Packet8d& b) {
638  __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_EQ_OQ);
639  return _mm512_castsi512_pd(
640  _mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, 0xffffffffffffffffu));
641 }
642 template <>
643 EIGEN_STRONG_INLINE Packet8d pcmp_le(const Packet8d& a, const Packet8d& b) {
644  __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_LE_OQ);
645  return _mm512_castsi512_pd(
646  _mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, 0xffffffffffffffffu));
647 }
648 template <>
649 EIGEN_STRONG_INLINE Packet8d pcmp_lt(const Packet8d& a, const Packet8d& b) {
650  __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_LT_OQ);
651  return _mm512_castsi512_pd(
652  _mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, 0xffffffffffffffffu));
653 }
654 template <>
655 EIGEN_STRONG_INLINE Packet8d pcmp_lt_or_nan(const Packet8d& a, const Packet8d& b) {
656  __mmask8 mask = _mm512_cmp_pd_mask(a, b, _CMP_NGE_UQ);
657  return _mm512_castsi512_pd(
658  _mm512_mask_set1_epi64(_mm512_setzero_epi32(), mask, 0xffffffffffffffffu));
659 }
660 
661 template<> EIGEN_STRONG_INLINE Packet16f print<Packet16f>(const Packet16f& a) { return _mm512_roundscale_ps(a, _MM_FROUND_CUR_DIRECTION); }
662 template<> EIGEN_STRONG_INLINE Packet8d print<Packet8d>(const Packet8d& a) { return _mm512_roundscale_pd(a, _MM_FROUND_CUR_DIRECTION); }
663 
664 template<> EIGEN_STRONG_INLINE Packet16f pceil<Packet16f>(const Packet16f& a) { return _mm512_roundscale_ps(a, _MM_FROUND_TO_POS_INF); }
665 template<> EIGEN_STRONG_INLINE Packet8d pceil<Packet8d>(const Packet8d& a) { return _mm512_roundscale_pd(a, _MM_FROUND_TO_POS_INF); }
666 
667 template<> EIGEN_STRONG_INLINE Packet16f pfloor<Packet16f>(const Packet16f& a) { return _mm512_roundscale_ps(a, _MM_FROUND_TO_NEG_INF); }
668 template<> EIGEN_STRONG_INLINE Packet8d pfloor<Packet8d>(const Packet8d& a) { return _mm512_roundscale_pd(a, _MM_FROUND_TO_NEG_INF); }
669 
670 template <>
671 EIGEN_STRONG_INLINE Packet16i ptrue<Packet16i>(const Packet16i& /*a*/) {
672  return _mm512_set1_epi32(0xffffffffu);
673 }
674 
675 template <>
676 EIGEN_STRONG_INLINE Packet16f ptrue<Packet16f>(const Packet16f& a) {
677  return _mm512_castsi512_ps(ptrue<Packet16i>(_mm512_castps_si512(a)));
678 }
679 
680 template <>
681 EIGEN_STRONG_INLINE Packet8d ptrue<Packet8d>(const Packet8d& a) {
682  return _mm512_castsi512_pd(ptrue<Packet16i>(_mm512_castpd_si512(a)));
683 }
684 
685 template <>
686 EIGEN_STRONG_INLINE Packet16i pand<Packet16i>(const Packet16i& a,
687  const Packet16i& b) {
688  return _mm512_and_si512(a,b);
689 }
690 
691 template <>
692 EIGEN_STRONG_INLINE Packet16f pand<Packet16f>(const Packet16f& a,
693  const Packet16f& b) {
694 #ifdef EIGEN_VECTORIZE_AVX512DQ
695  return _mm512_and_ps(a, b);
696 #else
697  return _mm512_castsi512_ps(pand(_mm512_castps_si512(a),_mm512_castps_si512(b)));
698 #endif
699 }
700 template <>
701 EIGEN_STRONG_INLINE Packet8d pand<Packet8d>(const Packet8d& a,
702  const Packet8d& b) {
703 #ifdef EIGEN_VECTORIZE_AVX512DQ
704  return _mm512_and_pd(a, b);
705 #else
706  Packet8d res = _mm512_undefined_pd();
707  Packet4d lane0_a = _mm512_extractf64x4_pd(a, 0);
708  Packet4d lane0_b = _mm512_extractf64x4_pd(b, 0);
709  res = _mm512_insertf64x4(res, _mm256_and_pd(lane0_a, lane0_b), 0);
710 
711  Packet4d lane1_a = _mm512_extractf64x4_pd(a, 1);
712  Packet4d lane1_b = _mm512_extractf64x4_pd(b, 1);
713  return _mm512_insertf64x4(res, _mm256_and_pd(lane1_a, lane1_b), 1);
714 #endif
715 }
716 
717 template <>
718 EIGEN_STRONG_INLINE Packet16i por<Packet16i>(const Packet16i& a, const Packet16i& b) {
719  return _mm512_or_si512(a, b);
720 }
721 
722 template <>
723 EIGEN_STRONG_INLINE Packet16f por<Packet16f>(const Packet16f& a, const Packet16f& b) {
724 #ifdef EIGEN_VECTORIZE_AVX512DQ
725  return _mm512_or_ps(a, b);
726 #else
727  return _mm512_castsi512_ps(por(_mm512_castps_si512(a),_mm512_castps_si512(b)));
728 #endif
729 }
730 
731 template <>
732 EIGEN_STRONG_INLINE Packet8d por<Packet8d>(const Packet8d& a,
733  const Packet8d& b) {
734 #ifdef EIGEN_VECTORIZE_AVX512DQ
735  return _mm512_or_pd(a, b);
736 #else
737  return _mm512_castsi512_pd(por(_mm512_castpd_si512(a),_mm512_castpd_si512(b)));
738 #endif
739 }
740 
741 template <>
742 EIGEN_STRONG_INLINE Packet16i pxor<Packet16i>(const Packet16i& a, const Packet16i& b) {
743  return _mm512_xor_si512(a, b);
744 }
745 
746 template <>
747 EIGEN_STRONG_INLINE Packet16f pxor<Packet16f>(const Packet16f& a, const Packet16f& b) {
748 #ifdef EIGEN_VECTORIZE_AVX512DQ
749  return _mm512_xor_ps(a, b);
750 #else
751  return _mm512_castsi512_ps(pxor(_mm512_castps_si512(a),_mm512_castps_si512(b)));
752 #endif
753 }
754 
755 template <>
756 EIGEN_STRONG_INLINE Packet8d pxor<Packet8d>(const Packet8d& a, const Packet8d& b) {
757 #ifdef EIGEN_VECTORIZE_AVX512DQ
758  return _mm512_xor_pd(a, b);
759 #else
760  return _mm512_castsi512_pd(pxor(_mm512_castpd_si512(a),_mm512_castpd_si512(b)));
761 #endif
762 }
763 
764 template <>
765 EIGEN_STRONG_INLINE Packet16i pandnot<Packet16i>(const Packet16i& a, const Packet16i& b) {
766  return _mm512_andnot_si512(b, a);
767 }
768 
769 template <>
770 EIGEN_STRONG_INLINE Packet16f pandnot<Packet16f>(const Packet16f& a, const Packet16f& b) {
771 #ifdef EIGEN_VECTORIZE_AVX512DQ
772  return _mm512_andnot_ps(b, a);
773 #else
774  return _mm512_castsi512_ps(pandnot(_mm512_castps_si512(a),_mm512_castps_si512(b)));
775 #endif
776 }
777 template <>
778 EIGEN_STRONG_INLINE Packet8d pandnot<Packet8d>(const Packet8d& a,const Packet8d& b) {
779 #ifdef EIGEN_VECTORIZE_AVX512DQ
780  return _mm512_andnot_pd(b, a);
781 #else
782  return _mm512_castsi512_pd(pandnot(_mm512_castpd_si512(a),_mm512_castpd_si512(b)));
783 #endif
784 }
785 
786 template<> EIGEN_STRONG_INLINE Packet16f pround<Packet16f>(const Packet16f& a)
787 {
788  // Work-around for default std::round rounding mode.
789  const Packet16f mask = pset1frombits<Packet16f>(static_cast<numext::uint32_t>(0x80000000u));
790  const Packet16f prev0dot5 = pset1frombits<Packet16f>(static_cast<numext::uint32_t>(0x3EFFFFFFu));
791  return _mm512_roundscale_ps(padd(por(pand(a, mask), prev0dot5), a), _MM_FROUND_TO_ZERO);
792 }
793 template<> EIGEN_STRONG_INLINE Packet8d pround<Packet8d>(const Packet8d& a)
794 {
795  // Work-around for default std::round rounding mode.
796  const Packet8d mask = pset1frombits<Packet8d>(static_cast<numext::uint64_t>(0x8000000000000000ull));
797  const Packet8d prev0dot5 = pset1frombits<Packet8d>(static_cast<numext::uint64_t>(0x3FDFFFFFFFFFFFFFull));
798  return _mm512_roundscale_pd(padd(por(pand(a, mask), prev0dot5), a), _MM_FROUND_TO_ZERO);
799 }
800 
801 template<int N> EIGEN_STRONG_INLINE Packet16i parithmetic_shift_right(Packet16i a) {
802  return _mm512_srai_epi32(a, N);
803 }
804 
805 template<int N> EIGEN_STRONG_INLINE Packet16i plogical_shift_right(Packet16i a) {
806  return _mm512_srli_epi32(a, N);
807 }
808 
809 template<int N> EIGEN_STRONG_INLINE Packet16i plogical_shift_left(Packet16i a) {
810  return _mm512_slli_epi32(a, N);
811 }
812 
813 template <>
814 EIGEN_STRONG_INLINE Packet16f pload<Packet16f>(const float* from) {
815  EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_ps(from);
816 }
817 template <>
818 EIGEN_STRONG_INLINE Packet8d pload<Packet8d>(const double* from) {
819  EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_pd(from);
820 }
821 template <>
822 EIGEN_STRONG_INLINE Packet16i pload<Packet16i>(const int* from) {
823  EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_si512(
824  reinterpret_cast<const __m512i*>(from));
825 }
826 
827 template <>
828 EIGEN_STRONG_INLINE Packet16f ploadu<Packet16f>(const float* from) {
829  EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_ps(from);
830 }
831 template <>
832 EIGEN_STRONG_INLINE Packet8d ploadu<Packet8d>(const double* from) {
833  EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_pd(from);
834 }
835 template <>
836 EIGEN_STRONG_INLINE Packet16i ploadu<Packet16i>(const int* from) {
837  EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_si512(
838  reinterpret_cast<const __m512i*>(from));
839 }
840 
841 template <>
842 EIGEN_STRONG_INLINE Packet16f ploadu<Packet16f>(const float* from, uint16_t umask) {
843  __mmask16 mask = static_cast<__mmask16>(umask);
844  EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_maskz_loadu_ps(mask, from);
845 }
846 template <>
847 EIGEN_STRONG_INLINE Packet8d ploadu<Packet8d>(const double* from, uint8_t umask) {
848  __mmask8 mask = static_cast<__mmask8>(umask);
849  EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_maskz_loadu_pd(mask, from);
850 }
851 
852 // Loads 8 floats from memory a returns the packet
853 // {a0, a0 a1, a1, a2, a2, a3, a3, a4, a4, a5, a5, a6, a6, a7, a7}
854 template <>
855 EIGEN_STRONG_INLINE Packet16f ploaddup<Packet16f>(const float* from) {
856  // an unaligned load is required here as there is no requirement
857  // on the alignment of input pointer 'from'
858  __m256i low_half = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(from));
859  __m512 even_elements = _mm512_castsi512_ps(_mm512_cvtepu32_epi64(low_half));
860  __m512 pairs = _mm512_permute_ps(even_elements, _MM_SHUFFLE(2, 2, 0, 0));
861  return pairs;
862 }
863 
864 #ifdef EIGEN_VECTORIZE_AVX512DQ
865 // FIXME: this does not look optimal, better load a Packet4d and shuffle...
866 // Loads 4 doubles from memory a returns the packet {a0, a0 a1, a1, a2, a2, a3,
867 // a3}
868 template <>
869 EIGEN_STRONG_INLINE Packet8d ploaddup<Packet8d>(const double* from) {
870  __m512d x = _mm512_setzero_pd();
871  x = _mm512_insertf64x2(x, _mm_loaddup_pd(&from[0]), 0);
872  x = _mm512_insertf64x2(x, _mm_loaddup_pd(&from[1]), 1);
873  x = _mm512_insertf64x2(x, _mm_loaddup_pd(&from[2]), 2);
874  x = _mm512_insertf64x2(x, _mm_loaddup_pd(&from[3]), 3);
875  return x;
876 }
877 #else
878 template <>
879 EIGEN_STRONG_INLINE Packet8d ploaddup<Packet8d>(const double* from) {
880  __m512d x = _mm512_setzero_pd();
881  x = _mm512_mask_broadcastsd_pd(x, 0x3<<0, _mm_load_sd(from+0));
882  x = _mm512_mask_broadcastsd_pd(x, 0x3<<2, _mm_load_sd(from+1));
883  x = _mm512_mask_broadcastsd_pd(x, 0x3<<4, _mm_load_sd(from+2));
884  x = _mm512_mask_broadcastsd_pd(x, 0x3<<6, _mm_load_sd(from+3));
885  return x;
886 }
887 #endif
888 
889 // Loads 8 integers from memory and returns the packet
890 // {a0, a0 a1, a1, a2, a2, a3, a3, a4, a4, a5, a5, a6, a6, a7, a7}
891 template <>
892 EIGEN_STRONG_INLINE Packet16i ploaddup<Packet16i>(const int* from) {
893  __m256i low_half = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(from));
894  __m512 even_elements = _mm512_castsi512_ps(_mm512_cvtepu32_epi64(low_half));
895  __m512 pairs = _mm512_permute_ps(even_elements, _MM_SHUFFLE(2, 2, 0, 0));
896  return _mm512_castps_si512(pairs);
897 }
898 
899 // Loads 4 floats from memory a returns the packet
900 // {a0, a0 a0, a0, a1, a1, a1, a1, a2, a2, a2, a2, a3, a3, a3, a3}
901 template <>
902 EIGEN_STRONG_INLINE Packet16f ploadquad<Packet16f>(const float* from) {
903  Packet16f tmp = _mm512_castps128_ps512(ploadu<Packet4f>(from));
904  const Packet16i scatter_mask = _mm512_set_epi32(3,3,3,3, 2,2,2,2, 1,1,1,1, 0,0,0,0);
905  return _mm512_permutexvar_ps(scatter_mask, tmp);
906 }
907 
908 // Loads 2 doubles from memory a returns the packet
909 // {a0, a0 a0, a0, a1, a1, a1, a1}
910 template <>
911 EIGEN_STRONG_INLINE Packet8d ploadquad<Packet8d>(const double* from) {
912  __m256d lane0 = _mm256_set1_pd(*from);
913  __m256d lane1 = _mm256_set1_pd(*(from+1));
914  __m512d tmp = _mm512_undefined_pd();
915  tmp = _mm512_insertf64x4(tmp, lane0, 0);
916  return _mm512_insertf64x4(tmp, lane1, 1);
917 }
918 
919 // Loads 4 integers from memory and returns the packet
920 // {a0, a0 a0, a0, a1, a1, a1, a1, a2, a2, a2, a2, a3, a3, a3, a3}
921 template <>
922 EIGEN_STRONG_INLINE Packet16i ploadquad<Packet16i>(const int* from) {
923  Packet16i tmp = _mm512_castsi128_si512(ploadu<Packet4i>(from));
924  const Packet16i scatter_mask = _mm512_set_epi32(3,3,3,3, 2,2,2,2, 1,1,1,1, 0,0,0,0);
925  return _mm512_permutexvar_epi32(scatter_mask, tmp);
926 }
927 
928 template <>
929 EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet16f& from) {
930  EIGEN_DEBUG_ALIGNED_STORE _mm512_store_ps(to, from);
931 }
932 template <>
933 EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet8d& from) {
934  EIGEN_DEBUG_ALIGNED_STORE _mm512_store_pd(to, from);
935 }
936 template <>
937 EIGEN_STRONG_INLINE void pstore<int>(int* to, const Packet16i& from) {
938  EIGEN_DEBUG_ALIGNED_STORE _mm512_storeu_si512(reinterpret_cast<__m512i*>(to),
939  from);
940 }
941 
942 template <>
943 EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet16f& from) {
944  EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_ps(to, from);
945 }
946 template <>
947 EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet8d& from) {
948  EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_pd(to, from);
949 }
950 template <>
951 EIGEN_STRONG_INLINE void pstoreu<int>(int* to, const Packet16i& from) {
952  EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_si512(
953  reinterpret_cast<__m512i*>(to), from);
954 }
955 template <>
956 EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet16f& from, uint16_t umask) {
957  __mmask16 mask = static_cast<__mmask16>(umask);
958  EIGEN_DEBUG_UNALIGNED_STORE return _mm512_mask_storeu_ps(to, mask, from);
959 }
960 template <>
961 EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet8d& from, uint8_t umask) {
962  __mmask8 mask = static_cast<__mmask8>(umask);
963  EIGEN_DEBUG_UNALIGNED_STORE return _mm512_mask_storeu_pd(to, mask, from);
964 }
965 
966 template <typename Scalar, typename Packet>
967 EIGEN_DEVICE_FUNC inline Packet pgather(const Packet& src, const Scalar* from,
968  Index stride, typename unpacket_traits<Packet>::mask_t umask);
969 template <>
970 EIGEN_DEVICE_FUNC inline Packet16f pgather<float, Packet16f>(const Packet16f& src,
971  const float* from,
972  Index stride,
973  uint16_t umask) {
974  Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
975  Packet16i stride_multiplier =
976  _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
977  Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
978  __mmask16 mask = static_cast<__mmask16>(umask);
979 
980  return _mm512_mask_i32gather_ps(src, mask, indices, from, 4);
981 }
982 template <>
983 EIGEN_DEVICE_FUNC inline Packet8d pgather<double, Packet8d>(const Packet8d& src,
984  const double* from,
985  Index stride,
986  uint8_t umask) {
987  Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
988  Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
989  Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
990  __mmask8 mask = static_cast<__mmask8>(umask);
991 
992  return _mm512_mask_i32gather_pd(src, mask, indices, from, 8);
993 }
994 
995 template <>
996 EIGEN_DEVICE_FUNC inline Packet16f pgather<float, Packet16f>(const float* from,
997  Index stride) {
998  Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
999  Packet16i stride_multiplier =
1000  _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1001  Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1002 
1003  return _mm512_i32gather_ps(indices, from, 4);
1004 }
1005 template <>
1006 EIGEN_DEVICE_FUNC inline Packet8d pgather<double, Packet8d>(const double* from,
1007  Index stride) {
1008  Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
1009  Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
1010  Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
1011 
1012  return _mm512_i32gather_pd(indices, from, 8);
1013 }
1014 template <>
1015 EIGEN_DEVICE_FUNC inline Packet16i pgather<int, Packet16i>(const int* from,
1016  Index stride) {
1017  Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1018  Packet16i stride_multiplier =
1019  _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1020  Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1021  return _mm512_i32gather_epi32(indices, from, 4);
1022 }
1023 
1024 template <typename Scalar, typename Packet>
1025 EIGEN_DEVICE_FUNC inline void pscatter(Scalar* to, const Packet& from,
1026  Index stride, typename unpacket_traits<Packet>::mask_t umask);
1027 template <>
1028 EIGEN_DEVICE_FUNC inline void pscatter<float, Packet16f>(float* to,
1029  const Packet16f& from,
1030  Index stride,
1031  uint16_t umask) {
1032  Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1033  Packet16i stride_multiplier =
1034  _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1035  Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1036  __mmask16 mask = static_cast<__mmask16>(umask);
1037  _mm512_mask_i32scatter_ps(to, mask, indices, from, 4);
1038 }
1039 template <>
1040 EIGEN_DEVICE_FUNC inline void pscatter<double, Packet8d>(double* to,
1041  const Packet8d& from,
1042  Index stride,
1043  uint8_t umask) {
1044  Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
1045  Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
1046  Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
1047  __mmask8 mask = static_cast<__mmask8>(umask);
1048  _mm512_mask_i32scatter_pd(to, mask, indices, from, 8);
1049 }
1050 
1051 template <>
1052 EIGEN_DEVICE_FUNC inline void pscatter<float, Packet16f>(float* to,
1053  const Packet16f& from,
1054  Index stride) {
1055  Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1056  Packet16i stride_multiplier =
1057  _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1058  Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1059  _mm512_i32scatter_ps(to, indices, from, 4);
1060 }
1061 template <>
1062 EIGEN_DEVICE_FUNC inline void pscatter<double, Packet8d>(double* to,
1063  const Packet8d& from,
1064  Index stride) {
1065  Packet8i stride_vector = _mm256_set1_epi32(convert_index<int>(stride));
1066  Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
1067  Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
1068  _mm512_i32scatter_pd(to, indices, from, 8);
1069 }
1070 template <>
1071 EIGEN_DEVICE_FUNC inline void pscatter<int, Packet16i>(int* to,
1072  const Packet16i& from,
1073  Index stride) {
1074  Packet16i stride_vector = _mm512_set1_epi32(convert_index<int>(stride));
1075  Packet16i stride_multiplier =
1076  _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
1077  Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
1078  _mm512_i32scatter_epi32(to, indices, from, 4);
1079 }
1080 
1081 template <>
1082 EIGEN_STRONG_INLINE void pstore1<Packet16f>(float* to, const float& a) {
1083  Packet16f pa = pset1<Packet16f>(a);
1084  pstore(to, pa);
1085 }
1086 template <>
1087 EIGEN_STRONG_INLINE void pstore1<Packet8d>(double* to, const double& a) {
1088  Packet8d pa = pset1<Packet8d>(a);
1089  pstore(to, pa);
1090 }
1091 template <>
1092 EIGEN_STRONG_INLINE void pstore1<Packet16i>(int* to, const int& a) {
1093  Packet16i pa = pset1<Packet16i>(a);
1094  pstore(to, pa);
1095 }
1096 
1097 template<> EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); }
1098 template<> EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); }
1099 template<> EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) { _mm_prefetch((SsePrefetchPtrType)(addr), _MM_HINT_T0); }
1100 
1101 template <>
1102 EIGEN_STRONG_INLINE float pfirst<Packet16f>(const Packet16f& a) {
1103  return _mm_cvtss_f32(_mm512_extractf32x4_ps(a, 0));
1104 }
1105 template <>
1106 EIGEN_STRONG_INLINE double pfirst<Packet8d>(const Packet8d& a) {
1107  return _mm_cvtsd_f64(_mm256_extractf128_pd(_mm512_extractf64x4_pd(a, 0), 0));
1108 }
1109 template <>
1110 EIGEN_STRONG_INLINE int pfirst<Packet16i>(const Packet16i& a) {
1111  return _mm_extract_epi32(_mm512_extracti32x4_epi32(a, 0), 0);
1112 }
1113 
1114 template<> EIGEN_STRONG_INLINE Packet16f preverse(const Packet16f& a)
1115 {
1116  return _mm512_permutexvar_ps(_mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), a);
1117 }
1118 
1119 template<> EIGEN_STRONG_INLINE Packet8d preverse(const Packet8d& a)
1120 {
1121  return _mm512_permutexvar_pd(_mm512_set_epi32(0, 0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7), a);
1122 }
1123 
1124 template<> EIGEN_STRONG_INLINE Packet16i preverse(const Packet16i& a)
1125 {
1126  return _mm512_permutexvar_epi32(_mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), a);
1127 }
1128 
1129 template<> EIGEN_STRONG_INLINE Packet16f pabs(const Packet16f& a)
1130 {
1131  // _mm512_abs_ps intrinsic not found, so hack around it
1132  return _mm512_castsi512_ps(_mm512_and_si512(_mm512_castps_si512(a), _mm512_set1_epi32(0x7fffffff)));
1133 }
1134 template <>
1135 EIGEN_STRONG_INLINE Packet8d pabs(const Packet8d& a) {
1136  // _mm512_abs_ps intrinsic not found, so hack around it
1137  return _mm512_castsi512_pd(_mm512_and_si512(_mm512_castpd_si512(a),
1138  _mm512_set1_epi64(0x7fffffffffffffff)));
1139 }
1140 template<> EIGEN_STRONG_INLINE Packet16i pabs(const Packet16i& a)
1141 {
1142  return _mm512_abs_epi32(a);
1143 }
1144 
1145 template<> EIGEN_STRONG_INLINE Packet16h psignbit(const Packet16h& a) { return _mm256_srai_epi16(a, 15); }
1146 template<> EIGEN_STRONG_INLINE Packet16bf psignbit(const Packet16bf& a) { return _mm256_srai_epi16(a, 15); }
1147 template<> EIGEN_STRONG_INLINE Packet16f psignbit(const Packet16f& a) { return _mm512_castsi512_ps(_mm512_srai_epi32(_mm512_castps_si512(a), 31)); }
1148 template<> EIGEN_STRONG_INLINE Packet8d psignbit(const Packet8d& a) { return _mm512_castsi512_pd(_mm512_srai_epi64(_mm512_castpd_si512(a), 63)); }
1149 
1150 template<>
1151 EIGEN_STRONG_INLINE Packet16f pfrexp<Packet16f>(const Packet16f& a, Packet16f& exponent){
1152  return pfrexp_generic(a, exponent);
1153 }
1154 
1155 // Extract exponent without existence of Packet8l.
1156 template<>
1157 EIGEN_STRONG_INLINE
1158 Packet8d pfrexp_generic_get_biased_exponent(const Packet8d& a) {
1159  const Packet8d cst_exp_mask = pset1frombits<Packet8d>(static_cast<uint64_t>(0x7ff0000000000000ull));
1160  #ifdef EIGEN_VECTORIZE_AVX512DQ
1161  return _mm512_cvtepi64_pd(_mm512_srli_epi64(_mm512_castpd_si512(pand(a, cst_exp_mask)), 52));
1162  #else
1163  return _mm512_cvtepi32_pd(_mm512_cvtepi64_epi32(_mm512_srli_epi64(_mm512_castpd_si512(pand(a, cst_exp_mask)), 52)));
1164  #endif
1165 }
1166 
1167 template<>
1168 EIGEN_STRONG_INLINE Packet8d pfrexp<Packet8d>(const Packet8d& a, Packet8d& exponent) {
1169  return pfrexp_generic(a, exponent);
1170 }
1171 
1172 template<> EIGEN_STRONG_INLINE Packet16f pldexp<Packet16f>(const Packet16f& a, const Packet16f& exponent) {
1173  return pldexp_generic(a, exponent);
1174 }
1175 
1176 template<> EIGEN_STRONG_INLINE Packet8d pldexp<Packet8d>(const Packet8d& a, const Packet8d& exponent) {
1177  // Clamp exponent to [-2099, 2099]
1178  const Packet8d max_exponent = pset1<Packet8d>(2099.0);
1179  const Packet8i e = _mm512_cvtpd_epi32(pmin(pmax(exponent, pnegate(max_exponent)), max_exponent));
1180 
1181  // Split 2^e into four factors and multiply.
1182  const Packet8i bias = pset1<Packet8i>(1023);
1183  Packet8i b = parithmetic_shift_right<2>(e); // floor(e/4)
1184 
1185  // 2^b
1186  const Packet8i permute_idx = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7);
1187  Packet8i hi = _mm256_permutevar8x32_epi32(padd(b, bias), permute_idx);
1188  Packet8i lo = _mm256_slli_epi64(hi, 52);
1189  hi = _mm256_slli_epi64(_mm256_srli_epi64(hi, 32), 52);
1190  Packet8d c = _mm512_castsi512_pd(_mm512_inserti64x4(_mm512_castsi256_si512(lo), hi, 1));
1191  Packet8d out = pmul(pmul(pmul(a, c), c), c); // a * 2^(3b)
1192 
1193  // 2^(e - 3b)
1194  b = psub(psub(psub(e, b), b), b); // e - 3b
1195  hi = _mm256_permutevar8x32_epi32(padd(b, bias), permute_idx);
1196  lo = _mm256_slli_epi64(hi, 52);
1197  hi = _mm256_slli_epi64(_mm256_srli_epi64(hi, 32), 52);
1198  c = _mm512_castsi512_pd(_mm512_inserti64x4(_mm512_castsi256_si512(lo), hi, 1));
1199  out = pmul(out, c); // a * 2^e
1200  return out;
1201 }
1202 
1203 #ifdef EIGEN_VECTORIZE_AVX512DQ
1204 // AVX512F does not define _mm512_extractf32x8_ps to extract _m256 from _m512
1205 #define EIGEN_EXTRACT_8f_FROM_16f(INPUT, OUTPUT) \
1206  __m256 OUTPUT##_0 = _mm512_extractf32x8_ps(INPUT, 0); \
1207  __m256 OUTPUT##_1 = _mm512_extractf32x8_ps(INPUT, 1)
1208 
1209 // AVX512F does not define _mm512_extracti32x8_epi32 to extract _m256i from _m512i
1210 #define EIGEN_EXTRACT_8i_FROM_16i(INPUT, OUTPUT) \
1211  __m256i OUTPUT##_0 = _mm512_extracti32x8_epi32(INPUT, 0); \
1212  __m256i OUTPUT##_1 = _mm512_extracti32x8_epi32(INPUT, 1)
1213 #else
1214 #define EIGEN_EXTRACT_8f_FROM_16f(INPUT, OUTPUT) \
1215  __m256 OUTPUT##_0 = _mm256_insertf128_ps( \
1216  _mm256_castps128_ps256(_mm512_extractf32x4_ps(INPUT, 0)), \
1217  _mm512_extractf32x4_ps(INPUT, 1), 1); \
1218  __m256 OUTPUT##_1 = _mm256_insertf128_ps( \
1219  _mm256_castps128_ps256(_mm512_extractf32x4_ps(INPUT, 2)), \
1220  _mm512_extractf32x4_ps(INPUT, 3), 1)
1221 
1222 #define EIGEN_EXTRACT_8i_FROM_16i(INPUT, OUTPUT) \
1223  __m256i OUTPUT##_0 = _mm256_insertf128_si256( \
1224  _mm256_castsi128_si256(_mm512_extracti32x4_epi32(INPUT, 0)), \
1225  _mm512_extracti32x4_epi32(INPUT, 1), 1); \
1226  __m256i OUTPUT##_1 = _mm256_insertf128_si256( \
1227  _mm256_castsi128_si256(_mm512_extracti32x4_epi32(INPUT, 2)), \
1228  _mm512_extracti32x4_epi32(INPUT, 3), 1)
1229 #endif
1230 
1231 #ifdef EIGEN_VECTORIZE_AVX512DQ
1232 #define EIGEN_INSERT_8f_INTO_16f(OUTPUT, INPUTA, INPUTB) \
1233  OUTPUT = _mm512_insertf32x8(_mm512_castps256_ps512(INPUTA), INPUTB, 1);
1234 
1235 #define EIGEN_INSERT_8i_INTO_16i(OUTPUT, INPUTA, INPUTB) \
1236  OUTPUT = _mm512_inserti32x8(_mm512_castsi256_si512(INPUTA), INPUTB, 1);
1237 #else
1238 #define EIGEN_INSERT_8f_INTO_16f(OUTPUT, INPUTA, INPUTB) \
1239  OUTPUT = _mm512_undefined_ps(); \
1240  OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTA, 0), 0); \
1241  OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTA, 1), 1); \
1242  OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTB, 0), 2); \
1243  OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTB, 1), 3);
1244 
1245 #define EIGEN_INSERT_8i_INTO_16i(OUTPUT, INPUTA, INPUTB) \
1246  OUTPUT = _mm512_undefined_epi32(); \
1247  OUTPUT = _mm512_inserti32x4(OUTPUT, _mm256_extractf128_si256(INPUTA, 0), 0); \
1248  OUTPUT = _mm512_inserti32x4(OUTPUT, _mm256_extractf128_si256(INPUTA, 1), 1); \
1249  OUTPUT = _mm512_inserti32x4(OUTPUT, _mm256_extractf128_si256(INPUTB, 0), 2); \
1250  OUTPUT = _mm512_inserti32x4(OUTPUT, _mm256_extractf128_si256(INPUTB, 1), 3);
1251 #endif
1252 
1253 template <>
1254 EIGEN_STRONG_INLINE float predux<Packet16f>(const Packet16f& a) {
1255 #ifdef EIGEN_VECTORIZE_AVX512DQ
1256  __m256 lane0 = _mm512_extractf32x8_ps(a, 0);
1257  __m256 lane1 = _mm512_extractf32x8_ps(a, 1);
1258  Packet8f x = _mm256_add_ps(lane0, lane1);
1259  return predux<Packet8f>(x);
1260 #else
1261  __m128 lane0 = _mm512_extractf32x4_ps(a, 0);
1262  __m128 lane1 = _mm512_extractf32x4_ps(a, 1);
1263  __m128 lane2 = _mm512_extractf32x4_ps(a, 2);
1264  __m128 lane3 = _mm512_extractf32x4_ps(a, 3);
1265  __m128 sum = _mm_add_ps(_mm_add_ps(lane0, lane1), _mm_add_ps(lane2, lane3));
1266  sum = _mm_hadd_ps(sum, sum);
1267  sum = _mm_hadd_ps(sum, _mm_permute_ps(sum, 1));
1268  return _mm_cvtss_f32(sum);
1269 #endif
1270 }
1271 template <>
1272 EIGEN_STRONG_INLINE double predux<Packet8d>(const Packet8d& a) {
1273  __m256d lane0 = _mm512_extractf64x4_pd(a, 0);
1274  __m256d lane1 = _mm512_extractf64x4_pd(a, 1);
1275  __m256d sum = _mm256_add_pd(lane0, lane1);
1276  __m256d tmp0 = _mm256_hadd_pd(sum, _mm256_permute2f128_pd(sum, sum, 1));
1277  return _mm_cvtsd_f64(_mm256_castpd256_pd128(_mm256_hadd_pd(tmp0, tmp0)));
1278 }
1279 template <>
1280 EIGEN_STRONG_INLINE int predux<Packet16i>(const Packet16i& a) {
1281 #ifdef EIGEN_VECTORIZE_AVX512DQ
1282  __m256i lane0 = _mm512_extracti32x8_epi32(a, 0);
1283  __m256i lane1 = _mm512_extracti32x8_epi32(a, 1);
1284  Packet8i x = _mm256_add_epi32(lane0, lane1);
1285  return predux<Packet8i>(x);
1286 #else
1287  __m128i lane0 = _mm512_extracti32x4_epi32(a, 0);
1288  __m128i lane1 = _mm512_extracti32x4_epi32(a, 1);
1289  __m128i lane2 = _mm512_extracti32x4_epi32(a, 2);
1290  __m128i lane3 = _mm512_extracti32x4_epi32(a, 3);
1291  __m128i sum = _mm_add_epi32(_mm_add_epi32(lane0, lane1), _mm_add_epi32(lane2, lane3));
1292  sum = _mm_hadd_epi32(sum, sum);
1293  sum = _mm_hadd_epi32(sum, _mm_castps_si128(_mm_permute_ps(_mm_castsi128_ps(sum), 1)));
1294  return _mm_cvtsi128_si32(sum);
1295 #endif
1296 }
1297 
1298 template <>
1299 EIGEN_STRONG_INLINE Packet8f predux_half_dowto4<Packet16f>(const Packet16f& a) {
1300 #ifdef EIGEN_VECTORIZE_AVX512DQ
1301  __m256 lane0 = _mm512_extractf32x8_ps(a, 0);
1302  __m256 lane1 = _mm512_extractf32x8_ps(a, 1);
1303  return _mm256_add_ps(lane0, lane1);
1304 #else
1305  __m128 lane0 = _mm512_extractf32x4_ps(a, 0);
1306  __m128 lane1 = _mm512_extractf32x4_ps(a, 1);
1307  __m128 lane2 = _mm512_extractf32x4_ps(a, 2);
1308  __m128 lane3 = _mm512_extractf32x4_ps(a, 3);
1309  __m128 sum0 = _mm_add_ps(lane0, lane2);
1310  __m128 sum1 = _mm_add_ps(lane1, lane3);
1311  return _mm256_insertf128_ps(_mm256_castps128_ps256(sum0), sum1, 1);
1312 #endif
1313 }
1314 template <>
1315 EIGEN_STRONG_INLINE Packet4d predux_half_dowto4<Packet8d>(const Packet8d& a) {
1316  __m256d lane0 = _mm512_extractf64x4_pd(a, 0);
1317  __m256d lane1 = _mm512_extractf64x4_pd(a, 1);
1318  return _mm256_add_pd(lane0, lane1);
1319 }
1320 template <>
1321 EIGEN_STRONG_INLINE Packet8i predux_half_dowto4<Packet16i>(const Packet16i& a) {
1322 #ifdef EIGEN_VECTORIZE_AVX512DQ
1323  __m256i lane0 = _mm512_extracti32x8_epi32(a, 0);
1324  __m256i lane1 = _mm512_extracti32x8_epi32(a, 1);
1325  return _mm256_add_epi32(lane0, lane1);
1326 #else
1327  __m128i lane0 = _mm512_extracti32x4_epi32(a, 0);
1328  __m128i lane1 = _mm512_extracti32x4_epi32(a, 1);
1329  __m128i lane2 = _mm512_extracti32x4_epi32(a, 2);
1330  __m128i lane3 = _mm512_extracti32x4_epi32(a, 3);
1331  __m128i sum0 = _mm_add_epi32(lane0, lane2);
1332  __m128i sum1 = _mm_add_epi32(lane1, lane3);
1333  return _mm256_inserti128_si256(_mm256_castsi128_si256(sum0), sum1, 1);
1334 #endif
1335 }
1336 
1337 template <>
1338 EIGEN_STRONG_INLINE float predux_mul<Packet16f>(const Packet16f& a) {
1339 //#ifdef EIGEN_VECTORIZE_AVX512DQ
1340 #if 0
1341  Packet8f lane0 = _mm512_extractf32x8_ps(a, 0);
1342  Packet8f lane1 = _mm512_extractf32x8_ps(a, 1);
1343  Packet8f res = pmul(lane0, lane1);
1344  res = pmul(res, _mm256_permute2f128_ps(res, res, 1));
1345  res = pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2)));
1346  return pfirst(pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1))));
1347 #else
1348  __m128 lane0 = _mm512_extractf32x4_ps(a, 0);
1349  __m128 lane1 = _mm512_extractf32x4_ps(a, 1);
1350  __m128 lane2 = _mm512_extractf32x4_ps(a, 2);
1351  __m128 lane3 = _mm512_extractf32x4_ps(a, 3);
1352  __m128 res = pmul(pmul(lane0, lane1), pmul(lane2, lane3));
1353  res = pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2)));
1354  return pfirst(pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1))));
1355 #endif
1356 }
1357 template <>
1358 EIGEN_STRONG_INLINE double predux_mul<Packet8d>(const Packet8d& a) {
1359  __m256d lane0 = _mm512_extractf64x4_pd(a, 0);
1360  __m256d lane1 = _mm512_extractf64x4_pd(a, 1);
1361  __m256d res = pmul(lane0, lane1);
1362  res = pmul(res, _mm256_permute2f128_pd(res, res, 1));
1363  return pfirst(pmul(res, _mm256_shuffle_pd(res, res, 1)));
1364 }
1365 
1366 template <>
1367 EIGEN_STRONG_INLINE float predux_min<Packet16f>(const Packet16f& a) {
1368  __m128 lane0 = _mm512_extractf32x4_ps(a, 0);
1369  __m128 lane1 = _mm512_extractf32x4_ps(a, 1);
1370  __m128 lane2 = _mm512_extractf32x4_ps(a, 2);
1371  __m128 lane3 = _mm512_extractf32x4_ps(a, 3);
1372  __m128 res = _mm_min_ps(_mm_min_ps(lane0, lane1), _mm_min_ps(lane2, lane3));
1373  res = _mm_min_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2)));
1374  return pfirst(_mm_min_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1))));
1375 }
1376 template <>
1377 EIGEN_STRONG_INLINE double predux_min<Packet8d>(const Packet8d& a) {
1378  __m256d lane0 = _mm512_extractf64x4_pd(a, 0);
1379  __m256d lane1 = _mm512_extractf64x4_pd(a, 1);
1380  __m256d res = _mm256_min_pd(lane0, lane1);
1381  res = _mm256_min_pd(res, _mm256_permute2f128_pd(res, res, 1));
1382  return pfirst(_mm256_min_pd(res, _mm256_shuffle_pd(res, res, 1)));
1383 }
1384 
1385 template <>
1386 EIGEN_STRONG_INLINE float predux_max<Packet16f>(const Packet16f& a) {
1387  __m128 lane0 = _mm512_extractf32x4_ps(a, 0);
1388  __m128 lane1 = _mm512_extractf32x4_ps(a, 1);
1389  __m128 lane2 = _mm512_extractf32x4_ps(a, 2);
1390  __m128 lane3 = _mm512_extractf32x4_ps(a, 3);
1391  __m128 res = _mm_max_ps(_mm_max_ps(lane0, lane1), _mm_max_ps(lane2, lane3));
1392  res = _mm_max_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2)));
1393  return pfirst(_mm_max_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1))));
1394 }
1395 
1396 template <>
1397 EIGEN_STRONG_INLINE double predux_max<Packet8d>(const Packet8d& a) {
1398  __m256d lane0 = _mm512_extractf64x4_pd(a, 0);
1399  __m256d lane1 = _mm512_extractf64x4_pd(a, 1);
1400  __m256d res = _mm256_max_pd(lane0, lane1);
1401  res = _mm256_max_pd(res, _mm256_permute2f128_pd(res, res, 1));
1402  return pfirst(_mm256_max_pd(res, _mm256_shuffle_pd(res, res, 1)));
1403 }
1404 
1405 template<> EIGEN_STRONG_INLINE bool predux_any(const Packet16f& x)
1406 {
1407  Packet16i xi = _mm512_castps_si512(x);
1408  __mmask16 tmp = _mm512_test_epi32_mask(xi,xi);
1409  return !_mm512_kortestz(tmp,tmp);
1410 }
1411 
1412 template<> EIGEN_STRONG_INLINE bool predux_any(const Packet16i& x)
1413 {
1414  __mmask16 tmp = _mm512_test_epi32_mask(x,x);
1415  return !_mm512_kortestz(tmp,tmp);
1416 }
1417 
1418 #define PACK_OUTPUT(OUTPUT, INPUT, INDEX, STRIDE) \
1419  EIGEN_INSERT_8f_INTO_16f(OUTPUT[INDEX], INPUT[INDEX], INPUT[INDEX + STRIDE]);
1420 
1421 EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16f, 16>& kernel) {
1422  __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]);
1423  __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]);
1424  __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]);
1425  __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]);
1426  __m512 T4 = _mm512_unpacklo_ps(kernel.packet[4], kernel.packet[5]);
1427  __m512 T5 = _mm512_unpackhi_ps(kernel.packet[4], kernel.packet[5]);
1428  __m512 T6 = _mm512_unpacklo_ps(kernel.packet[6], kernel.packet[7]);
1429  __m512 T7 = _mm512_unpackhi_ps(kernel.packet[6], kernel.packet[7]);
1430  __m512 T8 = _mm512_unpacklo_ps(kernel.packet[8], kernel.packet[9]);
1431  __m512 T9 = _mm512_unpackhi_ps(kernel.packet[8], kernel.packet[9]);
1432  __m512 T10 = _mm512_unpacklo_ps(kernel.packet[10], kernel.packet[11]);
1433  __m512 T11 = _mm512_unpackhi_ps(kernel.packet[10], kernel.packet[11]);
1434  __m512 T12 = _mm512_unpacklo_ps(kernel.packet[12], kernel.packet[13]);
1435  __m512 T13 = _mm512_unpackhi_ps(kernel.packet[12], kernel.packet[13]);
1436  __m512 T14 = _mm512_unpacklo_ps(kernel.packet[14], kernel.packet[15]);
1437  __m512 T15 = _mm512_unpackhi_ps(kernel.packet[14], kernel.packet[15]);
1438  __m512 S0 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
1439  __m512 S1 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
1440  __m512 S2 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
1441  __m512 S3 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
1442  __m512 S4 = _mm512_shuffle_ps(T4, T6, _MM_SHUFFLE(1, 0, 1, 0));
1443  __m512 S5 = _mm512_shuffle_ps(T4, T6, _MM_SHUFFLE(3, 2, 3, 2));
1444  __m512 S6 = _mm512_shuffle_ps(T5, T7, _MM_SHUFFLE(1, 0, 1, 0));
1445  __m512 S7 = _mm512_shuffle_ps(T5, T7, _MM_SHUFFLE(3, 2, 3, 2));
1446  __m512 S8 = _mm512_shuffle_ps(T8, T10, _MM_SHUFFLE(1, 0, 1, 0));
1447  __m512 S9 = _mm512_shuffle_ps(T8, T10, _MM_SHUFFLE(3, 2, 3, 2));
1448  __m512 S10 = _mm512_shuffle_ps(T9, T11, _MM_SHUFFLE(1, 0, 1, 0));
1449  __m512 S11 = _mm512_shuffle_ps(T9, T11, _MM_SHUFFLE(3, 2, 3, 2));
1450  __m512 S12 = _mm512_shuffle_ps(T12, T14, _MM_SHUFFLE(1, 0, 1, 0));
1451  __m512 S13 = _mm512_shuffle_ps(T12, T14, _MM_SHUFFLE(3, 2, 3, 2));
1452  __m512 S14 = _mm512_shuffle_ps(T13, T15, _MM_SHUFFLE(1, 0, 1, 0));
1453  __m512 S15 = _mm512_shuffle_ps(T13, T15, _MM_SHUFFLE(3, 2, 3, 2));
1454 
1455  EIGEN_EXTRACT_8f_FROM_16f(S0, S0);
1456  EIGEN_EXTRACT_8f_FROM_16f(S1, S1);
1457  EIGEN_EXTRACT_8f_FROM_16f(S2, S2);
1458  EIGEN_EXTRACT_8f_FROM_16f(S3, S3);
1459  EIGEN_EXTRACT_8f_FROM_16f(S4, S4);
1460  EIGEN_EXTRACT_8f_FROM_16f(S5, S5);
1461  EIGEN_EXTRACT_8f_FROM_16f(S6, S6);
1462  EIGEN_EXTRACT_8f_FROM_16f(S7, S7);
1463  EIGEN_EXTRACT_8f_FROM_16f(S8, S8);
1464  EIGEN_EXTRACT_8f_FROM_16f(S9, S9);
1465  EIGEN_EXTRACT_8f_FROM_16f(S10, S10);
1466  EIGEN_EXTRACT_8f_FROM_16f(S11, S11);
1467  EIGEN_EXTRACT_8f_FROM_16f(S12, S12);
1468  EIGEN_EXTRACT_8f_FROM_16f(S13, S13);
1469  EIGEN_EXTRACT_8f_FROM_16f(S14, S14);
1470  EIGEN_EXTRACT_8f_FROM_16f(S15, S15);
1471 
1472  PacketBlock<Packet8f, 32> tmp;
1473 
1474  tmp.packet[0] = _mm256_permute2f128_ps(S0_0, S4_0, 0x20);
1475  tmp.packet[1] = _mm256_permute2f128_ps(S1_0, S5_0, 0x20);
1476  tmp.packet[2] = _mm256_permute2f128_ps(S2_0, S6_0, 0x20);
1477  tmp.packet[3] = _mm256_permute2f128_ps(S3_0, S7_0, 0x20);
1478  tmp.packet[4] = _mm256_permute2f128_ps(S0_0, S4_0, 0x31);
1479  tmp.packet[5] = _mm256_permute2f128_ps(S1_0, S5_0, 0x31);
1480  tmp.packet[6] = _mm256_permute2f128_ps(S2_0, S6_0, 0x31);
1481  tmp.packet[7] = _mm256_permute2f128_ps(S3_0, S7_0, 0x31);
1482 
1483  tmp.packet[8] = _mm256_permute2f128_ps(S0_1, S4_1, 0x20);
1484  tmp.packet[9] = _mm256_permute2f128_ps(S1_1, S5_1, 0x20);
1485  tmp.packet[10] = _mm256_permute2f128_ps(S2_1, S6_1, 0x20);
1486  tmp.packet[11] = _mm256_permute2f128_ps(S3_1, S7_1, 0x20);
1487  tmp.packet[12] = _mm256_permute2f128_ps(S0_1, S4_1, 0x31);
1488  tmp.packet[13] = _mm256_permute2f128_ps(S1_1, S5_1, 0x31);
1489  tmp.packet[14] = _mm256_permute2f128_ps(S2_1, S6_1, 0x31);
1490  tmp.packet[15] = _mm256_permute2f128_ps(S3_1, S7_1, 0x31);
1491 
1492  // Second set of _m256 outputs
1493  tmp.packet[16] = _mm256_permute2f128_ps(S8_0, S12_0, 0x20);
1494  tmp.packet[17] = _mm256_permute2f128_ps(S9_0, S13_0, 0x20);
1495  tmp.packet[18] = _mm256_permute2f128_ps(S10_0, S14_0, 0x20);
1496  tmp.packet[19] = _mm256_permute2f128_ps(S11_0, S15_0, 0x20);
1497  tmp.packet[20] = _mm256_permute2f128_ps(S8_0, S12_0, 0x31);
1498  tmp.packet[21] = _mm256_permute2f128_ps(S9_0, S13_0, 0x31);
1499  tmp.packet[22] = _mm256_permute2f128_ps(S10_0, S14_0, 0x31);
1500  tmp.packet[23] = _mm256_permute2f128_ps(S11_0, S15_0, 0x31);
1501 
1502  tmp.packet[24] = _mm256_permute2f128_ps(S8_1, S12_1, 0x20);
1503  tmp.packet[25] = _mm256_permute2f128_ps(S9_1, S13_1, 0x20);
1504  tmp.packet[26] = _mm256_permute2f128_ps(S10_1, S14_1, 0x20);
1505  tmp.packet[27] = _mm256_permute2f128_ps(S11_1, S15_1, 0x20);
1506  tmp.packet[28] = _mm256_permute2f128_ps(S8_1, S12_1, 0x31);
1507  tmp.packet[29] = _mm256_permute2f128_ps(S9_1, S13_1, 0x31);
1508  tmp.packet[30] = _mm256_permute2f128_ps(S10_1, S14_1, 0x31);
1509  tmp.packet[31] = _mm256_permute2f128_ps(S11_1, S15_1, 0x31);
1510 
1511  // Pack them into the output
1512  PACK_OUTPUT(kernel.packet, tmp.packet, 0, 16);
1513  PACK_OUTPUT(kernel.packet, tmp.packet, 1, 16);
1514  PACK_OUTPUT(kernel.packet, tmp.packet, 2, 16);
1515  PACK_OUTPUT(kernel.packet, tmp.packet, 3, 16);
1516 
1517  PACK_OUTPUT(kernel.packet, tmp.packet, 4, 16);
1518  PACK_OUTPUT(kernel.packet, tmp.packet, 5, 16);
1519  PACK_OUTPUT(kernel.packet, tmp.packet, 6, 16);
1520  PACK_OUTPUT(kernel.packet, tmp.packet, 7, 16);
1521 
1522  PACK_OUTPUT(kernel.packet, tmp.packet, 8, 16);
1523  PACK_OUTPUT(kernel.packet, tmp.packet, 9, 16);
1524  PACK_OUTPUT(kernel.packet, tmp.packet, 10, 16);
1525  PACK_OUTPUT(kernel.packet, tmp.packet, 11, 16);
1526 
1527  PACK_OUTPUT(kernel.packet, tmp.packet, 12, 16);
1528  PACK_OUTPUT(kernel.packet, tmp.packet, 13, 16);
1529  PACK_OUTPUT(kernel.packet, tmp.packet, 14, 16);
1530  PACK_OUTPUT(kernel.packet, tmp.packet, 15, 16);
1531 }
1532 #define PACK_OUTPUT_2(OUTPUT, INPUT, INDEX, STRIDE) \
1533  EIGEN_INSERT_8f_INTO_16f(OUTPUT[INDEX], INPUT[2 * INDEX], \
1534  INPUT[2 * INDEX + STRIDE]);
1535 
1536 EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16f, 8>& kernel) {
1537  __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0],kernel.packet[1]);
1538  __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0],kernel.packet[1]);
1539  __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2],kernel.packet[3]);
1540  __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2],kernel.packet[3]);
1541  __m512 T4 = _mm512_unpacklo_ps(kernel.packet[4],kernel.packet[5]);
1542  __m512 T5 = _mm512_unpackhi_ps(kernel.packet[4],kernel.packet[5]);
1543  __m512 T6 = _mm512_unpacklo_ps(kernel.packet[6],kernel.packet[7]);
1544  __m512 T7 = _mm512_unpackhi_ps(kernel.packet[6],kernel.packet[7]);
1545 
1546  kernel.packet[0] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T0),_mm512_castps_pd(T2)));
1547  kernel.packet[1] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T0),_mm512_castps_pd(T2)));
1548  kernel.packet[2] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T1),_mm512_castps_pd(T3)));
1549  kernel.packet[3] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T1),_mm512_castps_pd(T3)));
1550  kernel.packet[4] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T4),_mm512_castps_pd(T6)));
1551  kernel.packet[5] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T4),_mm512_castps_pd(T6)));
1552  kernel.packet[6] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T5),_mm512_castps_pd(T7)));
1553  kernel.packet[7] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T5),_mm512_castps_pd(T7)));
1554 
1555  T0 = _mm512_shuffle_f32x4(kernel.packet[0], kernel.packet[4], 0x44);
1556  T1 = _mm512_shuffle_f32x4(kernel.packet[0], kernel.packet[4], 0xee);
1557  T2 = _mm512_shuffle_f32x4(kernel.packet[1], kernel.packet[5], 0x44);
1558  T3 = _mm512_shuffle_f32x4(kernel.packet[1], kernel.packet[5], 0xee);
1559  T4 = _mm512_shuffle_f32x4(kernel.packet[2], kernel.packet[6], 0x44);
1560  T5 = _mm512_shuffle_f32x4(kernel.packet[2], kernel.packet[6], 0xee);
1561  T6 = _mm512_shuffle_f32x4(kernel.packet[3], kernel.packet[7], 0x44);
1562  T7 = _mm512_shuffle_f32x4(kernel.packet[3], kernel.packet[7], 0xee);
1563 
1564  kernel.packet[0] = _mm512_shuffle_f32x4(T0, T2, 0x88);
1565  kernel.packet[2] = _mm512_shuffle_f32x4(T0, T2, 0xdd);
1566  kernel.packet[1] = _mm512_shuffle_f32x4(T4, T6, 0x88);
1567  kernel.packet[3] = _mm512_shuffle_f32x4(T4, T6, 0xdd);
1568  kernel.packet[4] = _mm512_shuffle_f32x4(T1, T3, 0x88);
1569  kernel.packet[6] = _mm512_shuffle_f32x4(T1, T3, 0xdd);
1570  kernel.packet[5] = _mm512_shuffle_f32x4(T5, T7, 0x88);
1571  kernel.packet[7] = _mm512_shuffle_f32x4(T5, T7, 0xdd);
1572 }
1573 
1574 EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16f, 4>& kernel) {
1575  __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]);
1576  __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]);
1577  __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]);
1578  __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]);
1579 
1580  __m512 S0 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
1581  __m512 S1 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
1582  __m512 S2 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
1583  __m512 S3 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
1584 
1585  EIGEN_EXTRACT_8f_FROM_16f(S0, S0);
1586  EIGEN_EXTRACT_8f_FROM_16f(S1, S1);
1587  EIGEN_EXTRACT_8f_FROM_16f(S2, S2);
1588  EIGEN_EXTRACT_8f_FROM_16f(S3, S3);
1589 
1590  PacketBlock<Packet8f, 8> tmp;
1591 
1592  tmp.packet[0] = _mm256_permute2f128_ps(S0_0, S1_0, 0x20);
1593  tmp.packet[1] = _mm256_permute2f128_ps(S2_0, S3_0, 0x20);
1594  tmp.packet[2] = _mm256_permute2f128_ps(S0_0, S1_0, 0x31);
1595  tmp.packet[3] = _mm256_permute2f128_ps(S2_0, S3_0, 0x31);
1596 
1597  tmp.packet[4] = _mm256_permute2f128_ps(S0_1, S1_1, 0x20);
1598  tmp.packet[5] = _mm256_permute2f128_ps(S2_1, S3_1, 0x20);
1599  tmp.packet[6] = _mm256_permute2f128_ps(S0_1, S1_1, 0x31);
1600  tmp.packet[7] = _mm256_permute2f128_ps(S2_1, S3_1, 0x31);
1601 
1602  PACK_OUTPUT_2(kernel.packet, tmp.packet, 0, 1);
1603  PACK_OUTPUT_2(kernel.packet, tmp.packet, 1, 1);
1604  PACK_OUTPUT_2(kernel.packet, tmp.packet, 2, 1);
1605  PACK_OUTPUT_2(kernel.packet, tmp.packet, 3, 1);
1606 }
1607 
1608 #define PACK_OUTPUT_SQ_D(OUTPUT, INPUT, INDEX, STRIDE) \
1609  OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[INDEX], 0); \
1610  OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[INDEX + STRIDE], 1);
1611 
1612 #define PACK_OUTPUT_D(OUTPUT, INPUT, INDEX, STRIDE) \
1613  OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[(2 * INDEX)], 0); \
1614  OUTPUT[INDEX] = \
1615  _mm512_insertf64x4(OUTPUT[INDEX], INPUT[(2 * INDEX) + STRIDE], 1);
1616 
1617 EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8d, 4>& kernel) {
1618  __m512d T0 = _mm512_shuffle_pd(kernel.packet[0], kernel.packet[1], 0);
1619  __m512d T1 = _mm512_shuffle_pd(kernel.packet[0], kernel.packet[1], 0xff);
1620  __m512d T2 = _mm512_shuffle_pd(kernel.packet[2], kernel.packet[3], 0);
1621  __m512d T3 = _mm512_shuffle_pd(kernel.packet[2], kernel.packet[3], 0xff);
1622 
1623  PacketBlock<Packet4d, 8> tmp;
1624 
1625  tmp.packet[0] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0),
1626  _mm512_extractf64x4_pd(T2, 0), 0x20);
1627  tmp.packet[1] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0),
1628  _mm512_extractf64x4_pd(T3, 0), 0x20);
1629  tmp.packet[2] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0),
1630  _mm512_extractf64x4_pd(T2, 0), 0x31);
1631  tmp.packet[3] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0),
1632  _mm512_extractf64x4_pd(T3, 0), 0x31);
1633 
1634  tmp.packet[4] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1),
1635  _mm512_extractf64x4_pd(T2, 1), 0x20);
1636  tmp.packet[5] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1),
1637  _mm512_extractf64x4_pd(T3, 1), 0x20);
1638  tmp.packet[6] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1),
1639  _mm512_extractf64x4_pd(T2, 1), 0x31);
1640  tmp.packet[7] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1),
1641  _mm512_extractf64x4_pd(T3, 1), 0x31);
1642 
1643  PACK_OUTPUT_D(kernel.packet, tmp.packet, 0, 1);
1644  PACK_OUTPUT_D(kernel.packet, tmp.packet, 1, 1);
1645  PACK_OUTPUT_D(kernel.packet, tmp.packet, 2, 1);
1646  PACK_OUTPUT_D(kernel.packet, tmp.packet, 3, 1);
1647 }
1648 
1649 EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8d, 8>& kernel) {
1650  __m512d T0 = _mm512_unpacklo_pd(kernel.packet[0],kernel.packet[1]);
1651  __m512d T1 = _mm512_unpackhi_pd(kernel.packet[0],kernel.packet[1]);
1652  __m512d T2 = _mm512_unpacklo_pd(kernel.packet[2],kernel.packet[3]);
1653  __m512d T3 = _mm512_unpackhi_pd(kernel.packet[2],kernel.packet[3]);
1654  __m512d T4 = _mm512_unpacklo_pd(kernel.packet[4],kernel.packet[5]);
1655  __m512d T5 = _mm512_unpackhi_pd(kernel.packet[4],kernel.packet[5]);
1656  __m512d T6 = _mm512_unpacklo_pd(kernel.packet[6],kernel.packet[7]);
1657  __m512d T7 = _mm512_unpackhi_pd(kernel.packet[6],kernel.packet[7]);
1658 
1659  kernel.packet[0] = _mm512_permutex_pd(T2, 0x4E);
1660  kernel.packet[0] = _mm512_mask_blend_pd(0xCC, T0, kernel.packet[0]);
1661  kernel.packet[2] = _mm512_permutex_pd(T0, 0x4E);
1662  kernel.packet[2] = _mm512_mask_blend_pd(0xCC, kernel.packet[2], T2);
1663  kernel.packet[1] = _mm512_permutex_pd(T3, 0x4E);
1664  kernel.packet[1] = _mm512_mask_blend_pd(0xCC, T1, kernel.packet[1]);
1665  kernel.packet[3] = _mm512_permutex_pd(T1, 0x4E);
1666  kernel.packet[3] = _mm512_mask_blend_pd(0xCC, kernel.packet[3], T3);
1667  kernel.packet[4] = _mm512_permutex_pd(T6, 0x4E);
1668  kernel.packet[4] = _mm512_mask_blend_pd(0xCC, T4, kernel.packet[4]);
1669  kernel.packet[6] = _mm512_permutex_pd(T4, 0x4E);
1670  kernel.packet[6] = _mm512_mask_blend_pd(0xCC, kernel.packet[6], T6);
1671  kernel.packet[5] = _mm512_permutex_pd(T7, 0x4E);
1672  kernel.packet[5] = _mm512_mask_blend_pd(0xCC, T5, kernel.packet[5]);
1673  kernel.packet[7] = _mm512_permutex_pd(T5, 0x4E);
1674  kernel.packet[7] = _mm512_mask_blend_pd(0xCC, kernel.packet[7], T7);
1675 
1676  T0 = _mm512_shuffle_f64x2(kernel.packet[4], kernel.packet[4], 0x4E);
1677  T0 = _mm512_mask_blend_pd(0xF0, kernel.packet[0], T0);
1678  T4 = _mm512_shuffle_f64x2(kernel.packet[0], kernel.packet[0], 0x4E);
1679  T4 = _mm512_mask_blend_pd(0xF0, T4, kernel.packet[4]);
1680  T1 = _mm512_shuffle_f64x2(kernel.packet[5], kernel.packet[5], 0x4E);
1681  T1 = _mm512_mask_blend_pd(0xF0, kernel.packet[1], T1);
1682  T5 = _mm512_shuffle_f64x2(kernel.packet[1], kernel.packet[1], 0x4E);
1683  T5 = _mm512_mask_blend_pd(0xF0, T5, kernel.packet[5]);
1684  T2 = _mm512_shuffle_f64x2(kernel.packet[6], kernel.packet[6], 0x4E);
1685  T2 = _mm512_mask_blend_pd(0xF0, kernel.packet[2], T2);
1686  T6 = _mm512_shuffle_f64x2(kernel.packet[2], kernel.packet[2], 0x4E);
1687  T6 = _mm512_mask_blend_pd(0xF0, T6, kernel.packet[6]);
1688  T3 = _mm512_shuffle_f64x2(kernel.packet[7], kernel.packet[7], 0x4E);
1689  T3 = _mm512_mask_blend_pd(0xF0, kernel.packet[3], T3);
1690  T7 = _mm512_shuffle_f64x2(kernel.packet[3], kernel.packet[3], 0x4E);
1691  T7 = _mm512_mask_blend_pd(0xF0, T7, kernel.packet[7]);
1692 
1693  kernel.packet[0] = T0; kernel.packet[1] = T1;
1694  kernel.packet[2] = T2; kernel.packet[3] = T3;
1695  kernel.packet[4] = T4; kernel.packet[5] = T5;
1696  kernel.packet[6] = T6; kernel.packet[7] = T7;
1697 }
1698 
1699 #define PACK_OUTPUT_I32(OUTPUT, INPUT, INDEX, STRIDE) \
1700  EIGEN_INSERT_8i_INTO_16i(OUTPUT[INDEX], INPUT[INDEX], INPUT[INDEX + STRIDE]);
1701 
1702 #define PACK_OUTPUT_I32_2(OUTPUT, INPUT, INDEX, STRIDE) \
1703  EIGEN_INSERT_8i_INTO_16i(OUTPUT[INDEX], INPUT[2 * INDEX], \
1704  INPUT[2 * INDEX + STRIDE]);
1705 
1706 #define SHUFFLE_EPI32(A, B, M) \
1707  _mm512_castps_si512(_mm512_shuffle_ps(_mm512_castsi512_ps(A), _mm512_castsi512_ps(B), M))
1708 
1709 EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16i, 16>& kernel) {
1710  __m512i T0 = _mm512_unpacklo_epi32(kernel.packet[0], kernel.packet[1]);
1711  __m512i T1 = _mm512_unpackhi_epi32(kernel.packet[0], kernel.packet[1]);
1712  __m512i T2 = _mm512_unpacklo_epi32(kernel.packet[2], kernel.packet[3]);
1713  __m512i T3 = _mm512_unpackhi_epi32(kernel.packet[2], kernel.packet[3]);
1714  __m512i T4 = _mm512_unpacklo_epi32(kernel.packet[4], kernel.packet[5]);
1715  __m512i T5 = _mm512_unpackhi_epi32(kernel.packet[4], kernel.packet[5]);
1716  __m512i T6 = _mm512_unpacklo_epi32(kernel.packet[6], kernel.packet[7]);
1717  __m512i T7 = _mm512_unpackhi_epi32(kernel.packet[6], kernel.packet[7]);
1718  __m512i T8 = _mm512_unpacklo_epi32(kernel.packet[8], kernel.packet[9]);
1719  __m512i T9 = _mm512_unpackhi_epi32(kernel.packet[8], kernel.packet[9]);
1720  __m512i T10 = _mm512_unpacklo_epi32(kernel.packet[10], kernel.packet[11]);
1721  __m512i T11 = _mm512_unpackhi_epi32(kernel.packet[10], kernel.packet[11]);
1722  __m512i T12 = _mm512_unpacklo_epi32(kernel.packet[12], kernel.packet[13]);
1723  __m512i T13 = _mm512_unpackhi_epi32(kernel.packet[12], kernel.packet[13]);
1724  __m512i T14 = _mm512_unpacklo_epi32(kernel.packet[14], kernel.packet[15]);
1725  __m512i T15 = _mm512_unpackhi_epi32(kernel.packet[14], kernel.packet[15]);
1726  __m512i S0 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
1727  __m512i S1 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
1728  __m512i S2 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
1729  __m512i S3 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
1730  __m512i S4 = SHUFFLE_EPI32(T4, T6, _MM_SHUFFLE(1, 0, 1, 0));
1731  __m512i S5 = SHUFFLE_EPI32(T4, T6, _MM_SHUFFLE(3, 2, 3, 2));
1732  __m512i S6 = SHUFFLE_EPI32(T5, T7, _MM_SHUFFLE(1, 0, 1, 0));
1733  __m512i S7 = SHUFFLE_EPI32(T5, T7, _MM_SHUFFLE(3, 2, 3, 2));
1734  __m512i S8 = SHUFFLE_EPI32(T8, T10, _MM_SHUFFLE(1, 0, 1, 0));
1735  __m512i S9 = SHUFFLE_EPI32(T8, T10, _MM_SHUFFLE(3, 2, 3, 2));
1736  __m512i S10 = SHUFFLE_EPI32(T9, T11, _MM_SHUFFLE(1, 0, 1, 0));
1737  __m512i S11 = SHUFFLE_EPI32(T9, T11, _MM_SHUFFLE(3, 2, 3, 2));
1738  __m512i S12 = SHUFFLE_EPI32(T12, T14, _MM_SHUFFLE(1, 0, 1, 0));
1739  __m512i S13 = SHUFFLE_EPI32(T12, T14, _MM_SHUFFLE(3, 2, 3, 2));
1740  __m512i S14 = SHUFFLE_EPI32(T13, T15, _MM_SHUFFLE(1, 0, 1, 0));
1741  __m512i S15 = SHUFFLE_EPI32(T13, T15, _MM_SHUFFLE(3, 2, 3, 2));
1742 
1743  EIGEN_EXTRACT_8i_FROM_16i(S0, S0);
1744  EIGEN_EXTRACT_8i_FROM_16i(S1, S1);
1745  EIGEN_EXTRACT_8i_FROM_16i(S2, S2);
1746  EIGEN_EXTRACT_8i_FROM_16i(S3, S3);
1747  EIGEN_EXTRACT_8i_FROM_16i(S4, S4);
1748  EIGEN_EXTRACT_8i_FROM_16i(S5, S5);
1749  EIGEN_EXTRACT_8i_FROM_16i(S6, S6);
1750  EIGEN_EXTRACT_8i_FROM_16i(S7, S7);
1751  EIGEN_EXTRACT_8i_FROM_16i(S8, S8);
1752  EIGEN_EXTRACT_8i_FROM_16i(S9, S9);
1753  EIGEN_EXTRACT_8i_FROM_16i(S10, S10);
1754  EIGEN_EXTRACT_8i_FROM_16i(S11, S11);
1755  EIGEN_EXTRACT_8i_FROM_16i(S12, S12);
1756  EIGEN_EXTRACT_8i_FROM_16i(S13, S13);
1757  EIGEN_EXTRACT_8i_FROM_16i(S14, S14);
1758  EIGEN_EXTRACT_8i_FROM_16i(S15, S15);
1759 
1760  PacketBlock<Packet8i, 32> tmp;
1761 
1762  tmp.packet[0] = _mm256_permute2f128_si256(S0_0, S4_0, 0x20);
1763  tmp.packet[1] = _mm256_permute2f128_si256(S1_0, S5_0, 0x20);
1764  tmp.packet[2] = _mm256_permute2f128_si256(S2_0, S6_0, 0x20);
1765  tmp.packet[3] = _mm256_permute2f128_si256(S3_0, S7_0, 0x20);
1766  tmp.packet[4] = _mm256_permute2f128_si256(S0_0, S4_0, 0x31);
1767  tmp.packet[5] = _mm256_permute2f128_si256(S1_0, S5_0, 0x31);
1768  tmp.packet[6] = _mm256_permute2f128_si256(S2_0, S6_0, 0x31);
1769  tmp.packet[7] = _mm256_permute2f128_si256(S3_0, S7_0, 0x31);
1770 
1771  tmp.packet[8] = _mm256_permute2f128_si256(S0_1, S4_1, 0x20);
1772  tmp.packet[9] = _mm256_permute2f128_si256(S1_1, S5_1, 0x20);
1773  tmp.packet[10] = _mm256_permute2f128_si256(S2_1, S6_1, 0x20);
1774  tmp.packet[11] = _mm256_permute2f128_si256(S3_1, S7_1, 0x20);
1775  tmp.packet[12] = _mm256_permute2f128_si256(S0_1, S4_1, 0x31);
1776  tmp.packet[13] = _mm256_permute2f128_si256(S1_1, S5_1, 0x31);
1777  tmp.packet[14] = _mm256_permute2f128_si256(S2_1, S6_1, 0x31);
1778  tmp.packet[15] = _mm256_permute2f128_si256(S3_1, S7_1, 0x31);
1779 
1780  // Second set of _m256 outputs
1781  tmp.packet[16] = _mm256_permute2f128_si256(S8_0, S12_0, 0x20);
1782  tmp.packet[17] = _mm256_permute2f128_si256(S9_0, S13_0, 0x20);
1783  tmp.packet[18] = _mm256_permute2f128_si256(S10_0, S14_0, 0x20);
1784  tmp.packet[19] = _mm256_permute2f128_si256(S11_0, S15_0, 0x20);
1785  tmp.packet[20] = _mm256_permute2f128_si256(S8_0, S12_0, 0x31);
1786  tmp.packet[21] = _mm256_permute2f128_si256(S9_0, S13_0, 0x31);
1787  tmp.packet[22] = _mm256_permute2f128_si256(S10_0, S14_0, 0x31);
1788  tmp.packet[23] = _mm256_permute2f128_si256(S11_0, S15_0, 0x31);
1789 
1790  tmp.packet[24] = _mm256_permute2f128_si256(S8_1, S12_1, 0x20);
1791  tmp.packet[25] = _mm256_permute2f128_si256(S9_1, S13_1, 0x20);
1792  tmp.packet[26] = _mm256_permute2f128_si256(S10_1, S14_1, 0x20);
1793  tmp.packet[27] = _mm256_permute2f128_si256(S11_1, S15_1, 0x20);
1794  tmp.packet[28] = _mm256_permute2f128_si256(S8_1, S12_1, 0x31);
1795  tmp.packet[29] = _mm256_permute2f128_si256(S9_1, S13_1, 0x31);
1796  tmp.packet[30] = _mm256_permute2f128_si256(S10_1, S14_1, 0x31);
1797  tmp.packet[31] = _mm256_permute2f128_si256(S11_1, S15_1, 0x31);
1798 
1799  // Pack them into the output
1800  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 0, 16);
1801  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 1, 16);
1802  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 2, 16);
1803  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 3, 16);
1804 
1805  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 4, 16);
1806  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 5, 16);
1807  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 6, 16);
1808  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 7, 16);
1809 
1810  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 8, 16);
1811  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 9, 16);
1812  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 10, 16);
1813  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 11, 16);
1814 
1815  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 12, 16);
1816  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 13, 16);
1817  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 14, 16);
1818  PACK_OUTPUT_I32(kernel.packet, tmp.packet, 15, 16);
1819 }
1820 
1821 EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16i, 4>& kernel) {
1822  __m512i T0 = _mm512_unpacklo_epi32(kernel.packet[0], kernel.packet[1]);
1823  __m512i T1 = _mm512_unpackhi_epi32(kernel.packet[0], kernel.packet[1]);
1824  __m512i T2 = _mm512_unpacklo_epi32(kernel.packet[2], kernel.packet[3]);
1825  __m512i T3 = _mm512_unpackhi_epi32(kernel.packet[2], kernel.packet[3]);
1826 
1827  __m512i S0 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
1828  __m512i S1 = SHUFFLE_EPI32(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
1829  __m512i S2 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
1830  __m512i S3 = SHUFFLE_EPI32(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
1831 
1832  EIGEN_EXTRACT_8i_FROM_16i(S0, S0);
1833  EIGEN_EXTRACT_8i_FROM_16i(S1, S1);
1834  EIGEN_EXTRACT_8i_FROM_16i(S2, S2);
1835  EIGEN_EXTRACT_8i_FROM_16i(S3, S3);
1836 
1837  PacketBlock<Packet8i, 8> tmp;
1838 
1839  tmp.packet[0] = _mm256_permute2f128_si256(S0_0, S1_0, 0x20);
1840  tmp.packet[1] = _mm256_permute2f128_si256(S2_0, S3_0, 0x20);
1841  tmp.packet[2] = _mm256_permute2f128_si256(S0_0, S1_0, 0x31);
1842  tmp.packet[3] = _mm256_permute2f128_si256(S2_0, S3_0, 0x31);
1843 
1844  tmp.packet[4] = _mm256_permute2f128_si256(S0_1, S1_1, 0x20);
1845  tmp.packet[5] = _mm256_permute2f128_si256(S2_1, S3_1, 0x20);
1846  tmp.packet[6] = _mm256_permute2f128_si256(S0_1, S1_1, 0x31);
1847  tmp.packet[7] = _mm256_permute2f128_si256(S2_1, S3_1, 0x31);
1848 
1849  PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 0, 1);
1850  PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 1, 1);
1851  PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 2, 1);
1852  PACK_OUTPUT_I32_2(kernel.packet, tmp.packet, 3, 1);
1853 }
1854 
1855 template <>
1856 EIGEN_STRONG_INLINE Packet16f pblend(const Selector<16>& ifPacket,
1857  const Packet16f& thenPacket,
1858  const Packet16f& elsePacket) {
1859  __mmask16 m = (ifPacket.select[0]) | (ifPacket.select[1] << 1) | (ifPacket.select[2] << 2) |
1860  (ifPacket.select[3] << 3) | (ifPacket.select[4] << 4) | (ifPacket.select[5] << 5) |
1861  (ifPacket.select[6] << 6) | (ifPacket.select[7] << 7) | (ifPacket.select[8] << 8) |
1862  (ifPacket.select[9] << 9) | (ifPacket.select[10] << 10) | (ifPacket.select[11] << 11) |
1863  (ifPacket.select[12] << 12) | (ifPacket.select[13] << 13) | (ifPacket.select[14] << 14) |
1864  (ifPacket.select[15] << 15);
1865  return _mm512_mask_blend_ps(m, elsePacket, thenPacket);
1866 }
1867 template <>
1868 EIGEN_STRONG_INLINE Packet8d pblend(const Selector<8>& ifPacket,
1869  const Packet8d& thenPacket,
1870  const Packet8d& elsePacket) {
1871  __mmask8 m = (ifPacket.select[0] )
1872  | (ifPacket.select[1]<<1)
1873  | (ifPacket.select[2]<<2)
1874  | (ifPacket.select[3]<<3)
1875  | (ifPacket.select[4]<<4)
1876  | (ifPacket.select[5]<<5)
1877  | (ifPacket.select[6]<<6)
1878  | (ifPacket.select[7]<<7);
1879  return _mm512_mask_blend_pd(m, elsePacket, thenPacket);
1880 }
1881 
1882 // Packet math for Eigen::half
1883 template<> EIGEN_STRONG_INLINE Packet16h pset1<Packet16h>(const Eigen::half& from) {
1884  return _mm256_set1_epi16(from.x);
1885 }
1886 
1887 template<> EIGEN_STRONG_INLINE Eigen::half pfirst<Packet16h>(const Packet16h& from) {
1888  return half_impl::raw_uint16_to_half(static_cast<unsigned short>(_mm256_extract_epi16(from, 0)));
1889 }
1890 
1891 template<> EIGEN_STRONG_INLINE Packet16h pload<Packet16h>(const Eigen::half* from) {
1892  return _mm256_load_si256(reinterpret_cast<const __m256i*>(from));
1893 }
1894 
1895 template<> EIGEN_STRONG_INLINE Packet16h ploadu<Packet16h>(const Eigen::half* from) {
1896  return _mm256_loadu_si256(reinterpret_cast<const __m256i*>(from));
1897 }
1898 
1899 template<> EIGEN_STRONG_INLINE void pstore<half>(Eigen::half* to, const Packet16h& from) {
1900  // (void*) -> workaround clang warning:
1901  // cast from 'Eigen::half *' to '__m256i *' increases required alignment from 2 to 32
1902  _mm256_store_si256((__m256i*)(void*)to, from);
1903 }
1904 
1905 template<> EIGEN_STRONG_INLINE void pstoreu<half>(Eigen::half* to, const Packet16h& from) {
1906  // (void*) -> workaround clang warning:
1907  // cast from 'Eigen::half *' to '__m256i *' increases required alignment from 2 to 32
1908  _mm256_storeu_si256((__m256i*)(void*)to, from);
1909 }
1910 
1911 template<> EIGEN_STRONG_INLINE Packet16h
1912 ploaddup<Packet16h>(const Eigen::half* from) {
1913  unsigned short a = from[0].x;
1914  unsigned short b = from[1].x;
1915  unsigned short c = from[2].x;
1916  unsigned short d = from[3].x;
1917  unsigned short e = from[4].x;
1918  unsigned short f = from[5].x;
1919  unsigned short g = from[6].x;
1920  unsigned short h = from[7].x;
1921  return _mm256_set_epi16(h, h, g, g, f, f, e, e, d, d, c, c, b, b, a, a);
1922 }
1923 
1924 template<> EIGEN_STRONG_INLINE Packet16h
1925 ploadquad(const Eigen::half* from) {
1926  unsigned short a = from[0].x;
1927  unsigned short b = from[1].x;
1928  unsigned short c = from[2].x;
1929  unsigned short d = from[3].x;
1930  return _mm256_set_epi16(d, d, d, d, c, c, c, c, b, b, b, b, a, a, a, a);
1931 }
1932 
1933 EIGEN_STRONG_INLINE Packet16f half2float(const Packet16h& a) {
1934  return _mm512_cvtph_ps(a);
1935 }
1936 
1937 EIGEN_STRONG_INLINE Packet16h float2half(const Packet16f& a) {
1938  return _mm512_cvtps_ph(a, _MM_FROUND_TO_NEAREST_INT|_MM_FROUND_NO_EXC);
1939 }
1940 
1941 template<> EIGEN_STRONG_INLINE Packet16h ptrue(const Packet16h& a) {
1942  return Packet16h(ptrue(Packet8i(a)));
1943 }
1944 
1945 template <>
1946 EIGEN_STRONG_INLINE Packet16h pabs(const Packet16h& a) {
1947  const __m256i sign_mask = _mm256_set1_epi16(static_cast<numext::uint16_t>(0x8000));
1948  return _mm256_andnot_si256(sign_mask, a);
1949 }
1950 
1951 template <>
1952 EIGEN_STRONG_INLINE Packet16h pmin<Packet16h>(const Packet16h& a,
1953  const Packet16h& b) {
1954  return float2half(pmin<Packet16f>(half2float(a), half2float(b)));
1955 }
1956 
1957 template <>
1958 EIGEN_STRONG_INLINE Packet16h pmax<Packet16h>(const Packet16h& a,
1959  const Packet16h& b) {
1960  return float2half(pmax<Packet16f>(half2float(a), half2float(b)));
1961 }
1962 
1963 template <>
1964 EIGEN_STRONG_INLINE Packet16h plset<Packet16h>(const half& a) {
1965  return float2half(plset<Packet16f>(static_cast<float>(a)));
1966 }
1967 
1968 template<> EIGEN_STRONG_INLINE Packet16h por(const Packet16h& a,const Packet16h& b) {
1969  // in some cases Packet8i is a wrapper around __m256i, so we need to
1970  // cast to Packet8i to call the correct overload.
1971  return Packet16h(por(Packet8i(a),Packet8i(b)));
1972 }
1973 template<> EIGEN_STRONG_INLINE Packet16h pxor(const Packet16h& a,const Packet16h& b) {
1974  return Packet16h(pxor(Packet8i(a),Packet8i(b)));
1975 }
1976 template<> EIGEN_STRONG_INLINE Packet16h pand(const Packet16h& a,const Packet16h& b) {
1977  return Packet16h(pand(Packet8i(a),Packet8i(b)));
1978 }
1979 template<> EIGEN_STRONG_INLINE Packet16h pandnot(const Packet16h& a,const Packet16h& b) {
1980  return Packet16h(pandnot(Packet8i(a),Packet8i(b)));
1981 }
1982 
1983 template<> EIGEN_STRONG_INLINE Packet16h pselect(const Packet16h& mask, const Packet16h& a, const Packet16h& b) {
1984  return _mm256_blendv_epi8(b, a, mask);
1985 }
1986 
1987 template<> EIGEN_STRONG_INLINE Packet16h pround<Packet16h>(const Packet16h& a) {
1988  return float2half(pround<Packet16f>(half2float(a)));
1989 }
1990 
1991 template<> EIGEN_STRONG_INLINE Packet16h print<Packet16h>(const Packet16h& a) {
1992  return float2half(print<Packet16f>(half2float(a)));
1993 }
1994 
1995 template<> EIGEN_STRONG_INLINE Packet16h pceil<Packet16h>(const Packet16h& a) {
1996  return float2half(pceil<Packet16f>(half2float(a)));
1997 }
1998 
1999 template<> EIGEN_STRONG_INLINE Packet16h pfloor<Packet16h>(const Packet16h& a) {
2000  return float2half(pfloor<Packet16f>(half2float(a)));
2001 }
2002 
2003 template<> EIGEN_STRONG_INLINE Packet16h pcmp_eq(const Packet16h& a,const Packet16h& b) {
2004  Packet16f af = half2float(a);
2005  Packet16f bf = half2float(b);
2006  return Pack32To16(pcmp_eq(af, bf));
2007 }
2008 
2009 template<> EIGEN_STRONG_INLINE Packet16h pcmp_le(const Packet16h& a,const Packet16h& b) {
2010  return Pack32To16(pcmp_le(half2float(a), half2float(b)));
2011 }
2012 
2013 template<> EIGEN_STRONG_INLINE Packet16h pcmp_lt(const Packet16h& a,const Packet16h& b) {
2014  return Pack32To16(pcmp_lt(half2float(a), half2float(b)));
2015 }
2016 
2017 template<> EIGEN_STRONG_INLINE Packet16h pcmp_lt_or_nan(const Packet16h& a,const Packet16h& b) {
2018  return Pack32To16(pcmp_lt_or_nan(half2float(a), half2float(b)));
2019 }
2020 
2021 template<> EIGEN_STRONG_INLINE Packet16h pconj(const Packet16h& a) { return a; }
2022 
2023 template<> EIGEN_STRONG_INLINE Packet16h pnegate(const Packet16h& a) {
2024  Packet16h sign_mask = _mm256_set1_epi16(static_cast<unsigned short>(0x8000));
2025  return _mm256_xor_si256(a, sign_mask);
2026 }
2027 
2028 #ifndef EIGEN_VECTORIZE_AVX512FP16
2029 template<> EIGEN_STRONG_INLINE Packet16h padd<Packet16h>(const Packet16h& a, const Packet16h& b) {
2030  Packet16f af = half2float(a);
2031  Packet16f bf = half2float(b);
2032  Packet16f rf = padd(af, bf);
2033  return float2half(rf);
2034 }
2035 
2036 template<> EIGEN_STRONG_INLINE Packet16h psub<Packet16h>(const Packet16h& a, const Packet16h& b) {
2037  Packet16f af = half2float(a);
2038  Packet16f bf = half2float(b);
2039  Packet16f rf = psub(af, bf);
2040  return float2half(rf);
2041 }
2042 
2043 template<> EIGEN_STRONG_INLINE Packet16h pmul<Packet16h>(const Packet16h& a, const Packet16h& b) {
2044  Packet16f af = half2float(a);
2045  Packet16f bf = half2float(b);
2046  Packet16f rf = pmul(af, bf);
2047  return float2half(rf);
2048 }
2049 
2050 template<> EIGEN_STRONG_INLINE Packet16h pdiv<Packet16h>(const Packet16h& a, const Packet16h& b) {
2051  Packet16f af = half2float(a);
2052  Packet16f bf = half2float(b);
2053  Packet16f rf = pdiv(af, bf);
2054  return float2half(rf);
2055 }
2056 
2057 template<> EIGEN_STRONG_INLINE half predux<Packet16h>(const Packet16h& from) {
2058  Packet16f from_float = half2float(from);
2059  return half(predux(from_float));
2060 }
2061 
2062 #endif
2063 
2064 template <>
2065 EIGEN_STRONG_INLINE Packet8h predux_half_dowto4<Packet16h>(const Packet16h& a) {
2066  Packet8h lane0 = _mm256_extractf128_si256(a, 0);
2067  Packet8h lane1 = _mm256_extractf128_si256(a, 1);
2068  return padd<Packet8h>(lane0, lane1);
2069 }
2070 
2071 template<> EIGEN_STRONG_INLINE Eigen::half predux_max<Packet16h>(const Packet16h& a) {
2072  Packet16f af = half2float(a);
2073  float reduced = predux_max<Packet16f>(af);
2074  return Eigen::half(reduced);
2075 }
2076 
2077 template<> EIGEN_STRONG_INLINE Eigen::half predux_min<Packet16h>(const Packet16h& a) {
2078  Packet16f af = half2float(a);
2079  float reduced = predux_min<Packet16f>(af);
2080  return Eigen::half(reduced);
2081 }
2082 
2083 template<> EIGEN_STRONG_INLINE half predux_mul<Packet16h>(const Packet16h& from) {
2084  Packet16f from_float = half2float(from);
2085  return half(predux_mul(from_float));
2086 }
2087 
2088 template<> EIGEN_STRONG_INLINE Packet16h preverse(const Packet16h& a)
2089 {
2090  __m128i m = _mm_setr_epi8(14,15,12,13,10,11,8,9,6,7,4,5,2,3,0,1);
2091  return _mm256_insertf128_si256(
2092  _mm256_castsi128_si256(_mm_shuffle_epi8(_mm256_extractf128_si256(a,1),m)),
2093  _mm_shuffle_epi8(_mm256_extractf128_si256(a,0),m), 1);
2094 }
2095 
2096 template<> EIGEN_STRONG_INLINE Packet16h pgather<Eigen::half, Packet16h>(const Eigen::half* from, Index stride)
2097 {
2098  return _mm256_set_epi16(
2099  from[15*stride].x, from[14*stride].x, from[13*stride].x, from[12*stride].x,
2100  from[11*stride].x, from[10*stride].x, from[9*stride].x, from[8*stride].x,
2101  from[7*stride].x, from[6*stride].x, from[5*stride].x, from[4*stride].x,
2102  from[3*stride].x, from[2*stride].x, from[1*stride].x, from[0*stride].x);
2103 }
2104 
2105 template<> EIGEN_STRONG_INLINE void pscatter<half, Packet16h>(half* to, const Packet16h& from, Index stride)
2106 {
2107  EIGEN_ALIGN64 half aux[16];
2108  pstore(aux, from);
2109  to[stride*0] = aux[0];
2110  to[stride*1] = aux[1];
2111  to[stride*2] = aux[2];
2112  to[stride*3] = aux[3];
2113  to[stride*4] = aux[4];
2114  to[stride*5] = aux[5];
2115  to[stride*6] = aux[6];
2116  to[stride*7] = aux[7];
2117  to[stride*8] = aux[8];
2118  to[stride*9] = aux[9];
2119  to[stride*10] = aux[10];
2120  to[stride*11] = aux[11];
2121  to[stride*12] = aux[12];
2122  to[stride*13] = aux[13];
2123  to[stride*14] = aux[14];
2124  to[stride*15] = aux[15];
2125 }
2126 
2127 EIGEN_STRONG_INLINE void
2128 ptranspose(PacketBlock<Packet16h,16>& kernel) {
2129  __m256i a = kernel.packet[0];
2130  __m256i b = kernel.packet[1];
2131  __m256i c = kernel.packet[2];
2132  __m256i d = kernel.packet[3];
2133  __m256i e = kernel.packet[4];
2134  __m256i f = kernel.packet[5];
2135  __m256i g = kernel.packet[6];
2136  __m256i h = kernel.packet[7];
2137  __m256i i = kernel.packet[8];
2138  __m256i j = kernel.packet[9];
2139  __m256i k = kernel.packet[10];
2140  __m256i l = kernel.packet[11];
2141  __m256i m = kernel.packet[12];
2142  __m256i n = kernel.packet[13];
2143  __m256i o = kernel.packet[14];
2144  __m256i p = kernel.packet[15];
2145 
2146  __m256i ab_07 = _mm256_unpacklo_epi16(a, b);
2147  __m256i cd_07 = _mm256_unpacklo_epi16(c, d);
2148  __m256i ef_07 = _mm256_unpacklo_epi16(e, f);
2149  __m256i gh_07 = _mm256_unpacklo_epi16(g, h);
2150  __m256i ij_07 = _mm256_unpacklo_epi16(i, j);
2151  __m256i kl_07 = _mm256_unpacklo_epi16(k, l);
2152  __m256i mn_07 = _mm256_unpacklo_epi16(m, n);
2153  __m256i op_07 = _mm256_unpacklo_epi16(o, p);
2154 
2155  __m256i ab_8f = _mm256_unpackhi_epi16(a, b);
2156  __m256i cd_8f = _mm256_unpackhi_epi16(c, d);
2157  __m256i ef_8f = _mm256_unpackhi_epi16(e, f);
2158  __m256i gh_8f = _mm256_unpackhi_epi16(g, h);
2159  __m256i ij_8f = _mm256_unpackhi_epi16(i, j);
2160  __m256i kl_8f = _mm256_unpackhi_epi16(k, l);
2161  __m256i mn_8f = _mm256_unpackhi_epi16(m, n);
2162  __m256i op_8f = _mm256_unpackhi_epi16(o, p);
2163 
2164  __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07);
2165  __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07);
2166  __m256i efgh_03 = _mm256_unpacklo_epi32(ef_07, gh_07);
2167  __m256i efgh_47 = _mm256_unpackhi_epi32(ef_07, gh_07);
2168  __m256i ijkl_03 = _mm256_unpacklo_epi32(ij_07, kl_07);
2169  __m256i ijkl_47 = _mm256_unpackhi_epi32(ij_07, kl_07);
2170  __m256i mnop_03 = _mm256_unpacklo_epi32(mn_07, op_07);
2171  __m256i mnop_47 = _mm256_unpackhi_epi32(mn_07, op_07);
2172 
2173  __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f);
2174  __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f);
2175  __m256i efgh_8b = _mm256_unpacklo_epi32(ef_8f, gh_8f);
2176  __m256i efgh_cf = _mm256_unpackhi_epi32(ef_8f, gh_8f);
2177  __m256i ijkl_8b = _mm256_unpacklo_epi32(ij_8f, kl_8f);
2178  __m256i ijkl_cf = _mm256_unpackhi_epi32(ij_8f, kl_8f);
2179  __m256i mnop_8b = _mm256_unpacklo_epi32(mn_8f, op_8f);
2180  __m256i mnop_cf = _mm256_unpackhi_epi32(mn_8f, op_8f);
2181 
2182  __m256i abcdefgh_01 = _mm256_unpacklo_epi64(abcd_03, efgh_03);
2183  __m256i abcdefgh_23 = _mm256_unpackhi_epi64(abcd_03, efgh_03);
2184  __m256i ijklmnop_01 = _mm256_unpacklo_epi64(ijkl_03, mnop_03);
2185  __m256i ijklmnop_23 = _mm256_unpackhi_epi64(ijkl_03, mnop_03);
2186  __m256i abcdefgh_45 = _mm256_unpacklo_epi64(abcd_47, efgh_47);
2187  __m256i abcdefgh_67 = _mm256_unpackhi_epi64(abcd_47, efgh_47);
2188  __m256i ijklmnop_45 = _mm256_unpacklo_epi64(ijkl_47, mnop_47);
2189  __m256i ijklmnop_67 = _mm256_unpackhi_epi64(ijkl_47, mnop_47);
2190  __m256i abcdefgh_89 = _mm256_unpacklo_epi64(abcd_8b, efgh_8b);
2191  __m256i abcdefgh_ab = _mm256_unpackhi_epi64(abcd_8b, efgh_8b);
2192  __m256i ijklmnop_89 = _mm256_unpacklo_epi64(ijkl_8b, mnop_8b);
2193  __m256i ijklmnop_ab = _mm256_unpackhi_epi64(ijkl_8b, mnop_8b);
2194  __m256i abcdefgh_cd = _mm256_unpacklo_epi64(abcd_cf, efgh_cf);
2195  __m256i abcdefgh_ef = _mm256_unpackhi_epi64(abcd_cf, efgh_cf);
2196  __m256i ijklmnop_cd = _mm256_unpacklo_epi64(ijkl_cf, mnop_cf);
2197  __m256i ijklmnop_ef = _mm256_unpackhi_epi64(ijkl_cf, mnop_cf);
2198 
2199  // NOTE: no unpacklo/hi instr in this case, so using permute instr.
2200  __m256i a_p_0 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x20);
2201  __m256i a_p_1 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x20);
2202  __m256i a_p_2 = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x20);
2203  __m256i a_p_3 = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x20);
2204  __m256i a_p_4 = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x20);
2205  __m256i a_p_5 = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x20);
2206  __m256i a_p_6 = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x20);
2207  __m256i a_p_7 = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x20);
2208  __m256i a_p_8 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x31);
2209  __m256i a_p_9 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x31);
2210  __m256i a_p_a = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x31);
2211  __m256i a_p_b = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x31);
2212  __m256i a_p_c = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x31);
2213  __m256i a_p_d = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x31);
2214  __m256i a_p_e = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x31);
2215  __m256i a_p_f = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x31);
2216 
2217  kernel.packet[0] = a_p_0;
2218  kernel.packet[1] = a_p_1;
2219  kernel.packet[2] = a_p_2;
2220  kernel.packet[3] = a_p_3;
2221  kernel.packet[4] = a_p_4;
2222  kernel.packet[5] = a_p_5;
2223  kernel.packet[6] = a_p_6;
2224  kernel.packet[7] = a_p_7;
2225  kernel.packet[8] = a_p_8;
2226  kernel.packet[9] = a_p_9;
2227  kernel.packet[10] = a_p_a;
2228  kernel.packet[11] = a_p_b;
2229  kernel.packet[12] = a_p_c;
2230  kernel.packet[13] = a_p_d;
2231  kernel.packet[14] = a_p_e;
2232  kernel.packet[15] = a_p_f;
2233 }
2234 
2235 EIGEN_STRONG_INLINE void
2236 ptranspose(PacketBlock<Packet16h,8>& kernel) {
2237  EIGEN_ALIGN64 half in[8][16];
2238  pstore<half>(in[0], kernel.packet[0]);
2239  pstore<half>(in[1], kernel.packet[1]);
2240  pstore<half>(in[2], kernel.packet[2]);
2241  pstore<half>(in[3], kernel.packet[3]);
2242  pstore<half>(in[4], kernel.packet[4]);
2243  pstore<half>(in[5], kernel.packet[5]);
2244  pstore<half>(in[6], kernel.packet[6]);
2245  pstore<half>(in[7], kernel.packet[7]);
2246 
2247  EIGEN_ALIGN64 half out[8][16];
2248 
2249  for (int i = 0; i < 8; ++i) {
2250  for (int j = 0; j < 8; ++j) {
2251  out[i][j] = in[j][2*i];
2252  }
2253  for (int j = 0; j < 8; ++j) {
2254  out[i][j+8] = in[j][2*i+1];
2255  }
2256  }
2257 
2258  kernel.packet[0] = pload<Packet16h>(out[0]);
2259  kernel.packet[1] = pload<Packet16h>(out[1]);
2260  kernel.packet[2] = pload<Packet16h>(out[2]);
2261  kernel.packet[3] = pload<Packet16h>(out[3]);
2262  kernel.packet[4] = pload<Packet16h>(out[4]);
2263  kernel.packet[5] = pload<Packet16h>(out[5]);
2264  kernel.packet[6] = pload<Packet16h>(out[6]);
2265  kernel.packet[7] = pload<Packet16h>(out[7]);
2266 }
2267 
2268 EIGEN_STRONG_INLINE void
2269 ptranspose(PacketBlock<Packet16h,4>& kernel) {
2270  EIGEN_ALIGN64 half in[4][16];
2271  pstore<half>(in[0], kernel.packet[0]);
2272  pstore<half>(in[1], kernel.packet[1]);
2273  pstore<half>(in[2], kernel.packet[2]);
2274  pstore<half>(in[3], kernel.packet[3]);
2275 
2276  EIGEN_ALIGN64 half out[4][16];
2277 
2278  for (int i = 0; i < 4; ++i) {
2279  for (int j = 0; j < 4; ++j) {
2280  out[i][j] = in[j][4*i];
2281  }
2282  for (int j = 0; j < 4; ++j) {
2283  out[i][j+4] = in[j][4*i+1];
2284  }
2285  for (int j = 0; j < 4; ++j) {
2286  out[i][j+8] = in[j][4*i+2];
2287  }
2288  for (int j = 0; j < 4; ++j) {
2289  out[i][j+12] = in[j][4*i+3];
2290  }
2291  }
2292 
2293  kernel.packet[0] = pload<Packet16h>(out[0]);
2294  kernel.packet[1] = pload<Packet16h>(out[1]);
2295  kernel.packet[2] = pload<Packet16h>(out[2]);
2296  kernel.packet[3] = pload<Packet16h>(out[3]);
2297 }
2298 
2299 template <> struct is_arithmetic<Packet16bf> { enum { value = true }; };
2300 
2301 template <>
2302 struct packet_traits<bfloat16> : default_packet_traits {
2303  typedef Packet16bf type;
2304  typedef Packet8bf half;
2305  enum {
2306  Vectorizable = 1,
2307  AlignedOnScalar = 1,
2308  size = 16,
2309  HasBlend = 0,
2310  HasInsert = 1,
2311  HasSin = EIGEN_FAST_MATH,
2312  HasCos = EIGEN_FAST_MATH,
2313  HasSqrt = 1,
2314  HasRsqrt = 1,
2315 #ifdef EIGEN_VECTORIZE_AVX512DQ
2316  HasLog = 1, // Currently fails test with bad accuracy.
2317  HasLog1p = 1,
2318  HasExpm1 = 1,
2319  HasNdtri = 1,
2320  HasBessel = 1,
2321 #endif
2322  HasExp = 1,
2323  HasTanh = EIGEN_FAST_MATH,
2324  HasErf = EIGEN_FAST_MATH,
2325  HasCmp = 1,
2326  HasDiv = 1
2327  };
2328 };
2329 
2330 template <>
2331 struct unpacket_traits<Packet16bf>
2332 {
2333  typedef bfloat16 type;
2334  enum {size=16, alignment=Aligned32, vectorizable=true, masked_load_available=false, masked_store_available=false};
2335  typedef Packet8bf half;
2336 };
2337 
2338 template <>
2339 EIGEN_STRONG_INLINE Packet16bf pset1<Packet16bf>(const bfloat16& from) {
2340  return _mm256_set1_epi16(from.value);
2341 }
2342 
2343 template <>
2344 EIGEN_STRONG_INLINE bfloat16 pfirst<Packet16bf>(const Packet16bf& from) {
2345  bfloat16 t;
2346  t.value = static_cast<unsigned short>(_mm256_extract_epi16(from, 0));
2347  return t;
2348 }
2349 
2350 template <>
2351 EIGEN_STRONG_INLINE Packet16bf pload<Packet16bf>(const bfloat16* from) {
2352  return _mm256_load_si256(reinterpret_cast<const __m256i*>(from));
2353 }
2354 
2355 template <>
2356 EIGEN_STRONG_INLINE Packet16bf ploadu<Packet16bf>(const bfloat16* from) {
2357  return _mm256_loadu_si256(reinterpret_cast<const __m256i*>(from));
2358 }
2359 
2360 template <>
2361 EIGEN_STRONG_INLINE void pstore<bfloat16>(bfloat16* to,
2362  const Packet16bf& from) {
2363  _mm256_store_si256(reinterpret_cast<__m256i*>(to), from);
2364 }
2365 
2366 template <>
2367 EIGEN_STRONG_INLINE void pstoreu<bfloat16>(bfloat16* to,
2368  const Packet16bf& from) {
2369  _mm256_storeu_si256(reinterpret_cast<__m256i*>(to), from);
2370 }
2371 
2372 template<> EIGEN_STRONG_INLINE Packet16bf
2373 ploaddup<Packet16bf>(const bfloat16* from) {
2374  unsigned short a = from[0].value;
2375  unsigned short b = from[1].value;
2376  unsigned short c = from[2].value;
2377  unsigned short d = from[3].value;
2378  unsigned short e = from[4].value;
2379  unsigned short f = from[5].value;
2380  unsigned short g = from[6].value;
2381  unsigned short h = from[7].value;
2382  return _mm256_set_epi16(h, h, g, g, f, f, e, e, d, d, c, c, b, b, a, a);
2383 }
2384 
2385 template<> EIGEN_STRONG_INLINE Packet16bf
2386 ploadquad(const bfloat16* from) {
2387  unsigned short a = from[0].value;
2388  unsigned short b = from[1].value;
2389  unsigned short c = from[2].value;
2390  unsigned short d = from[3].value;
2391  return _mm256_set_epi16(d, d, d, d, c, c, c, c, b, b, b, b, a, a, a, a);
2392 }
2393 
2394 EIGEN_STRONG_INLINE Packet16f Bf16ToF32(const Packet16bf& a) {
2395  return _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(a), 16));
2396 }
2397 
2398 // Convert float to bfloat16 according to round-to-nearest-even/denormals algorithm.
2399 EIGEN_STRONG_INLINE Packet16bf F32ToBf16(const Packet16f& a) {
2400  Packet16bf r;
2401 
2402 #if defined(EIGEN_VECTORIZE_AVX512BF16) && EIGEN_GNUC_STRICT_AT_LEAST(10,1,0)
2403  // Since GCC 10.1 supports avx512bf16 and C style explicit cast
2404  // (C++ static_cast is not supported yet), do conversion via intrinsic
2405  // and register path for performance.
2406  r = (__m256i)(_mm512_cvtneps_pbh(a));
2407 
2408 #else
2409  __m512i t;
2410  __m512i input = _mm512_castps_si512(a);
2411  __m512i nan = _mm512_set1_epi32(0x7fc0);
2412 
2413  // uint32_t lsb = (input >> 16) & 1;
2414  t = _mm512_and_si512(_mm512_srli_epi32(input, 16), _mm512_set1_epi32(1));
2415  // uint32_t rounding_bias = 0x7fff + lsb;
2416  t = _mm512_add_epi32(t, _mm512_set1_epi32(0x7fff));
2417  // input += rounding_bias;
2418  t = _mm512_add_epi32(t, input);
2419  // input = input >> 16;
2420  t = _mm512_srli_epi32(t, 16);
2421 
2422  // Check NaN before converting back to bf16
2423  __mmask16 mask = _mm512_cmp_ps_mask(a, a, _CMP_ORD_Q);
2424 
2425  t = _mm512_mask_blend_epi32(mask, nan, t);
2426  // output.value = static_cast<uint16_t>(input);
2427  r = _mm512_cvtepi32_epi16(t);
2428 #endif // EIGEN_VECTORIZE_AVX512BF16
2429 
2430  return r;
2431 }
2432 
2433 template <>
2434 EIGEN_STRONG_INLINE Packet16bf ptrue(const Packet16bf& a) {
2435  return Packet16bf(ptrue<Packet8i>(Packet8i(a)));
2436 }
2437 
2438 template <>
2439 EIGEN_STRONG_INLINE Packet16bf por(const Packet16bf& a, const Packet16bf& b) {
2440  return Packet16bf(por<Packet8i>(Packet8i(a), Packet8i(b)));
2441 }
2442 
2443 template <>
2444 EIGEN_STRONG_INLINE Packet16bf pxor(const Packet16bf& a, const Packet16bf& b) {
2445  return Packet16bf(pxor<Packet8i>(Packet8i(a), Packet8i(b)));
2446 }
2447 
2448 template <>
2449 EIGEN_STRONG_INLINE Packet16bf pand(const Packet16bf& a, const Packet16bf& b) {
2450  return Packet16bf(pand<Packet8i>(Packet8i(a), Packet8i(b)));
2451 }
2452 
2453 template <>
2454 EIGEN_STRONG_INLINE Packet16bf pandnot(const Packet16bf& a,
2455  const Packet16bf& b) {
2456  return Packet16bf(pandnot<Packet8i>(Packet8i(a), Packet8i(b)));
2457 }
2458 
2459 template <>
2460 EIGEN_STRONG_INLINE Packet16bf pselect(const Packet16bf& mask,
2461  const Packet16bf& a,
2462  const Packet16bf& b) {
2463  // Input mask is expected to be all 0/1, handle it with 8-bit
2464  // intrinsic for performance.
2465  return _mm256_blendv_epi8(b, a, mask);
2466 }
2467 
2468 template<> EIGEN_STRONG_INLINE Packet16bf pround<Packet16bf>(const Packet16bf& a)
2469 {
2470  return F32ToBf16(pround<Packet16f>(Bf16ToF32(a)));
2471 }
2472 
2473 template<> EIGEN_STRONG_INLINE Packet16bf print<Packet16bf>(const Packet16bf& a) {
2474  return F32ToBf16(print<Packet16f>(Bf16ToF32(a)));
2475 }
2476 
2477 template<> EIGEN_STRONG_INLINE Packet16bf pceil<Packet16bf>(const Packet16bf& a) {
2478  return F32ToBf16(pceil<Packet16f>(Bf16ToF32(a)));
2479 }
2480 
2481 template<> EIGEN_STRONG_INLINE Packet16bf pfloor<Packet16bf>(const Packet16bf& a) {
2482  return F32ToBf16(pfloor<Packet16f>(Bf16ToF32(a)));
2483 }
2484 
2485 template <>
2486 EIGEN_STRONG_INLINE Packet16bf pcmp_eq(const Packet16bf& a,
2487  const Packet16bf& b) {
2488  return Pack32To16(pcmp_eq(Bf16ToF32(a), Bf16ToF32(b)));
2489 }
2490 
2491 template <>
2492 EIGEN_STRONG_INLINE Packet16bf pcmp_le(const Packet16bf& a,
2493  const Packet16bf& b) {
2494  return Pack32To16(pcmp_le(Bf16ToF32(a), Bf16ToF32(b)));
2495 }
2496 
2497 template <>
2498 EIGEN_STRONG_INLINE Packet16bf pcmp_lt(const Packet16bf& a,
2499  const Packet16bf& b) {
2500  return Pack32To16(pcmp_lt(Bf16ToF32(a), Bf16ToF32(b)));
2501 }
2502 
2503 template <>
2504 EIGEN_STRONG_INLINE Packet16bf pcmp_lt_or_nan(const Packet16bf& a,
2505  const Packet16bf& b) {
2506  return Pack32To16(pcmp_lt_or_nan(Bf16ToF32(a), Bf16ToF32(b)));
2507 }
2508 
2509 template <>
2510 EIGEN_STRONG_INLINE Packet16bf pnegate(const Packet16bf& a) {
2511  Packet16bf sign_mask = _mm256_set1_epi16(static_cast<unsigned short>(0x8000));
2512  return _mm256_xor_si256(a, sign_mask);
2513 }
2514 
2515 template <>
2516 EIGEN_STRONG_INLINE Packet16bf pconj(const Packet16bf& a) {
2517  return a;
2518 }
2519 
2520 template <>
2521 EIGEN_STRONG_INLINE Packet16bf pabs(const Packet16bf& a) {
2522  const __m256i sign_mask = _mm256_set1_epi16(static_cast<numext::uint16_t>(0x8000));
2523  return _mm256_andnot_si256(sign_mask, a);
2524 }
2525 
2526 template <>
2527 EIGEN_STRONG_INLINE Packet16bf padd<Packet16bf>(const Packet16bf& a,
2528  const Packet16bf& b) {
2529  return F32ToBf16(padd<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2530 }
2531 
2532 template <>
2533 EIGEN_STRONG_INLINE Packet16bf psub<Packet16bf>(const Packet16bf& a,
2534  const Packet16bf& b) {
2535  return F32ToBf16(psub<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2536 }
2537 
2538 template <>
2539 EIGEN_STRONG_INLINE Packet16bf pmul<Packet16bf>(const Packet16bf& a,
2540  const Packet16bf& b) {
2541  return F32ToBf16(pmul<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2542 }
2543 
2544 template <>
2545 EIGEN_STRONG_INLINE Packet16bf pdiv<Packet16bf>(const Packet16bf& a,
2546  const Packet16bf& b) {
2547  return F32ToBf16(pdiv<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2548 }
2549 
2550 template <>
2551 EIGEN_STRONG_INLINE Packet16bf pmin<Packet16bf>(const Packet16bf& a,
2552  const Packet16bf& b) {
2553  return F32ToBf16(pmin<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2554 }
2555 
2556 template <>
2557 EIGEN_STRONG_INLINE Packet16bf pmax<Packet16bf>(const Packet16bf& a,
2558  const Packet16bf& b) {
2559  return F32ToBf16(pmax<Packet16f>(Bf16ToF32(a), Bf16ToF32(b)));
2560 }
2561 
2562 template <>
2563 EIGEN_STRONG_INLINE Packet16bf plset<Packet16bf>(const bfloat16& a) {
2564  return F32ToBf16(plset<Packet16f>(static_cast<float>(a)));
2565 }
2566 
2567 template <>
2568 EIGEN_STRONG_INLINE Packet8bf predux_half_dowto4<Packet16bf>(const Packet16bf& a) {
2569  Packet8bf lane0 = _mm256_extractf128_si256(a, 0);
2570  Packet8bf lane1 = _mm256_extractf128_si256(a, 1);
2571  return padd<Packet8bf>(lane0, lane1);
2572 }
2573 
2574 template <>
2575 EIGEN_STRONG_INLINE bfloat16 predux<Packet16bf>(const Packet16bf& p) {
2576  return static_cast<bfloat16>(predux<Packet16f>(Bf16ToF32(p)));
2577 }
2578 
2579 template <>
2580 EIGEN_STRONG_INLINE bfloat16 predux_mul<Packet16bf>(const Packet16bf& from) {
2581  return static_cast<bfloat16>(predux_mul<Packet16f>(Bf16ToF32(from)));
2582 }
2583 
2584 template <>
2585 EIGEN_STRONG_INLINE bfloat16 predux_min<Packet16bf>(const Packet16bf& from) {
2586  return static_cast<bfloat16>(predux_min<Packet16f>(Bf16ToF32(from)));
2587 }
2588 
2589 template <>
2590 EIGEN_STRONG_INLINE bfloat16 predux_max<Packet16bf>(const Packet16bf& from) {
2591  return static_cast<bfloat16>(predux_max<Packet16f>(Bf16ToF32(from)));
2592 }
2593 
2594 template <>
2595 EIGEN_STRONG_INLINE Packet16bf preverse(const Packet16bf& a) {
2596  __m256i m = _mm256_setr_epi8(14,15,12,13,10,11,8,9,6,7,4,5,2,3,0,1,
2597  14,15,12,13,10,11,8,9,6,7,4,5,2,3,0,1);
2598 
2599  Packet16bf res;
2600  // Swap hi and lo first because shuffle is in 128-bit lanes.
2601  res = _mm256_permute2x128_si256(a, a, 1);
2602  // Shuffle 8-bit values in src within 2*128-bit lanes.
2603  return _mm256_shuffle_epi8(res, m);
2604 }
2605 
2606 template <>
2607 EIGEN_STRONG_INLINE Packet16bf pgather<bfloat16, Packet16bf>(const bfloat16* from,
2608  Index stride) {
2609  return _mm256_set_epi16(
2610  from[15*stride].value, from[14*stride].value, from[13*stride].value, from[12*stride].value,
2611  from[11*stride].value, from[10*stride].value, from[9*stride].value, from[8*stride].value,
2612  from[7*stride].value, from[6*stride].value, from[5*stride].value, from[4*stride].value,
2613  from[3*stride].value, from[2*stride].value, from[1*stride].value, from[0*stride].value);
2614 }
2615 
2616 template <>
2617 EIGEN_STRONG_INLINE void pscatter<bfloat16, Packet16bf>(bfloat16* to,
2618  const Packet16bf& from,
2619  Index stride) {
2620  EIGEN_ALIGN64 bfloat16 aux[16];
2621  pstore(aux, from);
2622  to[stride*0] = aux[0];
2623  to[stride*1] = aux[1];
2624  to[stride*2] = aux[2];
2625  to[stride*3] = aux[3];
2626  to[stride*4] = aux[4];
2627  to[stride*5] = aux[5];
2628  to[stride*6] = aux[6];
2629  to[stride*7] = aux[7];
2630  to[stride*8] = aux[8];
2631  to[stride*9] = aux[9];
2632  to[stride*10] = aux[10];
2633  to[stride*11] = aux[11];
2634  to[stride*12] = aux[12];
2635  to[stride*13] = aux[13];
2636  to[stride*14] = aux[14];
2637  to[stride*15] = aux[15];
2638 }
2639 
2640 EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16bf,16>& kernel) {
2641  __m256i a = kernel.packet[0];
2642  __m256i b = kernel.packet[1];
2643  __m256i c = kernel.packet[2];
2644  __m256i d = kernel.packet[3];
2645  __m256i e = kernel.packet[4];
2646  __m256i f = kernel.packet[5];
2647  __m256i g = kernel.packet[6];
2648  __m256i h = kernel.packet[7];
2649  __m256i i = kernel.packet[8];
2650  __m256i j = kernel.packet[9];
2651  __m256i k = kernel.packet[10];
2652  __m256i l = kernel.packet[11];
2653  __m256i m = kernel.packet[12];
2654  __m256i n = kernel.packet[13];
2655  __m256i o = kernel.packet[14];
2656  __m256i p = kernel.packet[15];
2657 
2658  __m256i ab_07 = _mm256_unpacklo_epi16(a, b);
2659  __m256i cd_07 = _mm256_unpacklo_epi16(c, d);
2660  __m256i ef_07 = _mm256_unpacklo_epi16(e, f);
2661  __m256i gh_07 = _mm256_unpacklo_epi16(g, h);
2662  __m256i ij_07 = _mm256_unpacklo_epi16(i, j);
2663  __m256i kl_07 = _mm256_unpacklo_epi16(k, l);
2664  __m256i mn_07 = _mm256_unpacklo_epi16(m, n);
2665  __m256i op_07 = _mm256_unpacklo_epi16(o, p);
2666 
2667  __m256i ab_8f = _mm256_unpackhi_epi16(a, b);
2668  __m256i cd_8f = _mm256_unpackhi_epi16(c, d);
2669  __m256i ef_8f = _mm256_unpackhi_epi16(e, f);
2670  __m256i gh_8f = _mm256_unpackhi_epi16(g, h);
2671  __m256i ij_8f = _mm256_unpackhi_epi16(i, j);
2672  __m256i kl_8f = _mm256_unpackhi_epi16(k, l);
2673  __m256i mn_8f = _mm256_unpackhi_epi16(m, n);
2674  __m256i op_8f = _mm256_unpackhi_epi16(o, p);
2675 
2676  __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07);
2677  __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07);
2678  __m256i efgh_03 = _mm256_unpacklo_epi32(ef_07, gh_07);
2679  __m256i efgh_47 = _mm256_unpackhi_epi32(ef_07, gh_07);
2680  __m256i ijkl_03 = _mm256_unpacklo_epi32(ij_07, kl_07);
2681  __m256i ijkl_47 = _mm256_unpackhi_epi32(ij_07, kl_07);
2682  __m256i mnop_03 = _mm256_unpacklo_epi32(mn_07, op_07);
2683  __m256i mnop_47 = _mm256_unpackhi_epi32(mn_07, op_07);
2684 
2685  __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f);
2686  __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f);
2687  __m256i efgh_8b = _mm256_unpacklo_epi32(ef_8f, gh_8f);
2688  __m256i efgh_cf = _mm256_unpackhi_epi32(ef_8f, gh_8f);
2689  __m256i ijkl_8b = _mm256_unpacklo_epi32(ij_8f, kl_8f);
2690  __m256i ijkl_cf = _mm256_unpackhi_epi32(ij_8f, kl_8f);
2691  __m256i mnop_8b = _mm256_unpacklo_epi32(mn_8f, op_8f);
2692  __m256i mnop_cf = _mm256_unpackhi_epi32(mn_8f, op_8f);
2693 
2694  __m256i abcdefgh_01 = _mm256_unpacklo_epi64(abcd_03, efgh_03);
2695  __m256i abcdefgh_23 = _mm256_unpackhi_epi64(abcd_03, efgh_03);
2696  __m256i ijklmnop_01 = _mm256_unpacklo_epi64(ijkl_03, mnop_03);
2697  __m256i ijklmnop_23 = _mm256_unpackhi_epi64(ijkl_03, mnop_03);
2698  __m256i abcdefgh_45 = _mm256_unpacklo_epi64(abcd_47, efgh_47);
2699  __m256i abcdefgh_67 = _mm256_unpackhi_epi64(abcd_47, efgh_47);
2700  __m256i ijklmnop_45 = _mm256_unpacklo_epi64(ijkl_47, mnop_47);
2701  __m256i ijklmnop_67 = _mm256_unpackhi_epi64(ijkl_47, mnop_47);
2702  __m256i abcdefgh_89 = _mm256_unpacklo_epi64(abcd_8b, efgh_8b);
2703  __m256i abcdefgh_ab = _mm256_unpackhi_epi64(abcd_8b, efgh_8b);
2704  __m256i ijklmnop_89 = _mm256_unpacklo_epi64(ijkl_8b, mnop_8b);
2705  __m256i ijklmnop_ab = _mm256_unpackhi_epi64(ijkl_8b, mnop_8b);
2706  __m256i abcdefgh_cd = _mm256_unpacklo_epi64(abcd_cf, efgh_cf);
2707  __m256i abcdefgh_ef = _mm256_unpackhi_epi64(abcd_cf, efgh_cf);
2708  __m256i ijklmnop_cd = _mm256_unpacklo_epi64(ijkl_cf, mnop_cf);
2709  __m256i ijklmnop_ef = _mm256_unpackhi_epi64(ijkl_cf, mnop_cf);
2710 
2711  // NOTE: no unpacklo/hi instr in this case, so using permute instr.
2712  kernel.packet[0] = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x20);
2713  kernel.packet[1] = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x20);
2714  kernel.packet[2] = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x20);
2715  kernel.packet[3] = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x20);
2716  kernel.packet[4] = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x20);
2717  kernel.packet[5] = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x20);
2718  kernel.packet[6] = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x20);
2719  kernel.packet[7] = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x20);
2720  kernel.packet[8] = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x31);
2721  kernel.packet[9] = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x31);
2722  kernel.packet[10] = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x31);
2723  kernel.packet[11] = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x31);
2724  kernel.packet[12] = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x31);
2725  kernel.packet[13] = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x31);
2726  kernel.packet[14] = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x31);
2727  kernel.packet[15] = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x31);
2728 }
2729 
2730 EIGEN_STRONG_INLINE void ptranspose(PacketBlock<Packet16bf,4>& kernel) {
2731  __m256i a = kernel.packet[0];
2732  __m256i b = kernel.packet[1];
2733  __m256i c = kernel.packet[2];
2734  __m256i d = kernel.packet[3];
2735 
2736  __m256i ab_07 = _mm256_unpacklo_epi16(a, b);
2737  __m256i cd_07 = _mm256_unpacklo_epi16(c, d);
2738  __m256i ab_8f = _mm256_unpackhi_epi16(a, b);
2739  __m256i cd_8f = _mm256_unpackhi_epi16(c, d);
2740 
2741  __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07);
2742  __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07);
2743  __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f);
2744  __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f);
2745 
2746  // NOTE: no unpacklo/hi instr in this case, so using permute instr.
2747  kernel.packet[0] = _mm256_permute2x128_si256(abcd_03, abcd_47, 0x20);
2748  kernel.packet[1] = _mm256_permute2x128_si256(abcd_8b, abcd_cf, 0x20);
2749  kernel.packet[2] = _mm256_permute2x128_si256(abcd_03, abcd_47, 0x31);
2750  kernel.packet[3] = _mm256_permute2x128_si256(abcd_8b, abcd_cf, 0x31);
2751 }
2752 
2753 } // end namespace internal
2754 
2755 } // end namespace Eigen
2756 
2757 #endif // EIGEN_PACKET_MATH_AVX512_H
EIGEN_EXTRACT_8i_FROM_16i
#define EIGEN_EXTRACT_8i_FROM_16i(INPUT, OUTPUT)
Definition: AVX512/PacketMath.h:1222
PACK_OUTPUT_2
#define PACK_OUTPUT_2(OUTPUT, INPUT, INDEX, STRIDE)
Definition: AVX512/PacketMath.h:1532
SHUFFLE_EPI32
#define SHUFFLE_EPI32(A, B, M)
Definition: AVX512/PacketMath.h:1706
PACK_OUTPUT_D
#define PACK_OUTPUT_D(OUTPUT, INPUT, INDEX, STRIDE)
Definition: AVX512/PacketMath.h:1612
PACK_OUTPUT_I32_2
#define PACK_OUTPUT_I32_2(OUTPUT, INPUT, INDEX, STRIDE)
Definition: AVX512/PacketMath.h:1702
EIGEN_EXTRACT_8f_FROM_16f
#define EIGEN_EXTRACT_8f_FROM_16f(INPUT, OUTPUT)
Definition: AVX512/PacketMath.h:1214
PACK_OUTPUT
#define PACK_OUTPUT(OUTPUT, INPUT, INDEX, STRIDE)
Definition: AVX512/PacketMath.h:1418
PACK_OUTPUT_I32
#define PACK_OUTPUT_I32(OUTPUT, INPUT, INDEX, STRIDE)
Definition: AVX512/PacketMath.h:1699
m
Matrix3f m
Definition: AngleAxis_mimic_euler.cpp:1
a
ArrayXXi a
Definition: Array_initializer_list_23_cxx11.cpp:1
b
Array< int, 3, 1 > b
Definition: Array_variadic_ctor_cxx11.cpp:2
n
int n
Definition: BiCGSTAB_simple.cpp:1
x
x
Definition: BiCGSTAB_simple.cpp:7
i
int i
Definition: BiCGSTAB_step_by_step.cpp:9
EIGEN_ALIGN64
#define EIGEN_ALIGN64
Definition: ConfigureVectorization.h:135
e
Array< double, 1, 3 > e(1./3., 0.5, 2.)
c
Array33i c
Definition: Cwise_product.cpp:2
EIGEN_DEBUG_ALIGNED_STORE
#define EIGEN_DEBUG_ALIGNED_STORE
Definition: GenericPacketMath.h:37
EIGEN_DEBUG_ALIGNED_LOAD
#define EIGEN_DEBUG_ALIGNED_LOAD
Definition: GenericPacketMath.h:29
EIGEN_DEBUG_UNALIGNED_STORE
#define EIGEN_DEBUG_UNALIGNED_STORE
Definition: GenericPacketMath.h:41
EIGEN_DEBUG_UNALIGNED_LOAD
#define EIGEN_DEBUG_UNALIGNED_LOAD
Definition: GenericPacketMath.h:33
EIGEN_DEVICE_FUNC
#define EIGEN_DEVICE_FUNC
Definition: Macros.h:883
EIGEN_FAST_MATH
#define EIGEN_FAST_MATH
Definition: Macros.h:50
res
cout<< "Here is the matrix m:"<< endl<< m<< endl;Matrix< ptrdiff_t, 3, 1 > res
Definition: PartialRedux_count.cpp:3
size
const int size
Definition: Tutorial_AdvancedInitialization_ThreeWays.cpp:1
p
float * p
Definition: Tutorial_Map_using.cpp:9
Eigen::Aligned64
@ Aligned64
Definition: Constants.h:239
Eigen::Aligned32
@ Aligned32
Definition: Constants.h:238
Eigen::half_impl::raw_uint16_to_half
EIGEN_CONSTEXPR __half_raw raw_uint16_to_half(numext::uint16_t x)
Definition: Half.h:551
Eigen::internal::pmin
Packet pmin(const Packet &a, const Packet &b)
Definition: GenericPacketMath.h:620
Eigen::internal::pnmsub
Packet pnmsub(const Packet &a, const Packet &b, const Packet &c)
Definition: GenericPacketMath.h:1172
Eigen::internal::padd
Packet padd(const Packet &a, const Packet &b)
Definition: GenericPacketMath.h:308
Eigen::internal::pround< Packet16bf >
Packet16bf pround< Packet16bf >(const Packet16bf &a)
Definition: AVX512/PacketMath.h:2468
Eigen::internal::pzero
Packet8f pzero(const Packet8f &)
Definition: AVX/PacketMath.h:677
Eigen::internal::pmax< Packet16h >
Packet16h pmax< Packet16h >(const Packet16h &a, const Packet16h &b)
Definition: AVX512/PacketMath.h:1958
Eigen::internal::pstore
void pstore(Scalar *to, const Packet &from)
Definition: GenericPacketMath.h:836
Eigen::internal::pstore< float >
void pstore< float >(float *to, const Packet4f &from)
Definition: AltiVec/PacketMath.h:611
Eigen::internal::pscatter< float, Packet16f >
void pscatter< float, Packet16f >(float *to, const Packet16f &from, Index stride, uint16_t umask)
Definition: AVX512/PacketMath.h:1028
Eigen::internal::predux_mul< Packet16h >
half predux_mul< Packet16h >(const Packet16h &from)
Definition: AVX512/PacketMath.h:2083
Eigen::internal::pload1< Packet8d >
Packet8d pload1< Packet8d >(const double *from)
Definition: AVX512/PacketMath.h:260
Eigen::internal::pstoreu< half >
void pstoreu< half >(Eigen::half *to, const Packet16h &from)
Definition: AVX512/PacketMath.h:1905
Eigen::internal::pfrexp< Packet8d >
Packet8d pfrexp< Packet8d >(const Packet8d &a, Packet8d &exponent)
Definition: AVX512/PacketMath.h:1168
Eigen::internal::pand< Packet16i >
Packet16i pand< Packet16i >(const Packet16i &a, const Packet16i &b)
Definition: AVX512/PacketMath.h:686
Eigen::internal::predux_half_dowto4< Packet8d >
Packet4d predux_half_dowto4< Packet8d >(const Packet8d &a)
Definition: AVX512/PacketMath.h:1315
Eigen::internal::predux_max< Packet16h >
Eigen::half predux_max< Packet16h >(const Packet16h &a)
Definition: AVX512/PacketMath.h:2071
Eigen::internal::psub< Packet16i >
Packet16i psub< Packet16i >(const Packet16i &a, const Packet16i &b)
Definition: AVX512/PacketMath.h:331
Eigen::internal::pdiv< Packet8d >
Packet8d pdiv< Packet8d >(const Packet8d &a, const Packet8d &b)
Definition: AVX512/PacketMath.h:394
Eigen::internal::pmin< PropagateNaN, Packet8d >
Packet8d pmin< PropagateNaN, Packet8d >(const Packet8d &a, const Packet8d &b)
Definition: AVX512/PacketMath.h:536
Eigen::internal::predux
unpacket_traits< Packet >::type predux(const Packet &a)
Definition: GenericPacketMath.h:1077
Eigen::internal::predux< Packet16bf >
bfloat16 predux< Packet16bf >(const Packet16bf &p)
Definition: AVX512/PacketMath.h:2575
Eigen::internal::pscatter< half, Packet16h >
void pscatter< half, Packet16h >(half *to, const Packet16h &from, Index stride)
Definition: AVX512/PacketMath.h:2105
Eigen::internal::ptrue
Packet8h ptrue(const Packet8h &a)
Definition: AVX/PacketMath.h:1828
Eigen::internal::pcmp_lt_or_nan
Packet4f pcmp_lt_or_nan(const Packet4f &a, const Packet4f &b)
Definition: AltiVec/PacketMath.h:1123
Eigen::internal::por< Packet8d >
Packet8d por< Packet8d >(const Packet8d &a, const Packet8d &b)
Definition: AVX512/PacketMath.h:732
Eigen::internal::ploadquad< Packet8d >
Packet8d ploadquad< Packet8d >(const double *from)
Definition: AVX512/PacketMath.h:911
Eigen::internal::pstore< int >
void pstore< int >(int *to, const Packet4i &from)
Definition: AltiVec/PacketMath.h:616
Eigen::internal::por< Packet16f >
Packet16f por< Packet16f >(const Packet16f &a, const Packet16f &b)
Definition: AVX512/PacketMath.h:723
Eigen::internal::ptrue< Packet16i >
Packet16i ptrue< Packet16i >(const Packet16i &)
Definition: AVX512/PacketMath.h:671
Eigen::internal::ploadu< Packet8d >
Packet8d ploadu< Packet8d >(const double *from)
Definition: AVX512/PacketMath.h:832
Eigen::internal::F32ToBf16
Packet8bf F32ToBf16(Packet4f p4f)
Definition: AltiVec/PacketMath.h:1675
Eigen::internal::pmax< Packet8d >
Packet8d pmax< Packet8d >(const Packet8d &a, const Packet8d &b)
Definition: AVX512/PacketMath.h:503
Eigen::internal::pmin< Packet16i >
Packet16i pmin< Packet16i >(const Packet16i &a, const Packet16i &b)
Definition: AVX512/PacketMath.h:491
Eigen::internal::pmul< Packet16i >
Packet16i pmul< Packet16i >(const Packet16i &a, const Packet16i &b)
Definition: AVX512/PacketMath.h:382
Eigen::internal::pfloor< Packet16h >
Packet16h pfloor< Packet16h >(const Packet16h &a)
Definition: AVX512/PacketMath.h:1999
Eigen::internal::plset< Packet16h >
Packet16h plset< Packet16h >(const half &a)
Definition: AVX512/PacketMath.h:1964
Eigen::internal::predux_min< Packet16bf >
bfloat16 predux_min< Packet16bf >(const Packet16bf &from)
Definition: AVX512/PacketMath.h:2585
Eigen::internal::predux_mul< Packet16bf >
bfloat16 predux_mul< Packet16bf >(const Packet16bf &from)
Definition: AVX512/PacketMath.h:2580
Eigen::internal::pdiv< Packet8i >
Packet8i pdiv< Packet8i >(const Packet8i &a, const Packet8i &b)
Definition: AVX/PacketMath.h:794
Eigen::internal::print< Packet16f >
Packet16f print< Packet16f >(const Packet16f &a)
Definition: AVX512/PacketMath.h:661
Eigen::internal::extract256
Packet8f extract256(Packet16f x)
Definition: AVX512/PacketMath.h:556
Eigen::internal::ptrue< Packet8d >
Packet8d ptrue< Packet8d >(const Packet8d &a)
Definition: AVX512/PacketMath.h:681
Eigen::internal::padd< Packet8h >
Packet8h padd< Packet8h >(const Packet8h &a, const Packet8h &b)
Definition: AVX/PacketMath.h:1934
Eigen::internal::pand< Packet16f >
Packet16f pand< Packet16f >(const Packet16f &a, const Packet16f &b)
Definition: AVX512/PacketMath.h:692
Eigen::internal::pmin< PropagateNaN, Packet16f >
Packet16f pmin< PropagateNaN, Packet16f >(const Packet16f &a, const Packet16f &b)
Definition: AVX512/PacketMath.h:532
Eigen::internal::pmin< PropagateNumbers, Packet8d >
Packet8d pmin< PropagateNumbers, Packet8d >(const Packet8d &a, const Packet8d &b)
Definition: AVX512/PacketMath.h:520
Eigen::internal::pxor< Packet8d >
Packet8d pxor< Packet8d >(const Packet8d &a, const Packet8d &b)
Definition: AVX512/PacketMath.h:756
Eigen::internal::plset< Packet16f >
Packet16f plset< Packet16f >(const float &a)
Definition: AVX512/PacketMath.h:271
Eigen::internal::predux_half_dowto4< Packet16h >
Packet8h predux_half_dowto4< Packet16h >(const Packet16h &a)
Definition: AVX512/PacketMath.h:2065
Eigen::internal::ploadu< Packet16h >
Packet16h ploadu< Packet16h >(const Eigen::half *from)
Definition: AVX512/PacketMath.h:1895
Eigen::internal::pfirst< Packet8d >
double pfirst< Packet8d >(const Packet8d &a)
Definition: AVX512/PacketMath.h:1106
Eigen::internal::predux_max< Packet8d >
double predux_max< Packet8d >(const Packet8d &a)
Definition: AVX512/PacketMath.h:1397
Eigen::internal::predux_mul< Packet16f >
float predux_mul< Packet16f >(const Packet16f &a)
Definition: AVX512/PacketMath.h:1338
Eigen::internal::pandnot
Packet8h pandnot(const Packet8h &a, const Packet8h &b)
Definition: AVX/PacketMath.h:1886
Eigen::internal::ploaddup< Packet16i >
Packet16i ploaddup< Packet16i >(const int *from)
Definition: AVX512/PacketMath.h:892
Eigen::internal::pmax< PropagateNumbers, Packet16f >
Packet16f pmax< PropagateNumbers, Packet16f >(const Packet16f &a, const Packet16f &b)
Definition: AVX512/PacketMath.h:524
Eigen::internal::pxor< Packet8i >
Packet8i pxor< Packet8i >(const Packet8i &a, const Packet8i &b)
Definition: AVX/PacketMath.h:1098
Eigen::internal::pabs
Packet4f pabs(const Packet4f &a)
Definition: AltiVec/PacketMath.h:1579
Eigen::internal::pdiv< Packet16i >
Packet16i pdiv< Packet16i >(const Packet16i &a, const Packet16i &b)
Definition: AVX512/PacketMath.h:400
Eigen::internal::pmax
Packet pmax(const Packet &a, const Packet &b)
Definition: GenericPacketMath.h:632
Eigen::internal::Bf16ToF32
Packet8f Bf16ToF32(const Packet8bf &a)
Definition: AVX/PacketMath.h:2098
Eigen::internal::predux< Packet8d >
double predux< Packet8d >(const Packet8d &a)
Definition: AVX512/PacketMath.h:1272
Eigen::internal::predux_max< Packet16bf >
bfloat16 predux_max< Packet16bf >(const Packet16bf &from)
Definition: AVX512/PacketMath.h:2590
Eigen::internal::pnegate
Packet2cf pnegate(const Packet2cf &a)
Definition: AltiVec/Complex.h:214
Eigen::internal::predux_max< Packet16f >
float predux_max< Packet16f >(const Packet16f &a)
Definition: AVX512/PacketMath.h:1386
Eigen::internal::pscatter< double, Packet8d >
void pscatter< double, Packet8d >(double *to, const Packet8d &from, Index stride, uint8_t umask)
Definition: AVX512/PacketMath.h:1040
Eigen::internal::pmin< Packet16h >
Packet16h pmin< Packet16h >(const Packet16h &a, const Packet16h &b)
Definition: AVX512/PacketMath.h:1952
Eigen::internal::ploadu< Packet16i >
Packet16i ploadu< Packet16i >(const int *from)
Definition: AVX512/PacketMath.h:836
Eigen::internal::print< Packet16h >
Packet16h print< Packet16h >(const Packet16h &a)
Definition: AVX512/PacketMath.h:1991
Eigen::internal::plogical_shift_right
Packet4i plogical_shift_right(const Packet4i &a)
Definition: AltiVec/PacketMath.h:1595
Eigen::internal::pandnot< Packet16f >
Packet16f pandnot< Packet16f >(const Packet16f &a, const Packet16f &b)
Definition: AVX512/PacketMath.h:770
Eigen::internal::predux< Packet8i >
int predux< Packet8i >(const Packet8i &a)
Definition: AVX/PacketMath.h:1551
Eigen::internal::predux_mul< Packet8d >
double predux_mul< Packet8d >(const Packet8d &a)
Definition: AVX512/PacketMath.h:1358
Eigen::internal::pminmax_propagate_nan
Packet pminmax_propagate_nan(const Packet &a, const Packet &b, Op op)
Definition: SSE/PacketMath.h:688
Eigen::internal::pldexp< Packet8d >
Packet8d pldexp< Packet8d >(const Packet8d &a, const Packet8d &exponent)
Definition: AVX512/PacketMath.h:1176
Eigen::internal::pand< Packet8d >
Packet8d pand< Packet8d >(const Packet8d &a, const Packet8d &b)
Definition: AVX512/PacketMath.h:701
Eigen::internal::pgather< bfloat16, Packet16bf >
Packet16bf pgather< bfloat16, Packet16bf >(const bfloat16 *from, Index stride)
Definition: AVX512/PacketMath.h:2607
Eigen::internal::plset< Packet8d >
Packet8d plset< Packet8d >(const double &a)
Definition: AVX512/PacketMath.h:278
Eigen::internal::predux< Packet8f >
float predux< Packet8f >(const Packet8f &a)
Definition: AVX/PacketMath.h:1543
Eigen::internal::Pack32To16
__m256i Pack32To16(Packet16f rf)
Definition: AVX512/PacketMath.h:576
Eigen::internal::pmax< Packet16bf >
Packet16bf pmax< Packet16bf >(const Packet16bf &a, const Packet16bf &b)
Definition: AVX512/PacketMath.h:2557
Eigen::internal::pset1< Packet16h >
Packet16h pset1< Packet16h >(const Eigen::half &from)
Definition: AVX512/PacketMath.h:1883
Eigen::internal::pmadd
Packet4f pmadd(const Packet4f &a, const Packet4f &b, const Packet4f &c)
Definition: AltiVec/PacketMath.h:1071
Eigen::internal::print< Packet16bf >
Packet16bf print< Packet16bf >(const Packet16bf &a)
Definition: AVX512/PacketMath.h:2473
Eigen::internal::por< Packet16i >
Packet16i por< Packet16i >(const Packet16i &a, const Packet16i &b)
Definition: AVX512/PacketMath.h:718
Eigen::internal::pcmp_eq
Packet2cf pcmp_eq(const Packet2cf &a, const Packet2cf &b)
Definition: AltiVec/Complex.h:281
Eigen::internal::Packet16bf
eigen_packet_wrapper< __m256i, 2 > Packet16bf
Definition: AVX512/PacketMath.h:39
Eigen::internal::padd< Packet16bf >
Packet16bf padd< Packet16bf >(const Packet16bf &a, const Packet16bf &b)
Definition: AVX512/PacketMath.h:2527
Eigen::internal::ptrue< Packet16f >
Packet16f ptrue< Packet16f >(const Packet16f &a)
Definition: AVX512/PacketMath.h:676
Eigen::internal::pfirst
bfloat16 pfirst(const Packet8bf &a)
Definition: AltiVec/PacketMath.h:2011
Eigen::internal::pload< Packet16h >
Packet16h pload< Packet16h >(const Eigen::half *from)
Definition: AVX512/PacketMath.h:1891
Eigen::internal::pscatter< bfloat16, Packet16bf >
void pscatter< bfloat16, Packet16bf >(bfloat16 *to, const Packet16bf &from, Index stride)
Definition: AVX512/PacketMath.h:2617
Eigen::internal::pstoreu< double >
void pstoreu< double >(double *to, const Packet4d &from)
Definition: AVX/PacketMath.h:1293
Eigen::internal::pselect
Packet4f pselect(const Packet4f &mask, const Packet4f &a, const Packet4f &b)
Definition: AltiVec/PacketMath.h:1181
Eigen::internal::pmul
Packet pmul(const Packet &a, const Packet &b)
Definition: GenericPacketMath.h:339
Eigen::internal::pscatter
void pscatter(Scalar *to, const Packet &from, Index stride, typename unpacket_traits< Packet >::mask_t umask)
Eigen::internal::pxor< Packet16i >
Packet16i pxor< Packet16i >(const Packet16i &a, const Packet16i &b)
Definition: AVX512/PacketMath.h:742
Eigen::internal::pfrexp< Packet16f >
Packet16f pfrexp< Packet16f >(const Packet16f &a, Packet16f &exponent)
Definition: AVX512/PacketMath.h:1151
Eigen::internal::pgather< int, Packet16i >
Packet16i pgather< int, Packet16i >(const int *from, Index stride)
Definition: AVX512/PacketMath.h:1015
Eigen::internal::ptranspose
void ptranspose(PacketBlock< Packet2cf, 2 > &kernel)
Definition: AltiVec/Complex.h:269
Eigen::internal::pfrexp_generic_get_biased_exponent
Packet4d pfrexp_generic_get_biased_exponent(const Packet4d &a)
Definition: AVX/PacketMath.h:1488
Eigen::internal::pmsub
Packet pmsub(const Packet &a, const Packet &b, const Packet &c)
Definition: GenericPacketMath.h:1158
Eigen::internal::pandnot< Packet8i >
Packet8i pandnot< Packet8i >(const Packet8i &a, const Packet8i &b)
Definition: AVX/PacketMath.h:1115
Eigen::internal::ploaddup< Packet16f >
Packet16f ploaddup< Packet16f >(const float *from)
Definition: AVX512/PacketMath.h:855
Eigen::internal::pmax< Packet16f >
Packet16f pmax< Packet16f >(const Packet16f &a, const Packet16f &b)
Definition: AVX512/PacketMath.h:497
Eigen::internal::ploadu< Packet4i >
Packet4i ploadu< Packet4i >(const int *from)
Definition: AltiVec/PacketMath.h:1238
Eigen::internal::pfrexp_generic
Packet pfrexp_generic(const Packet &a, Packet &exponent)
Definition: GenericPacketMathFunctions.h:42
Eigen::internal::pldexp_generic
Packet pldexp_generic(const Packet &a, const Packet &exponent)
Definition: GenericPacketMathFunctions.h:86
Eigen::internal::ploadu< Packet16f >
Packet16f ploadu< Packet16f >(const float *from)
Definition: AVX512/PacketMath.h:828
Eigen::internal::psub< Packet8d >
Packet8d psub< Packet8d >(const Packet8d &a, const Packet8d &b)
Definition: AVX512/PacketMath.h:326
Eigen::internal::pstoreu< bfloat16 >
void pstoreu< bfloat16 >(bfloat16 *to, const Packet8bf &from)
Definition: AltiVec/PacketMath.h:1436
Eigen::internal::Packet8bf
eigen_packet_wrapper< __vector unsigned short int, 0 > Packet8bf
Definition: AltiVec/PacketMath.h:41
Eigen::internal::pfirst< Packet16h >
Eigen::half pfirst< Packet16h >(const Packet16h &from)
Definition: AVX512/PacketMath.h:1887
Eigen::internal::pminmax_propagate_numbers
Packet pminmax_propagate_numbers(const Packet &a, const Packet &b, Op op)
Definition: SSE/PacketMath.h:679
Eigen::internal::pandnot< Packet16i >
Packet16i pandnot< Packet16i >(const Packet16i &a, const Packet16i &b)
Definition: AVX512/PacketMath.h:765
Eigen::internal::pldexp< Packet16f >
Packet16f pldexp< Packet16f >(const Packet16f &a, const Packet16f &exponent)
Definition: AVX512/PacketMath.h:1172
Eigen::internal::pand< Packet8i >
Packet8i pand< Packet8i >(const Packet8i &a, const Packet8i &b)
Definition: AVX/PacketMath.h:1064
Eigen::internal::float2half
Packet8h float2half(const Packet8f &a)
Definition: AVX/PacketMath.h:1848
Eigen::internal::pceil< Packet8d >
Packet8d pceil< Packet8d >(const Packet8d &a)
Definition: AVX512/PacketMath.h:665
Eigen::internal::ploadu< Packet4f >
Packet4f ploadu< Packet4f >(const float *from)
Definition: AltiVec/PacketMath.h:1234
Eigen::internal::ploaddup< Packet16bf >
Packet16bf ploaddup< Packet16bf >(const bfloat16 *from)
Definition: AVX512/PacketMath.h:2373
Eigen::internal::pmax< Packet16i >
Packet16i pmax< Packet16i >(const Packet16i &a, const Packet16i &b)
Definition: AVX512/PacketMath.h:509
Eigen::internal::predux_half_dowto4< Packet16bf >
Packet8bf predux_half_dowto4< Packet16bf >(const Packet16bf &a)
Definition: AVX512/PacketMath.h:2568
Eigen::internal::pmax< PropagateNaN, Packet16f >
Packet16f pmax< PropagateNaN, Packet16f >(const Packet16f &a, const Packet16f &b)
Definition: AVX512/PacketMath.h:540
Eigen::internal::pload< Packet16i >
Packet16i pload< Packet16i >(const int *from)
Definition: AVX512/PacketMath.h:822
Eigen::internal::pnmadd
Packet pnmadd(const Packet &a, const Packet &b, const Packet &c)
Definition: GenericPacketMath.h:1165
Eigen::internal::pceil< Packet16bf >
Packet16bf pceil< Packet16bf >(const Packet16bf &a)
Definition: AVX512/PacketMath.h:2477
Eigen::internal::pmin< PropagateNumbers, Packet16f >
Packet16f pmin< PropagateNumbers, Packet16f >(const Packet16f &a, const Packet16f &b)
Definition: AVX512/PacketMath.h:516
Eigen::internal::psub
Packet psub(const Packet &a, const Packet &b)
Definition: GenericPacketMath.h:324
Eigen::internal::pstore< half >
void pstore< half >(Eigen::half *to, const Packet16h &from)
Definition: AVX512/PacketMath.h:1899
Eigen::internal::pmul< Packet16bf >
Packet16bf pmul< Packet16bf >(const Packet16bf &a, const Packet16bf &b)
Definition: AVX512/PacketMath.h:2539
Eigen::internal::pceil< Packet16f >
Packet16f pceil< Packet16f >(const Packet16f &a)
Definition: AVX512/PacketMath.h:664
Eigen::internal::pset1frombits< Packet8d >
Packet8d pset1frombits< Packet8d >(const numext::uint64_t from)
Definition: AVX512/PacketMath.h:226
Eigen::internal::prefetch< float >
void prefetch< float >(const float *addr)
Definition: AltiVec/PacketMath.h:1512
Eigen::internal::prefetch< double >
void prefetch< double >(const double *addr)
Definition: AVX/PacketMath.h:1396
Eigen::internal::pgather
Packet pgather(const Packet &src, const Scalar *from, Index stride, typename unpacket_traits< Packet >::mask_t umask)
Eigen::internal::predux_mul
unpacket_traits< Packet >::type predux_mul(const Packet &a)
Definition: GenericPacketMath.h:1084
Eigen::internal::pload< Packet8d >
Packet8d pload< Packet8d >(const double *from)
Definition: AVX512/PacketMath.h:818
Eigen::internal::half2float
Packet8f half2float(const Packet8h &a)
Definition: AVX/PacketMath.h:1838
Eigen::internal::padd< Packet16h >
Packet16h padd< Packet16h >(const Packet16h &a, const Packet16h &b)
Definition: AVX512/PacketMath.h:2029
Eigen::internal::pand
Packet8h pand(const Packet8h &a, const Packet8h &b)
Definition: AVX/PacketMath.h:1883
Eigen::internal::ploadquad
Packet16h ploadquad(const Eigen::half *from)
Definition: AVX512/PacketMath.h:1925
Eigen::internal::pset1< Packet16i >
Packet16i pset1< Packet16i >(const int &from)
Definition: AVX512/PacketMath.h:216
Eigen::internal::SsePrefetchPtrType
const char * SsePrefetchPtrType
Definition: SSE/PacketMath.h:1049
Eigen::internal::predux< Packet16f >
float predux< Packet16f >(const Packet16f &a)
Definition: AVX512/PacketMath.h:1254
Eigen::internal::pstoreu< float >
void pstoreu< float >(float *to, const Packet4f &from)
Definition: AltiVec/PacketMath.h:1420
Eigen::internal::predux_min< Packet8d >
double predux_min< Packet8d >(const Packet8d &a)
Definition: AVX512/PacketMath.h:1377
Eigen::internal::pfloor< Packet16f >
Packet16f pfloor< Packet16f >(const Packet16f &a)
Definition: AVX512/PacketMath.h:667
Eigen::internal::pmul< Packet16h >
Packet16h pmul< Packet16h >(const Packet16h &a, const Packet16h &b)
Definition: AVX512/PacketMath.h:2043
Eigen::internal::padd< Packet8d >
Packet8d padd< Packet8d >(const Packet8d &a, const Packet8d &b)
Definition: AVX512/PacketMath.h:295
Eigen::internal::pmin< Packet16bf >
Packet16bf pmin< Packet16bf >(const Packet16bf &a, const Packet16bf &b)
Definition: AVX512/PacketMath.h:2551
Eigen::internal::pmax< PropagateNaN, Packet8d >
Packet8d pmax< PropagateNaN, Packet8d >(const Packet8d &a, const Packet8d &b)
Definition: AVX512/PacketMath.h:544
Eigen::internal::pround< Packet16f >
Packet16f pround< Packet16f >(const Packet16f &a)
Definition: AVX512/PacketMath.h:786
Eigen::internal::Packet8i
eigen_packet_wrapper< __m256i, 0 > Packet8i
Definition: AVX/PacketMath.h:34
Eigen::internal::pset1< Packet16bf >
Packet16bf pset1< Packet16bf >(const bfloat16 &from)
Definition: AVX512/PacketMath.h:2339
Eigen::internal::predux_half_dowto4< Packet16i >
Packet8i predux_half_dowto4< Packet16i >(const Packet16i &a)
Definition: AVX512/PacketMath.h:1321
Eigen::internal::plset< Packet16bf >
Packet16bf plset< Packet16bf >(const bfloat16 &a)
Definition: AVX512/PacketMath.h:2563
Eigen::internal::predux< Packet16h >
half predux< Packet16h >(const Packet16h &from)
Definition: AVX512/PacketMath.h:2057
Eigen::internal::print< Packet8d >
Packet8d print< Packet8d >(const Packet8d &a)
Definition: AVX512/PacketMath.h:662
Eigen::internal::pload< Packet16bf >
Packet16bf pload< Packet16bf >(const bfloat16 *from)
Definition: AVX512/PacketMath.h:2351
Eigen::internal::pmin< Packet8d >
Packet8d pmin< Packet8d >(const Packet8d &a, const Packet8d &b)
Definition: AVX512/PacketMath.h:485
Eigen::internal::pxor
Packet8h pxor(const Packet8h &a, const Packet8h &b)
Definition: AVX/PacketMath.h:1880
Eigen::internal::padd< Packet8bf >
Packet8bf padd< Packet8bf >(const Packet8bf &a, const Packet8bf &b)
Definition: AltiVec/PacketMath.h:1909
Eigen::internal::predux_half_dowto4< Packet16f >
Packet8f predux_half_dowto4< Packet16f >(const Packet16f &a)
Definition: AVX512/PacketMath.h:1299
Eigen::internal::cat256i
Packet16i cat256i(Packet8i a, Packet8i b)
Definition: AVX512/PacketMath.h:569
Eigen::internal::pset1< Packet8d >
Packet8d pset1< Packet8d >(const double &from)
Definition: AVX512/PacketMath.h:212
Eigen::internal::pdiv< Packet16f >
Packet16f pdiv< Packet16f >(const Packet16f &a, const Packet16f &b)
Definition: AVX512/PacketMath.h:388
Eigen::internal::pfloor< Packet16bf >
Packet16bf pfloor< Packet16bf >(const Packet16bf &a)
Definition: AVX512/PacketMath.h:2481
Eigen::internal::padd< Packet16f >
Packet16f padd< Packet16f >(const Packet16f &a, const Packet16f &b)
Definition: AVX512/PacketMath.h:290
Eigen::internal::pset1< Packet8i >
Packet8i pset1< Packet8i >(const int &from)
Definition: AVX/PacketMath.h:671
Eigen::internal::pxor< Packet16f >
Packet16f pxor< Packet16f >(const Packet16f &a, const Packet16f &b)
Definition: AVX512/PacketMath.h:747
Eigen::internal::psignbit
Packet8bf psignbit(const Packet8bf &a)
Definition: AltiVec/PacketMath.h:1590
Eigen::internal::pdiv
Packet pdiv(const Packet &a, const Packet &b)
Definition: GenericPacketMath.h:346
Eigen::internal::pblend
Packet4i pblend(const Selector< 4 > &ifPacket, const Packet4i &thenPacket, const Packet4i &elsePacket)
Definition: AltiVec/PacketMath.h:2689
Eigen::internal::pfirst< Packet16i >
int pfirst< Packet16i >(const Packet16i &a)
Definition: AVX512/PacketMath.h:1110
Eigen::internal::pfirst< Packet16bf >
bfloat16 pfirst< Packet16bf >(const Packet16bf &from)
Definition: AVX512/PacketMath.h:2344
Eigen::internal::extract128
Packet2d extract128(Packet8d x)
Definition: AVX512/PacketMath.h:561
Eigen::internal::ploaddup< Packet8d >
Packet8d ploaddup< Packet8d >(const double *from)
Definition: AVX512/PacketMath.h:879
Eigen::internal::cat256
Packet16f cat256(Packet8f a, Packet8f b)
Definition: AVX512/PacketMath.h:565
Eigen::internal::pconj
Packet2cf pconj(const Packet2cf &a)
Definition: AltiVec/Complex.h:215
Eigen::internal::pmin< Packet16f >
Packet16f pmin< Packet16f >(const Packet16f &a, const Packet16f &b)
Definition: AVX512/PacketMath.h:479
Eigen::internal::pstoreu< int >
void pstoreu< int >(int *to, const Packet4i &from)
Definition: AltiVec/PacketMath.h:1424
Eigen::internal::ploadquad< Packet16f >
Packet16f ploadquad< Packet16f >(const float *from)
Definition: AVX512/PacketMath.h:902
Eigen::internal::pscatter< int, Packet16i >
void pscatter< int, Packet16i >(int *to, const Packet16i &from, Index stride)
Definition: AVX512/PacketMath.h:1071
Eigen::internal::pgather< float, Packet16f >
Packet16f pgather< float, Packet16f >(const Packet16f &src, const float *from, Index stride, uint16_t umask)
Definition: AVX512/PacketMath.h:970
Eigen::internal::psub< Packet16h >
Packet16h psub< Packet16h >(const Packet16h &a, const Packet16h &b)
Definition: AVX512/PacketMath.h:2036
Eigen::internal::plogical_shift_left
Packet4i plogical_shift_left(const Packet4i &a)
Definition: AltiVec/PacketMath.h:1597
Eigen::internal::ptrue< Packet8i >
Packet8i ptrue< Packet8i >(const Packet8i &a)
Definition: AVX/PacketMath.h:1032
Eigen::internal::preverse
Packet2cf preverse(const Packet2cf &a)
Definition: AltiVec/Complex.h:232
Eigen::internal::parithmetic_shift_right
Packet4i parithmetic_shift_right(const Packet4i &a)
Definition: AltiVec/PacketMath.h:1593
Eigen::internal::psub< Packet16f >
Packet16f psub< Packet16f >(const Packet16f &a, const Packet16f &b)
Definition: AVX512/PacketMath.h:321
Eigen::internal::ploadquad< Packet16i >
Packet16i ploadquad< Packet16i >(const int *from)
Definition: AVX512/PacketMath.h:922
Eigen::internal::ploaddup< Packet16h >
Packet16h ploaddup< Packet16h >(const Eigen::half *from)
Definition: AVX512/PacketMath.h:1912
Eigen::internal::pgather< double, Packet8d >
Packet8d pgather< double, Packet8d >(const Packet8d &src, const double *from, Index stride, uint8_t umask)
Definition: AVX512/PacketMath.h:983
Eigen::internal::pstore1< Packet16f >
void pstore1< Packet16f >(float *to, const float &a)
Definition: AVX512/PacketMath.h:1082
Eigen::internal::pfirst< Packet16f >
float pfirst< Packet16f >(const Packet16f &a)
Definition: AVX512/PacketMath.h:1102
Eigen::internal::por
Packet8h por(const Packet8h &a, const Packet8h &b)
Definition: AVX/PacketMath.h:1875
Eigen::internal::pload1< Packet16f >
Packet16f pload1< Packet16f >(const float *from)
Definition: AVX512/PacketMath.h:248
Eigen::internal::pcmp_lt
Packet4i pcmp_lt(const Packet4i &a, const Packet4i &b)
Definition: AltiVec/PacketMath.h:1131
Eigen::internal::pload< Packet16f >
Packet16f pload< Packet16f >(const float *from)
Definition: AVX512/PacketMath.h:814
Eigen::internal::pandnot< Packet8d >
Packet8d pandnot< Packet8d >(const Packet8d &a, const Packet8d &b)
Definition: AVX512/PacketMath.h:778
Eigen::internal::pisnan
Packet8f pisnan(const Packet8f &a)
Definition: AVX/PacketMath.h:851
Eigen::internal::pset1< Packet16f >
Packet16f pset1< Packet16f >(const float &from)
Definition: AVX512/PacketMath.h:208
Eigen::internal::Packet16h
eigen_packet_wrapper< __m256i, 1 > Packet16h
Definition: AVX512/PacketMath.h:37
Eigen::internal::plset< Packet16i >
Packet16i plset< Packet16i >(const int &a)
Definition: AVX512/PacketMath.h:283
Eigen::internal::pmul< Packet8d >
Packet8d pmul< Packet8d >(const Packet8d &a, const Packet8d &b)
Definition: AVX512/PacketMath.h:377
Eigen::internal::por< Packet8i >
Packet8i por< Packet8i >(const Packet8i &a, const Packet8i &b)
Definition: AVX/PacketMath.h:1081
Eigen::internal::pstore< bfloat16 >
void pstore< bfloat16 >(bfloat16 *to, const Packet8bf &from)
Definition: AltiVec/PacketMath.h:631
Eigen::internal::prefetch< int >
void prefetch< int >(const int *addr)
Definition: AltiVec/PacketMath.h:1513
Eigen::internal::pround< Packet8d >
Packet8d pround< Packet8d >(const Packet8d &a)
Definition: AVX512/PacketMath.h:793
Eigen::internal::pdiv< Packet16h >
Packet16h pdiv< Packet16h >(const Packet16h &a, const Packet16h &b)
Definition: AVX512/PacketMath.h:2050
Eigen::internal::pset1frombits< Packet16f >
Packet16f pset1frombits< Packet16f >(unsigned int from)
Definition: AVX512/PacketMath.h:221
Eigen::internal::padd< Packet16i >
Packet16i padd< Packet16i >(const Packet16i &a, const Packet16i &b)
Definition: AVX512/PacketMath.h:300
Eigen::internal::pstore1< Packet16i >
void pstore1< Packet16i >(int *to, const int &a)
Definition: AVX512/PacketMath.h:1092
Eigen::internal::psub< Packet16bf >
Packet16bf psub< Packet16bf >(const Packet16bf &a, const Packet16bf &b)
Definition: AVX512/PacketMath.h:2533
Eigen::internal::pmax< PropagateNumbers, Packet8d >
Packet8d pmax< PropagateNumbers, Packet8d >(const Packet8d &a, const Packet8d &b)
Definition: AVX512/PacketMath.h:528
Eigen::internal::pmul< Packet16f >
Packet16f pmul< Packet16f >(const Packet16f &a, const Packet16f &b)
Definition: AVX512/PacketMath.h:372
Eigen::internal::ploadu< Packet16bf >
Packet16bf ploadu< Packet16bf >(const bfloat16 *from)
Definition: AVX512/PacketMath.h:2356
Eigen::internal::predux_min< Packet16h >
Eigen::half predux_min< Packet16h >(const Packet16h &a)
Definition: AVX512/PacketMath.h:2077
Eigen::internal::predux_any
bool predux_any(const Packet4f &x)
Definition: AltiVec/PacketMath.h:2347
Eigen::internal::pdiv< Packet16bf >
Packet16bf pdiv< Packet16bf >(const Packet16bf &a, const Packet16bf &b)
Definition: AVX512/PacketMath.h:2545
Eigen::internal::pround< Packet16h >
Packet16h pround< Packet16h >(const Packet16h &a)
Definition: AVX512/PacketMath.h:1987
Eigen::internal::Packet8h
eigen_packet_wrapper< __m128i, 2 > Packet8h
Definition: AVX/PacketMath.h:37
Eigen::internal::pfloor< Packet8d >
Packet8d pfloor< Packet8d >(const Packet8d &a)
Definition: AVX512/PacketMath.h:668
Eigen::internal::pstore< double >
void pstore< double >(double *to, const Packet4d &from)
Definition: AVX/PacketMath.h:1288
Eigen::internal::pcmp_le
Packet4f pcmp_le(const Packet4f &a, const Packet4f &b)
Definition: AltiVec/PacketMath.h:1117
Eigen::internal::pceil< Packet16h >
Packet16h pceil< Packet16h >(const Packet16h &a)
Definition: AVX512/PacketMath.h:1995
Eigen::internal::pstore1< Packet8d >
void pstore1< Packet8d >(double *to, const double &a)
Definition: AVX512/PacketMath.h:1087
Eigen::internal::predux< Packet16i >
int predux< Packet16i >(const Packet16i &a)
Definition: AVX512/PacketMath.h:1280
Eigen::internal::predux_min< Packet16f >
float predux_min< Packet16f >(const Packet16f &a)
Definition: AVX512/PacketMath.h:1367
Eigen::internal::peven_mask
Packet8f peven_mask(const Packet8f &)
Definition: AVX/PacketMath.h:683
Eigen::numext::uint8_t
std::uint8_t uint8_t
Definition: Meta.h:35
Eigen::numext::uint16_t
std::uint16_t uint16_t
Definition: Meta.h:37
Eigen::numext::uint32_t
std::uint32_t uint32_t
Definition: Meta.h:39
Eigen::numext::uint64_t
std::uint64_t uint64_t
Definition: Meta.h:41
Eigen
: InteropHeaders
Definition: Core:139
Eigen::Index
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition: Meta.h:82
Eigen::bfloat16_impl::__bfloat16_raw::value
unsigned short value
Definition: BFloat16.h:76
Eigen::half_impl::__half_raw::x
numext::uint16_t x
Definition: Half.h:104
j
std::ptrdiff_t j
Definition: tut_arithmetic_redux_minmax.cpp:2
Generated on Tue Jul 18 2023 15:28:17 for Eigen by Doxygen 1.9.1 and the doxyYoda theme. This is an unofficial themed variant maintained by Rohit Goswami, based on the official documentation.