1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
|
#ifndef CXX11_SRC_FIXEDPOINT_PACKETMATHAVX512_H_
#define CXX11_SRC_FIXEDPOINT_PACKETMATHAVX512_H_
#include "PacketMathAVX2.h"
namespace Eigen {
namespace internal {
typedef struct Packet64q8i {
__m512i val;
operator __m512i() const { return val; }
Packet64q8i();
Packet64q8i(__m512i val) : val(val) {}
} Packet64q8i;
typedef struct Packet32q16i {
__m512i val;
operator __m512i() const { return val; }
Packet32q16i();
Packet32q16i(__m512i val) : val(val) {}
} Packet32q16i;
typedef struct Packet64q8u {
__m512i val;
operator __m512i() const { return val; }
Packet64q8u();
Packet64q8u(__m512i val) : val(val) {}
} Packet64q8u;
typedef struct Packet16q32i {
__m512i val;
operator __m512i() const { return val; }
Packet16q32i();
Packet16q32i(__m512i val) : val(val) {}
} Packet16q32i;
template <>
struct packet_traits<QInt8> : default_packet_traits {
typedef Packet64q8i type;
typedef Packet32q8i half;
enum {
Vectorizable = 1,
AlignedOnScalar = 1,
size = 64,
};
enum {
HasAdd = 0,
HasSub = 0,
HasMul = 0,
HasNegate = 0,
HasAbs = 0,
HasAbs2 = 0,
HasMin = 1,
HasMax = 1,
HasConj = 0,
HasSetLinear = 0
};
};
template <>
struct packet_traits<QUInt8> : default_packet_traits {
typedef Packet64q8u type;
typedef Packet32q8u half;
enum {
Vectorizable = 1,
AlignedOnScalar = 1,
size = 64,
};
enum {
HasAdd = 0,
HasSub = 0,
HasMul = 0,
HasNegate = 0,
HasAbs = 0,
HasAbs2 = 0,
HasMin = 1,
HasMax = 1,
HasConj = 0,
HasSetLinear = 0
};
};
template <>
struct packet_traits<QInt16> : default_packet_traits {
typedef Packet32q16i type;
typedef Packet16q16i half;
enum {
Vectorizable = 1,
AlignedOnScalar = 1,
size = 32,
};
enum {
HasAdd = 0,
HasSub = 0,
HasMul = 0,
HasNegate = 0,
HasAbs = 0,
HasAbs2 = 0,
HasMin = 1,
HasMax = 1,
HasConj = 0,
HasSetLinear = 0
};
};
template <>
struct packet_traits<QInt32> : default_packet_traits {
typedef Packet16q32i type;
typedef Packet8q32i half;
enum {
Vectorizable = 1,
AlignedOnScalar = 1,
size = 16,
};
enum {
HasAdd = 1,
HasSub = 1,
HasMul = 1,
HasNegate = 1,
HasAbs = 0,
HasAbs2 = 0,
HasMin = 1,
HasMax = 1,
HasConj = 0,
HasSetLinear = 0
};
};
template <>
struct unpacket_traits<Packet64q8i> {
typedef QInt8 type;
typedef Packet32q8i half;
enum { size = 64, alignment=Aligned64 };
};
template <>
struct unpacket_traits<Packet32q16i> {
typedef QInt16 type;
typedef Packet16q16i half;
enum { size = 32, alignment=Aligned64 };
};
template <>
struct unpacket_traits<Packet64q8u> {
typedef QUInt8 type;
typedef Packet32q8u half;
enum { size = 64, alignment=Aligned64 };
};
template <>
struct unpacket_traits<Packet16q32i> {
typedef QInt32 type;
typedef Packet8q32i half;
enum { size = 16, alignment=Aligned64 };
};
// Unaligned load
template <>
EIGEN_STRONG_INLINE Packet64q8i ploadu<Packet64q8i>(const QInt8* from) {
EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_si512(
reinterpret_cast<const __m512i*>(from));
}
template <>
EIGEN_STRONG_INLINE Packet32q16i ploadu<Packet32q16i>(const QInt16* from) {
EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_si512(
reinterpret_cast<const __m512i*>(from));
}
template <>
EIGEN_STRONG_INLINE Packet64q8u ploadu<Packet64q8u>(const QUInt8* from) {
EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_si512(
reinterpret_cast<const __m512i*>(from));
}
template <>
EIGEN_STRONG_INLINE Packet16q32i ploadu<Packet16q32i>(const QInt32* from) {
EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_si512(
reinterpret_cast<const __m512i*>(from));
}
// Aligned load
template <>
EIGEN_STRONG_INLINE Packet64q8i pload<Packet64q8i>(const QInt8* from) {
EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_si512(
reinterpret_cast<const __m512i*>(from));
}
template <>
EIGEN_STRONG_INLINE Packet32q16i pload<Packet32q16i>(const QInt16* from) {
EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_si512(
reinterpret_cast<const __m512i*>(from));
}
template <>
EIGEN_STRONG_INLINE Packet64q8u pload<Packet64q8u>(const QUInt8* from) {
EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_si512(
reinterpret_cast<const __m512i*>(from));
}
template <>
EIGEN_STRONG_INLINE Packet16q32i pload<Packet16q32i>(const QInt32* from) {
EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_si512(
reinterpret_cast<const __m512i*>(from));
}
// Unaligned store
template <>
EIGEN_STRONG_INLINE void pstoreu<QInt8>(QInt8* to, const Packet64q8i& from) {
EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_si512(
reinterpret_cast<__m512i*>(to), from.val);
}
template <>
EIGEN_STRONG_INLINE void pstoreu<QInt16>(QInt16* to, const Packet32q16i& from) {
EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_si512(
reinterpret_cast<__m512i*>(to), from.val);
}
template <>
EIGEN_STRONG_INLINE void pstoreu<QUInt8>(QUInt8* to, const Packet64q8u& from) {
EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_si512(
reinterpret_cast<__m512i*>(to), from.val);
}
template <>
EIGEN_STRONG_INLINE void pstoreu<QInt32>(QInt32* to, const Packet16q32i& from) {
EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_si512(
reinterpret_cast<__m512i*>(to), from.val);
}
// Aligned store
template <>
EIGEN_STRONG_INLINE void pstore<QInt32>(QInt32* to, const Packet16q32i& from) {
EIGEN_DEBUG_ALIGNED_STORE _mm512_store_si512(reinterpret_cast<__m512i*>(to),
from.val);
}
template <>
EIGEN_STRONG_INLINE void pstore<QUInt8>(QUInt8* to, const Packet64q8u& from) {
EIGEN_DEBUG_ALIGNED_STORE _mm512_store_si512(reinterpret_cast<__m512i*>(to),
from.val);
}
template <>
EIGEN_STRONG_INLINE void pstore<QInt8>(QInt8* to, const Packet64q8i& from) {
EIGEN_DEBUG_ALIGNED_STORE _mm512_store_si512(reinterpret_cast<__m512i*>(to),
from.val);
}
template <>
EIGEN_STRONG_INLINE void pstore<QInt16>(QInt16* to, const Packet32q16i& from) {
EIGEN_DEBUG_ALIGNED_STORE _mm512_store_si512(reinterpret_cast<__m512i*>(to),
from.val);
}
// Extract first element.
template <>
EIGEN_STRONG_INLINE QInt32 pfirst<Packet16q32i>(const Packet16q32i& a) {
return _mm_cvtsi128_si32(_mm512_extracti32x4_epi32(a, 0));
}
template <>
EIGEN_STRONG_INLINE QUInt8 pfirst<Packet64q8u>(const Packet64q8u& a) {
return static_cast<uint8_t>(
_mm_extract_epi8(_mm512_extracti32x4_epi32(a.val, 0), 0));
}
template <>
EIGEN_STRONG_INLINE QInt8 pfirst<Packet64q8i>(const Packet64q8i& a) {
return _mm_extract_epi8(_mm512_extracti32x4_epi32(a.val, 0), 0);
}
template <>
EIGEN_STRONG_INLINE QInt16 pfirst<Packet32q16i>(const Packet32q16i& a) {
return _mm_extract_epi16(_mm512_extracti32x4_epi32(a.val, 0), 0);
}
// Initialize to constant value.
template <>
EIGEN_STRONG_INLINE Packet64q8i pset1<Packet64q8i>(const QInt8& from) {
return _mm512_set1_epi8(from.value);
}
template <>
EIGEN_STRONG_INLINE Packet32q16i pset1<Packet32q16i>(const QInt16& from) {
return _mm512_set1_epi16(from.value);
}
template <>
EIGEN_STRONG_INLINE Packet64q8u pset1<Packet64q8u>(const QUInt8& from) {
return _mm512_set1_epi8(static_cast<uint8_t>(from.value));
}
template <>
EIGEN_STRONG_INLINE Packet16q32i pset1<Packet16q32i>(const QInt32& from) {
return _mm512_set1_epi32(from.value);
}
// Basic arithmetic packet ops for QInt32.
template <>
EIGEN_STRONG_INLINE Packet16q32i padd<Packet16q32i>(const Packet16q32i& a,
const Packet16q32i& b) {
return _mm512_add_epi32(a.val, b.val);
}
template <>
EIGEN_STRONG_INLINE Packet16q32i psub<Packet16q32i>(const Packet16q32i& a,
const Packet16q32i& b) {
return _mm512_sub_epi32(a.val, b.val);
}
// Note: mullo truncates the result to 32 bits.
template <>
EIGEN_STRONG_INLINE Packet16q32i pmul<Packet16q32i>(const Packet16q32i& a,
const Packet16q32i& b) {
return _mm512_mullo_epi32(a.val, b.val);
}
template <>
EIGEN_STRONG_INLINE Packet16q32i pnegate<Packet16q32i>(const Packet16q32i& a) {
return _mm512_sub_epi32(_mm512_setzero_si512(), a.val);
}
// Min and max.
template <>
EIGEN_STRONG_INLINE Packet16q32i pmin<Packet16q32i>(const Packet16q32i& a,
const Packet16q32i& b) {
return _mm512_min_epi32(a.val, b.val);
}
template <>
EIGEN_STRONG_INLINE Packet16q32i pmax<Packet16q32i>(const Packet16q32i& a,
const Packet16q32i& b) {
return _mm512_max_epi32(a.val, b.val);
}
template <>
EIGEN_STRONG_INLINE Packet64q8u pmin<Packet64q8u>(const Packet64q8u& a,
const Packet64q8u& b) {
#ifdef EIGEN_VECTORIZE_AVX512BW
return _mm512_min_epu8(a.val, b.val);
#else
__m256i ap0 = _mm512_extracti32x8_epi32(a.val, 0);
__m256i ap1 = _mm512_extracti32x8_epi32(a.val, 1);
__m256i bp0 = _mm512_extracti32x8_epi32(b.val, 0);
__m256i bp1 = _mm512_extracti32x8_epi32(b.val, 1);
__m256i r0 = _mm256_min_epu8(ap0, bp0);
__m256i r1 = _mm256_min_epu8(ap1, bp1);
return _mm512_inserti32x8(_mm512_castsi256_si512(r0), r1, 1);
#endif
}
template <>
EIGEN_STRONG_INLINE Packet64q8u pmax<Packet64q8u>(const Packet64q8u& a,
const Packet64q8u& b) {
#ifdef EIGEN_VECTORIZE_AVX512BW
return _mm512_max_epu8(a.val, b.val);
#else
__m256i ap0 = _mm512_extracti32x8_epi32(a.val, 0);
__m256i ap1 = _mm512_extracti32x8_epi32(a.val, 1);
__m256i bp0 = _mm512_extracti32x8_epi32(b.val, 0);
__m256i bp1 = _mm512_extracti32x8_epi32(b.val, 1);
__m256i r0 = _mm256_max_epu8(ap0, bp0);
__m256i r1 = _mm256_max_epu8(ap1, bp1);
return _mm512_inserti32x8(_mm512_castsi256_si512(r0), r1, 1);
#endif
}
template <>
EIGEN_STRONG_INLINE Packet64q8i pmin<Packet64q8i>(const Packet64q8i& a,
const Packet64q8i& b) {
#ifdef EIGEN_VECTORIZE_AVX512BW
return _mm512_min_epi8(a.val, b.val);
#else
__m256i ap0 = _mm512_extracti32x8_epi32(a.val, 0);
__m256i ap1 = _mm512_extracti32x8_epi32(a.val, 1);
__m256i bp0 = _mm512_extracti32x8_epi32(b.val, 0);
__m256i bp1 = _mm512_extracti32x8_epi32(b.val, 1);
__m256i r0 = _mm256_min_epi8(ap0, bp0);
__m256i r1 = _mm256_min_epi8(ap1, bp1);
return _mm512_inserti32x8(_mm512_castsi256_si512(r0), r1, 1);
#endif
}
template <>
EIGEN_STRONG_INLINE Packet32q16i pmin<Packet32q16i>(const Packet32q16i& a,
const Packet32q16i& b) {
#ifdef EIGEN_VECTORIZE_AVX512BW
return _mm512_min_epi16(a.val, b.val);
#else
__m256i ap0 = _mm512_extracti32x8_epi32(a.val, 0);
__m256i ap1 = _mm512_extracti32x8_epi32(a.val, 1);
__m256i bp0 = _mm512_extracti32x8_epi32(b.val, 0);
__m256i bp1 = _mm512_extracti32x8_epi32(b.val, 1);
__m256i r0 = _mm256_min_epi16(ap0, bp0);
__m256i r1 = _mm256_min_epi16(ap1, bp1);
return _mm512_inserti32x8(_mm512_castsi256_si512(r0), r1, 1);
#endif
}
template <>
EIGEN_STRONG_INLINE Packet64q8i pmax<Packet64q8i>(const Packet64q8i& a,
const Packet64q8i& b) {
#ifdef EIGEN_VECTORIZE_AVX512BW
return _mm512_max_epi8(a.val, b.val);
#else
__m256i ap0 = _mm512_extracti32x8_epi32(a.val, 0);
__m256i ap1 = _mm512_extracti32x8_epi32(a.val, 1);
__m256i bp0 = _mm512_extracti32x8_epi32(b.val, 0);
__m256i bp1 = _mm512_extracti32x8_epi32(b.val, 1);
__m256i r0 = _mm256_max_epi8(ap0, bp0);
__m256i r1 = _mm256_max_epi8(ap1, bp1);
return _mm512_inserti32x8(_mm512_castsi256_si512(r0), r1, 1);
#endif
}
template <>
EIGEN_STRONG_INLINE Packet32q16i pmax<Packet32q16i>(const Packet32q16i& a,
const Packet32q16i& b) {
#ifdef EIGEN_VECTORIZE_AVX512BW
return _mm512_max_epi16(a.val, b.val);
#else
__m256i ap0 = _mm512_extracti32x8_epi32(a.val, 0);
__m256i ap1 = _mm512_extracti32x8_epi32(a.val, 1);
__m256i bp0 = _mm512_extracti32x8_epi32(b.val, 0);
__m256i bp1 = _mm512_extracti32x8_epi32(b.val, 1);
__m256i r0 = _mm256_max_epi16(ap0, bp0);
__m256i r1 = _mm256_max_epi16(ap1, bp1);
return _mm512_inserti32x8(_mm512_castsi256_si512(r0), r1, 1);
#endif
}
// Reductions.
template <>
EIGEN_STRONG_INLINE QInt32 predux_min<Packet16q32i>(const Packet16q32i& a) {
Packet4i lane0 = _mm512_extracti32x4_epi32(a.val, 0);
Packet4i lane1 = _mm512_extracti32x4_epi32(a.val, 1);
Packet4i lane2 = _mm512_extracti32x4_epi32(a.val, 2);
Packet4i lane3 = _mm512_extracti32x4_epi32(a.val, 3);
Packet4i res =
_mm_min_epi32(_mm_min_epi32(lane0, lane1), _mm_min_epi32(lane2, lane3));
res = _mm_min_epi32(res, _mm_shuffle_epi32(res, _MM_SHUFFLE(0, 0, 3, 2)));
return pfirst(
_mm_min_epi32(
res,
_mm_shuffle_epi32(res, _MM_SHUFFLE(0, 0, 0, 1))));
}
template <>
EIGEN_STRONG_INLINE QInt32 predux_max<Packet16q32i>(const Packet16q32i& a) {
Packet4i lane0 = _mm512_extracti32x4_epi32(a.val, 0);
Packet4i lane1 = _mm512_extracti32x4_epi32(a.val, 1);
Packet4i lane2 = _mm512_extracti32x4_epi32(a.val, 2);
Packet4i lane3 = _mm512_extracti32x4_epi32(a.val, 3);
Packet4i res =
_mm_max_epi32(_mm_max_epi32(lane0, lane1), _mm_max_epi32(lane2, lane3));
res = _mm_max_epi32(res, _mm_shuffle_epi32(res, _MM_SHUFFLE(0, 0, 3, 2)));
return pfirst(
_mm_max_epi32(
res,
_mm_shuffle_epi32(res, _MM_SHUFFLE(0, 0, 0, 1))));
}
template <>
EIGEN_STRONG_INLINE QInt16 predux_min<Packet32q16i>(const Packet32q16i& a) {
Packet4i lane0 = _mm512_extracti32x4_epi32(a.val, 0);
Packet4i lane1 = _mm512_extracti32x4_epi32(a.val, 1);
Packet4i lane2 = _mm512_extracti32x4_epi32(a.val, 2);
Packet4i lane3 = _mm512_extracti32x4_epi32(a.val, 3);
Packet4i res =
_mm_min_epi16(_mm_min_epi16(lane0, lane1), _mm_min_epi16(lane2, lane3));
res = _mm_min_epi16(res, _mm_shuffle_epi32(res, _MM_SHUFFLE(0, 0, 3, 2)));
std::uint32_t w =
pfirst(
_mm_min_epi16(res, _mm_shuffle_epi32(res, _MM_SHUFFLE(0, 0, 0, 1))));
return std::min({
static_cast<std::int16_t>(w >> 16),
static_cast<std::int16_t>(w)
});
}
template <>
EIGEN_STRONG_INLINE QInt16 predux_max<Packet32q16i>(const Packet32q16i& a) {
Packet4i lane0 = _mm512_extracti32x4_epi32(a.val, 0);
Packet4i lane1 = _mm512_extracti32x4_epi32(a.val, 1);
Packet4i lane2 = _mm512_extracti32x4_epi32(a.val, 2);
Packet4i lane3 = _mm512_extracti32x4_epi32(a.val, 3);
Packet4i res =
_mm_max_epi16(_mm_max_epi16(lane0, lane1), _mm_max_epi16(lane2, lane3));
res = _mm_max_epi16(res, _mm_shuffle_epi32(res, _MM_SHUFFLE(0, 0, 3, 2)));
std::uint32_t w =
pfirst(
_mm_max_epi16(res, _mm_shuffle_epi32(res, _MM_SHUFFLE(0, 0, 0, 1))));
return std::max({
static_cast<std::int16_t>(w >> 16),
static_cast<std::int16_t>(w)
});
}
template <>
EIGEN_STRONG_INLINE QUInt8 predux_min<Packet64q8u>(const Packet64q8u& a) {
Packet4i lane0 = _mm512_extracti32x4_epi32(a.val, 0);
Packet4i lane1 = _mm512_extracti32x4_epi32(a.val, 1);
Packet4i lane2 = _mm512_extracti32x4_epi32(a.val, 2);
Packet4i lane3 = _mm512_extracti32x4_epi32(a.val, 3);
Packet4i res =
_mm_min_epu8(_mm_min_epu8(lane0, lane1), _mm_min_epu8(lane2, lane3));
res = _mm_min_epu8(res, _mm_shuffle_epi32(res, _MM_SHUFFLE(0, 0, 3, 2)));
std::uint32_t w =
pfirst(
_mm_min_epu8(res, _mm_shuffle_epi32(res, _MM_SHUFFLE(0, 0, 0, 1))));
return std::min({
static_cast<std::uint8_t>(w >> 24),
static_cast<std::uint8_t>(w >> 16),
static_cast<std::uint8_t>(w >> 8),
static_cast<std::uint8_t>(w)
});
}
template <>
EIGEN_STRONG_INLINE QUInt8 predux_max<Packet64q8u>(const Packet64q8u& a) {
Packet4i lane0 = _mm512_extracti32x4_epi32(a.val, 0);
Packet4i lane1 = _mm512_extracti32x4_epi32(a.val, 1);
Packet4i lane2 = _mm512_extracti32x4_epi32(a.val, 2);
Packet4i lane3 = _mm512_extracti32x4_epi32(a.val, 3);
Packet4i res =
_mm_max_epu8(_mm_max_epu8(lane0, lane1), _mm_max_epu8(lane2, lane3));
res = _mm_max_epu8(res, _mm_shuffle_epi32(res, _MM_SHUFFLE(0, 0, 3, 2)));
std::uint32_t w =
pfirst(
_mm_max_epu8(res, _mm_shuffle_epi32(res, _MM_SHUFFLE(0, 0, 0, 1))));
return std::max({
static_cast<std::uint8_t>(w >> 24),
static_cast<std::uint8_t>(w >> 16),
static_cast<std::uint8_t>(w >> 8),
static_cast<std::uint8_t>(w)
});
}
template <>
EIGEN_STRONG_INLINE QInt8 predux_min<Packet64q8i>(const Packet64q8i& a) {
Packet4i lane0 = _mm512_extracti32x4_epi32(a.val, 0);
Packet4i lane1 = _mm512_extracti32x4_epi32(a.val, 1);
Packet4i lane2 = _mm512_extracti32x4_epi32(a.val, 2);
Packet4i lane3 = _mm512_extracti32x4_epi32(a.val, 3);
Packet4i res =
_mm_min_epi8(_mm_min_epi8(lane0, lane1), _mm_min_epi8(lane2, lane3));
res = _mm_min_epi8(res, _mm_shuffle_epi32(res, _MM_SHUFFLE(0, 0, 3, 2)));
std::uint32_t w =
pfirst(
_mm_min_epi8(res, _mm_shuffle_epi32(res, _MM_SHUFFLE(0, 0, 0, 1))));
return std::min({
static_cast<std::int8_t>(w >> 24),
static_cast<std::int8_t>(w >> 16),
static_cast<std::int8_t>(w >> 8),
static_cast<std::int8_t>(w)
});
}
template <>
EIGEN_STRONG_INLINE QInt8 predux_max<Packet64q8i>(const Packet64q8i& a) {
Packet4i lane0 = _mm512_extracti32x4_epi32(a.val, 0);
Packet4i lane1 = _mm512_extracti32x4_epi32(a.val, 1);
Packet4i lane2 = _mm512_extracti32x4_epi32(a.val, 2);
Packet4i lane3 = _mm512_extracti32x4_epi32(a.val, 3);
Packet4i res =
_mm_max_epi8(_mm_max_epi8(lane0, lane1), _mm_max_epi8(lane2, lane3));
res = _mm_max_epi8(res, _mm_shuffle_epi32(res, _MM_SHUFFLE(0, 0, 3, 2)));
std::uint32_t w =
pfirst(
_mm_max_epi8(res, _mm_shuffle_epi32(res, _MM_SHUFFLE(0, 0, 0, 1))));
return std::min({
static_cast<std::int8_t>(w >> 24),
static_cast<std::int8_t>(w >> 16),
static_cast<std::int8_t>(w >> 8),
static_cast<std::int8_t>(w)
});
}
} // end namespace internal
} // end namespace Eigen
#endif // CXX11_SRC_FIXEDPOINT_PACKETMATHAVX512_H_
|
