core/stdarch/crates/core_arch/src/s390x/
vector.rs

1//! s390x vector intrinsics.
2//!
3//! For more info see the [Reference Summary] or the online [IBM docs].
4//!
5//! [Reference Summary]: https://www.ibm.com/support/pages/sites/default/files/2021-05/SA22-7871-10.pdf
6//! [IBM docs]: https://www.ibm.com/docs/en/zos/2.4.0?topic=support-vector-built-in-functions
7
8#![allow(non_camel_case_types)]
9
10use crate::{core_arch::simd::*, intrinsics::simd::*, mem::MaybeUninit, mem::transmute};
11
12#[cfg(test)]
13use stdarch_test::assert_instr;
14
15use super::macros::*;
16
17types! {
18    #![unstable(feature = "stdarch_s390x", issue = "135681")]
19
20    /// s390x-specific 128-bit wide vector of sixteen packed `i8`
21    pub struct vector_signed_char(16 x i8);
22    /// s390x-specific 128-bit wide vector of sixteen packed `u8`
23    pub struct vector_unsigned_char(16 x u8);
24    /// s390x-specific 128-bit wide vector mask of sixteen packed elements
25    pub struct vector_bool_char(16 x i8);
26
27    /// s390x-specific 128-bit wide vector of eight packed `i16`
28    pub struct vector_signed_short(8 x i16);
29    /// s390x-specific 128-bit wide vector of eight packed `u16`
30    pub struct vector_unsigned_short(8 x u16);
31    /// s390x-specific 128-bit wide vector mask of eight packed elements
32    pub struct vector_bool_short(8 x i16);
33
34    /// s390x-specific 128-bit wide vector of four packed `i32`
35    pub struct vector_signed_int(4 x i32);
36    /// s390x-specific 128-bit wide vector of four packed `u32`
37    pub struct vector_unsigned_int(4 x u32);
38    /// s390x-specific 128-bit wide vector mask of four packed elements
39    pub struct vector_bool_int(4 x i32);
40
41    /// s390x-specific 128-bit wide vector of two packed `i64`
42    pub struct vector_signed_long_long(2 x i64);
43    /// s390x-specific 128-bit wide vector of two packed `u64`
44    pub struct vector_unsigned_long_long(2 x u64);
45    /// s390x-specific 128-bit wide vector mask of two packed elements
46    pub struct vector_bool_long_long(2 x i64);
47
48    /// s390x-specific 128-bit wide vector of four packed `f32`
49    pub struct vector_float(4 x f32);
50    /// s390x-specific 128-bit wide vector of two packed `f64`
51    pub struct vector_double(2 x f64);
52}
53
54#[repr(C, packed)]
55struct PackedTuple<T, U> {
56    x: T,
57    y: U,
58}
59
60#[allow(improper_ctypes)]
61#[rustfmt::skip]
62unsafe extern "unadjusted" {
63    #[link_name = "llvm.smax.v16i8"] fn vmxb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
64    #[link_name = "llvm.smax.v8i16"] fn vmxh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short;
65    #[link_name = "llvm.smax.v4i32"] fn vmxf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int;
66    #[link_name = "llvm.smax.v2i64"] fn vmxg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long;
67
68    #[link_name = "llvm.umax.v16i8"] fn vmxlb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
69    #[link_name = "llvm.umax.v8i16"] fn vmxlh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
70    #[link_name = "llvm.umax.v4i32"] fn vmxlf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
71    #[link_name = "llvm.umax.v2i64"] fn vmxlg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long;
72
73    #[link_name = "llvm.smin.v16i8"] fn vmnb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
74    #[link_name = "llvm.smin.v8i16"] fn vmnh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short;
75    #[link_name = "llvm.smin.v4i32"] fn vmnf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int;
76    #[link_name = "llvm.smin.v2i64"] fn vmng(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long;
77
78    #[link_name = "llvm.umin.v16i8"] fn vmnlb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
79    #[link_name = "llvm.umin.v8i16"] fn vmnlh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
80    #[link_name = "llvm.umin.v4i32"] fn vmnlf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
81    #[link_name = "llvm.umin.v2i64"] fn vmnlg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long;
82
83    #[link_name = "llvm.nearbyint.v4f32"] fn nearbyint_v4f32(a: vector_float) -> vector_float;
84    #[link_name = "llvm.nearbyint.v2f64"] fn nearbyint_v2f64(a: vector_double) -> vector_double;
85
86    #[link_name = "llvm.roundeven.v4f32"] fn roundeven_v4f32(a: vector_float) -> vector_float;
87    #[link_name = "llvm.roundeven.v2f64"] fn roundeven_v2f64(a: vector_double) -> vector_double;
88
89    #[link_name = "llvm.s390.vsra"] fn vsra(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
90    #[link_name = "llvm.s390.vsrl"] fn vsrl(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
91    #[link_name = "llvm.s390.vsl"] fn vsl(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
92
93    #[link_name = "llvm.s390.vsrab"] fn vsrab(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
94    #[link_name = "llvm.s390.vsrlb"] fn vsrlb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
95    #[link_name = "llvm.s390.vslb"] fn vslb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
96
97    #[link_name = "llvm.s390.vsldb"] fn vsldb(a: i8x16, b: i8x16, c: u32) -> i8x16;
98    #[link_name = "llvm.s390.vsld"] fn vsld(a: i8x16, b: i8x16, c: u32) -> i8x16;
99    #[link_name = "llvm.s390.vsrd"] fn vsrd(a: i8x16, b: i8x16, c: u32) -> i8x16;
100
101    #[link_name = "llvm.s390.verimb"] fn verimb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char, d: i32) -> vector_signed_char;
102    #[link_name = "llvm.s390.verimh"] fn verimh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short, d: i32) -> vector_signed_short;
103    #[link_name = "llvm.s390.verimf"] fn verimf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int, d: i32) -> vector_signed_int;
104    #[link_name = "llvm.s390.verimg"] fn verimg(a: vector_signed_long_long, b: vector_signed_long_long, c: vector_signed_long_long, d: i32) -> vector_signed_long_long;
105
106    #[link_name = "llvm.s390.vperm"] fn vperm(a: vector_signed_char, b: vector_signed_char, c: vector_unsigned_char) -> vector_signed_char;
107
108    #[link_name = "llvm.s390.vsumb"] fn vsumb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_int;
109    #[link_name = "llvm.s390.vsumh"] fn vsumh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int;
110
111    #[link_name = "llvm.s390.vsumgh"] fn vsumgh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_long_long;
112    #[link_name = "llvm.s390.vsumgf"] fn vsumgf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long;
113
114    #[link_name = "llvm.s390.vsumqf"] fn vsumqf(a: vector_unsigned_int, b: vector_unsigned_int) -> u128;
115    #[link_name = "llvm.s390.vsumqg"] fn vsumqg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> u128;
116
117    #[link_name = "llvm.s390.vscbiq"] fn vscbiq(a: u128, b: u128) -> u128;
118    #[link_name = "llvm.s390.vsbiq"] fn vsbiq(a: u128, b: u128, c: u128) -> u128;
119    #[link_name = "llvm.s390.vsbcbiq"] fn vsbcbiq(a: u128, b: u128, c: u128) -> u128;
120
121    #[link_name = "llvm.s390.vacq"] fn vacq(a: u128, b: u128, c: u128) -> u128;
122
123    #[link_name = "llvm.s390.vscbib"] fn vscbib(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
124    #[link_name = "llvm.s390.vscbih"] fn vscbih(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
125    #[link_name = "llvm.s390.vscbif"] fn vscbif(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
126    #[link_name = "llvm.s390.vscbig"] fn vscbig(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long;
127
128    #[link_name = "llvm.s390.vfaeb"] fn vfaeb(a: vector_signed_char, b: vector_signed_char, c: i32) -> vector_signed_char;
129    #[link_name = "llvm.s390.vfaeh"] fn vfaeh(a: vector_signed_short, b: vector_signed_short, c: i32) -> vector_signed_short;
130    #[link_name = "llvm.s390.vfaef"] fn vfaef(a: vector_signed_int, b: vector_signed_int, c: i32) -> vector_signed_int;
131
132    #[link_name = "llvm.s390.vfaezb"] fn vfaezb(a: vector_signed_char, b: vector_signed_char, c: i32) -> vector_signed_char;
133    #[link_name = "llvm.s390.vfaezh"] fn vfaezh(a: vector_signed_short, b: vector_signed_short, c: i32) -> vector_signed_short;
134    #[link_name = "llvm.s390.vfaezf"] fn vfaezf(a: vector_signed_int, b: vector_signed_int, c: i32) -> vector_signed_int;
135
136    #[link_name = "llvm.s390.vfaebs"] fn vfaebs(a: vector_signed_char, b: vector_signed_char, c: i32) -> PackedTuple<vector_signed_char, i32>;
137    #[link_name = "llvm.s390.vfaehs"] fn vfaehs(a: vector_signed_short, b: vector_signed_short, c: i32) -> PackedTuple<vector_signed_short, i32>;
138    #[link_name = "llvm.s390.vfaefs"] fn vfaefs(a: vector_signed_int, b: vector_signed_int, c: i32) -> PackedTuple<vector_signed_int, i32>;
139
140    #[link_name = "llvm.s390.vfaezbs"] fn vfaezbs(a: vector_signed_char, b: vector_signed_char, c: i32) -> PackedTuple<vector_signed_char, i32>;
141    #[link_name = "llvm.s390.vfaezhs"] fn vfaezhs(a: vector_signed_short, b: vector_signed_short, c: i32) -> PackedTuple<vector_signed_short, i32>;
142    #[link_name = "llvm.s390.vfaezfs"] fn vfaezfs(a: vector_signed_int, b: vector_signed_int, c: i32) -> PackedTuple<vector_signed_int, i32>;
143
144    #[link_name = "llvm.s390.vll"] fn vll(a: u32, b: *const u8) -> vector_signed_char;
145    #[link_name = "llvm.s390.vstl"] fn vstl(a: vector_signed_char, b: u32, c: *mut u8);
146
147    #[link_name = "llvm.s390.vlrl"] fn vlrl(a: u32, b: *const u8) -> vector_unsigned_char;
148    #[link_name = "llvm.s390.vstrl"] fn vstrl(a: vector_unsigned_char, b: u32, c: *mut u8);
149
150    #[link_name = "llvm.s390.lcbb"] fn lcbb(a: *const u8, b: u32) -> u32;
151    #[link_name = "llvm.s390.vlbb"] fn vlbb(a: *const u8, b: u32) -> MaybeUninit<vector_signed_char>;
152
153    #[link_name = "llvm.s390.vpksh"] fn vpksh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_char;
154    #[link_name = "llvm.s390.vpksf"] fn vpksf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_short;
155    #[link_name = "llvm.s390.vpksg"] fn vpksg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_int;
156
157    #[link_name = "llvm.s390.vpklsh"] fn vpklsh(a: vector_signed_short, b: vector_signed_short) -> vector_unsigned_char;
158    #[link_name = "llvm.s390.vpklsf"] fn vpklsf(a: vector_signed_int, b: vector_signed_int) -> vector_unsigned_short;
159    #[link_name = "llvm.s390.vpklsg"] fn vpklsg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_unsigned_int;
160
161    #[link_name = "llvm.s390.vpkshs"] fn vpkshs(a: vector_signed_short, b: vector_signed_short) -> PackedTuple<vector_signed_char, i32>;
162    #[link_name = "llvm.s390.vpksfs"] fn vpksfs(a: vector_signed_int, b: vector_signed_int) -> PackedTuple<vector_signed_short, i32>;
163    #[link_name = "llvm.s390.vpksgs"] fn vpksgs(a: vector_signed_long_long, b: vector_signed_long_long) -> PackedTuple<vector_signed_int, i32>;
164
165    #[link_name = "llvm.s390.vpklshs"] fn vpklshs(a: vector_unsigned_short, b: vector_unsigned_short) -> PackedTuple<vector_unsigned_char, i32>;
166    #[link_name = "llvm.s390.vpklsfs"] fn vpklsfs(a: vector_unsigned_int, b: vector_unsigned_int) -> PackedTuple<vector_unsigned_short, i32>;
167    #[link_name = "llvm.s390.vpklsgs"] fn vpklsgs(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> PackedTuple<vector_unsigned_int, i32>;
168
169    #[link_name = "llvm.s390.vuplb"] fn vuplb (a: vector_signed_char) -> vector_signed_short;
170    #[link_name = "llvm.s390.vuplhw"] fn vuplhw (a: vector_signed_short) -> vector_signed_int;
171    #[link_name = "llvm.s390.vuplf"] fn vuplf (a: vector_signed_int) -> vector_signed_long_long;
172    #[link_name = "llvm.s390.vupllb"] fn vupllb (a: vector_unsigned_char) -> vector_unsigned_short;
173    #[link_name = "llvm.s390.vupllh"] fn vupllh (a: vector_unsigned_short) -> vector_unsigned_int;
174    #[link_name = "llvm.s390.vupllf"] fn vupllf (a: vector_unsigned_int) -> vector_unsigned_long_long;
175
176    #[link_name = "llvm.s390.vavgb"] fn vavgb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
177    #[link_name = "llvm.s390.vavgh"] fn vavgh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short;
178    #[link_name = "llvm.s390.vavgf"] fn vavgf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int;
179    #[link_name = "llvm.s390.vavgg"] fn vavgg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long;
180
181    #[link_name = "llvm.s390.vavglb"] fn vavglb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
182    #[link_name = "llvm.s390.vavglh"] fn vavglh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
183    #[link_name = "llvm.s390.vavglf"] fn vavglf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
184    #[link_name = "llvm.s390.vavglg"] fn vavglg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long;
185
186    #[link_name = "llvm.s390.vcksm"] fn vcksm(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
187
188    #[link_name = "llvm.s390.vmeb"] fn vmeb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_short;
189    #[link_name = "llvm.s390.vmeh"] fn vmeh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_int;
190    #[link_name = "llvm.s390.vmef"] fn vmef(a: vector_signed_int, b: vector_signed_int) -> vector_signed_long_long;
191
192    #[link_name = "llvm.s390.vmleb"] fn vmleb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short;
193    #[link_name = "llvm.s390.vmleh"] fn vmleh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int;
194    #[link_name = "llvm.s390.vmlef"] fn vmlef(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long;
195
196    #[link_name = "llvm.s390.vmob"] fn vmob(a: vector_signed_char, b: vector_signed_char) -> vector_signed_short;
197    #[link_name = "llvm.s390.vmoh"] fn vmoh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_int;
198    #[link_name = "llvm.s390.vmof"] fn vmof(a: vector_signed_int, b: vector_signed_int) -> vector_signed_long_long;
199
200    #[link_name = "llvm.s390.vmlob"] fn vmlob(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short;
201    #[link_name = "llvm.s390.vmloh"] fn vmloh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int;
202    #[link_name = "llvm.s390.vmlof"] fn vmlof(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long;
203
204    #[link_name = "llvm.s390.vmhb"] fn vmhb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char;
205    #[link_name = "llvm.s390.vmhh"] fn vmhh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short;
206    #[link_name = "llvm.s390.vmhf"] fn vmhf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int;
207
208    #[link_name = "llvm.s390.vmlhb"] fn vmlhb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char;
209    #[link_name = "llvm.s390.vmlhh"] fn vmlhh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short;
210    #[link_name = "llvm.s390.vmlhf"] fn vmlhf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int;
211
212    #[link_name = "llvm.s390.vmaeb"] fn vmaeb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short;
213    #[link_name = "llvm.s390.vmaeh"] fn vmaeh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int;
214    #[link_name = "llvm.s390.vmaef"] fn vmaef(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long;
215
216    #[link_name = "llvm.s390.vmaleb"] fn vmaleb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short;
217    #[link_name = "llvm.s390.vmaleh"] fn vmaleh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int;
218    #[link_name = "llvm.s390.vmalef"] fn vmalef(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long;
219
220    #[link_name = "llvm.s390.vmaob"] fn vmaob(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short;
221    #[link_name = "llvm.s390.vmaoh"] fn vmaoh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int;
222    #[link_name = "llvm.s390.vmaof"] fn vmaof(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long;
223
224    #[link_name = "llvm.s390.vmalob"] fn vmalob(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short;
225    #[link_name = "llvm.s390.vmaloh"] fn vmaloh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int;
226    #[link_name = "llvm.s390.vmalof"] fn vmalof(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long;
227
228    #[link_name = "llvm.s390.vmahb"] fn vmahb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char;
229    #[link_name = "llvm.s390.vmahh"] fn vmahh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short;
230    #[link_name = "llvm.s390.vmahf"] fn vmahf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int;
231
232    #[link_name = "llvm.s390.vmalhb"] fn vmalhb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char;
233    #[link_name = "llvm.s390.vmalhh"] fn vmalhh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short;
234    #[link_name = "llvm.s390.vmalhf"] fn vmalhf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int;
235
236    #[link_name = "llvm.s390.vmalb"] fn vmalb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char;
237    #[link_name = "llvm.s390.vmalh"] fn vmalh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short;
238    #[link_name = "llvm.s390.vmalf"] fn vmalf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int;
239
240    #[link_name = "llvm.s390.vmallb"] fn vmallb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char;
241    #[link_name = "llvm.s390.vmallh"] fn vmallh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short;
242    #[link_name = "llvm.s390.vmallf"] fn vmallf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int;
243
244    #[link_name = "llvm.s390.vgfmb"] fn vgfmb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short;
245    #[link_name = "llvm.s390.vgfmh"] fn vgfmh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int;
246    #[link_name = "llvm.s390.vgfmf"] fn vgfmf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long;
247    #[link_name = "llvm.s390.vgfmg"] fn vgfmg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> u128;
248
249    #[link_name = "llvm.s390.vgfmab"] fn vgfmab(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short;
250    #[link_name = "llvm.s390.vgfmah"] fn vgfmah(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int;
251    #[link_name = "llvm.s390.vgfmaf"] fn vgfmaf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long;
252    #[link_name = "llvm.s390.vgfmag"] fn vgfmag(a: vector_unsigned_long_long, b: vector_unsigned_long_long, c: u128) -> u128;
253
254    #[link_name = "llvm.s390.vbperm"] fn vbperm(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_long_long;
255
256    #[link_name = "llvm.s390.vftcisb"] fn vftcisb(a: vector_float, b: u32) -> PackedTuple<vector_bool_int, i32>;
257    #[link_name = "llvm.s390.vftcidb"] fn vftcidb(a: vector_double, b: u32) -> PackedTuple<vector_bool_long_long, i32>;
258
259    #[link_name = "llvm.s390.vtm"] fn vtm(a: i8x16, b: i8x16) -> i32;
260
261    #[link_name = "llvm.s390.vstrsb"] fn vstrsb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
262    #[link_name = "llvm.s390.vstrsh"] fn vstrsh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
263    #[link_name = "llvm.s390.vstrsf"] fn vstrsf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
264
265    #[link_name = "llvm.s390.vstrszb"] fn vstrszb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
266    #[link_name = "llvm.s390.vstrszh"] fn vstrszh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
267    #[link_name = "llvm.s390.vstrszf"] fn vstrszf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
268
269    #[link_name = "llvm.s390.vistrb"] fn vistrb(a: vector_unsigned_char) -> vector_unsigned_char;
270    #[link_name = "llvm.s390.vistrh"] fn vistrh(a: vector_unsigned_short) -> vector_unsigned_short;
271    #[link_name = "llvm.s390.vistrf"] fn vistrf(a: vector_unsigned_int) -> vector_unsigned_int;
272
273    #[link_name = "llvm.s390.vistrbs"] fn vistrbs(a: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32>;
274    #[link_name = "llvm.s390.vistrhs"] fn vistrhs(a: vector_unsigned_short) -> PackedTuple<vector_unsigned_short, i32>;
275    #[link_name = "llvm.s390.vistrfs"] fn vistrfs(a: vector_unsigned_int) -> PackedTuple<vector_unsigned_int, i32>;
276
277    #[link_name = "llvm.s390.vmslg"] fn vmslg(a: vector_unsigned_long_long, b: vector_unsigned_long_long, c: u128, d: u32) -> u128;
278
279    #[link_name = "llvm.s390.vstrcb"] fn vstrcb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> vector_bool_char;
280    #[link_name = "llvm.s390.vstrch"] fn vstrch(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> vector_bool_short;
281    #[link_name = "llvm.s390.vstrcf"] fn vstrcf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> vector_bool_int;
282
283    #[link_name = "llvm.s390.vstrcbs"] fn vstrcbs(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> PackedTuple<vector_bool_char, i32>;
284    #[link_name = "llvm.s390.vstrchs"] fn vstrchs(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> PackedTuple<vector_bool_short, i32>;
285    #[link_name = "llvm.s390.vstrcfs"] fn vstrcfs(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> PackedTuple<vector_bool_int, i32>;
286
287    #[link_name = "llvm.s390.vstrczb"] fn vstrczb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> vector_bool_char;
288    #[link_name = "llvm.s390.vstrczh"] fn vstrczh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> vector_bool_short;
289    #[link_name = "llvm.s390.vstrczf"] fn vstrczf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> vector_bool_int;
290
291    #[link_name = "llvm.s390.vstrczbs"] fn vstrczbs(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char, d: u32) -> PackedTuple<vector_bool_char, i32>;
292    #[link_name = "llvm.s390.vstrczhs"] fn vstrczhs(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short, d: u32) -> PackedTuple<vector_bool_short, i32>;
293    #[link_name = "llvm.s390.vstrczfs"] fn vstrczfs(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int, d: u32) -> PackedTuple<vector_bool_int, i32>;
294
295    #[link_name = "llvm.s390.vfeeb"] fn vfeeb(a: i8x16, b: i8x16) -> i8x16;
296    #[link_name = "llvm.s390.vfeeh"] fn vfeeh(a: i16x8, b: i16x8) -> i16x8;
297    #[link_name = "llvm.s390.vfeef"] fn vfeef(a: i32x4, b: i32x4) -> i32x4;
298
299    #[link_name = "llvm.s390.vfeezb"] fn vfeezb(a: i8x16, b: i8x16) -> i8x16;
300    #[link_name = "llvm.s390.vfeezh"] fn vfeezh(a: i16x8, b: i16x8) -> i16x8;
301    #[link_name = "llvm.s390.vfeezf"] fn vfeezf(a: i32x4, b: i32x4) -> i32x4;
302
303    #[link_name = "llvm.s390.vfeebs"] fn vfeebs(a: i8x16, b: i8x16) -> PackedTuple<i8x16, i32>;
304    #[link_name = "llvm.s390.vfeehs"] fn vfeehs(a: i16x8, b: i16x8) -> PackedTuple<i16x8, i32>;
305    #[link_name = "llvm.s390.vfeefs"] fn vfeefs(a: i32x4, b: i32x4) -> PackedTuple<i32x4, i32>;
306
307    #[link_name = "llvm.s390.vfeezbs"] fn vfeezbs(a: i8x16, b: i8x16) -> PackedTuple<i8x16, i32>;
308    #[link_name = "llvm.s390.vfeezhs"] fn vfeezhs(a: i16x8, b: i16x8) -> PackedTuple<i16x8, i32>;
309    #[link_name = "llvm.s390.vfeezfs"] fn vfeezfs(a: i32x4, b: i32x4) -> PackedTuple<i32x4, i32>;
310
311    #[link_name = "llvm.s390.vfeneb"] fn vfeneb(a: i8x16, b: i8x16) -> i8x16;
312    #[link_name = "llvm.s390.vfeneh"] fn vfeneh(a: i16x8, b: i16x8) -> i16x8;
313    #[link_name = "llvm.s390.vfenef"] fn vfenef(a: i32x4, b: i32x4) -> i32x4;
314
315    #[link_name = "llvm.s390.vfenezb"] fn vfenezb(a: i8x16, b: i8x16) -> i8x16;
316    #[link_name = "llvm.s390.vfenezh"] fn vfenezh(a: i16x8, b: i16x8) -> i16x8;
317    #[link_name = "llvm.s390.vfenezf"] fn vfenezf(a: i32x4, b: i32x4) -> i32x4;
318
319    #[link_name = "llvm.s390.vfenebs"] fn vfenebs(a: i8x16, b: i8x16) -> PackedTuple<i8x16, i32>;
320    #[link_name = "llvm.s390.vfenehs"] fn vfenehs(a: i16x8, b: i16x8) -> PackedTuple<i16x8, i32>;
321    #[link_name = "llvm.s390.vfenefs"] fn vfenefs(a: i32x4, b: i32x4) -> PackedTuple<i32x4, i32>;
322
323    #[link_name = "llvm.s390.vfenezbs"] fn vfenezbs(a: i8x16, b: i8x16) -> PackedTuple<i8x16, i32>;
324    #[link_name = "llvm.s390.vfenezhs"] fn vfenezhs(a: i16x8, b: i16x8) -> PackedTuple<i16x8, i32>;
325    #[link_name = "llvm.s390.vfenezfs"] fn vfenezfs(a: i32x4, b: i32x4) -> PackedTuple<i32x4, i32>;
326}
327
328impl_from! { i8x16, u8x16,  i16x8, u16x8, i32x4, u32x4, i64x2, u64x2, f32x4, f64x2 }
329
330impl_neg! { i8x16 : 0 }
331impl_neg! { i16x8 : 0 }
332impl_neg! { i32x4 : 0 }
333impl_neg! { i64x2 : 0 }
334impl_neg! { f32x4 : 0f32 }
335impl_neg! { f64x2 : 0f64 }
336
337#[repr(simd)]
338struct ShuffleMask<const N: usize>([u32; N]);
339
340impl<const N: usize> ShuffleMask<N> {
341    const fn reverse() -> Self {
342        let mut index = [0; N];
343        let mut i = 0;
344        while i < N {
345            index[i] = (N - i - 1) as u32;
346            i += 1;
347        }
348        ShuffleMask(index)
349    }
350
351    const fn merge_low() -> Self {
352        let mut mask = [0; N];
353        let mut i = N / 2;
354        let mut index = 0;
355        while index < N {
356            mask[index] = i as u32;
357            mask[index + 1] = (i + N) as u32;
358
359            i += 1;
360            index += 2;
361        }
362        ShuffleMask(mask)
363    }
364
365    const fn merge_high() -> Self {
366        let mut mask = [0; N];
367        let mut i = 0;
368        let mut index = 0;
369        while index < N {
370            mask[index] = i as u32;
371            mask[index + 1] = (i + N) as u32;
372
373            i += 1;
374            index += 2;
375        }
376        ShuffleMask(mask)
377    }
378
379    const fn pack() -> Self {
380        let mut mask = [0; N];
381        let mut i = 1;
382        let mut index = 0;
383        while index < N {
384            mask[index] = i as u32;
385
386            i += 2;
387            index += 1;
388        }
389        ShuffleMask(mask)
390    }
391
392    const fn unpack_low() -> Self {
393        let mut mask = [0; N];
394        let mut i = 0;
395        while i < N {
396            mask[i] = (N + i) as u32;
397            i += 1;
398        }
399        ShuffleMask(mask)
400    }
401
402    const fn unpack_high() -> Self {
403        let mut mask = [0; N];
404        let mut i = 0;
405        while i < N {
406            mask[i] = i as u32;
407            i += 1;
408        }
409        ShuffleMask(mask)
410    }
411}
412
413const fn genmask<const MASK: u16>() -> [u8; 16] {
414    let mut bits = MASK;
415    let mut elements = [0u8; 16];
416
417    let mut i = 0;
418    while i < 16 {
419        elements[i] = match bits & (1u16 << 15) {
420            0 => 0,
421            _ => 0xFF,
422        };
423
424        bits <<= 1;
425        i += 1;
426    }
427
428    elements
429}
430
431const fn genmasks(bit_width: u32, a: u8, b: u8) -> u64 {
432    let bit_width = bit_width as u8;
433    let a = a % bit_width;
434    let mut b = b % bit_width;
435    if a > b {
436        b = bit_width - 1;
437    }
438
439    // of course these indices start from the left
440    let a = (bit_width - 1) - a;
441    let b = (bit_width - 1) - b;
442
443    ((1u64.wrapping_shl(a as u32 + 1)) - 1) & !((1u64.wrapping_shl(b as u32)) - 1)
444}
445
446const fn validate_block_boundary(block_boundary: u16) -> u32 {
447    assert!(
448        block_boundary.is_power_of_two() && block_boundary >= 64 && block_boundary <= 4096,
449        "block boundary must be a constant power of 2 from 64 to 4096",
450    );
451
452    // so that 64 is encoded as 0, 128 as 1, ect.
453    block_boundary as u32 >> 7
454}
455
456enum FindImm {
457    Eq = 4,
458    Ne = 12,
459    EqIdx = 0,
460    NeIdx = 8,
461}
462
463#[macro_use]
464mod sealed {
465    use super::*;
466
467    #[unstable(feature = "stdarch_s390x", issue = "135681")]
468    pub trait VectorAdd<Other> {
469        type Result;
470        unsafe fn vec_add(self, other: Other) -> Self::Result;
471    }
472
473    macro_rules! impl_add {
474        ($name:ident, $a:ty, $instr:ident) => {
475            impl_add!($name, $a, $a, $a, $instr);
476        };
477        ($name:ident, $a:ty, $b:ty, $c:ty, $instr:ident) => {
478            #[inline]
479            #[target_feature(enable = "vector")]
480            #[cfg_attr(test, assert_instr($instr))]
481            pub unsafe fn $name(a: $a, b: $b) -> $c {
482                transmute(simd_add(transmute(a), b))
483            }
484
485            #[unstable(feature = "stdarch_s390x", issue = "135681")]
486            impl VectorAdd<$b> for $a {
487                type Result = $c;
488
489                #[inline]
490                #[target_feature(enable = "vector")]
491                unsafe fn vec_add(self, other: $b) -> Self::Result {
492                    $name(self, other)
493                }
494            }
495        };
496    }
497
498    #[rustfmt::skip]
499    mod impl_add {
500        use super::*;
501
502        impl_add!(va_sc, vector_signed_char, vab);
503        impl_add!(va_uc, vector_unsigned_char, vab);
504        impl_add!(va_sh, vector_signed_short, vah);
505        impl_add!(va_uh, vector_unsigned_short, vah);
506        impl_add!(va_sf, vector_signed_int, vaf);
507        impl_add!(va_uf, vector_unsigned_int, vaf);
508        impl_add!(va_sg, vector_signed_long_long, vag);
509        impl_add!(va_ug, vector_unsigned_long_long, vag);
510
511        impl_add!(va_sc_bc, vector_signed_char, vector_bool_char, vector_signed_char, vab);
512        impl_add!(va_uc_bc, vector_unsigned_char, vector_bool_char, vector_unsigned_char, vab);
513        impl_add!(va_sh_bh, vector_signed_short, vector_bool_short, vector_signed_short, vah);
514        impl_add!(va_uh_bh, vector_unsigned_short, vector_bool_short, vector_unsigned_short, vah);
515        impl_add!(va_sf_bf, vector_signed_int, vector_bool_int, vector_signed_int, vaf);
516        impl_add!(va_uf_bf, vector_unsigned_int, vector_bool_int, vector_unsigned_int, vaf);
517        impl_add!(va_sg_bg, vector_signed_long_long, vector_bool_long_long, vector_signed_long_long, vag);
518        impl_add!(va_ug_bg, vector_unsigned_long_long, vector_bool_long_long, vector_unsigned_long_long, vag);
519
520        impl_add!(va_bc_sc, vector_bool_char, vector_signed_char, vector_signed_char, vab);
521        impl_add!(va_bc_uc, vector_bool_char, vector_unsigned_char, vector_unsigned_char, vab);
522        impl_add!(va_bh_sh, vector_bool_short, vector_signed_short, vector_signed_short, vah);
523        impl_add!(va_bh_uh, vector_bool_short, vector_unsigned_short, vector_unsigned_short, vah);
524        impl_add!(va_bf_sf, vector_bool_int, vector_signed_int, vector_signed_int, vaf);
525        impl_add!(va_bf_uf, vector_bool_int, vector_unsigned_int, vector_unsigned_int, vaf);
526        impl_add!(va_bg_sg, vector_bool_long_long, vector_signed_long_long, vector_signed_long_long, vag);
527        impl_add!(va_bg_ug, vector_bool_long_long, vector_unsigned_long_long, vector_unsigned_long_long, vag);
528
529        impl_add!(va_double, vector_double, vfadb);
530
531        #[inline]
532        #[target_feature(enable = "vector")]
533        #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vfasb))]
534        pub unsafe fn va_float(a: vector_float, b: vector_float) -> vector_float {
535            transmute(simd_add(a, b))
536        }
537
538        #[unstable(feature = "stdarch_s390x", issue = "135681")]
539        impl VectorAdd<Self> for vector_float {
540            type Result = Self;
541
542            #[inline]
543            #[target_feature(enable = "vector")]
544            unsafe fn vec_add(self, other: Self) -> Self::Result {
545                va_float(self, other)
546            }
547        }
548    }
549
550    #[unstable(feature = "stdarch_s390x", issue = "135681")]
551    pub trait VectorSub<Other> {
552        type Result;
553        unsafe fn vec_sub(self, other: Other) -> Self::Result;
554    }
555
556    macro_rules! impl_sub {
557        ($name:ident, $a:ty, $instr:ident) => {
558            impl_sub!($name, $a, $a, $a, $instr);
559        };
560        ($name:ident, $a:ty, $b:ty, $c:ty, $instr:ident) => {
561            #[inline]
562            #[target_feature(enable = "vector")]
563            #[cfg_attr(test, assert_instr($instr))]
564            pub unsafe fn $name(a: $a, b: $b) -> $c {
565                transmute(simd_sub(transmute(a), b))
566            }
567
568            #[unstable(feature = "stdarch_s390x", issue = "135681")]
569            impl VectorSub<$b> for $a {
570                type Result = $c;
571
572                #[inline]
573                #[target_feature(enable = "vector")]
574                unsafe fn vec_sub(self, other: $b) -> Self::Result {
575                    $name(self, other)
576                }
577            }
578        };
579    }
580
581    #[rustfmt::skip]
582    mod impl_sub {
583        use super::*;
584
585        impl_sub!(vs_sc, vector_signed_char, vsb);
586        impl_sub!(vs_uc, vector_unsigned_char, vsb);
587        impl_sub!(vs_sh, vector_signed_short, vsh);
588        impl_sub!(vs_uh, vector_unsigned_short, vsh);
589        impl_sub!(vs_sf, vector_signed_int, vsf);
590        impl_sub!(vs_uf, vector_unsigned_int, vsf);
591        impl_sub!(vs_sg, vector_signed_long_long, vsg);
592        impl_sub!(vs_ug, vector_unsigned_long_long, vsg);
593
594        impl_sub!(vs_sc_bc, vector_signed_char, vector_bool_char, vector_signed_char, vsb);
595        impl_sub!(vs_uc_bc, vector_unsigned_char, vector_bool_char, vector_unsigned_char, vsb);
596        impl_sub!(vs_sh_bh, vector_signed_short, vector_bool_short, vector_signed_short, vsh);
597        impl_sub!(vs_uh_bh, vector_unsigned_short, vector_bool_short, vector_unsigned_short, vsh);
598        impl_sub!(vs_sf_bf, vector_signed_int, vector_bool_int, vector_signed_int, vsf);
599        impl_sub!(vs_uf_bf, vector_unsigned_int, vector_bool_int, vector_unsigned_int, vsf);
600        impl_sub!(vs_sg_bg, vector_signed_long_long, vector_bool_long_long, vector_signed_long_long, vsg);
601        impl_sub!(vs_ug_bg, vector_unsigned_long_long, vector_bool_long_long, vector_unsigned_long_long, vsg);
602
603        impl_sub!(vs_bc_sc, vector_bool_char, vector_signed_char, vector_signed_char, vsb);
604        impl_sub!(vs_bc_uc, vector_bool_char, vector_unsigned_char, vector_unsigned_char, vsb);
605        impl_sub!(vs_bh_sh, vector_bool_short, vector_signed_short, vector_signed_short, vsh);
606        impl_sub!(vs_bh_uh, vector_bool_short, vector_unsigned_short, vector_unsigned_short, vsh);
607        impl_sub!(vs_bf_sf, vector_bool_int, vector_signed_int, vector_signed_int, vsf);
608        impl_sub!(vs_bf_uf, vector_bool_int, vector_unsigned_int, vector_unsigned_int, vsf);
609        impl_sub!(vs_bg_sg, vector_bool_long_long, vector_signed_long_long, vector_signed_long_long, vsg);
610        impl_sub!(vs_bg_ug, vector_bool_long_long, vector_unsigned_long_long, vector_unsigned_long_long, vsg);
611
612        impl_sub!(vs_double, vector_double, vfsdb);
613
614        #[inline]
615        #[target_feature(enable = "vector")]
616        #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vfssb))]
617        pub unsafe fn vs_float(a: vector_float, b: vector_float) -> vector_float {
618            transmute(simd_sub(a, b))
619        }
620
621        #[unstable(feature = "stdarch_s390x", issue = "135681")]
622        impl VectorSub<Self> for vector_float {
623            type Result = Self;
624
625            #[inline]
626            #[target_feature(enable = "vector")]
627            unsafe fn vec_sub(self, other: Self) -> Self::Result {
628                vs_float(self, other)
629            }
630        }
631    }
632
633    #[unstable(feature = "stdarch_s390x", issue = "135681")]
634    pub trait VectorMul {
635        unsafe fn vec_mul(self, b: Self) -> Self;
636    }
637
638    macro_rules! impl_mul {
639        ($name:ident, $a:ty, std_simd) => {
640            #[unstable(feature = "stdarch_s390x", issue = "135681")]
641            impl VectorMul for $a {
642                #[inline]
643                #[target_feature(enable = "vector")]
644                unsafe fn vec_mul(self, other: Self) -> Self {
645                    transmute(simd_mul(transmute(self), other))
646                }
647            }
648        };
649        ($name:ident, $a:ty, $instr:ident) => {
650            #[inline]
651            #[target_feature(enable = "vector")]
652            #[cfg_attr(test, assert_instr($instr))]
653            pub unsafe fn $name(a: $a, b: $a) -> $a {
654                transmute(simd_mul(transmute(a), b))
655            }
656
657            #[unstable(feature = "stdarch_s390x", issue = "135681")]
658            impl VectorMul for $a {
659                #[inline]
660                #[target_feature(enable = "vector")]
661                unsafe fn vec_mul(self, other: Self) -> Self {
662                    $name(self, other)
663                }
664            }
665        };
666    }
667
668    #[rustfmt::skip]
669    mod impl_mul {
670        use super::*;
671
672        impl_mul!(vml_sc, vector_signed_char, vmlb);
673        impl_mul!(vml_uc, vector_unsigned_char, vmlb);
674        impl_mul!(vml_sh, vector_signed_short, vmlhw);
675        impl_mul!(vml_uh, vector_unsigned_short, vmlhw);
676        impl_mul!(vml_sf, vector_signed_int, vmlf);
677        impl_mul!(vml_uf, vector_unsigned_int, vmlf);
678        impl_mul!(vml_sg, vector_signed_long_long, std_simd);
679        impl_mul!(vml_ug, vector_unsigned_long_long, std_simd);
680
681        impl_mul!(vml_float, vector_float, std_simd);
682        impl_mul!(vml_double, vector_double, vfmdb);
683    }
684
685    #[unstable(feature = "stdarch_s390x", issue = "135681")]
686    pub trait VectorMax<Other> {
687        type Result;
688        unsafe fn vec_max(self, b: Other) -> Self::Result;
689    }
690
691    test_impl! { vec_vmxsb (a: vector_signed_char, b: vector_signed_char) -> vector_signed_char [vmxb, vmxb] }
692    test_impl! { vec_vmxsh (a: vector_signed_short, b: vector_signed_short) -> vector_signed_short [vmxh, vmxh] }
693    test_impl! { vec_vmxsf (a: vector_signed_int, b: vector_signed_int) -> vector_signed_int [vmxf, vmxf] }
694    test_impl! { vec_vmxsg (a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long [vmxg, vmxg] }
695
696    test_impl! { vec_vmxslb (a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [vmxlb, vmxlb] }
697    test_impl! { vec_vmxslh (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [vmxlh, vmxlh] }
698    test_impl! { vec_vmxslf (a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [vmxlf, vmxlf] }
699    test_impl! { vec_vmxslg (a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long [vmxlg, vmxlg] }
700
701    impl_vec_trait! { [VectorMax vec_max] ~(vmxlb, vmxb, vmxlh, vmxh, vmxlf, vmxf, vmxlg, vmxg) }
702
703    test_impl! { vec_vfmaxsb (a: vector_float, b: vector_float) -> vector_float [simd_fmax, "vector-enhancements-1" vfmaxsb ] }
704    test_impl! { vec_vfmaxdb (a: vector_double, b: vector_double) -> vector_double [simd_fmax, "vector-enhancements-1" vfmaxdb] }
705
706    impl_vec_trait!([VectorMax vec_max] vec_vfmaxsb (vector_float, vector_float) -> vector_float);
707    impl_vec_trait!([VectorMax vec_max] vec_vfmaxdb (vector_double, vector_double) -> vector_double);
708
709    #[unstable(feature = "stdarch_s390x", issue = "135681")]
710    pub trait VectorMin<Other> {
711        type Result;
712        unsafe fn vec_min(self, b: Other) -> Self::Result;
713    }
714
715    test_impl! { vec_vmnsb (a: vector_signed_char, b: vector_signed_char) -> vector_signed_char [vmnb, vmnb] }
716    test_impl! { vec_vmnsh (a: vector_signed_short, b: vector_signed_short) -> vector_signed_short [vmnh, vmnh] }
717    test_impl! { vec_vmnsf (a: vector_signed_int, b: vector_signed_int) -> vector_signed_int [vmnf, vmnf] }
718    test_impl! { vec_vmnsg (a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long [vmng, vmng] }
719
720    test_impl! { vec_vmnslb (a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [vmnlb, vmnlb] }
721    test_impl! { vec_vmnslh (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [vmnlh, vmnlh] }
722    test_impl! { vec_vmnslf (a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [vmnlf, vmnlf] }
723    test_impl! { vec_vmnslg (a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long [vmnlg, vmnlg] }
724
725    impl_vec_trait! { [VectorMin vec_min] ~(vmxlb, vmxb, vmxlh, vmxh, vmxlf, vmxf, vmxlg, vmxg) }
726
727    test_impl! { vec_vfminsb (a: vector_float, b: vector_float) -> vector_float [simd_fmin, "vector-enhancements-1" vfminsb]  }
728    test_impl! { vec_vfmindb (a: vector_double, b: vector_double) -> vector_double [simd_fmin, "vector-enhancements-1" vfmindb]  }
729
730    impl_vec_trait!([VectorMin vec_min] vec_vfminsb (vector_float, vector_float) -> vector_float);
731    impl_vec_trait!([VectorMin vec_min] vec_vfmindb (vector_double, vector_double) -> vector_double);
732
733    #[unstable(feature = "stdarch_s390x", issue = "135681")]
734    pub trait VectorAbs {
735        unsafe fn vec_abs(self) -> Self;
736    }
737
738    macro_rules! impl_abs {
739        ($name:ident, $ty:ident) => {
740            #[inline]
741            #[target_feature(enable = "vector")]
742            unsafe fn $name(v: s_t_l!($ty)) -> s_t_l!($ty) {
743                v.vec_max(-v)
744            }
745
746            impl_vec_trait! { [VectorAbs vec_abs] $name (s_t_l!($ty)) }
747        };
748    }
749
750    impl_abs! { vec_abs_i8, i8x16 }
751    impl_abs! { vec_abs_i16, i16x8 }
752    impl_abs! { vec_abs_i32, i32x4 }
753    impl_abs! { vec_abs_i64, i64x2 }
754
755    test_impl! { vec_abs_f32 (v: vector_float) -> vector_float [ simd_fabs, "vector-enhancements-1" vflpsb ] }
756    test_impl! { vec_abs_f64 (v: vector_double) -> vector_double [ simd_fabs, vflpdb ] }
757
758    impl_vec_trait! { [VectorAbs vec_abs] vec_abs_f32 (vector_float) }
759    impl_vec_trait! { [VectorAbs vec_abs] vec_abs_f64 (vector_double) }
760
761    #[unstable(feature = "stdarch_s390x", issue = "135681")]
762    pub trait VectorNabs {
763        unsafe fn vec_nabs(self) -> Self;
764    }
765
766    #[inline]
767    #[target_feature(enable = "vector")]
768    #[cfg_attr(
769        all(test, target_feature = "vector-enhancements-1"),
770        assert_instr(vflnsb)
771    )]
772    unsafe fn vec_nabs_f32(a: vector_float) -> vector_float {
773        simd_neg(simd_fabs(a))
774    }
775
776    #[inline]
777    #[target_feature(enable = "vector")]
778    #[cfg_attr(test, assert_instr(vflndb))]
779    unsafe fn vec_nabs_f64(a: vector_double) -> vector_double {
780        simd_neg(simd_fabs(a))
781    }
782
783    impl_vec_trait! { [VectorNabs vec_nabs] vec_nabs_f32 (vector_float) }
784    impl_vec_trait! { [VectorNabs vec_nabs] vec_nabs_f64 (vector_double) }
785
786    #[unstable(feature = "stdarch_s390x", issue = "135681")]
787    pub trait VectorNmsub {
788        unsafe fn vec_nmsub(self, b: Self, c: Self) -> Self;
789    }
790
791    #[inline]
792    #[target_feature(enable = "vector")]
793    #[cfg_attr(
794        all(test, target_feature = "vector-enhancements-2"),
795        assert_instr(vfnmssb)
796    )]
797    unsafe fn vec_nmsub_f32(a: vector_float, b: vector_float, c: vector_float) -> vector_float {
798        simd_neg(simd_fma(a, b, simd_neg(c)))
799    }
800
801    #[unstable(feature = "stdarch_s390x", issue = "135681")]
802    impl VectorNmsub for vector_float {
803        #[target_feature(enable = "vector")]
804        unsafe fn vec_nmsub(self, b: Self, c: Self) -> Self {
805            vec_nmsub_f32(self, b, c)
806        }
807    }
808
809    #[inline]
810    #[target_feature(enable = "vector")]
811    #[cfg_attr(
812        all(test, target_feature = "vector-enhancements-2"),
813        assert_instr(vfnmsdb)
814    )]
815    unsafe fn vec_nmsub_f64(a: vector_double, b: vector_double, c: vector_double) -> vector_double {
816        simd_neg(simd_fma(a, b, simd_neg(c)))
817    }
818
819    #[unstable(feature = "stdarch_s390x", issue = "135681")]
820    impl VectorNmsub for vector_double {
821        #[target_feature(enable = "vector")]
822        unsafe fn vec_nmsub(self, b: Self, c: Self) -> Self {
823            vec_nmsub_f64(self, b, c)
824        }
825    }
826
827    #[unstable(feature = "stdarch_s390x", issue = "135681")]
828    pub trait VectorNmadd {
829        unsafe fn vec_nmadd(self, b: Self, c: Self) -> Self;
830    }
831
832    #[inline]
833    #[target_feature(enable = "vector")]
834    #[cfg_attr(
835        all(test, target_feature = "vector-enhancements-2"),
836        assert_instr(vfnmasb)
837    )]
838    unsafe fn vec_nmadd_f32(a: vector_float, b: vector_float, c: vector_float) -> vector_float {
839        simd_neg(simd_fma(a, b, c))
840    }
841
842    #[unstable(feature = "stdarch_s390x", issue = "135681")]
843    impl VectorNmadd for vector_float {
844        #[target_feature(enable = "vector")]
845        unsafe fn vec_nmadd(self, b: Self, c: Self) -> Self {
846            vec_nmadd_f32(self, b, c)
847        }
848    }
849
850    #[inline]
851    #[target_feature(enable = "vector")]
852    #[cfg_attr(
853        all(test, target_feature = "vector-enhancements-2"),
854        assert_instr(vfnmadb)
855    )]
856    unsafe fn vec_nmadd_f64(a: vector_double, b: vector_double, c: vector_double) -> vector_double {
857        simd_neg(simd_fma(a, b, c))
858    }
859
860    #[unstable(feature = "stdarch_s390x", issue = "135681")]
861    impl VectorNmadd for vector_double {
862        #[target_feature(enable = "vector")]
863        unsafe fn vec_nmadd(self, b: Self, c: Self) -> Self {
864            vec_nmadd_f64(self, b, c)
865        }
866    }
867
868    #[unstable(feature = "stdarch_s390x", issue = "135681")]
869    pub trait VectorSplat {
870        unsafe fn vec_splat<const IMM: u32>(self) -> Self;
871    }
872
873    #[inline]
874    #[target_feature(enable = "vector")]
875    #[cfg_attr(test, assert_instr(vrepb, IMM2 = 1))]
876    unsafe fn vrepb<const IMM2: u32>(a: vector_signed_char) -> vector_signed_char {
877        static_assert_uimm_bits!(IMM2, 4);
878        simd_shuffle(a, a, const { u32x16::from_array([IMM2; 16]) })
879    }
880
881    #[inline]
882    #[target_feature(enable = "vector")]
883    #[cfg_attr(test, assert_instr(vreph, IMM2 = 1))]
884    unsafe fn vreph<const IMM2: u32>(a: vector_signed_short) -> vector_signed_short {
885        static_assert_uimm_bits!(IMM2, 3);
886        simd_shuffle(a, a, const { u32x8::from_array([IMM2; 8]) })
887    }
888
889    #[inline]
890    #[target_feature(enable = "vector")]
891    #[cfg_attr(test, assert_instr(vrepf, IMM2 = 1))]
892    unsafe fn vrepf<const IMM2: u32>(a: vector_signed_int) -> vector_signed_int {
893        static_assert_uimm_bits!(IMM2, 2);
894        simd_shuffle(a, a, const { u32x4::from_array([IMM2; 4]) })
895    }
896
897    #[inline]
898    #[target_feature(enable = "vector")]
899    #[cfg_attr(test, assert_instr(vrepg, IMM2 = 1))]
900    unsafe fn vrepg<const IMM2: u32>(a: vector_signed_long_long) -> vector_signed_long_long {
901        static_assert_uimm_bits!(IMM2, 1);
902        simd_shuffle(a, a, const { u32x2::from_array([IMM2; 2]) })
903    }
904
905    macro_rules! impl_vec_splat {
906        ($ty:ty, $fun:ident) => {
907            #[unstable(feature = "stdarch_s390x", issue = "135681")]
908            impl VectorSplat for $ty {
909                #[inline]
910                #[target_feature(enable = "vector")]
911                unsafe fn vec_splat<const IMM: u32>(self) -> Self {
912                    transmute($fun::<IMM>(transmute(self)))
913                }
914            }
915        };
916    }
917
918    impl_vec_splat! { vector_signed_char, vrepb }
919    impl_vec_splat! { vector_unsigned_char, vrepb }
920    impl_vec_splat! { vector_bool_char, vrepb }
921    impl_vec_splat! { vector_signed_short, vreph }
922    impl_vec_splat! { vector_unsigned_short, vreph }
923    impl_vec_splat! { vector_bool_short, vreph }
924    impl_vec_splat! { vector_signed_int, vrepf }
925    impl_vec_splat! { vector_unsigned_int, vrepf }
926    impl_vec_splat! { vector_bool_int, vrepf }
927    impl_vec_splat! { vector_signed_long_long, vrepg }
928    impl_vec_splat! { vector_unsigned_long_long, vrepg }
929    impl_vec_splat! { vector_bool_long_long, vrepg }
930
931    impl_vec_splat! { vector_float, vrepf }
932    impl_vec_splat! { vector_double, vrepg }
933
934    #[unstable(feature = "stdarch_s390x", issue = "135681")]
935    pub trait VectorSplats<Output> {
936        unsafe fn vec_splats(self) -> Output;
937    }
938
939    macro_rules! impl_vec_splats {
940        ($(($fn:ident ($ty:ty, $shortty:tt) $instr:ident)),*) => {
941            $(
942                #[inline]
943                #[target_feature(enable = "vector")]
944                #[cfg_attr(test, assert_instr($instr))]
945                pub unsafe fn $fn(v: $ty) -> s_t_l!($shortty) {
946                    transmute($shortty::splat(v))
947                }
948
949                #[unstable(feature = "stdarch_s390x", issue = "135681")]
950                impl VectorSplats<s_t_l!($shortty)> for $ty {
951                    #[inline]
952                    #[target_feature(enable = "vector")]
953                    unsafe fn vec_splats(self) -> s_t_l!($shortty) {
954                        $fn (self)
955                    }
956                }
957            )*
958        }
959    }
960
961    impl_vec_splats! {
962        (vec_splats_u8 (u8, u8x16) vrepb),
963        (vec_splats_i8 (i8, i8x16) vrepb),
964        (vec_splats_u16 (u16, u16x8) vreph),
965        (vec_splats_i16 (i16, i16x8) vreph),
966        (vec_splats_u32 (u32, u32x4) vrepf),
967        (vec_splats_i32 (i32, i32x4) vrepf),
968        (vec_splats_u64 (u64, u64x2) vlvgp),
969        (vec_splats_i64 (i64, i64x2) vlvgp),
970        (vec_splats_f32 (f32, f32x4) vrepf),
971        (vec_splats_f64 (f64, f64x2) vrepg)
972    }
973
974    macro_rules! impl_bool_vec_splats {
975        ($(($ty:ty, $shortty:tt, $boolty:ty)),*) => {
976            $(
977                #[unstable(feature = "stdarch_s390x", issue = "135681")]
978                impl VectorSplats<$boolty> for $ty {
979                    #[inline]
980                    #[target_feature(enable = "vector")]
981                    unsafe fn vec_splats(self) -> $boolty {
982                        transmute($shortty::splat(self))
983                    }
984                }
985            )*
986        }
987    }
988
989    impl_bool_vec_splats! {
990        (u8, u8x16, vector_bool_char),
991        (i8, i8x16, vector_bool_char),
992        (u16, u16x8, vector_bool_short),
993        (i16, i16x8, vector_bool_short),
994        (u32, u32x4, vector_bool_int),
995        (i32, i32x4, vector_bool_int),
996        (u64, u64x2, vector_bool_long_long),
997        (i64, i64x2, vector_bool_long_long)
998    }
999
1000    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1001    pub trait CountBits {
1002        type Result;
1003
1004        unsafe fn vec_cntlz(self) -> Self::Result;
1005        unsafe fn vec_cnttz(self) -> Self::Result;
1006        unsafe fn vec_popcnt(self) -> Self::Result;
1007    }
1008
1009    macro_rules! impl_count_bits {
1010        ($ty:tt) => {
1011            #[unstable(feature = "stdarch_s390x", issue = "135681")]
1012            impl CountBits for $ty {
1013                type Result = t_u!($ty);
1014
1015                #[inline]
1016                #[target_feature(enable = "vector")]
1017                unsafe fn vec_cntlz(self) -> Self::Result {
1018                    transmute(simd_ctlz(self))
1019                }
1020
1021                #[inline]
1022                #[target_feature(enable = "vector")]
1023                unsafe fn vec_cnttz(self) -> Self::Result {
1024                    transmute(simd_cttz(self))
1025                }
1026
1027                #[inline]
1028                #[target_feature(enable = "vector")]
1029                unsafe fn vec_popcnt(self) -> Self::Result {
1030                    transmute(simd_ctpop(self))
1031                }
1032            }
1033        };
1034    }
1035
1036    impl_count_bits!(vector_signed_char);
1037    impl_count_bits!(vector_unsigned_char);
1038    impl_count_bits!(vector_signed_short);
1039    impl_count_bits!(vector_unsigned_short);
1040    impl_count_bits!(vector_signed_int);
1041    impl_count_bits!(vector_unsigned_int);
1042    impl_count_bits!(vector_signed_long_long);
1043    impl_count_bits!(vector_unsigned_long_long);
1044
1045    test_impl! { vec_clzb_signed +(a: vector_signed_char) -> vector_unsigned_char [simd_ctlz, vclzb] }
1046    test_impl! { vec_clzh_signed +(a: vector_signed_short) -> vector_unsigned_short [simd_ctlz, vclzh] }
1047    test_impl! { vec_clzf_signed +(a: vector_signed_int) -> vector_unsigned_int [simd_ctlz, vclzf] }
1048    test_impl! { vec_clzg_signed +(a: vector_signed_long_long) -> vector_unsigned_long_long [simd_ctlz, vclzg] }
1049
1050    test_impl! { vec_clzb_unsigned +(a: vector_unsigned_char) -> vector_unsigned_char [simd_ctlz, vclzb] }
1051    test_impl! { vec_clzh_unsigned +(a: vector_unsigned_short) -> vector_unsigned_short [simd_ctlz, vclzh] }
1052    test_impl! { vec_clzf_unsigned +(a: vector_unsigned_int) -> vector_unsigned_int [simd_ctlz, vclzf] }
1053    test_impl! { vec_clzg_unsigned +(a: vector_unsigned_long_long) -> vector_unsigned_long_long [simd_ctlz, vclzg] }
1054
1055    test_impl! { vec_ctzb_signed +(a: vector_signed_char) -> vector_unsigned_char [simd_cttz, vctzb] }
1056    test_impl! { vec_ctzh_signed +(a: vector_signed_short) -> vector_unsigned_short [simd_cttz, vctzh] }
1057    test_impl! { vec_ctzf_signed +(a: vector_signed_int) -> vector_unsigned_int [simd_cttz, vctzf] }
1058    test_impl! { vec_ctzg_signed +(a: vector_signed_long_long) -> vector_unsigned_long_long [simd_cttz, vctzg] }
1059
1060    test_impl! { vec_ctzb_unsigned +(a: vector_unsigned_char) -> vector_unsigned_char [simd_cttz, vctzb] }
1061    test_impl! { vec_ctzh_unsigned +(a: vector_unsigned_short) -> vector_unsigned_short [simd_cttz, vctzh] }
1062    test_impl! { vec_ctzf_unsigned +(a: vector_unsigned_int) -> vector_unsigned_int [simd_cttz, vctzf] }
1063    test_impl! { vec_ctzg_unsigned +(a: vector_unsigned_long_long) -> vector_unsigned_long_long [simd_cttz, vctzg] }
1064
1065    test_impl! { vec_vpopctb_signed +(a: vector_signed_char) -> vector_signed_char [simd_ctpop, vpopctb] }
1066    test_impl! { vec_vpopcth_signed +(a: vector_signed_short) -> vector_signed_short [simd_ctpop, "vector-enhancements-1" vpopcth] }
1067    test_impl! { vec_vpopctf_signed +(a: vector_signed_int) -> vector_signed_int [simd_ctpop, "vector-enhancements-1" vpopctf] }
1068    test_impl! { vec_vpopctg_signed +(a: vector_signed_long_long) -> vector_signed_long_long [simd_ctpop, "vector-enhancements-1" vpopctg] }
1069
1070    test_impl! { vec_vpopctb_unsigned +(a: vector_unsigned_char) -> vector_unsigned_char [simd_ctpop, vpopctb] }
1071    test_impl! { vec_vpopcth_unsigned +(a: vector_unsigned_short) -> vector_unsigned_short [simd_ctpop, "vector-enhancements-1" vpopcth] }
1072    test_impl! { vec_vpopctf_unsigned +(a: vector_unsigned_int) -> vector_unsigned_int [simd_ctpop, "vector-enhancements-1" vpopctf] }
1073    test_impl! { vec_vpopctg_unsigned +(a: vector_unsigned_long_long) -> vector_unsigned_long_long [simd_ctpop, "vector-enhancements-1" vpopctg] }
1074
1075    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1076    pub trait VectorAnd<Other> {
1077        type Result;
1078        unsafe fn vec_and(self, b: Other) -> Self::Result;
1079    }
1080
1081    impl_vec_trait! { [VectorAnd vec_and] ~(simd_and) }
1082
1083    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1084    pub trait VectorOr<Other> {
1085        type Result;
1086        unsafe fn vec_or(self, b: Other) -> Self::Result;
1087    }
1088
1089    impl_vec_trait! { [VectorOr vec_or] ~(simd_or) }
1090
1091    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1092    pub trait VectorXor<Other> {
1093        type Result;
1094        unsafe fn vec_xor(self, b: Other) -> Self::Result;
1095    }
1096
1097    impl_vec_trait! { [VectorXor vec_xor] ~(simd_xor) }
1098
1099    #[inline]
1100    #[target_feature(enable = "vector")]
1101    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vno))]
1102    unsafe fn nor(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1103        let a: u8x16 = transmute(a);
1104        let b: u8x16 = transmute(b);
1105        transmute(simd_xor(simd_or(a, b), u8x16::splat(0xff)))
1106    }
1107
1108    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1109    pub trait VectorNor<Other> {
1110        type Result;
1111        unsafe fn vec_nor(self, b: Other) -> Self::Result;
1112    }
1113
1114    impl_vec_trait! { [VectorNor vec_nor]+ 2c (nor) }
1115
1116    #[inline]
1117    #[target_feature(enable = "vector")]
1118    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vnn))]
1119    unsafe fn nand(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1120        let a: u8x16 = transmute(a);
1121        let b: u8x16 = transmute(b);
1122        transmute(simd_xor(simd_and(a, b), u8x16::splat(0xff)))
1123    }
1124
1125    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1126    pub trait VectorNand<Other> {
1127        type Result;
1128        unsafe fn vec_nand(self, b: Other) -> Self::Result;
1129    }
1130
1131    impl_vec_trait! { [VectorNand vec_nand]+ 2c (nand) }
1132
1133    #[inline]
1134    #[target_feature(enable = "vector")]
1135    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vnx))]
1136    unsafe fn eqv(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1137        let a: u8x16 = transmute(a);
1138        let b: u8x16 = transmute(b);
1139        transmute(simd_xor(simd_xor(a, b), u8x16::splat(0xff)))
1140    }
1141
1142    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1143    pub trait VectorEqv<Other> {
1144        type Result;
1145        unsafe fn vec_eqv(self, b: Other) -> Self::Result;
1146    }
1147
1148    impl_vec_trait! { [VectorEqv vec_eqv]+ 2c (eqv) }
1149
1150    #[inline]
1151    #[target_feature(enable = "vector")]
1152    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vnc))]
1153    unsafe fn andc(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1154        let a = transmute(a);
1155        let b = transmute(b);
1156        transmute(simd_and(simd_xor(u8x16::splat(0xff), b), a))
1157    }
1158
1159    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1160    pub trait VectorAndc<Other> {
1161        type Result;
1162        unsafe fn vec_andc(self, b: Other) -> Self::Result;
1163    }
1164
1165    impl_vec_trait! { [VectorAndc vec_andc]+ 2c (andc) }
1166
1167    #[inline]
1168    #[target_feature(enable = "vector")]
1169    #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(voc))]
1170    unsafe fn orc(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char {
1171        let a = transmute(a);
1172        let b = transmute(b);
1173        transmute(simd_or(simd_xor(u8x16::splat(0xff), b), a))
1174    }
1175
1176    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1177    pub trait VectorOrc<Other> {
1178        type Result;
1179        unsafe fn vec_orc(self, b: Other) -> Self::Result;
1180    }
1181
1182    impl_vec_trait! { [VectorOrc vec_orc]+ 2c (orc) }
1183
1184    test_impl! { vec_roundc_f32 (a: vector_float) -> vector_float [nearbyint_v4f32,  "vector-enhancements-1" vfisb] }
1185    test_impl! { vec_roundc_f64 (a: vector_double) -> vector_double [nearbyint_v2f64, vfidb] }
1186
1187    // FIXME(llvm) llvm trunk already lowers roundeven to vfidb, but rust does not use it yet
1188    // use https://godbolt.org/z/cWq95fexe to check, and enable the instruction test when it works
1189    test_impl! { vec_round_f32 (a: vector_float) -> vector_float [roundeven_v4f32, _] }
1190    test_impl! { vec_round_f64 (a: vector_double) -> vector_double [roundeven_v2f64, _] }
1191
1192    test_impl! { vec_rint_f32 (a: vector_float) -> vector_float [simd_round_ties_even, "vector-enhancements-1" vfisb] }
1193    test_impl! { vec_rint_f64 (a: vector_double) -> vector_double [simd_round_ties_even, vfidb] }
1194
1195    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1196    pub trait VectorRoundc {
1197        unsafe fn vec_roundc(self) -> Self;
1198    }
1199
1200    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1201    pub trait VectorRound {
1202        unsafe fn vec_round(self) -> Self;
1203    }
1204
1205    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1206    pub trait VectorRint {
1207        unsafe fn vec_rint(self) -> Self;
1208    }
1209
1210    impl_vec_trait! { [VectorRoundc vec_roundc] vec_roundc_f32 (vector_float) }
1211    impl_vec_trait! { [VectorRoundc vec_roundc] vec_roundc_f64 (vector_double) }
1212
1213    impl_vec_trait! { [VectorRound vec_round] vec_round_f32 (vector_float) }
1214    impl_vec_trait! { [VectorRound vec_round] vec_round_f64 (vector_double) }
1215
1216    impl_vec_trait! { [VectorRint vec_rint] simd_round_ties_even (vector_float) }
1217    impl_vec_trait! { [VectorRint vec_rint] simd_round_ties_even (vector_double) }
1218
1219    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1220    pub trait VectorTrunc {
1221        // same as vec_roundz
1222        unsafe fn vec_trunc(self) -> Self;
1223    }
1224
1225    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1226    pub trait VectorCeil {
1227        // same as vec_roundp
1228        unsafe fn vec_ceil(self) -> Self;
1229    }
1230
1231    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1232    pub trait VectorFloor {
1233        // same as vec_roundm
1234        unsafe fn vec_floor(self) -> Self;
1235    }
1236
1237    impl_vec_trait! { [VectorTrunc vec_trunc] simd_trunc (vector_float) }
1238    impl_vec_trait! { [VectorTrunc vec_trunc] simd_trunc (vector_double) }
1239
1240    impl_vec_trait! { [VectorCeil vec_ceil] simd_ceil (vector_float) }
1241    impl_vec_trait! { [VectorCeil vec_ceil] simd_ceil (vector_double) }
1242
1243    impl_vec_trait! { [VectorFloor vec_floor] simd_floor (vector_float) }
1244    impl_vec_trait! { [VectorFloor vec_floor] simd_floor (vector_double) }
1245
1246    macro_rules! impl_vec_shift {
1247        ([$Trait:ident $m:ident] ($b:ident, $h:ident, $w:ident, $g:ident)) => {
1248            impl_vec_trait!{ [$Trait $m]+ $b (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char }
1249            impl_vec_trait!{ [$Trait $m]+ $b (vector_signed_char, vector_unsigned_char) -> vector_signed_char }
1250            impl_vec_trait!{ [$Trait $m]+ $h (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short }
1251            impl_vec_trait!{ [$Trait $m]+ $h (vector_signed_short, vector_unsigned_short) -> vector_signed_short }
1252            impl_vec_trait!{ [$Trait $m]+ $w (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int }
1253            impl_vec_trait!{ [$Trait $m]+ $w (vector_signed_int, vector_unsigned_int) -> vector_signed_int }
1254            impl_vec_trait!{ [$Trait $m]+ $g (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_long_long }
1255            impl_vec_trait!{ [$Trait $m]+ $g (vector_signed_long_long, vector_unsigned_long_long) -> vector_signed_long_long }
1256        };
1257    }
1258
1259    macro_rules! impl_shift {
1260        ($fun:ident $intr:ident $ty:ident) => {
1261            #[inline]
1262            #[target_feature(enable = "vector")]
1263            #[cfg_attr(test, assert_instr($fun))]
1264            unsafe fn $fun(a: t_t_l!($ty), b: t_t_l!($ty)) -> t_t_l!($ty) {
1265                let a = transmute(a);
1266                // use the remainder of b by the width of a's elements to prevent UB
1267                let b = simd_rem(transmute(b), <t_t_s!($ty)>::splat($ty::BITS as $ty));
1268
1269                transmute($intr(a, b))
1270            }
1271        };
1272    }
1273
1274    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1275    pub trait VectorSl<Other> {
1276        type Result;
1277        unsafe fn vec_sl(self, b: Other) -> Self::Result;
1278    }
1279
1280    impl_shift! { veslvb simd_shl u8 }
1281    impl_shift! { veslvh simd_shl u16 }
1282    impl_shift! { veslvf simd_shl u32 }
1283    impl_shift! { veslvg simd_shl u64 }
1284
1285    impl_vec_shift! { [VectorSl vec_sl] (veslvb, veslvh, veslvf, veslvg) }
1286
1287    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1288    pub trait VectorSr<Other> {
1289        type Result;
1290        unsafe fn vec_sr(self, b: Other) -> Self::Result;
1291    }
1292
1293    impl_shift! { vesrlvb simd_shr u8 }
1294    impl_shift! { vesrlvh simd_shr u16 }
1295    impl_shift! { vesrlvf simd_shr u32 }
1296    impl_shift! { vesrlvg simd_shr u64 }
1297
1298    impl_vec_shift! { [VectorSr vec_sr] (vesrlvb, vesrlvh, vesrlvf, vesrlvg) }
1299
1300    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1301    pub trait VectorSra<Other> {
1302        type Result;
1303        unsafe fn vec_sra(self, b: Other) -> Self::Result;
1304    }
1305
1306    impl_shift! { vesravb simd_shr i8 }
1307    impl_shift! { vesravh simd_shr i16 }
1308    impl_shift! { vesravf simd_shr i32 }
1309    impl_shift! { vesravg simd_shr i64 }
1310
1311    impl_vec_shift! { [VectorSra vec_sra] (vesravb, vesravh, vesravf, vesravg) }
1312
1313    macro_rules! impl_vec_shift_byte {
1314        ([$trait:ident $m:ident] ($f:ident)) => {
1315            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_char, vector_signed_char) -> vector_unsigned_char }
1316            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char }
1317            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_char, vector_signed_char) -> vector_signed_char }
1318            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_char, vector_unsigned_char) -> vector_signed_char }
1319            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_short, vector_signed_short) -> vector_unsigned_short }
1320            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short }
1321            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_short, vector_signed_short) -> vector_signed_short }
1322            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_short, vector_unsigned_short) -> vector_signed_short }
1323            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_int, vector_signed_int) -> vector_unsigned_int }
1324            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int }
1325            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_int, vector_signed_int) -> vector_signed_int }
1326            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_int, vector_unsigned_int) -> vector_signed_int }
1327            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_long_long, vector_signed_long_long) -> vector_unsigned_long_long }
1328            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_long_long }
1329            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_long_long, vector_signed_long_long) -> vector_signed_long_long }
1330            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_long_long, vector_unsigned_long_long) -> vector_signed_long_long }
1331            impl_vec_trait!{ [$trait $m]+ $f (vector_float, vector_signed_int) -> vector_float }
1332            impl_vec_trait!{ [$trait $m]+ $f (vector_float, vector_unsigned_int) -> vector_float }
1333            impl_vec_trait!{ [$trait $m]+ $f (vector_double, vector_signed_long_long) -> vector_double }
1334            impl_vec_trait!{ [$trait $m]+ $f (vector_double, vector_unsigned_long_long) -> vector_double }
1335        };
1336    }
1337
1338    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1339    pub trait VectorSlb<Other> {
1340        type Result;
1341        unsafe fn vec_slb(self, b: Other) -> Self::Result;
1342    }
1343
1344    impl_vec_shift_byte! { [VectorSlb vec_slb] (vslb) }
1345
1346    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1347    pub trait VectorSrab<Other> {
1348        type Result;
1349        unsafe fn vec_srab(self, b: Other) -> Self::Result;
1350    }
1351
1352    impl_vec_shift_byte! { [VectorSrab vec_srab] (vsrab) }
1353
1354    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1355    pub trait VectorSrb<Other> {
1356        type Result;
1357        unsafe fn vec_srb(self, b: Other) -> Self::Result;
1358    }
1359
1360    impl_vec_shift_byte! { [VectorSrb vec_srb] (vsrlb) }
1361
1362    macro_rules! impl_vec_shift_long {
1363        ([$trait:ident $m:ident] ($f:ident)) => {
1364            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char }
1365            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_char, vector_unsigned_char) -> vector_signed_char }
1366            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_short, vector_unsigned_char) -> vector_unsigned_short }
1367            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_short, vector_unsigned_char) -> vector_signed_short }
1368            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_int, vector_unsigned_char) -> vector_unsigned_int }
1369            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_int, vector_unsigned_char) -> vector_signed_int }
1370            impl_vec_trait!{ [$trait $m]+ $f (vector_unsigned_long_long, vector_unsigned_char) -> vector_unsigned_long_long }
1371            impl_vec_trait!{ [$trait $m]+ $f (vector_signed_long_long, vector_unsigned_char) -> vector_signed_long_long }
1372        };
1373    }
1374
1375    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1376    pub trait VectorSrl<Other> {
1377        type Result;
1378        unsafe fn vec_srl(self, b: Other) -> Self::Result;
1379    }
1380
1381    impl_vec_shift_long! { [VectorSrl vec_srl] (vsrl) }
1382
1383    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1384    pub trait VectorSral<Other> {
1385        type Result;
1386        unsafe fn vec_sral(self, b: Other) -> Self::Result;
1387    }
1388
1389    impl_vec_shift_long! { [VectorSral vec_sral] (vsra) }
1390
1391    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1392    pub trait VectorSll<Other> {
1393        type Result;
1394        unsafe fn vec_sll(self, b: Other) -> Self::Result;
1395    }
1396
1397    impl_vec_shift_long! { [VectorSll vec_sll] (vsl) }
1398
1399    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1400    pub trait VectorRl<Other> {
1401        type Result;
1402        unsafe fn vec_rl(self, b: Other) -> Self::Result;
1403    }
1404
1405    macro_rules! impl_rot {
1406        ($fun:ident $ty:ident) => {
1407            #[inline]
1408            #[target_feature(enable = "vector")]
1409            #[cfg_attr(test, assert_instr($fun))]
1410            unsafe fn $fun(a: t_t_l!($ty), b: t_t_l!($ty)) -> t_t_l!($ty) {
1411                simd_funnel_shl(a, a, b)
1412            }
1413        };
1414    }
1415
1416    impl_rot! { verllvb u8 }
1417    impl_rot! { verllvh u16 }
1418    impl_rot! { verllvf u32 }
1419    impl_rot! { verllvg u64 }
1420
1421    impl_vec_shift! { [VectorRl vec_rl] (verllvb, verllvh, verllvf, verllvg) }
1422
1423    macro_rules! test_rot_imm {
1424        ($fun:ident $instr:ident $ty:ident) => {
1425            #[inline]
1426            #[target_feature(enable = "vector")]
1427            #[cfg_attr(test, assert_instr($instr))]
1428            unsafe fn $fun(a: t_t_l!($ty), bits: core::ffi::c_ulong) -> t_t_l!($ty) {
1429                // mod by the number of bits in a's element type to prevent UB
1430                let bits = (bits % $ty::BITS as core::ffi::c_ulong) as $ty;
1431                let b = <t_t_s!($ty)>::splat(bits);
1432
1433                simd_funnel_shl(a, a, transmute(b))
1434            }
1435        };
1436    }
1437
1438    test_rot_imm! { verllvb_imm verllb u8 }
1439    test_rot_imm! { verllvh_imm verllh u16 }
1440    test_rot_imm! { verllvf_imm verllf u32 }
1441    test_rot_imm! { verllvg_imm verllg u64 }
1442
1443    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1444    pub trait VectorRli {
1445        unsafe fn vec_rli(self, bits: core::ffi::c_ulong) -> Self;
1446    }
1447
1448    macro_rules! impl_rot_imm {
1449        ($($ty:ident, $intr:ident),*) => {
1450            $(
1451                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1452                impl VectorRli for $ty {
1453                    #[inline]
1454                    #[target_feature(enable = "vector")]
1455                    unsafe fn vec_rli(self, bits: core::ffi::c_ulong) -> Self {
1456                        transmute($intr(transmute(self), bits))
1457                    }
1458                }
1459
1460                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1461                impl VectorRli for t_u!($ty) {
1462                    #[inline]
1463                    #[target_feature(enable = "vector")]
1464                    unsafe fn vec_rli(self, bits: core::ffi::c_ulong) -> Self {
1465                        $intr(self, bits)
1466                    }
1467                }
1468            )*
1469        }
1470    }
1471
1472    impl_rot_imm! {
1473        vector_signed_char, verllvb_imm,
1474        vector_signed_short, verllvh_imm,
1475        vector_signed_int, verllvf_imm,
1476        vector_signed_long_long, verllvg_imm
1477    }
1478
1479    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1480    pub trait VectorRlMask<Other> {
1481        unsafe fn vec_rl_mask<const IMM8: u8>(self, other: Other) -> Self;
1482    }
1483
1484    macro_rules! impl_rl_mask {
1485        ($($ty:ident, $intr:ident, $fun:ident),*) => {
1486            $(
1487                #[inline]
1488                #[target_feature(enable = "vector")]
1489                #[cfg_attr(test, assert_instr($intr, IMM8 = 6))]
1490                unsafe fn $fun<const IMM8: u8>(a: $ty, b: t_u!($ty)) -> $ty {
1491                    // mod by the number of bits in a's element type to prevent UB
1492                    $intr(a, a, transmute(b), const { (IMM8 % <l_t_t!($ty)>::BITS as u8) as i32 })
1493                }
1494
1495                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1496                impl VectorRlMask<t_u!($ty)> for $ty {
1497                    #[inline]
1498                    #[target_feature(enable = "vector")]
1499                    unsafe fn vec_rl_mask<const IMM8: u8>(self, other: t_u!($ty)) -> Self {
1500                        $fun::<IMM8>(self, other)
1501                    }
1502                }
1503
1504                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1505                impl VectorRlMask<t_u!($ty)> for t_u!($ty) {
1506                    #[inline]
1507                    #[target_feature(enable = "vector")]
1508                    unsafe fn vec_rl_mask<const IMM8: u8>(self, other: t_u!($ty)) -> Self {
1509                        transmute($fun::<IMM8>(transmute(self), transmute(other)))
1510                    }
1511                }
1512            )*
1513        }
1514    }
1515
1516    impl_rl_mask! {
1517        vector_signed_char, verimb, test_verimb,
1518        vector_signed_short, verimh, test_verimh,
1519        vector_signed_int, verimf, test_verimf,
1520        vector_signed_long_long, verimg, test_verimg
1521    }
1522
1523    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1524    pub trait VectorReve {
1525        unsafe fn vec_reve(self) -> Self;
1526    }
1527
1528    #[repr(simd)]
1529    struct ReverseMask<const N: usize>([u32; N]);
1530
1531    impl<const N: usize> ReverseMask<N> {
1532        const fn new() -> Self {
1533            let mut index = [0; N];
1534            let mut i = 0;
1535            while i < N {
1536                index[i] = (N - i - 1) as u32;
1537                i += 1;
1538            }
1539            ReverseMask(index)
1540        }
1541    }
1542
1543    macro_rules! impl_reve {
1544        ($($ty:ident, $fun:ident, $instr:ident),*) => {
1545            $(
1546                #[inline]
1547                #[target_feature(enable = "vector")]
1548                #[cfg_attr(test, assert_instr($instr))]
1549                unsafe fn $fun(a: $ty) -> $ty {
1550                    const N: usize = core::mem::size_of::<$ty>() / core::mem::size_of::<l_t_t!($ty)>();
1551                    simd_shuffle(a, a, const { ShuffleMask::<N>::reverse() })
1552                }
1553
1554                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1555                impl VectorReve for $ty {
1556                    #[inline]
1557                    #[target_feature(enable = "vector")]
1558                    unsafe fn vec_reve(self) -> Self {
1559                        $fun(self)
1560                    }
1561                }
1562
1563                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1564                impl VectorReve for t_u!($ty) {
1565                    #[inline]
1566                    #[target_feature(enable = "vector")]
1567                    unsafe fn vec_reve(self) -> Self {
1568                        transmute($fun(transmute(self)))
1569                    }
1570                }
1571
1572                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1573                impl VectorReve for t_b!($ty) {
1574                    #[inline]
1575                    #[target_feature(enable = "vector")]
1576                    unsafe fn vec_reve(self) -> Self {
1577                        transmute($fun(transmute(self)))
1578                    }
1579                }
1580            )*
1581        }
1582    }
1583
1584    impl_reve! {
1585        vector_signed_char, reveb, vperm,
1586        vector_signed_short, reveh, vperm,
1587        vector_signed_int, revef, vperm,
1588        vector_signed_long_long, reveg, vpdi
1589    }
1590
1591    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1592    impl VectorReve for vector_float {
1593        #[inline]
1594        #[target_feature(enable = "vector")]
1595        unsafe fn vec_reve(self) -> Self {
1596            transmute(transmute::<_, vector_signed_int>(self).vec_reve())
1597        }
1598    }
1599
1600    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1601    impl VectorReve for vector_double {
1602        #[inline]
1603        #[target_feature(enable = "vector")]
1604        unsafe fn vec_reve(self) -> Self {
1605            transmute(transmute::<_, vector_signed_long_long>(self).vec_reve())
1606        }
1607    }
1608
1609    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1610    pub trait VectorRevb {
1611        unsafe fn vec_revb(self) -> Self;
1612    }
1613
1614    test_impl! { bswapb (a: vector_signed_char) -> vector_signed_char [simd_bswap, _] }
1615    test_impl! { bswaph (a: vector_signed_short) -> vector_signed_short [simd_bswap, vperm] }
1616    test_impl! { bswapf (a: vector_signed_int) -> vector_signed_int [simd_bswap, vperm] }
1617    test_impl! { bswapg (a: vector_signed_long_long) -> vector_signed_long_long [simd_bswap, vperm] }
1618
1619    impl_vec_trait! { [VectorRevb vec_revb]+ bswapb (vector_unsigned_char) }
1620    impl_vec_trait! { [VectorRevb vec_revb]+ bswapb (vector_signed_char) }
1621    impl_vec_trait! { [VectorRevb vec_revb]+ bswaph (vector_unsigned_short) }
1622    impl_vec_trait! { [VectorRevb vec_revb]+ bswaph (vector_signed_short) }
1623    impl_vec_trait! { [VectorRevb vec_revb]+ bswapf (vector_unsigned_int) }
1624    impl_vec_trait! { [VectorRevb vec_revb]+ bswapf (vector_signed_int) }
1625    impl_vec_trait! { [VectorRevb vec_revb]+ bswapg (vector_unsigned_long_long) }
1626    impl_vec_trait! { [VectorRevb vec_revb]+ bswapg (vector_signed_long_long) }
1627
1628    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1629    impl VectorRevb for vector_float {
1630        #[inline]
1631        #[target_feature(enable = "vector")]
1632        unsafe fn vec_revb(self) -> Self {
1633            transmute(transmute::<_, vector_signed_int>(self).vec_revb())
1634        }
1635    }
1636
1637    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1638    impl VectorRevb for vector_double {
1639        #[inline]
1640        #[target_feature(enable = "vector")]
1641        unsafe fn vec_revb(self) -> Self {
1642            transmute(transmute::<_, vector_signed_long_long>(self).vec_revb())
1643        }
1644    }
1645
1646    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1647    pub trait VectorMergel {
1648        unsafe fn vec_mergel(self, other: Self) -> Self;
1649    }
1650
1651    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1652    pub trait VectorMergeh {
1653        unsafe fn vec_mergeh(self, other: Self) -> Self;
1654    }
1655
1656    macro_rules! impl_merge {
1657        ($($ty:ident, $mergel:ident, $mergeh:ident),*) => {
1658            $(
1659                #[inline]
1660                #[target_feature(enable = "vector")]
1661                #[cfg_attr(test, assert_instr($mergel))]
1662                unsafe fn $mergel(a: $ty, b: $ty) -> $ty {
1663                    const N: usize = core::mem::size_of::<$ty>() / core::mem::size_of::<l_t_t!($ty)>();
1664                    simd_shuffle(a, b, const { ShuffleMask::<N>::merge_low() })
1665                }
1666
1667                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1668                impl VectorMergel for $ty {
1669                    #[inline]
1670                    #[target_feature(enable = "vector")]
1671                    unsafe fn vec_mergel(self, other: Self) -> Self {
1672                        $mergel(self, other)
1673                    }
1674                }
1675
1676                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1677                impl VectorMergel for t_u!($ty) {
1678                    #[inline]
1679                    #[target_feature(enable = "vector")]
1680                    unsafe fn vec_mergel(self, other: Self) -> Self {
1681                        transmute($mergel(transmute(self), transmute(other)))
1682                    }
1683                }
1684
1685                #[inline]
1686                #[target_feature(enable = "vector")]
1687                #[cfg_attr(test, assert_instr($mergeh))]
1688                unsafe fn $mergeh(a: $ty, b: $ty) -> $ty {
1689                    const N: usize = core::mem::size_of::<$ty>() / core::mem::size_of::<l_t_t!($ty)>();
1690                    simd_shuffle(a, b, const { ShuffleMask::<N>::merge_high() })
1691                }
1692
1693                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1694                impl VectorMergeh for $ty {
1695                    #[inline]
1696                    #[target_feature(enable = "vector")]
1697                    unsafe fn vec_mergeh(self, other: Self) -> Self {
1698                        $mergeh(self, other)
1699                    }
1700                }
1701
1702                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1703                impl VectorMergeh for t_u!($ty) {
1704                    #[inline]
1705                    #[target_feature(enable = "vector")]
1706                    unsafe fn vec_mergeh(self, other: Self) -> Self {
1707                        transmute($mergeh(transmute(self), transmute(other)))
1708                    }
1709                }
1710            )*
1711        }
1712    }
1713
1714    impl_merge! {
1715        vector_signed_char, vmrlb, vmrhb,
1716        vector_signed_short, vmrlh, vmrhh,
1717        vector_signed_int, vmrlf, vmrhf,
1718        vector_signed_long_long, vmrlg, vmrhg
1719    }
1720
1721    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1722    pub trait VectorPerm {
1723        unsafe fn vec_perm(self, other: Self, c: vector_unsigned_char) -> Self;
1724    }
1725
1726    macro_rules! impl_merge {
1727        ($($ty:ident),*) => {
1728            $(
1729                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1730                impl VectorPerm for $ty {
1731                    #[inline]
1732                    #[target_feature(enable = "vector")]
1733                    unsafe fn vec_perm(self, other: Self, c: vector_unsigned_char) -> Self {
1734                        transmute(vperm(transmute(self), transmute(other), c))
1735                    }
1736                }
1737            )*
1738        }
1739    }
1740
1741    impl_merge! {
1742        vector_signed_char,
1743        vector_signed_short,
1744        vector_signed_int,
1745        vector_signed_long_long,
1746        vector_unsigned_char,
1747        vector_unsigned_short,
1748        vector_unsigned_int,
1749        vector_unsigned_long_long,
1750        vector_bool_char,
1751        vector_bool_short,
1752        vector_bool_int,
1753        vector_bool_long_long,
1754        vector_float,
1755        vector_double
1756    }
1757
1758    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1759    pub trait VectorSumU128 {
1760        unsafe fn vec_sum_u128(self, other: Self) -> vector_unsigned_char;
1761    }
1762
1763    #[inline]
1764    #[target_feature(enable = "vector")]
1765    #[cfg_attr(test, assert_instr(vsumqf))]
1766    pub unsafe fn vec_vsumqf(a: vector_unsigned_int, b: vector_unsigned_int) -> u128 {
1767        transmute(vsumqf(a, b))
1768    }
1769
1770    #[inline]
1771    #[target_feature(enable = "vector")]
1772    #[cfg_attr(test, assert_instr(vsumqg))]
1773    pub unsafe fn vec_vsumqg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> u128 {
1774        transmute(vsumqg(a, b))
1775    }
1776
1777    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1778    impl VectorSumU128 for vector_unsigned_int {
1779        #[inline]
1780        #[target_feature(enable = "vector")]
1781        unsafe fn vec_sum_u128(self, other: Self) -> vector_unsigned_char {
1782            transmute(vec_vsumqf(self, other))
1783        }
1784    }
1785
1786    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1787    impl VectorSumU128 for vector_unsigned_long_long {
1788        #[inline]
1789        #[target_feature(enable = "vector")]
1790        unsafe fn vec_sum_u128(self, other: Self) -> vector_unsigned_char {
1791            transmute(vec_vsumqg(self, other))
1792        }
1793    }
1794
1795    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1796    pub trait VectorSum2 {
1797        unsafe fn vec_sum2(self, other: Self) -> vector_unsigned_long_long;
1798    }
1799
1800    test_impl! { vec_vsumgh (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_long_long [vsumgh, vsumgh] }
1801    test_impl! { vec_vsumgf (a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long [vsumgf, vsumgf] }
1802
1803    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1804    impl VectorSum2 for vector_unsigned_short {
1805        #[inline]
1806        #[target_feature(enable = "vector")]
1807        unsafe fn vec_sum2(self, other: Self) -> vector_unsigned_long_long {
1808            vec_vsumgh(self, other)
1809        }
1810    }
1811
1812    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1813    impl VectorSum2 for vector_unsigned_int {
1814        #[inline]
1815        #[target_feature(enable = "vector")]
1816        unsafe fn vec_sum2(self, other: Self) -> vector_unsigned_long_long {
1817            vec_vsumgf(self, other)
1818        }
1819    }
1820
1821    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1822    pub trait VectorSum4 {
1823        unsafe fn vec_sum4(self, other: Self) -> vector_unsigned_int;
1824    }
1825
1826    test_impl! { vec_vsumb (a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_int [vsumb, vsumb] }
1827    test_impl! { vec_vsumh (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int [vsumh, vsumh] }
1828
1829    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1830    impl VectorSum4 for vector_unsigned_char {
1831        #[inline]
1832        #[target_feature(enable = "vector")]
1833        unsafe fn vec_sum4(self, other: Self) -> vector_unsigned_int {
1834            vec_vsumb(self, other)
1835        }
1836    }
1837
1838    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1839    impl VectorSum4 for vector_unsigned_short {
1840        #[inline]
1841        #[target_feature(enable = "vector")]
1842        unsafe fn vec_sum4(self, other: Self) -> vector_unsigned_int {
1843            vec_vsumh(self, other)
1844        }
1845    }
1846
1847    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1848    pub trait VectorSubc<Other> {
1849        type Result;
1850        unsafe fn vec_subc(self, b: Other) -> Self::Result;
1851    }
1852
1853    test_impl! { vec_vscbib (a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [vscbib, vscbib] }
1854    test_impl! { vec_vscbih (a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [vscbih, vscbih] }
1855    test_impl! { vec_vscbif (a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [vscbif, vscbif] }
1856    test_impl! { vec_vscbig (a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long [vscbig, vscbig] }
1857
1858    impl_vec_trait! {[VectorSubc vec_subc] vec_vscbib (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char }
1859    impl_vec_trait! {[VectorSubc vec_subc] vec_vscbih (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short }
1860    impl_vec_trait! {[VectorSubc vec_subc] vec_vscbif (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int }
1861    impl_vec_trait! {[VectorSubc vec_subc] vec_vscbig (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_long_long }
1862
1863    #[unstable(feature = "stdarch_s390x", issue = "135681")]
1864    pub trait VectorSqrt {
1865        unsafe fn vec_sqrt(self) -> Self;
1866    }
1867
1868    test_impl! { vec_sqrt_f32 (v: vector_float) -> vector_float [ simd_fsqrt, "vector-enhancements-1" vfsqsb ] }
1869    test_impl! { vec_sqrt_f64 (v: vector_double) -> vector_double [ simd_fsqrt, vfsqdb ] }
1870
1871    impl_vec_trait! { [VectorSqrt vec_sqrt] vec_sqrt_f32 (vector_float) }
1872    impl_vec_trait! { [VectorSqrt vec_sqrt] vec_sqrt_f64 (vector_double) }
1873
1874    macro_rules! vfae_wrapper {
1875        ($($name:ident $ty:ident)*) => {
1876            $(
1877                #[inline]
1878                #[target_feature(enable = "vector")]
1879                #[cfg_attr(test, assert_instr($name, IMM = 0))]
1880                unsafe fn $name<const IMM: i32>(
1881                    a: $ty,
1882                    b: $ty,
1883                ) -> $ty {
1884                    super::$name(a, b, IMM)
1885                }
1886            )*
1887        }
1888     }
1889
1890    vfae_wrapper! {
1891       vfaeb vector_signed_char
1892       vfaeh vector_signed_short
1893       vfaef vector_signed_int
1894
1895       vfaezb vector_signed_char
1896       vfaezh vector_signed_short
1897       vfaezf vector_signed_int
1898    }
1899
1900    macro_rules! impl_vfae {
1901        ([idx_cc $Trait:ident $m:ident] $imm:ident $b:ident $h:ident $f:ident) => {
1902            impl_vfae! { [idx_cc $Trait $m] $imm
1903                $b vector_signed_char vector_signed_char
1904                $b vector_unsigned_char vector_unsigned_char
1905                $b vector_bool_char vector_unsigned_char
1906
1907                $h vector_signed_short vector_signed_short
1908                $h vector_unsigned_short vector_unsigned_short
1909                $h vector_bool_short vector_unsigned_short
1910
1911                $f vector_signed_int vector_signed_int
1912                $f vector_unsigned_int vector_unsigned_int
1913                $f vector_bool_int vector_unsigned_int
1914            }
1915        };
1916        ([idx_cc $Trait:ident $m:ident] $imm:ident $($fun:ident $ty:ident $r:ident)*) => {
1917            $(
1918                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1919                impl $Trait<Self> for $ty {
1920                    type Result = $r;
1921                    #[inline]
1922                    #[target_feature(enable = "vector")]
1923                    unsafe fn $m(self, b: Self) -> (Self::Result, i32) {
1924                        let PackedTuple { x, y } = $fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b));
1925                        (transmute(x), y)
1926                    }
1927                }
1928            )*
1929        };
1930        ([cc $Trait:ident $m:ident] $imm:ident $b:ident $h:ident $f:ident) => {
1931            impl_vfae! { [cc $Trait $m] $imm
1932                $b vector_signed_char
1933                $b vector_unsigned_char
1934                $b vector_bool_char
1935
1936                $h vector_signed_short
1937                $h vector_unsigned_short
1938                $h vector_bool_short
1939
1940                $f vector_signed_int
1941                $f vector_unsigned_int
1942                $f vector_bool_int
1943            }
1944        };
1945        ([cc $Trait:ident $m:ident] $imm:ident $($fun:ident $ty:ident)*) => {
1946            $(
1947                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1948                impl $Trait<Self> for $ty {
1949                    type Result = t_b!($ty);
1950                    #[inline]
1951                    #[target_feature(enable = "vector")]
1952                    unsafe fn $m(self, b: Self) -> (Self::Result, i32) {
1953                        let PackedTuple { x, y } = $fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b));
1954                        (transmute(x), y)
1955                    }
1956                }
1957            )*
1958        };
1959        ([idx $Trait:ident $m:ident] $imm:ident $b:ident $h:ident $f:ident) => {
1960            impl_vfae! { [idx $Trait $m] $imm
1961                $b vector_signed_char vector_signed_char
1962                $b vector_unsigned_char vector_unsigned_char
1963                $b vector_bool_char vector_unsigned_char
1964
1965                $h vector_signed_short vector_signed_short
1966                $h vector_unsigned_short vector_unsigned_short
1967                $h vector_bool_short vector_unsigned_short
1968
1969                $f vector_signed_int vector_signed_int
1970                $f vector_unsigned_int vector_unsigned_int
1971                $f vector_bool_int vector_unsigned_int
1972            }
1973        };
1974        ([idx $Trait:ident $m:ident] $imm:ident $($fun:ident $ty:ident $r:ident)*) => {
1975            $(
1976                #[unstable(feature = "stdarch_s390x", issue = "135681")]
1977                impl $Trait<Self> for $ty {
1978                    type Result = $r;
1979                    #[inline]
1980                    #[target_feature(enable = "vector")]
1981                    unsafe fn $m(self, b: Self) -> Self::Result {
1982                        transmute($fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b)))
1983                    }
1984                }
1985            )*
1986        };
1987        ([$Trait:ident $m:ident] $imm:ident $b:ident $h:ident $f:ident) => {
1988            impl_vfae! { [$Trait $m] $imm
1989                $b vector_signed_char
1990                $b vector_unsigned_char
1991                $b vector_bool_char
1992
1993                $h vector_signed_short
1994                $h vector_unsigned_short
1995                $h vector_bool_short
1996
1997                $f vector_signed_int
1998                $f vector_unsigned_int
1999                $f vector_bool_int
2000            }
2001        };
2002        ([$Trait:ident $m:ident] $imm:ident $($fun:ident $ty:ident)*) => {
2003            $(
2004                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2005                impl $Trait<Self> for $ty {
2006                    type Result = t_b!($ty);
2007                    #[inline]
2008                    #[target_feature(enable = "vector")]
2009                    unsafe fn $m(self, b: Self) -> Self::Result {
2010                        transmute($fun::<{ FindImm::$imm as i32 }>(transmute(self), transmute(b)))
2011                    }
2012                }
2013            )*
2014        };
2015    }
2016
2017    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2018    pub trait VectorFindAnyEq<Other> {
2019        type Result;
2020        unsafe fn vec_find_any_eq(self, other: Other) -> Self::Result;
2021    }
2022
2023    impl_vfae! { [VectorFindAnyEq vec_find_any_eq] Eq vfaeb vfaeh vfaef }
2024
2025    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2026    pub trait VectorFindAnyNe<Other> {
2027        type Result;
2028        unsafe fn vec_find_any_ne(self, other: Other) -> Self::Result;
2029    }
2030
2031    impl_vfae! { [VectorFindAnyNe vec_find_any_ne] Ne vfaeb vfaeh vfaef }
2032
2033    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2034    pub trait VectorFindAnyEqOrZeroIdx<Other> {
2035        type Result;
2036        unsafe fn vec_find_any_eq_or_0_idx(self, other: Other) -> Self::Result;
2037    }
2038
2039    impl_vfae! { [idx VectorFindAnyEqOrZeroIdx vec_find_any_eq_or_0_idx] EqIdx
2040        vfaezb vector_signed_char vector_signed_char
2041        vfaezb vector_unsigned_char vector_unsigned_char
2042        vfaezb vector_bool_char vector_unsigned_char
2043
2044        vfaezh vector_signed_short vector_signed_short
2045        vfaezh vector_unsigned_short vector_unsigned_short
2046        vfaezh vector_bool_short vector_unsigned_short
2047
2048        vfaezf vector_signed_int vector_signed_int
2049        vfaezf vector_unsigned_int vector_unsigned_int
2050        vfaezf vector_bool_int vector_unsigned_int
2051    }
2052
2053    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2054    pub trait VectorFindAnyNeOrZeroIdx<Other> {
2055        type Result;
2056        unsafe fn vec_find_any_ne_or_0_idx(self, other: Other) -> Self::Result;
2057    }
2058
2059    impl_vfae! { [idx VectorFindAnyNeOrZeroIdx vec_find_any_ne_or_0_idx] NeIdx
2060        vfaezb vector_signed_char vector_signed_char
2061        vfaezb vector_unsigned_char vector_unsigned_char
2062        vfaezb vector_bool_char vector_unsigned_char
2063
2064        vfaezh vector_signed_short vector_signed_short
2065        vfaezh vector_unsigned_short vector_unsigned_short
2066        vfaezh vector_bool_short vector_unsigned_short
2067
2068        vfaezf vector_signed_int vector_signed_int
2069        vfaezf vector_unsigned_int vector_unsigned_int
2070        vfaezf vector_bool_int vector_unsigned_int
2071    }
2072
2073    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2074    pub trait VectorFindAnyEqIdx<Other> {
2075        type Result;
2076        unsafe fn vec_find_any_eq_idx(self, other: Other) -> Self::Result;
2077    }
2078
2079    impl_vfae! { [idx VectorFindAnyEqIdx vec_find_any_eq_idx] EqIdx vfaeb vfaeh vfaef }
2080
2081    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2082    pub trait VectorFindAnyNeIdx<Other> {
2083        type Result;
2084        unsafe fn vec_find_any_ne_idx(self, other: Other) -> Self::Result;
2085    }
2086
2087    impl_vfae! { [idx VectorFindAnyNeIdx vec_find_any_ne_idx] NeIdx vfaeb vfaeh vfaef }
2088
2089    macro_rules! vfaes_wrapper {
2090        ($($name:ident $ty:ident)*) => {
2091            $(
2092                #[inline]
2093                #[target_feature(enable = "vector")]
2094                #[cfg_attr(test, assert_instr($name, IMM = 0))]
2095                unsafe fn $name<const IMM: i32>(
2096                    a: $ty,
2097                    b: $ty,
2098                ) -> PackedTuple<$ty, i32> {
2099                    super::$name(a, b, IMM)
2100                }
2101            )*
2102        }
2103     }
2104
2105    vfaes_wrapper! {
2106        vfaebs vector_signed_char
2107        vfaehs vector_signed_short
2108        vfaefs vector_signed_int
2109
2110        vfaezbs vector_signed_char
2111        vfaezhs vector_signed_short
2112        vfaezfs vector_signed_int
2113    }
2114
2115    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2116    pub trait VectorFindAnyEqCC<Other> {
2117        type Result;
2118        unsafe fn vec_find_any_eq_cc(self, other: Other) -> (Self::Result, i32);
2119    }
2120
2121    impl_vfae! { [cc VectorFindAnyEqCC vec_find_any_eq_cc] Eq vfaebs vfaehs vfaefs }
2122
2123    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2124    pub trait VectorFindAnyNeCC<Other> {
2125        type Result;
2126        unsafe fn vec_find_any_ne_cc(self, other: Other) -> (Self::Result, i32);
2127    }
2128
2129    impl_vfae! { [cc VectorFindAnyNeCC vec_find_any_ne_cc] Ne vfaebs vfaehs vfaefs }
2130
2131    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2132    pub trait VectorFindAnyEqIdxCC<Other> {
2133        type Result;
2134        unsafe fn vec_find_any_eq_idx_cc(self, other: Other) -> (Self::Result, i32);
2135    }
2136
2137    impl_vfae! { [idx_cc VectorFindAnyEqIdxCC vec_find_any_eq_idx_cc] EqIdx vfaebs vfaehs vfaefs }
2138
2139    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2140    pub trait VectorFindAnyNeIdxCC<Other> {
2141        type Result;
2142        unsafe fn vec_find_any_ne_idx_cc(self, other: Other) -> (Self::Result, i32);
2143    }
2144
2145    impl_vfae! { [idx_cc VectorFindAnyNeIdxCC vec_find_any_ne_idx_cc] NeIdx vfaebs vfaehs vfaefs }
2146
2147    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2148    pub trait VectorFindAnyEqOrZeroIdxCC<Other> {
2149        type Result;
2150        unsafe fn vec_find_any_eq_or_0_idx_cc(self, other: Other) -> (Self::Result, i32);
2151    }
2152
2153    impl_vfae! { [idx_cc VectorFindAnyEqOrZeroIdxCC vec_find_any_eq_or_0_idx_cc] EqIdx vfaezbs vfaezhs vfaezfs }
2154
2155    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2156    pub trait VectorFindAnyNeOrZeroIdxCC<Other> {
2157        type Result;
2158        unsafe fn vec_find_any_ne_or_0_idx_cc(self, other: Other) -> (Self::Result, i32);
2159    }
2160
2161    impl_vfae! { [idx_cc VectorFindAnyNeOrZeroIdxCC vec_find_any_ne_or_0_idx_cc] NeIdx vfaezbs vfaezhs vfaezfs }
2162
2163    #[inline]
2164    #[target_feature(enable = "vector")]
2165    #[cfg_attr(test, assert_instr(vl))]
2166    unsafe fn test_vector_load(offset: isize, ptr: *const i32) -> vector_signed_int {
2167        ptr.byte_offset(offset)
2168            .cast::<vector_signed_int>()
2169            .read_unaligned()
2170    }
2171
2172    #[inline]
2173    #[target_feature(enable = "vector")]
2174    #[cfg_attr(test, assert_instr(vst))]
2175    unsafe fn test_vector_store(vector: vector_signed_int, offset: isize, ptr: *mut i32) {
2176        ptr.byte_offset(offset)
2177            .cast::<vector_signed_int>()
2178            .write_unaligned(vector)
2179    }
2180
2181    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2182    pub trait VectorLoad: Sized {
2183        type ElementType;
2184
2185        #[inline]
2186        #[target_feature(enable = "vector")]
2187        unsafe fn vec_xl(offset: isize, ptr: *const Self::ElementType) -> Self {
2188            ptr.byte_offset(offset).cast::<Self>().read_unaligned()
2189        }
2190
2191        unsafe fn vec_load_len(ptr: *const Self::ElementType, byte_count: u32) -> Self;
2192
2193        unsafe fn vec_load_bndry<const BLOCK_BOUNDARY: u16>(
2194            ptr: *const Self::ElementType,
2195        ) -> MaybeUninit<Self>;
2196    }
2197
2198    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2199    pub trait VectorStore: Sized {
2200        type ElementType;
2201
2202        #[inline]
2203        #[target_feature(enable = "vector")]
2204        unsafe fn vec_xst(self, offset: isize, ptr: *mut Self::ElementType) {
2205            ptr.byte_offset(offset).cast::<Self>().write_unaligned(self)
2206        }
2207
2208        unsafe fn vec_store_len(self, ptr: *mut Self::ElementType, byte_count: u32);
2209    }
2210
2211    macro_rules! impl_load_store {
2212        ($($ty:ident)*) => {
2213            $(
2214                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2215                impl VectorLoad for t_t_l!($ty) {
2216                    type ElementType = $ty;
2217
2218                    #[inline]
2219                    #[target_feature(enable = "vector")]
2220                    unsafe fn vec_load_len(ptr: *const Self::ElementType, byte_count: u32) -> Self {
2221                        transmute(vll( byte_count, ptr.cast(),))
2222                    }
2223
2224                    #[inline]
2225                    #[target_feature(enable = "vector")]
2226                    unsafe fn vec_load_bndry<const BLOCK_BOUNDARY: u16>(ptr: *const Self::ElementType) -> MaybeUninit<Self> {
2227                        transmute(vlbb(ptr.cast(), const { validate_block_boundary(BLOCK_BOUNDARY) }))
2228                    }
2229
2230                }
2231
2232                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2233                impl VectorStore for t_t_l!($ty) {
2234                    type ElementType = $ty;
2235
2236                    #[inline]
2237                    #[target_feature(enable = "vector")]
2238                    unsafe fn vec_store_len(self, ptr: *mut Self::ElementType, byte_count: u32) {
2239                        vstl(transmute(self), byte_count, ptr.cast())
2240                    }
2241                }
2242            )*
2243        }
2244    }
2245
2246    impl_load_store! { i8 u8 i16 u16 i32 u32 i64 u64 f32 f64 }
2247
2248    #[inline]
2249    #[target_feature(enable = "vector")]
2250    #[cfg_attr(test, assert_instr(vll))]
2251    unsafe fn test_vec_load_len(ptr: *const i32, byte_count: u32) -> vector_signed_int {
2252        vector_signed_int::vec_load_len(ptr, byte_count)
2253    }
2254
2255    #[inline]
2256    #[target_feature(enable = "vector")]
2257    #[cfg_attr(test, assert_instr("vlbb"))]
2258    unsafe fn test_vec_load_bndry(ptr: *const i32) -> MaybeUninit<vector_signed_int> {
2259        vector_signed_int::vec_load_bndry::<512>(ptr)
2260    }
2261
2262    #[inline]
2263    #[target_feature(enable = "vector")]
2264    #[cfg_attr(test, assert_instr(vst))]
2265    unsafe fn test_vec_store_len(vector: vector_signed_int, ptr: *mut i32, byte_count: u32) {
2266        vector.vec_store_len(ptr, byte_count)
2267    }
2268
2269    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2270    pub trait VectorLoadPair: Sized {
2271        type ElementType;
2272
2273        unsafe fn vec_load_pair(a: Self::ElementType, b: Self::ElementType) -> Self;
2274    }
2275
2276    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2277    impl VectorLoadPair for vector_signed_long_long {
2278        type ElementType = i64;
2279
2280        #[inline]
2281        #[target_feature(enable = "vector")]
2282        unsafe fn vec_load_pair(a: i64, b: i64) -> Self {
2283            vector_signed_long_long([a, b])
2284        }
2285    }
2286
2287    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2288    impl VectorLoadPair for vector_unsigned_long_long {
2289        type ElementType = u64;
2290
2291        #[inline]
2292        #[target_feature(enable = "vector")]
2293        unsafe fn vec_load_pair(a: u64, b: u64) -> Self {
2294            vector_unsigned_long_long([a, b])
2295        }
2296    }
2297
2298    #[inline]
2299    #[target_feature(enable = "vector")]
2300    unsafe fn pack<T, const N: usize>(a: T, b: T) -> T {
2301        simd_shuffle(a, b, const { ShuffleMask::<N>::pack() })
2302    }
2303
2304    #[inline]
2305    #[target_feature(enable = "vector")]
2306    #[cfg_attr(test, assert_instr(vpkh))]
2307    unsafe fn vpkh(a: i16x8, b: i16x8) -> i8x16 {
2308        let a: i8x16 = transmute(a);
2309        let b: i8x16 = transmute(b);
2310        simd_shuffle(a, b, const { ShuffleMask::<16>::pack() })
2311    }
2312    #[inline]
2313    #[target_feature(enable = "vector")]
2314    #[cfg_attr(test, assert_instr(vpkf))]
2315    unsafe fn vpkf(a: i32x4, b: i32x4) -> i16x8 {
2316        let a: i16x8 = transmute(a);
2317        let b: i16x8 = transmute(b);
2318        simd_shuffle(a, b, const { ShuffleMask::<8>::pack() })
2319    }
2320    #[inline]
2321    #[target_feature(enable = "vector")]
2322    #[cfg_attr(test, assert_instr(vpkg))]
2323    unsafe fn vpkg(a: i64x2, b: i64x2) -> i32x4 {
2324        let a: i32x4 = transmute(a);
2325        let b: i32x4 = transmute(b);
2326        simd_shuffle(a, b, const { ShuffleMask::<4>::pack() })
2327    }
2328
2329    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2330    pub trait VectorPack<Other> {
2331        type Result;
2332        unsafe fn vec_pack(self, b: Other) -> Self::Result;
2333    }
2334
2335    impl_vec_trait! { [VectorPack vec_pack]+ vpkh (vector_signed_short, vector_signed_short) -> vector_signed_char }
2336    impl_vec_trait! { [VectorPack vec_pack]+ vpkh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_char }
2337    impl_vec_trait! { [VectorPack vec_pack]+ vpkh (vector_bool_short, vector_bool_short) -> vector_bool_char }
2338    impl_vec_trait! { [VectorPack vec_pack]+ vpkf (vector_signed_int, vector_signed_int) -> vector_signed_short }
2339    impl_vec_trait! { [VectorPack vec_pack]+ vpkf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_short }
2340    impl_vec_trait! { [VectorPack vec_pack]+ vpkf (vector_bool_int, vector_bool_int) -> vector_bool_short }
2341    impl_vec_trait! { [VectorPack vec_pack]+ vpkg (vector_signed_long_long, vector_signed_long_long) -> vector_signed_int }
2342    impl_vec_trait! { [VectorPack vec_pack]+ vpkg (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_int }
2343    impl_vec_trait! { [VectorPack vec_pack]+ vpkg (vector_bool_long_long, vector_bool_long_long) -> vector_bool_int }
2344
2345    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2346    pub trait VectorPacks<Other> {
2347        type Result;
2348        unsafe fn vec_packs(self, b: Other) -> Self::Result;
2349    }
2350
2351    impl_vec_trait! { [VectorPacks vec_packs] vpksh (vector_signed_short, vector_signed_short) -> vector_signed_char }
2352    impl_vec_trait! { [VectorPacks vec_packs] vpklsh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_char }
2353    impl_vec_trait! { [VectorPacks vec_packs] vpksf (vector_signed_int, vector_signed_int) -> vector_signed_short }
2354    impl_vec_trait! { [VectorPacks vec_packs] vpklsf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_short }
2355    impl_vec_trait! { [VectorPacks vec_packs] vpksg (vector_signed_long_long, vector_signed_long_long) -> vector_signed_int }
2356    impl_vec_trait! { [VectorPacks vec_packs] vpklsg (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_int }
2357
2358    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2359    pub trait VectorPacksu<Other> {
2360        type Result;
2361        unsafe fn vec_packsu(self, b: Other) -> Self::Result;
2362    }
2363
2364    unsafe fn simd_smax<T: Copy>(a: T, b: T) -> T {
2365        simd_select::<T, T>(simd_gt::<T, T>(a, b), a, b)
2366    }
2367
2368    #[inline]
2369    #[target_feature(enable = "vector")]
2370    #[cfg_attr(test, assert_instr(vpklsh))]
2371    unsafe fn vpacksuh(a: vector_signed_short, b: vector_signed_short) -> vector_unsigned_char {
2372        vpklsh(
2373            simd_smax(a, vector_signed_short([0; 8])),
2374            simd_smax(b, vector_signed_short([0; 8])),
2375        )
2376    }
2377    #[inline]
2378    #[target_feature(enable = "vector")]
2379    #[cfg_attr(test, assert_instr(vpklsf))]
2380    unsafe fn vpacksuf(a: vector_signed_int, b: vector_signed_int) -> vector_unsigned_short {
2381        vpklsf(
2382            simd_smax(a, vector_signed_int([0; 4])),
2383            simd_smax(b, vector_signed_int([0; 4])),
2384        )
2385    }
2386    #[inline]
2387    #[target_feature(enable = "vector")]
2388    #[cfg_attr(test, assert_instr(vpklsg))]
2389    unsafe fn vpacksug(
2390        a: vector_signed_long_long,
2391        b: vector_signed_long_long,
2392    ) -> vector_unsigned_int {
2393        vpklsg(
2394            simd_smax(a, vector_signed_long_long([0; 2])),
2395            simd_smax(b, vector_signed_long_long([0; 2])),
2396        )
2397    }
2398
2399    impl_vec_trait! { [VectorPacksu vec_packsu] vpacksuh (vector_signed_short, vector_signed_short) -> vector_unsigned_char }
2400    impl_vec_trait! { [VectorPacksu vec_packsu] vpklsh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_char }
2401    impl_vec_trait! { [VectorPacksu vec_packsu] vpacksuf (vector_signed_int, vector_signed_int) -> vector_unsigned_short }
2402    impl_vec_trait! { [VectorPacksu vec_packsu] vpklsf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_short }
2403    impl_vec_trait! { [VectorPacksu vec_packsu] vpacksug (vector_signed_long_long, vector_signed_long_long) -> vector_unsigned_int }
2404    impl_vec_trait! { [VectorPacksu vec_packsu] vpklsg (vector_unsigned_long_long, vector_unsigned_long_long) -> vector_unsigned_int }
2405
2406    macro_rules! impl_vector_packs_cc {
2407        ($($intr:ident $ty:ident $outty:ident)*) => {
2408            $(
2409                #[inline]
2410                #[target_feature(enable = "vector")]
2411                #[cfg_attr(test, assert_instr($intr))]
2412                unsafe fn $intr(
2413                    a: $ty,
2414                    b: $ty,
2415                ) -> ($outty, i32) {
2416                    let PackedTuple { x, y } = super::$intr(a, b);
2417                    (x, y)
2418                }
2419
2420                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2421                impl VectorPacksCC for $ty {
2422                    type Result = $outty;
2423
2424                    #[inline]
2425                    #[target_feature(enable = "vector")]
2426                    unsafe fn vec_packs_cc(self, b: Self) -> (Self::Result, i32) {
2427                        $intr(self, b)
2428                    }
2429                }
2430            )*
2431        }
2432    }
2433
2434    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2435    pub trait VectorPacksCC {
2436        type Result;
2437        unsafe fn vec_packs_cc(self, b: Self) -> (Self::Result, i32);
2438    }
2439
2440    impl_vector_packs_cc! {
2441        vpkshs vector_signed_short vector_signed_char
2442        vpklshs vector_unsigned_short vector_unsigned_char
2443        vpksfs vector_signed_int vector_signed_short
2444        vpklsfs vector_unsigned_int vector_unsigned_short
2445        vpksgs vector_signed_long_long vector_signed_int
2446        vpklsgs vector_unsigned_long_long vector_unsigned_int
2447    }
2448
2449    macro_rules! impl_vector_packsu_cc {
2450        ($($intr:ident $ty:ident $outty:ident)*) => {
2451            $(
2452                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2453                impl VectorPacksuCC for $ty {
2454                    type Result = $outty;
2455
2456                    #[inline]
2457                    #[target_feature(enable = "vector")]
2458                    unsafe fn vec_packsu_cc(self, b: Self) -> (Self::Result, i32) {
2459                        $intr(self, b)
2460                    }
2461                }
2462            )*
2463        }
2464    }
2465
2466    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2467    pub trait VectorPacksuCC {
2468        type Result;
2469        unsafe fn vec_packsu_cc(self, b: Self) -> (Self::Result, i32);
2470    }
2471
2472    impl_vector_packsu_cc! {
2473        vpklshs vector_unsigned_short vector_unsigned_char
2474        vpklsfs vector_unsigned_int vector_unsigned_short
2475        vpklsgs vector_unsigned_long_long vector_unsigned_int
2476    }
2477
2478    #[unstable(feature = "stdarch_powerpc", issue = "111145")]
2479    pub trait VectorMadd {
2480        unsafe fn vec_madd(self, b: Self, c: Self) -> Self;
2481        unsafe fn vec_msub(self, b: Self, c: Self) -> Self;
2482    }
2483
2484    test_impl! { vfmasb (a: vector_float, b: vector_float, c: vector_float) -> vector_float [simd_fma, "vector-enhancements-1" vfmasb] }
2485    test_impl! { vfmadb (a: vector_double, b: vector_double, c: vector_double) -> vector_double [simd_fma, vfmadb] }
2486
2487    #[inline]
2488    unsafe fn simd_fms<T>(a: T, b: T, c: T) -> T {
2489        simd_fma(a, b, simd_neg(c))
2490    }
2491
2492    test_impl! { vfmssb (a: vector_float, b: vector_float, c: vector_float) -> vector_float [simd_fms, "vector-enhancements-1" vfmssb] }
2493    test_impl! { vfmsdb (a: vector_double, b: vector_double, c: vector_double) -> vector_double [simd_fms, vfmsdb] }
2494
2495    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2496    impl VectorMadd for vector_float {
2497        #[inline]
2498        #[target_feature(enable = "vector")]
2499        unsafe fn vec_madd(self, b: Self, c: Self) -> Self {
2500            vfmasb(self, b, c)
2501        }
2502
2503        #[inline]
2504        #[target_feature(enable = "vector")]
2505        unsafe fn vec_msub(self, b: Self, c: Self) -> Self {
2506            vfmssb(self, b, c)
2507        }
2508    }
2509
2510    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2511    impl VectorMadd for vector_double {
2512        #[inline]
2513        #[target_feature(enable = "vector")]
2514        unsafe fn vec_madd(self, b: Self, c: Self) -> Self {
2515            vfmadb(self, b, c)
2516        }
2517
2518        #[inline]
2519        #[target_feature(enable = "vector")]
2520        unsafe fn vec_msub(self, b: Self, c: Self) -> Self {
2521            vfmsdb(self, b, c)
2522        }
2523    }
2524
2525    macro_rules! impl_vec_unpack {
2526        ($mask:ident $instr:ident $src:ident $shuffled:ident $dst:ident $width:literal) => {
2527            #[inline]
2528            #[target_feature(enable = "vector")]
2529            #[cfg_attr(test, assert_instr($instr))]
2530            unsafe fn $instr(a: $src) -> $dst {
2531                simd_as(simd_shuffle::<_, _, $shuffled>(
2532                    a,
2533                    a,
2534                    const { ShuffleMask::<$width>::$mask() },
2535                ))
2536            }
2537        };
2538    }
2539
2540    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2541    pub trait VectorUnpackh {
2542        type Result;
2543        unsafe fn vec_unpackh(self) -> Self::Result;
2544    }
2545
2546    impl_vec_unpack!(unpack_high vuphb vector_signed_char i8x8 vector_signed_short 8);
2547    impl_vec_unpack!(unpack_high vuphh vector_signed_short i16x4 vector_signed_int 4);
2548    impl_vec_unpack!(unpack_high vuphf vector_signed_int i32x2 vector_signed_long_long 2);
2549
2550    impl_vec_unpack!(unpack_high vuplhb vector_unsigned_char u8x8 vector_unsigned_short 8);
2551    impl_vec_unpack!(unpack_high vuplhh vector_unsigned_short u16x4 vector_unsigned_int 4);
2552    impl_vec_unpack!(unpack_high vuplhf vector_unsigned_int u32x2 vector_unsigned_long_long 2);
2553
2554    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuphb (vector_signed_char) -> vector_signed_short}
2555    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuphh (vector_signed_short) -> vector_signed_int}
2556    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuphf (vector_signed_int) -> vector_signed_long_long}
2557
2558    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuplhb (vector_unsigned_char) -> vector_unsigned_short}
2559    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuplhh (vector_unsigned_short) -> vector_unsigned_int}
2560    impl_vec_trait! {[VectorUnpackh vec_unpackh] vuplhf (vector_unsigned_int) -> vector_unsigned_long_long}
2561
2562    impl_vec_trait! {[VectorUnpackh vec_unpackh]+ vuplhb (vector_bool_char) -> vector_bool_short}
2563    impl_vec_trait! {[VectorUnpackh vec_unpackh]+ vuplhh (vector_bool_short) -> vector_bool_int}
2564    impl_vec_trait! {[VectorUnpackh vec_unpackh]+ vuplhf (vector_bool_int) -> vector_bool_long_long}
2565
2566    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2567    pub trait VectorUnpackl {
2568        type Result;
2569        unsafe fn vec_unpackl(self) -> Self::Result;
2570    }
2571
2572    // FIXME(llvm): a shuffle + simd_as does not currently optimize into a single instruction like
2573    // unpachk above. Tracked in https://github.com/llvm/llvm-project/issues/129576.
2574
2575    impl_vec_trait! {[VectorUnpackl vec_unpackl] vuplb (vector_signed_char) -> vector_signed_short}
2576    impl_vec_trait! {[VectorUnpackl vec_unpackl] vuplhw (vector_signed_short) -> vector_signed_int}
2577    impl_vec_trait! {[VectorUnpackl vec_unpackl] vuplf (vector_signed_int) -> vector_signed_long_long}
2578
2579    impl_vec_trait! {[VectorUnpackl vec_unpackl] vupllb (vector_unsigned_char) -> vector_unsigned_short}
2580    impl_vec_trait! {[VectorUnpackl vec_unpackl] vupllh (vector_unsigned_short) -> vector_unsigned_int}
2581    impl_vec_trait! {[VectorUnpackl vec_unpackl] vupllf (vector_unsigned_int) -> vector_unsigned_long_long}
2582
2583    impl_vec_trait! {[VectorUnpackl vec_unpackl]+ vupllb (vector_bool_char) -> vector_bool_short}
2584    impl_vec_trait! {[VectorUnpackl vec_unpackl]+ vupllh (vector_bool_short) -> vector_bool_int}
2585    impl_vec_trait! {[VectorUnpackl vec_unpackl]+ vupllf (vector_bool_int) -> vector_bool_long_long}
2586
2587    test_impl! { vec_vavgb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char [ vavgb, vavgb ] }
2588    test_impl! { vec_vavgh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short [ vavgh, vavgh ] }
2589    test_impl! { vec_vavgf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int [ vavgf, vavgf ] }
2590    test_impl! { vec_vavgg(a: vector_signed_long_long, b: vector_signed_long_long) -> vector_signed_long_long [ vavgg, vavgg ] }
2591
2592    test_impl! { vec_vavglb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [ vavglb, vavglb ] }
2593    test_impl! { vec_vavglh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [ vavglh, vavglh ] }
2594    test_impl! { vec_vavglf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [ vavglf, vavglf ] }
2595    test_impl! { vec_vavglg(a: vector_unsigned_long_long, b: vector_unsigned_long_long) -> vector_unsigned_long_long [ vavglg, vavglg ] }
2596
2597    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2598    pub trait VectorAvg<Other> {
2599        type Result;
2600        unsafe fn vec_avg(self, b: Other) -> Self::Result;
2601    }
2602
2603    impl_vec_trait! { [VectorAvg vec_avg] 2 (vec_vavglb, vec_vavgb, vec_vavglh, vec_vavgh, vec_vavglf, vec_vavgf, vec_vavglg, vec_vavgg) }
2604
2605    macro_rules! impl_mul {
2606        ([$Trait:ident $m:ident] $fun:ident ($a:ty, $b:ty) -> $r:ty) => {
2607            #[unstable(feature = "stdarch_s390x", issue = "135681")]
2608            impl $Trait<$r> for $a {
2609                #[inline]
2610                #[target_feature(enable = "vector")]
2611                unsafe fn $m(self, b: $b) -> $r {
2612                    $fun(transmute(self), transmute(b))
2613                }
2614            }
2615        };
2616        ([$Trait:ident $m:ident] $fun:ident ($a:ty, $b:ty, $c:ty) -> $r:ty) => {
2617            #[unstable(feature = "stdarch_s390x", issue = "135681")]
2618            impl $Trait for $a {
2619                type Result = $r;
2620                #[inline]
2621                #[target_feature(enable = "vector")]
2622                unsafe fn $m(self, b: $b, c: $c) -> $r {
2623                    $fun(self, b, c)
2624                }
2625            }
2626        };
2627    }
2628
2629    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2630    pub trait VectorMule<Result> {
2631        unsafe fn vec_mule(self, b: Self) -> Result;
2632    }
2633
2634    // FIXME(llvm) sadly this does not yet work https://github.com/llvm/llvm-project/issues/129705
2635    //    #[target_feature(enable = "vector")]
2636    //    #[cfg_attr(test, assert_instr(vmleh))]
2637    //    unsafe fn vec_vmleh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int {
2638    //        let even_a: vector_unsigned_int = simd_as(simd_shuffle::<_, _, u16x4>(
2639    //            a,
2640    //            a,
2641    //            const { ShuffleMask([0, 2, 4, 6]) },
2642    //        ));
2643    //
2644    //        let even_b: vector_unsigned_int = simd_as(simd_shuffle::<_, _, u16x4>(
2645    //            b,
2646    //            b,
2647    //            const { ShuffleMask([0, 2, 4, 6]) },
2648    //        ));
2649    //
2650    //        simd_mul(even_a, even_b)
2651    //    }
2652
2653    test_impl! { vec_vmeb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_short [ vmeb, vmeb ] }
2654    test_impl! { vec_vmeh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_int[ vmeh, vmeh ] }
2655    test_impl! { vec_vmef(a: vector_signed_int, b: vector_signed_int) -> vector_signed_long_long [ vmef, vmef ] }
2656
2657    test_impl! { vec_vmleb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short [ vmleb, vmleb ] }
2658    test_impl! { vec_vmleh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int[ vmleh, vmleh ] }
2659    test_impl! { vec_vmlef(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long [ vmlef, vmlef ] }
2660
2661    impl_mul!([VectorMule vec_mule] vec_vmeb (vector_signed_char, vector_signed_char) -> vector_signed_short );
2662    impl_mul!([VectorMule vec_mule] vec_vmeh (vector_signed_short, vector_signed_short) -> vector_signed_int);
2663    impl_mul!([VectorMule vec_mule] vec_vmef (vector_signed_int, vector_signed_int) -> vector_signed_long_long );
2664
2665    impl_mul!([VectorMule vec_mule] vec_vmleb (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_short );
2666    impl_mul!([VectorMule vec_mule] vec_vmleh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_int);
2667    impl_mul!([VectorMule vec_mule] vec_vmlef (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_long_long );
2668
2669    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2670    pub trait VectorMulo<Result> {
2671        unsafe fn vec_mulo(self, b: Self) -> Result;
2672    }
2673
2674    test_impl! { vec_vmob(a: vector_signed_char, b: vector_signed_char) -> vector_signed_short [ vmob, vmob ] }
2675    test_impl! { vec_vmoh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_int[ vmoh, vmoh ] }
2676    test_impl! { vec_vmof(a: vector_signed_int, b: vector_signed_int) -> vector_signed_long_long [ vmof, vmof ] }
2677
2678    test_impl! { vec_vmlob(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short [ vmlob, vmlob ] }
2679    test_impl! { vec_vmloh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int[ vmloh, vmloh ] }
2680    test_impl! { vec_vmlof(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long [ vmlof, vmlof ] }
2681
2682    impl_mul!([VectorMulo vec_mulo] vec_vmob (vector_signed_char, vector_signed_char) -> vector_signed_short );
2683    impl_mul!([VectorMulo vec_mulo] vec_vmoh (vector_signed_short, vector_signed_short) -> vector_signed_int);
2684    impl_mul!([VectorMulo vec_mulo] vec_vmof (vector_signed_int, vector_signed_int) -> vector_signed_long_long );
2685
2686    impl_mul!([VectorMulo vec_mulo] vec_vmlob (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_short );
2687    impl_mul!([VectorMulo vec_mulo] vec_vmloh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_int);
2688    impl_mul!([VectorMulo vec_mulo] vec_vmlof (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_long_long );
2689
2690    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2691    pub trait VectorMulh<Result> {
2692        unsafe fn vec_mulh(self, b: Self) -> Result;
2693    }
2694
2695    test_impl! { vec_vmhb(a: vector_signed_char, b: vector_signed_char) -> vector_signed_char [ vmhb, vmhb ] }
2696    test_impl! { vec_vmhh(a: vector_signed_short, b: vector_signed_short) -> vector_signed_short [ vmhh, vmhh ] }
2697    test_impl! { vec_vmhf(a: vector_signed_int, b: vector_signed_int) -> vector_signed_int [ vmhf, vmhf ] }
2698
2699    test_impl! { vec_vmlhb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_char [ vmlhb, vmlhb ] }
2700    test_impl! { vec_vmlhh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_short [ vmlhh, vmlhh ] }
2701    test_impl! { vec_vmlhf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int [ vmlhf, vmlhf ] }
2702
2703    impl_mul!([VectorMulh vec_mulh] vec_vmhb (vector_signed_char, vector_signed_char) -> vector_signed_char);
2704    impl_mul!([VectorMulh vec_mulh] vec_vmhh (vector_signed_short, vector_signed_short) -> vector_signed_short);
2705    impl_mul!([VectorMulh vec_mulh] vec_vmhf (vector_signed_int, vector_signed_int) -> vector_signed_int);
2706
2707    impl_mul!([VectorMulh vec_mulh] vec_vmlhb (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char);
2708    impl_mul!([VectorMulh vec_mulh] vec_vmlhh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short);
2709    impl_mul!([VectorMulh vec_mulh] vec_vmlhf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int);
2710
2711    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2712    pub trait VectorMeadd {
2713        type Result;
2714        unsafe fn vec_meadd(self, b: Self, c: Self::Result) -> Self::Result;
2715    }
2716
2717    test_impl! { vec_vmaeb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short [ vmaeb, vmaeb ] }
2718    test_impl! { vec_vmaeh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int[ vmaeh, vmaeh ] }
2719    test_impl! { vec_vmaef(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long [ vmaef, vmaef ] }
2720
2721    test_impl! { vec_vmaleb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short [ vmaleb, vmaleb ] }
2722    test_impl! { vec_vmaleh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int[ vmaleh, vmaleh ] }
2723    test_impl! { vec_vmalef(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long [ vmalef, vmalef ] }
2724
2725    impl_mul!([VectorMeadd vec_meadd] vec_vmaeb (vector_signed_char, vector_signed_char, vector_signed_short) -> vector_signed_short );
2726    impl_mul!([VectorMeadd vec_meadd] vec_vmaeh (vector_signed_short, vector_signed_short, vector_signed_int) -> vector_signed_int);
2727    impl_mul!([VectorMeadd vec_meadd] vec_vmaef (vector_signed_int, vector_signed_int, vector_signed_long_long) -> vector_signed_long_long );
2728
2729    impl_mul!([VectorMeadd vec_meadd] vec_vmaleb (vector_unsigned_char, vector_unsigned_char, vector_unsigned_short) -> vector_unsigned_short );
2730    impl_mul!([VectorMeadd vec_meadd] vec_vmaleh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_int) -> vector_unsigned_int);
2731    impl_mul!([VectorMeadd vec_meadd] vec_vmalef (vector_unsigned_int, vector_unsigned_int, vector_unsigned_long_long) -> vector_unsigned_long_long );
2732
2733    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2734    pub trait VectorMoadd {
2735        type Result;
2736        unsafe fn vec_moadd(self, b: Self, c: Self::Result) -> Self::Result;
2737    }
2738
2739    test_impl! { vec_vmaob(a: vector_signed_char, b: vector_signed_char, c: vector_signed_short) -> vector_signed_short [ vmaob, vmaob ] }
2740    test_impl! { vec_vmaoh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_int) -> vector_signed_int[ vmaoh, vmaoh ] }
2741    test_impl! { vec_vmaof(a: vector_signed_int, b: vector_signed_int, c: vector_signed_long_long) -> vector_signed_long_long [ vmaof, vmaof ] }
2742
2743    test_impl! { vec_vmalob(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short [ vmalob, vmalob ] }
2744    test_impl! { vec_vmaloh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int[ vmaloh, vmaloh ] }
2745    test_impl! { vec_vmalof(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long [ vmalof, vmalof ] }
2746
2747    impl_mul!([VectorMoadd vec_moadd] vec_vmaob (vector_signed_char, vector_signed_char, vector_signed_short) -> vector_signed_short );
2748    impl_mul!([VectorMoadd vec_moadd] vec_vmaoh (vector_signed_short, vector_signed_short, vector_signed_int) -> vector_signed_int);
2749    impl_mul!([VectorMoadd vec_moadd] vec_vmaof (vector_signed_int, vector_signed_int, vector_signed_long_long) -> vector_signed_long_long );
2750
2751    impl_mul!([VectorMoadd vec_moadd] vec_vmalob (vector_unsigned_char, vector_unsigned_char, vector_unsigned_short) -> vector_unsigned_short );
2752    impl_mul!([VectorMoadd vec_moadd] vec_vmaloh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_int) -> vector_unsigned_int);
2753    impl_mul!([VectorMoadd vec_moadd] vec_vmalof (vector_unsigned_int, vector_unsigned_int, vector_unsigned_long_long) -> vector_unsigned_long_long );
2754
2755    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2756    pub trait VectorMhadd {
2757        type Result;
2758        unsafe fn vec_mhadd(self, b: Self, c: Self::Result) -> Self::Result;
2759    }
2760
2761    test_impl! { vec_vmahb(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char [ vmahb, vmahb ] }
2762    test_impl! { vec_vmahh(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short[ vmahh, vmahh ] }
2763    test_impl! { vec_vmahf(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int [ vmahf, vmahf ] }
2764
2765    test_impl! { vec_vmalhb(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char [ vmalhb, vmalhb ] }
2766    test_impl! { vec_vmalhh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short[ vmalhh, vmalhh ] }
2767    test_impl! { vec_vmalhf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int [ vmalhf, vmalhf ] }
2768
2769    impl_mul!([VectorMhadd vec_mhadd] vec_vmahb (vector_signed_char, vector_signed_char, vector_signed_char) -> vector_signed_char );
2770    impl_mul!([VectorMhadd vec_mhadd] vec_vmahh (vector_signed_short, vector_signed_short, vector_signed_short) -> vector_signed_short);
2771    impl_mul!([VectorMhadd vec_mhadd] vec_vmahf (vector_signed_int, vector_signed_int, vector_signed_int) -> vector_signed_int );
2772
2773    impl_mul!([VectorMhadd vec_mhadd] vec_vmalhb (vector_unsigned_char, vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char );
2774    impl_mul!([VectorMhadd vec_mhadd] vec_vmalhh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short);
2775    impl_mul!([VectorMhadd vec_mhadd] vec_vmalhf (vector_unsigned_int, vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int );
2776
2777    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2778    pub trait VectorMladd {
2779        type Result;
2780        unsafe fn vec_mladd(self, b: Self, c: Self::Result) -> Self::Result;
2781    }
2782
2783    #[inline]
2784    #[target_feature(enable = "vector")]
2785    unsafe fn simd_mladd<T>(a: T, b: T, c: T) -> T {
2786        simd_add(simd_mul(a, b), c)
2787    }
2788
2789    test_impl! { vec_vmal_ib(a: vector_signed_char, b: vector_signed_char, c: vector_signed_char) -> vector_signed_char [simd_mladd, vmalb ] }
2790    test_impl! { vec_vmal_ih(a: vector_signed_short, b: vector_signed_short, c: vector_signed_short) -> vector_signed_short[simd_mladd, vmalh ] }
2791    test_impl! { vec_vmal_if(a: vector_signed_int, b: vector_signed_int, c: vector_signed_int) -> vector_signed_int [simd_mladd, vmalf ] }
2792
2793    test_impl! { vec_vmal_ub(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_char) -> vector_unsigned_char [simd_mladd, vmalb ] }
2794    test_impl! { vec_vmal_uh(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_short) -> vector_unsigned_short[simd_mladd, vmalh ] }
2795    test_impl! { vec_vmal_uf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_int) -> vector_unsigned_int [simd_mladd, vmalf ] }
2796
2797    impl_mul!([VectorMladd vec_mladd] vec_vmal_ib (vector_signed_char, vector_signed_char, vector_signed_char) -> vector_signed_char );
2798    impl_mul!([VectorMladd vec_mladd] vec_vmal_ih (vector_signed_short, vector_signed_short, vector_signed_short) -> vector_signed_short);
2799    impl_mul!([VectorMladd vec_mladd] vec_vmal_if (vector_signed_int, vector_signed_int, vector_signed_int) -> vector_signed_int );
2800
2801    impl_mul!([VectorMladd vec_mladd] vec_vmal_ub (vector_unsigned_char, vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_char );
2802    impl_mul!([VectorMladd vec_mladd] vec_vmal_uh (vector_unsigned_short, vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_short);
2803    impl_mul!([VectorMladd vec_mladd] vec_vmal_uf (vector_unsigned_int, vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_int );
2804
2805    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2806    pub trait VectorGfmsum<Result> {
2807        unsafe fn vec_gfmsum(self, b: Self) -> Result;
2808    }
2809
2810    test_impl! { vec_vgfmb(a: vector_unsigned_char, b: vector_unsigned_char) -> vector_unsigned_short [ vgfmb, vgfmb ] }
2811    test_impl! { vec_vgfmh(a: vector_unsigned_short, b: vector_unsigned_short) -> vector_unsigned_int[ vgfmh, vgfmh] }
2812    test_impl! { vec_vgfmf(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_long_long [ vgfmf, vgfmf ] }
2813
2814    impl_mul!([VectorGfmsum vec_gfmsum] vec_vgfmb (vector_unsigned_char, vector_unsigned_char) -> vector_unsigned_short );
2815    impl_mul!([VectorGfmsum vec_gfmsum] vec_vgfmh (vector_unsigned_short, vector_unsigned_short) -> vector_unsigned_int);
2816    impl_mul!([VectorGfmsum vec_gfmsum] vec_vgfmf (vector_unsigned_int, vector_unsigned_int) -> vector_unsigned_long_long );
2817
2818    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2819    pub trait VectorGfmsumAccum {
2820        type Result;
2821        unsafe fn vec_gfmsum_accum(self, b: Self, c: Self::Result) -> Self::Result;
2822    }
2823
2824    test_impl! { vec_vgfmab(a: vector_unsigned_char, b: vector_unsigned_char, c: vector_unsigned_short) -> vector_unsigned_short [ vgfmab, vgfmab ] }
2825    test_impl! { vec_vgfmah(a: vector_unsigned_short, b: vector_unsigned_short, c: vector_unsigned_int) -> vector_unsigned_int[ vgfmah, vgfmah] }
2826    test_impl! { vec_vgfmaf(a: vector_unsigned_int, b: vector_unsigned_int, c: vector_unsigned_long_long) -> vector_unsigned_long_long [ vgfmaf, vgfmaf ] }
2827
2828    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2829    impl VectorGfmsumAccum for vector_unsigned_char {
2830        type Result = vector_unsigned_short;
2831        #[inline]
2832        #[target_feature(enable = "vector")]
2833        unsafe fn vec_gfmsum_accum(self, b: Self, c: Self::Result) -> Self::Result {
2834            vec_vgfmab(self, b, c)
2835        }
2836    }
2837    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2838    impl VectorGfmsumAccum for vector_unsigned_short {
2839        type Result = vector_unsigned_int;
2840        #[inline]
2841        #[target_feature(enable = "vector")]
2842        unsafe fn vec_gfmsum_accum(self, b: Self, c: Self::Result) -> Self::Result {
2843            vec_vgfmah(self, b, c)
2844        }
2845    }
2846    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2847    impl VectorGfmsumAccum for vector_unsigned_int {
2848        type Result = vector_unsigned_long_long;
2849        #[inline]
2850        #[target_feature(enable = "vector")]
2851        unsafe fn vec_gfmsum_accum(self, b: Self, c: Self::Result) -> Self::Result {
2852            vec_vgfmaf(self, b, c)
2853        }
2854    }
2855
2856    #[inline]
2857    #[target_feature(enable = "vector")]
2858    #[cfg_attr(test, assert_instr(vgef, D = 3))]
2859    unsafe fn vgef<const D: u32>(
2860        a: vector_unsigned_int,
2861        b: vector_unsigned_int,
2862        c: *const u32,
2863    ) -> vector_unsigned_int {
2864        static_assert_uimm_bits!(D, 2);
2865        let offset: u32 = simd_extract(b, D);
2866        let ptr = c.byte_add(offset as usize);
2867        let value = ptr.read();
2868        simd_insert(a, D, value)
2869    }
2870
2871    #[inline]
2872    #[target_feature(enable = "vector")]
2873    #[cfg_attr(test, assert_instr(vgeg, D = 1))]
2874    unsafe fn vgeg<const D: u32>(
2875        a: vector_unsigned_long_long,
2876        b: vector_unsigned_long_long,
2877        c: *const u64,
2878    ) -> vector_unsigned_long_long {
2879        static_assert_uimm_bits!(D, 1);
2880        let offset: u64 = simd_extract(b, D);
2881        let ptr = c.byte_add(offset as usize);
2882        let value = ptr.read();
2883        simd_insert(a, D, value)
2884    }
2885
2886    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2887    pub trait VectorGatherElement {
2888        type Element;
2889        type Offset;
2890        unsafe fn vec_gather_element<const D: u32>(
2891            self,
2892            b: Self::Offset,
2893            c: *const Self::Element,
2894        ) -> Self;
2895    }
2896
2897    macro_rules! impl_vec_gather_element {
2898        ($($instr:ident $ty:ident)*) => {
2899            $(
2900                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2901                impl VectorGatherElement for $ty {
2902                    type Element = l_t_t!($ty);
2903                    type Offset = t_u!($ty);
2904
2905                    #[inline]
2906                    #[target_feature(enable = "vector")]
2907                    unsafe fn vec_gather_element<const D: u32>(self, b: Self::Offset, c: *const Self::Element) -> Self {
2908                        transmute($instr::<D>(transmute(self), b, c.cast()))
2909                    }
2910                }
2911            )*
2912        }
2913    }
2914
2915    impl_vec_gather_element! {
2916        vgef vector_signed_int
2917        vgef vector_bool_int
2918        vgef vector_unsigned_int
2919
2920        vgeg vector_signed_long_long
2921        vgeg vector_bool_long_long
2922        vgeg vector_unsigned_long_long
2923
2924        vgef vector_float
2925        vgeg vector_double
2926    }
2927
2928    #[inline]
2929    #[target_feature(enable = "vector")]
2930    #[cfg_attr(test, assert_instr(vscef, D = 3))]
2931    unsafe fn vscef<const D: u32>(a: vector_unsigned_int, b: vector_unsigned_int, c: *mut u32) {
2932        static_assert_uimm_bits!(D, 2);
2933        let value = simd_extract(a, D);
2934        let offset: u32 = simd_extract(b, D);
2935        let ptr = c.byte_add(offset as usize);
2936        ptr.write(value);
2937    }
2938
2939    #[inline]
2940    #[target_feature(enable = "vector")]
2941    #[cfg_attr(test, assert_instr(vsceg, D = 1))]
2942    unsafe fn vsceg<const D: u32>(
2943        a: vector_unsigned_long_long,
2944        b: vector_unsigned_long_long,
2945        c: *mut u64,
2946    ) {
2947        static_assert_uimm_bits!(D, 1);
2948        let value = simd_extract(a, D);
2949        let offset: u64 = simd_extract(b, D);
2950        let ptr = c.byte_add(offset as usize);
2951        ptr.write(value);
2952    }
2953
2954    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2955    pub trait VectorScatterElement {
2956        type Element;
2957        type Offset;
2958        unsafe fn vec_scatter_element<const D: u32>(self, b: Self::Offset, c: *mut Self::Element);
2959    }
2960
2961    macro_rules! impl_vec_scatter_element {
2962        ($($instr:ident $ty:ident)*) => {
2963            $(
2964                #[unstable(feature = "stdarch_s390x", issue = "135681")]
2965                impl VectorScatterElement for $ty {
2966                    type Element = l_t_t!($ty);
2967                    type Offset = t_u!($ty);
2968
2969                    #[inline]
2970                    #[target_feature(enable = "vector")]
2971                    unsafe fn vec_scatter_element<const D: u32>(self, b: Self::Offset, c: *mut Self::Element) {
2972                        $instr::<D>(transmute(self), b, c.cast())
2973                    }
2974                }
2975            )*
2976        }
2977    }
2978
2979    impl_vec_scatter_element! {
2980        vscef vector_signed_int
2981        vscef vector_bool_int
2982        vscef vector_unsigned_int
2983
2984        vsceg vector_signed_long_long
2985        vsceg vector_bool_long_long
2986        vsceg vector_unsigned_long_long
2987
2988        vscef vector_float
2989        vsceg vector_double
2990    }
2991
2992    #[unstable(feature = "stdarch_s390x", issue = "135681")]
2993    pub trait VectorSel<Mask>: Sized {
2994        unsafe fn vec_sel(self, b: Self, c: Mask) -> Self;
2995    }
2996
2997    macro_rules! impl_vec_sel {
2998        ($($ty:ident)*) => {
2999            $(
3000                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3001                impl VectorSel<t_u!($ty)> for $ty {
3002                    #[inline]
3003                    #[target_feature(enable = "vector")]
3004                    unsafe fn vec_sel(self, b: Self, c: t_u!($ty)) -> Self {
3005                        let b = simd_and(transmute(b), c);
3006                        let a = simd_and(transmute(self), simd_xor(c, transmute(vector_signed_char([!0; 16]))));
3007                        transmute(simd_or(a, b))
3008                    }
3009                }
3010
3011                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3012                impl VectorSel<t_b!($ty)> for $ty {
3013                    #[inline]
3014                    #[target_feature(enable = "vector")]
3015                    unsafe fn vec_sel(self, b: Self, c: t_b!($ty)) -> Self {
3016                        // defer to the implementation with an unsigned mask
3017                        self.vec_sel(b, transmute::<_, t_u!($ty)>(c))
3018                    }
3019                }
3020            )*
3021        }
3022    }
3023
3024    impl_vec_sel! {
3025        vector_signed_char
3026        vector_signed_short
3027        vector_signed_int
3028        vector_signed_long_long
3029
3030        vector_unsigned_char
3031        vector_unsigned_short
3032        vector_unsigned_int
3033        vector_unsigned_long_long
3034
3035        vector_bool_char
3036        vector_bool_short
3037        vector_bool_int
3038        vector_bool_long_long
3039
3040        vector_float
3041        vector_double
3042    }
3043
3044    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3045    pub trait VectorFpTestDataClass {
3046        type Result;
3047        unsafe fn vec_fp_test_data_class<const CLASS: u32>(self) -> (Self::Result, i32);
3048    }
3049
3050    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3051    impl VectorFpTestDataClass for vector_float {
3052        type Result = vector_bool_int;
3053
3054        #[inline]
3055        #[target_feature(enable = "vector")]
3056        unsafe fn vec_fp_test_data_class<const CLASS: u32>(self) -> (Self::Result, i32) {
3057            let PackedTuple { x, y } = vftcisb(self, CLASS);
3058            (x, y)
3059        }
3060    }
3061
3062    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3063    impl VectorFpTestDataClass for vector_double {
3064        type Result = vector_bool_long_long;
3065
3066        #[inline]
3067        #[target_feature(enable = "vector")]
3068        unsafe fn vec_fp_test_data_class<const CLASS: u32>(self) -> (Self::Result, i32) {
3069            let PackedTuple { x, y } = vftcidb(self, CLASS);
3070            (x, y)
3071        }
3072    }
3073
3074    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3075    pub trait VectorCompare {
3076        unsafe fn vec_all_lt(self, other: Self) -> i32;
3077        unsafe fn vec_all_le(self, other: Self) -> i32;
3078        unsafe fn vec_all_gt(self, other: Self) -> i32;
3079        unsafe fn vec_all_ge(self, other: Self) -> i32;
3080    }
3081
3082    // NOTE: this implementation is currently non-optimal, but it does work for floats even with
3083    // only `vector` enabled.
3084    //
3085    // - https://github.com/llvm/llvm-project/issues/129434
3086    // - https://github.com/llvm/llvm-project/issues/130424
3087    macro_rules! impl_vec_compare {
3088        ($($ty:ident)*) => {
3089            $(
3090                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3091                impl VectorCompare for $ty {
3092                    #[inline]
3093                    #[target_feature(enable = "vector")]
3094                    unsafe fn vec_all_lt(self, other: Self) -> i32 {
3095                        simd_reduce_all(simd_lt::<_, t_b!($ty)>(self, other)) as i32
3096                    }
3097                    #[inline]
3098                    #[target_feature(enable = "vector")]
3099                    unsafe fn vec_all_le(self, other: Self) -> i32 {
3100                        simd_reduce_all(simd_le::<_, t_b!($ty)>(self, other)) as i32
3101                    }
3102                    #[inline]
3103                    #[target_feature(enable = "vector")]
3104                    unsafe fn vec_all_gt(self, other: Self) -> i32 {
3105                        simd_reduce_all(simd_gt::<_, t_b!($ty)>(self, other)) as i32
3106                    }
3107                    #[inline]
3108                    #[target_feature(enable = "vector")]
3109                    unsafe fn vec_all_ge(self, other: Self) -> i32 {
3110                        simd_reduce_all(simd_ge::<_, t_b!($ty)>(self, other)) as i32
3111                    }
3112                }
3113            )*
3114        }
3115    }
3116
3117    impl_vec_compare! {
3118        vector_signed_char
3119        vector_unsigned_char
3120
3121        vector_signed_short
3122        vector_unsigned_short
3123
3124        vector_signed_int
3125        vector_unsigned_int
3126        vector_float
3127
3128        vector_signed_long_long
3129        vector_unsigned_long_long
3130        vector_double
3131    }
3132
3133    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3134    pub trait VectorTestMask {
3135        type Mask;
3136        unsafe fn vec_test_mask(self, other: Self::Mask) -> i32;
3137    }
3138
3139    macro_rules! impl_vec_test_mask {
3140        ($($instr:ident $ty:ident)*) => {
3141            $(
3142                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3143                impl VectorTestMask for $ty {
3144                    type Mask = t_u!($ty);
3145
3146                    #[inline]
3147                    #[target_feature(enable = "vector")]
3148                    unsafe fn vec_test_mask(self, other: Self::Mask) -> i32 {
3149                        vtm(transmute(self), transmute(other))
3150                    }
3151                }
3152            )*
3153        }
3154    }
3155
3156    impl_vec_test_mask! {
3157        vector_signed_char
3158        vector_signed_short
3159        vector_signed_int
3160        vector_signed_long_long
3161
3162        vector_unsigned_char
3163        vector_unsigned_short
3164        vector_unsigned_int
3165        vector_unsigned_long_long
3166
3167        vector_float
3168        vector_double
3169    }
3170
3171    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3172    pub trait VectorSearchString {
3173        unsafe fn vec_search_string_cc(
3174            self,
3175            b: Self,
3176            c: vector_unsigned_char,
3177        ) -> (vector_unsigned_char, i32);
3178
3179        unsafe fn vec_search_string_until_zero_cc(
3180            self,
3181            b: Self,
3182            c: vector_unsigned_char,
3183        ) -> (vector_unsigned_char, i32);
3184    }
3185
3186    macro_rules! impl_vec_search_string{
3187        ($($intr_s:ident $intr_sz:ident $ty:ident)*) => {
3188            $(
3189                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3190                impl VectorSearchString for $ty {
3191                    #[inline]
3192                    #[target_feature(enable = "vector-enhancements-2")]
3193                    unsafe fn vec_search_string_cc(self, b: Self, c: vector_unsigned_char) -> (vector_unsigned_char, i32) {
3194                        let PackedTuple { x,y } = $intr_s(transmute(self), transmute(b), c);
3195                        (x, y)
3196                    }
3197
3198                    #[inline]
3199                    #[target_feature(enable = "vector-enhancements-2")]
3200                    unsafe fn vec_search_string_until_zero_cc(self, b: Self, c: vector_unsigned_char) -> (vector_unsigned_char, i32) {
3201                        let PackedTuple { x,y } = $intr_sz(transmute(self), transmute(b), c);
3202                        (x, y)
3203                    }
3204                }
3205
3206            )*
3207        }
3208    }
3209
3210    impl_vec_search_string! {
3211        vstrsb vstrszb vector_signed_char
3212        vstrsb vstrszb vector_bool_char
3213        vstrsb vstrszb vector_unsigned_char
3214
3215        vstrsh vstrszh vector_signed_short
3216        vstrsh vstrszh vector_bool_short
3217        vstrsh vstrszh vector_unsigned_short
3218
3219        vstrsf vstrszf vector_signed_int
3220        vstrsf vstrszf vector_bool_int
3221        vstrsf vstrszf vector_unsigned_int
3222    }
3223
3224    #[inline]
3225    #[target_feature(enable = "vector")]
3226    #[cfg_attr(test, assert_instr(vcdgb))]
3227    pub unsafe fn vcdgb(a: vector_signed_long_long) -> vector_double {
3228        simd_as(a)
3229    }
3230
3231    #[inline]
3232    #[target_feature(enable = "vector")]
3233    #[cfg_attr(test, assert_instr(vcdlgb))]
3234    pub unsafe fn vcdlgb(a: vector_unsigned_long_long) -> vector_double {
3235        simd_as(a)
3236    }
3237
3238    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3239    pub trait VectorDouble {
3240        unsafe fn vec_double(self) -> vector_double;
3241    }
3242
3243    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3244    impl VectorDouble for vector_signed_long_long {
3245        #[inline]
3246        #[target_feature(enable = "vector")]
3247        unsafe fn vec_double(self) -> vector_double {
3248            vcdgb(self)
3249        }
3250    }
3251
3252    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3253    impl VectorDouble for vector_unsigned_long_long {
3254        #[inline]
3255        #[target_feature(enable = "vector")]
3256        unsafe fn vec_double(self) -> vector_double {
3257            vcdlgb(self)
3258        }
3259    }
3260
3261    #[inline]
3262    #[target_feature(enable = "vector")]
3263    #[cfg_attr(
3264        all(test, target_feature = "vector-enhancements-2"),
3265        assert_instr(vcefb)
3266    )]
3267    pub unsafe fn vcefb(a: vector_signed_int) -> vector_float {
3268        simd_as(a)
3269    }
3270
3271    #[inline]
3272    #[target_feature(enable = "vector")]
3273    #[cfg_attr(
3274        all(test, target_feature = "vector-enhancements-2"),
3275        assert_instr(vcelfb)
3276    )]
3277    pub unsafe fn vcelfb(a: vector_unsigned_int) -> vector_float {
3278        simd_as(a)
3279    }
3280
3281    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3282    pub trait VectorFloat {
3283        unsafe fn vec_float(self) -> vector_float;
3284    }
3285
3286    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3287    impl VectorFloat for vector_signed_int {
3288        #[inline]
3289        #[target_feature(enable = "vector")]
3290        unsafe fn vec_float(self) -> vector_float {
3291            vcefb(self)
3292        }
3293    }
3294
3295    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3296    impl VectorFloat for vector_unsigned_int {
3297        #[inline]
3298        #[target_feature(enable = "vector")]
3299        unsafe fn vec_float(self) -> vector_float {
3300            vcelfb(self)
3301        }
3302    }
3303
3304    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3305    pub trait VectorExtendSigned64 {
3306        unsafe fn vec_extend_s64(self) -> vector_signed_long_long;
3307    }
3308
3309    #[inline]
3310    #[target_feature(enable = "vector")]
3311    // FIXME(llvm): https://github.com/llvm/llvm-project/issues/129899
3312    // #[cfg_attr(test, assert_instr(vsegb))]
3313    pub unsafe fn vsegb(a: vector_signed_char) -> vector_signed_long_long {
3314        simd_as(simd_shuffle::<_, _, i8x2>(
3315            a,
3316            a,
3317            const { u32x2::from_array([7, 15]) },
3318        ))
3319    }
3320
3321    #[inline]
3322    #[target_feature(enable = "vector")]
3323    // FIXME(llvm): https://github.com/llvm/llvm-project/issues/129899
3324    // #[cfg_attr(test, assert_instr(vsegh))]
3325    pub unsafe fn vsegh(a: vector_signed_short) -> vector_signed_long_long {
3326        simd_as(simd_shuffle::<_, _, i16x2>(
3327            a,
3328            a,
3329            const { u32x2::from_array([3, 7]) },
3330        ))
3331    }
3332
3333    #[inline]
3334    #[target_feature(enable = "vector")]
3335    // FIXME(llvm): https://github.com/llvm/llvm-project/issues/129899
3336    // #[cfg_attr(test, assert_instr(vsegf))]
3337    pub unsafe fn vsegf(a: vector_signed_int) -> vector_signed_long_long {
3338        simd_as(simd_shuffle::<_, _, i32x2>(
3339            a,
3340            a,
3341            const { u32x2::from_array([1, 3]) },
3342        ))
3343    }
3344
3345    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3346    impl VectorExtendSigned64 for vector_signed_char {
3347        #[inline]
3348        #[target_feature(enable = "vector")]
3349        unsafe fn vec_extend_s64(self) -> vector_signed_long_long {
3350            vsegb(self)
3351        }
3352    }
3353    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3354    impl VectorExtendSigned64 for vector_signed_short {
3355        #[inline]
3356        #[target_feature(enable = "vector")]
3357        unsafe fn vec_extend_s64(self) -> vector_signed_long_long {
3358            vsegh(self)
3359        }
3360    }
3361    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3362    impl VectorExtendSigned64 for vector_signed_int {
3363        #[inline]
3364        #[target_feature(enable = "vector")]
3365        unsafe fn vec_extend_s64(self) -> vector_signed_long_long {
3366            vsegf(self)
3367        }
3368    }
3369
3370    // NOTE: VectorSigned and VectorUnsigned make strong safety assumptions around floats.
3371    // This is what C provides, but even IBM does not clearly document these constraints.
3372    //
3373    // https://doc.rust-lang.org/std/intrinsics/simd/fn.simd_cast.html
3374
3375    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3376    pub trait VectorSigned {
3377        type Result;
3378        unsafe fn vec_signed(self) -> Self::Result;
3379    }
3380
3381    test_impl! { vcgsb (a: vector_float) -> vector_signed_int [simd_cast, "vector-enhancements-2" vcgsb] }
3382    test_impl! { vcgdb (a: vector_double) -> vector_signed_long_long [simd_cast, vcgdb] }
3383
3384    impl_vec_trait! { [VectorSigned vec_signed] vcgsb (vector_float) -> vector_signed_int }
3385    impl_vec_trait! { [VectorSigned vec_signed] vcgdb (vector_double) -> vector_signed_long_long }
3386
3387    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3388    pub trait VectorUnsigned {
3389        type Result;
3390        unsafe fn vec_unsigned(self) -> Self::Result;
3391    }
3392
3393    test_impl! { vclgsb (a: vector_float) -> vector_unsigned_int [simd_cast, "vector-enhancements-2" vclgsb] }
3394    test_impl! { vclgdb (a: vector_double) -> vector_unsigned_long_long [simd_cast, vclgdb] }
3395
3396    impl_vec_trait! { [VectorUnsigned vec_unsigned] vclgsb (vector_float) -> vector_unsigned_int }
3397    impl_vec_trait! { [VectorUnsigned vec_unsigned] vclgdb (vector_double) -> vector_unsigned_long_long }
3398
3399    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3400    pub trait VectorCopyUntilZero {
3401        unsafe fn vec_cp_until_zero(self) -> Self;
3402    }
3403
3404    test_impl! { vec_vistrb (a: vector_unsigned_char) -> vector_unsigned_char [vistrb, vistrb] }
3405    test_impl! { vec_vistrh (a: vector_unsigned_short) -> vector_unsigned_short [vistrh, vistrh] }
3406    test_impl! { vec_vistrf (a: vector_unsigned_int) -> vector_unsigned_int [vistrf, vistrf] }
3407
3408    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrb (vector_signed_char) }
3409    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrb (vector_bool_char) }
3410    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrb (vector_unsigned_char) }
3411
3412    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrh (vector_signed_short) }
3413    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrh (vector_bool_short) }
3414    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrh (vector_unsigned_short) }
3415
3416    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrf (vector_signed_int) }
3417    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrf (vector_bool_int) }
3418    impl_vec_trait! { [VectorCopyUntilZero vec_cp_until_zero]+ vec_vistrf (vector_unsigned_int) }
3419
3420    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3421    pub trait VectorCopyUntilZeroCC: Sized {
3422        unsafe fn vec_cp_until_zero_cc(self) -> (Self, i32);
3423    }
3424
3425    test_impl! { vec_vistrbs (a: vector_unsigned_char) -> PackedTuple<vector_unsigned_char, i32> [vistrbs, vistrbs] }
3426    test_impl! { vec_vistrhs (a: vector_unsigned_short) -> PackedTuple<vector_unsigned_short, i32> [vistrhs, vistrhs] }
3427    test_impl! { vec_vistrfs (a: vector_unsigned_int) -> PackedTuple<vector_unsigned_int, i32> [vistrfs, vistrfs] }
3428
3429    macro_rules! impl_vec_copy_until_zero_cc {
3430        ($($intr:ident $ty:ident)*) => {
3431            $(
3432                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3433                impl VectorCopyUntilZeroCC for $ty {
3434                    #[inline]
3435                    #[target_feature(enable = "vector")]
3436                    unsafe fn vec_cp_until_zero_cc(self) -> (Self, i32) {
3437                        let PackedTuple { x,y } = $intr(transmute(self));
3438                        (transmute(x), y)
3439                    }
3440                }
3441
3442            )*
3443        }
3444    }
3445
3446    impl_vec_copy_until_zero_cc! {
3447        vec_vistrbs vector_signed_char
3448        vec_vistrbs vector_bool_char
3449        vec_vistrbs vector_unsigned_char
3450
3451        vec_vistrhs vector_signed_short
3452        vec_vistrhs vector_bool_short
3453        vec_vistrhs vector_unsigned_short
3454
3455        vec_vistrfs vector_signed_int
3456        vec_vistrfs vector_bool_int
3457        vec_vistrfs vector_unsigned_int
3458    }
3459
3460    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3461    pub trait VectorSrdb {
3462        unsafe fn vec_srdb<const C: u32>(self, b: Self) -> Self;
3463    }
3464
3465    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3466    pub trait VectorSld {
3467        unsafe fn vec_sld<const C: u32>(self, b: Self) -> Self;
3468
3469        unsafe fn vec_sldw<const C: u32>(self, b: Self) -> Self;
3470
3471        unsafe fn vec_sldb<const C: u32>(self, b: Self) -> Self;
3472    }
3473
3474    // FIXME(llvm) https://github.com/llvm/llvm-project/issues/129955
3475    // ideally we could implement this in terms of llvm.fshl.i128
3476    // #[link_name = "llvm.fshl.i128"] fn fshl_i128(a: u128, b: u128, c: u128) -> u128;
3477    // transmute(fshl_i128(transmute(a), transmute(b), const { C * 8 } ))
3478
3479    macro_rules! impl_vec_sld {
3480        ($($ty:ident)*) => {
3481            $(
3482                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3483                impl VectorSld for $ty {
3484                    #[inline]
3485                    #[target_feature(enable = "vector")]
3486                    unsafe fn vec_sld<const C: u32>(self, b: Self) -> Self {
3487                        static_assert_uimm_bits!(C, 4);
3488                        transmute(vsldb(transmute(self), transmute(b), C))
3489                    }
3490
3491                    #[inline]
3492                    #[target_feature(enable = "vector")]
3493                    unsafe fn vec_sldw<const C: u32>(self, b: Self) -> Self {
3494                        static_assert_uimm_bits!(C, 2);
3495                        transmute(vsldb(transmute(self), transmute(b), const { 4 * C }))
3496                    }
3497
3498                    #[inline]
3499                    #[target_feature(enable = "vector-enhancements-2")]
3500                    unsafe fn vec_sldb<const C: u32>(self, b: Self) -> Self {
3501                        static_assert_uimm_bits!(C, 3);
3502                        transmute(vsld(transmute(self), transmute(b), C))
3503                    }
3504                }
3505
3506                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3507                impl VectorSrdb for $ty {
3508                    #[inline]
3509                    #[target_feature(enable = "vector-enhancements-2")]
3510                    unsafe fn vec_srdb<const C: u32>(self, b: Self) -> Self {
3511                        static_assert_uimm_bits!(C, 3);
3512                        transmute(vsrd(transmute(self), transmute(b), C))
3513                    }
3514                }
3515            )*
3516        }
3517    }
3518
3519    impl_vec_sld! {
3520        vector_signed_char
3521        vector_bool_char
3522        vector_unsigned_char
3523
3524        vector_signed_short
3525        vector_bool_short
3526        vector_unsigned_short
3527
3528        vector_signed_int
3529        vector_bool_int
3530        vector_unsigned_int
3531
3532        vector_signed_long_long
3533        vector_bool_long_long
3534        vector_unsigned_long_long
3535
3536        vector_float
3537        vector_double
3538    }
3539
3540    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3541    pub trait VectorCompareRange: Sized {
3542        type Result;
3543
3544        unsafe fn vstrc<const IMM: u32>(self, b: Self, c: Self) -> Self::Result;
3545        unsafe fn vstrcz<const IMM: u32>(self, b: Self, c: Self) -> Self::Result;
3546        unsafe fn vstrcs<const IMM: u32>(self, b: Self, c: Self) -> (Self::Result, i32);
3547        unsafe fn vstrczs<const IMM: u32>(self, b: Self, c: Self) -> (Self::Result, i32);
3548    }
3549
3550    const fn validate_compare_range_imm(imm: u32) {
3551        if !matches!(imm, 0 | 4 | 8 | 12) {
3552            panic!("IMM needs to be one of 0, 4, 8, 12");
3553        }
3554    }
3555
3556    macro_rules! impl_compare_range {
3557        ($($ty:ident $vstrc:ident $vstrcs:ident $vstrcz:ident $vstrczs:ident)*) => {
3558            $(
3559                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3560                impl VectorCompareRange for $ty {
3561                    type Result = t_b!($ty);
3562
3563                    #[inline]
3564                    #[target_feature(enable = "vector")]
3565                    unsafe fn vstrc<const IMM: u32>(self, b: Self, c: Self) -> Self::Result {
3566                        const { validate_compare_range_imm };
3567                        $vstrc(self, b, c, IMM)
3568                    }
3569
3570                    #[inline]
3571                    #[target_feature(enable = "vector")]
3572                    unsafe fn vstrcz<const IMM: u32>(self, b: Self, c: Self) -> Self::Result {
3573                        const { validate_compare_range_imm };
3574                        $vstrcz(self, b, c, IMM)
3575                    }
3576
3577                    #[inline]
3578                    #[target_feature(enable = "vector")]
3579                    unsafe fn vstrcs<const IMM: u32>(self, b: Self, c: Self) -> (Self::Result, i32) {
3580                        const { validate_compare_range_imm };
3581                        let PackedTuple { x, y } = $vstrcs(self, b, c, IMM);
3582                        (x,y)
3583                    }
3584
3585                    #[inline]
3586                    #[target_feature(enable = "vector")]
3587                    unsafe fn vstrczs<const IMM: u32>(self, b: Self, c: Self) -> (Self::Result, i32) {
3588                        const { validate_compare_range_imm };
3589                        let PackedTuple { x, y } = $vstrczs(self, b, c, IMM);
3590                        (x,y)
3591                    }
3592                }
3593            )*
3594        }
3595    }
3596
3597    impl_compare_range! {
3598        vector_unsigned_char    vstrcb vstrcbs vstrczb vstrczbs
3599        vector_unsigned_short   vstrch vstrchs vstrczh vstrczhs
3600        vector_unsigned_int     vstrcf vstrcfs vstrczf vstrczfs
3601    }
3602
3603    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3604    pub trait VectorComparePredicate: Sized {
3605        type Result;
3606
3607        #[inline]
3608        #[target_feature(enable = "vector")]
3609        unsafe fn vec_cmpgt(self, other: Self) -> Self::Result {
3610            simd_gt(self, other)
3611        }
3612
3613        #[inline]
3614        #[target_feature(enable = "vector")]
3615        unsafe fn vec_cmpge(self, other: Self) -> Self::Result {
3616            simd_ge(self, other)
3617        }
3618
3619        #[inline]
3620        #[target_feature(enable = "vector")]
3621        unsafe fn vec_cmplt(self, other: Self) -> Self::Result {
3622            simd_lt(self, other)
3623        }
3624
3625        #[inline]
3626        #[target_feature(enable = "vector")]
3627        unsafe fn vec_cmple(self, other: Self) -> Self::Result {
3628            simd_le(self, other)
3629        }
3630    }
3631
3632    macro_rules! impl_compare_predicate {
3633        ($($ty:ident)*) => {
3634            $(
3635                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3636                impl VectorComparePredicate for $ty {
3637                    type Result = t_b!($ty);
3638                }
3639            )*
3640        }
3641    }
3642
3643    impl_compare_predicate! {
3644        vector_signed_char
3645        vector_unsigned_char
3646
3647        vector_signed_short
3648        vector_unsigned_short
3649
3650        vector_signed_int
3651        vector_unsigned_int
3652        vector_float
3653
3654        vector_signed_long_long
3655        vector_unsigned_long_long
3656        vector_double
3657    }
3658
3659    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3660    pub trait VectorEquality: Sized {
3661        type Result;
3662
3663        #[inline]
3664        #[target_feature(enable = "vector")]
3665        unsafe fn vec_cmpeq(self, other: Self) -> Self::Result {
3666            simd_eq(self, other)
3667        }
3668
3669        #[inline]
3670        #[target_feature(enable = "vector")]
3671        unsafe fn vec_cmpne(self, other: Self) -> Self::Result {
3672            simd_ne(self, other)
3673        }
3674    }
3675
3676    macro_rules! impl_compare_equality {
3677        ($($ty:ident)*) => {
3678            $(
3679                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3680                impl VectorEquality for $ty {
3681                    type Result = t_b!($ty);
3682                }
3683            )*
3684        }
3685    }
3686
3687    impl_compare_equality! {
3688        vector_bool_char
3689        vector_signed_char
3690        vector_unsigned_char
3691
3692        vector_bool_short
3693        vector_signed_short
3694        vector_unsigned_short
3695
3696        vector_bool_int
3697        vector_signed_int
3698        vector_unsigned_int
3699        vector_float
3700
3701        vector_bool_long_long
3702        vector_signed_long_long
3703        vector_unsigned_long_long
3704        vector_double
3705    }
3706
3707    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3708    pub trait VectorEqualityIdx: Sized {
3709        type Result;
3710
3711        unsafe fn vec_cmpeq_idx(self, other: Self) -> Self::Result;
3712        unsafe fn vec_cmpne_idx(self, other: Self) -> Self::Result;
3713
3714        unsafe fn vec_cmpeq_idx_cc(self, other: Self) -> (Self::Result, i32);
3715        unsafe fn vec_cmpne_idx_cc(self, other: Self) -> (Self::Result, i32);
3716
3717        unsafe fn vec_cmpeq_or_0_idx(self, other: Self) -> Self::Result;
3718        unsafe fn vec_cmpne_or_0_idx(self, other: Self) -> Self::Result;
3719
3720        unsafe fn vec_cmpeq_or_0_idx_cc(self, other: Self) -> (Self::Result, i32);
3721        unsafe fn vec_cmpne_or_0_idx_cc(self, other: Self) -> (Self::Result, i32);
3722    }
3723
3724    macro_rules! impl_compare_equality_idx {
3725        ($($ty:ident $ret:ident
3726                $cmpeq:ident $cmpne:ident
3727                $cmpeq_or_0:ident $cmpne_or_0:ident
3728                $cmpeq_cc:ident $cmpne_cc:ident
3729                $cmpeq_or_0_cc:ident $cmpne_or_0_cc:ident
3730        )*) => {
3731            $(
3732                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3733                impl VectorEqualityIdx for $ty {
3734                    type Result = $ret;
3735
3736                    #[inline]
3737                    #[target_feature(enable = "vector")]
3738                    unsafe fn vec_cmpeq_idx(self, other: Self) -> Self::Result {
3739                        transmute($cmpeq(transmute(self), transmute(other)))
3740                    }
3741
3742                    #[inline]
3743                    #[target_feature(enable = "vector")]
3744                    unsafe fn vec_cmpne_idx(self, other: Self) -> Self::Result {
3745                        transmute($cmpne(transmute(self), transmute(other)))
3746                    }
3747
3748                    #[inline]
3749                    #[target_feature(enable = "vector")]
3750                    unsafe fn vec_cmpeq_or_0_idx(self, other: Self) -> Self::Result {
3751                        transmute($cmpeq_or_0(transmute(self), transmute(other)))
3752                    }
3753
3754                    #[inline]
3755                    #[target_feature(enable = "vector")]
3756                    unsafe fn vec_cmpne_or_0_idx(self, other: Self) -> Self::Result {
3757                        transmute($cmpne_or_0(transmute(self), transmute(other)))
3758                    }
3759
3760                    #[inline]
3761                    #[target_feature(enable = "vector")]
3762                    unsafe fn vec_cmpeq_idx_cc(self, other: Self) -> (Self::Result, i32) {
3763                        let PackedTuple { x, y } = $cmpeq_cc(transmute(self), transmute(other));
3764                        (transmute(x), y)
3765                    }
3766
3767                    #[inline]
3768                    #[target_feature(enable = "vector")]
3769                    unsafe fn vec_cmpne_idx_cc(self, other: Self) -> (Self::Result, i32) {
3770                        let PackedTuple { x, y } = $cmpne_cc(transmute(self), transmute(other));
3771                        (transmute(x),y)
3772                    }
3773
3774                    #[inline]
3775                    #[target_feature(enable = "vector")]
3776                    unsafe fn vec_cmpeq_or_0_idx_cc(self, other: Self) -> (Self::Result, i32) {
3777                        let PackedTuple { x, y } = $cmpeq_or_0_cc(transmute(self), transmute(other));
3778                        (transmute(x), y)
3779                    }
3780
3781                    #[inline]
3782                    #[target_feature(enable = "vector")]
3783                    unsafe fn vec_cmpne_or_0_idx_cc(self, other: Self) -> (Self::Result, i32) {
3784                        let PackedTuple { x, y } = $cmpne_or_0_cc(transmute(self), transmute(other));
3785                        (transmute(x),y)
3786                    }
3787                }
3788            )*
3789        }
3790    }
3791
3792    impl_compare_equality_idx! {
3793        vector_signed_char vector_signed_char               vfeeb vfeneb vfeezb vfenezb vfeebs vfenebs vfeezbs vfenezbs
3794        vector_bool_char vector_unsigned_char               vfeeb vfeneb vfeezb vfenezb vfeebs vfenebs vfeezbs vfenezbs
3795        vector_unsigned_char vector_unsigned_char           vfeeb vfeneb vfeezb vfenezb vfeebs vfenebs vfeezbs vfenezbs
3796        vector_signed_short vector_signed_short             vfeeh vfeneh vfeezh vfenezh vfeehs vfenehs vfeezhs vfenezhs
3797        vector_bool_short  vector_unsigned_short            vfeeh vfeneh vfeezh vfenezh vfeehs vfenehs vfeezhs vfenezhs
3798        vector_unsigned_short vector_unsigned_short         vfeeh vfeneh vfeezh vfenezh vfeehs vfenehs vfeezhs vfenezhs
3799        vector_signed_int vector_signed_int                 vfeef vfenef vfeezf vfenezf vfeefs vfenefs vfeezfs vfenezfs
3800        vector_bool_int  vector_unsigned_int                vfeef vfenef vfeezf vfenezf vfeefs vfenefs vfeezfs vfenezfs
3801        vector_unsigned_int vector_unsigned_int             vfeef vfenef vfeezf vfenezf vfeefs vfenefs vfeezfs vfenezfs
3802    }
3803
3804    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3805    pub trait VectorExtract {
3806        type ElementType;
3807
3808        unsafe fn vec_extract(a: Self, b: i32) -> Self::ElementType;
3809    }
3810
3811    #[inline]
3812    #[target_feature(enable = "vector")]
3813    #[cfg_attr(test, assert_instr(vlgvb))]
3814    unsafe fn vlgvb(a: vector_unsigned_char, b: i32) -> u8 {
3815        simd_extract_dyn(a, b as u32 % 16)
3816    }
3817
3818    #[inline]
3819    #[target_feature(enable = "vector")]
3820    #[cfg_attr(test, assert_instr(vlgvh))]
3821    unsafe fn vlgvh(a: vector_unsigned_short, b: i32) -> u16 {
3822        simd_extract_dyn(a, b as u32 % 8)
3823    }
3824
3825    #[inline]
3826    #[target_feature(enable = "vector")]
3827    #[cfg_attr(test, assert_instr(vlgvf))]
3828    unsafe fn vlgvf(a: vector_unsigned_int, b: i32) -> u32 {
3829        simd_extract_dyn(a, b as u32 % 4)
3830    }
3831
3832    #[inline]
3833    #[target_feature(enable = "vector")]
3834    #[cfg_attr(test, assert_instr(vlgvg))]
3835    unsafe fn vlgvg(a: vector_unsigned_long_long, b: i32) -> u64 {
3836        simd_extract_dyn(a, b as u32 % 2)
3837    }
3838
3839    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3840    pub trait VectorInsert {
3841        type ElementType;
3842
3843        unsafe fn vec_insert(a: Self::ElementType, b: Self, c: i32) -> Self;
3844    }
3845
3846    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3847    pub trait VectorPromote: Sized {
3848        type ElementType;
3849
3850        unsafe fn vec_promote(a: Self::ElementType, b: i32) -> MaybeUninit<Self>;
3851    }
3852
3853    #[inline]
3854    #[target_feature(enable = "vector")]
3855    #[cfg_attr(test, assert_instr(vlvgb))]
3856    unsafe fn vlvgb(a: u8, b: vector_unsigned_char, c: i32) -> vector_unsigned_char {
3857        simd_insert_dyn(b, c as u32 % 16, a)
3858    }
3859
3860    #[inline]
3861    #[target_feature(enable = "vector")]
3862    #[cfg_attr(test, assert_instr(vlvgh))]
3863    unsafe fn vlvgh(a: u16, b: vector_unsigned_short, c: i32) -> vector_unsigned_short {
3864        simd_insert_dyn(b, c as u32 % 8, a)
3865    }
3866
3867    #[inline]
3868    #[target_feature(enable = "vector")]
3869    #[cfg_attr(test, assert_instr(vlvgf))]
3870    unsafe fn vlvgf(a: u32, b: vector_unsigned_int, c: i32) -> vector_unsigned_int {
3871        simd_insert_dyn(b, c as u32 % 4, a)
3872    }
3873
3874    #[inline]
3875    #[target_feature(enable = "vector")]
3876    #[cfg_attr(test, assert_instr(vlvgg))]
3877    unsafe fn vlvgg(a: u64, b: vector_unsigned_long_long, c: i32) -> vector_unsigned_long_long {
3878        simd_insert_dyn(b, c as u32 % 2, a)
3879    }
3880
3881    #[unstable(feature = "stdarch_s390x", issue = "135681")]
3882    pub trait VectorInsertAndZero {
3883        type ElementType;
3884
3885        unsafe fn vec_insert_and_zero(a: *const Self::ElementType) -> Self;
3886    }
3887
3888    #[inline]
3889    #[target_feature(enable = "vector")]
3890    #[cfg_attr(test, assert_instr(vllezb))]
3891    unsafe fn vllezb(x: *const u8) -> vector_unsigned_char {
3892        vector_unsigned_char([0, 0, 0, 0, 0, 0, 0, *x, 0, 0, 0, 0, 0, 0, 0, 0])
3893    }
3894
3895    #[inline]
3896    #[target_feature(enable = "vector")]
3897    #[cfg_attr(test, assert_instr(vllezh))]
3898    unsafe fn vllezh(x: *const u16) -> vector_unsigned_short {
3899        vector_unsigned_short([0, 0, 0, *x, 0, 0, 0, 0])
3900    }
3901
3902    #[inline]
3903    #[target_feature(enable = "vector")]
3904    #[cfg_attr(test, assert_instr(vllezf))]
3905    unsafe fn vllezf(x: *const u32) -> vector_unsigned_int {
3906        vector_unsigned_int([0, *x, 0, 0])
3907    }
3908
3909    #[inline]
3910    #[target_feature(enable = "vector")]
3911    #[cfg_attr(test, assert_instr(vllezg))]
3912    unsafe fn vllezg(x: *const u64) -> vector_unsigned_long_long {
3913        vector_unsigned_long_long([*x, 0])
3914    }
3915
3916    macro_rules! impl_extract_insert {
3917        ($($ty:ident $extract_intr:ident $insert_intr:ident $insert_and_zero_intr:ident)*) => {
3918            $(
3919                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3920                impl VectorExtract for $ty {
3921                    type ElementType = l_t_t!($ty);
3922
3923                    #[inline]
3924                    #[target_feature(enable = "vector")]
3925                    unsafe fn vec_extract(a: Self, b: i32) -> Self::ElementType {
3926                        transmute($extract_intr(transmute(a), b))
3927                    }
3928                }
3929
3930                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3931                impl VectorInsert for $ty {
3932                    type ElementType = l_t_t!($ty);
3933
3934                    #[inline]
3935                    #[target_feature(enable = "vector")]
3936                    unsafe fn vec_insert(a: Self::ElementType, b: Self, c: i32) -> Self {
3937                        transmute($insert_intr(transmute(a), transmute(b), c))
3938                    }
3939                }
3940
3941                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3942                impl VectorInsertAndZero for $ty {
3943                    type ElementType = l_t_t!($ty);
3944
3945                    #[inline]
3946                    #[target_feature(enable = "vector")]
3947                    unsafe fn vec_insert_and_zero(a: *const Self::ElementType) -> Self {
3948                        transmute($insert_and_zero_intr(a.cast()))
3949                    }
3950                }
3951
3952                #[unstable(feature = "stdarch_s390x", issue = "135681")]
3953                impl VectorPromote for $ty {
3954                    type ElementType = l_t_t!($ty);
3955
3956                    #[inline]
3957                    #[target_feature(enable = "vector")]
3958                    unsafe fn vec_promote(a: Self::ElementType, c: i32) -> MaybeUninit<Self> {
3959                        // Rust does not currently support `MaybeUninit` element types to simd
3960                        // vectors. In C/LLVM that is allowed (using poison values). So rust will
3961                        // use an extra instruction to zero the memory.
3962                        let b = MaybeUninit::<$ty>::zeroed();
3963                        MaybeUninit::new(transmute($insert_intr(transmute(a), transmute(b), c)))
3964                    }
3965                }
3966            )*
3967        }
3968
3969    }
3970
3971    impl_extract_insert! {
3972        vector_signed_char          vlgvb vlvgb vllezb
3973        vector_unsigned_char        vlgvb vlvgb vllezb
3974        vector_signed_short         vlgvh vlvgh vllezh
3975        vector_unsigned_short       vlgvh vlvgh vllezh
3976        vector_signed_int           vlgvf vlvgf vllezf
3977        vector_unsigned_int         vlgvf vlvgf vllezf
3978        vector_signed_long_long     vlgvg vlvgg vllezg
3979        vector_unsigned_long_long   vlgvg vlvgg vllezg
3980        vector_float                vlgvf vlvgf vllezf
3981        vector_double               vlgvg vlvgg vllezg
3982    }
3983}
3984
3985/// Load Count to Block Boundary
3986#[inline]
3987#[target_feature(enable = "vector")]
3988#[unstable(feature = "stdarch_s390x", issue = "135681")]
3989#[cfg_attr(test, assert_instr(lcbb, BLOCK_BOUNDARY = 512))]
3990unsafe fn __lcbb<const BLOCK_BOUNDARY: u16>(ptr: *const u8) -> u32 {
3991    lcbb(ptr, const { validate_block_boundary(BLOCK_BOUNDARY) })
3992}
3993
3994/// Vector Add
3995#[inline]
3996#[target_feature(enable = "vector")]
3997#[unstable(feature = "stdarch_s390x", issue = "135681")]
3998pub unsafe fn vec_add<T: sealed::VectorAdd<U>, U>(a: T, b: U) -> T::Result {
3999    a.vec_add(b)
4000}
4001
4002/// Vector Subtract
4003#[inline]
4004#[target_feature(enable = "vector")]
4005#[unstable(feature = "stdarch_s390x", issue = "135681")]
4006pub unsafe fn vec_sub<T: sealed::VectorSub<U>, U>(a: T, b: U) -> T::Result {
4007    a.vec_sub(b)
4008}
4009
4010/// Vector Multiply
4011///
4012/// ## Purpose
4013/// Compute the products of corresponding elements of two vectors.
4014///
4015/// ## Result value
4016/// Each element of r receives the product of the corresponding elements of a and b.
4017#[inline]
4018#[target_feature(enable = "vector")]
4019#[unstable(feature = "stdarch_s390x", issue = "135681")]
4020pub unsafe fn vec_mul<T: sealed::VectorMul>(a: T, b: T) -> T {
4021    a.vec_mul(b)
4022}
4023
4024/// Vector Count Leading Zeros
4025#[inline]
4026#[target_feature(enable = "vector")]
4027#[unstable(feature = "stdarch_s390x", issue = "135681")]
4028pub unsafe fn vec_cntlz<T: sealed::CountBits>(a: T) -> T::Result {
4029    a.vec_cntlz()
4030}
4031
4032/// Vector Count Trailing Zeros
4033#[inline]
4034#[target_feature(enable = "vector")]
4035#[unstable(feature = "stdarch_s390x", issue = "135681")]
4036pub unsafe fn vec_cnttz<T: sealed::CountBits>(a: T) -> T::Result {
4037    a.vec_cnttz()
4038}
4039
4040/// Vector Population Count
4041///
4042/// Computes the population count (number of set bits) in each element of the input.
4043#[inline]
4044#[target_feature(enable = "vector")]
4045#[unstable(feature = "stdarch_s390x", issue = "135681")]
4046pub unsafe fn vec_popcnt<T: sealed::CountBits>(a: T) -> T::Result {
4047    a.vec_popcnt()
4048}
4049
4050/// Vector Maximum
4051#[inline]
4052#[target_feature(enable = "vector")]
4053#[unstable(feature = "stdarch_s390x", issue = "135681")]
4054pub unsafe fn vec_max<T: sealed::VectorMax<U>, U>(a: T, b: U) -> T::Result {
4055    a.vec_max(b)
4056}
4057
4058/// Vector  Minimum
4059#[inline]
4060#[target_feature(enable = "vector")]
4061#[unstable(feature = "stdarch_s390x", issue = "135681")]
4062pub unsafe fn vec_min<T: sealed::VectorMin<U>, U>(a: T, b: U) -> T::Result {
4063    a.vec_min(b)
4064}
4065
4066/// Vector Absolute
4067#[inline]
4068#[target_feature(enable = "vector")]
4069#[unstable(feature = "stdarch_s390x", issue = "135681")]
4070pub unsafe fn vec_abs<T: sealed::VectorAbs>(a: T) -> T {
4071    a.vec_abs()
4072}
4073
4074/// Vector Negative Absolute
4075#[inline]
4076#[target_feature(enable = "vector")]
4077#[unstable(feature = "stdarch_s390x", issue = "135681")]
4078pub unsafe fn vec_nabs<T: sealed::VectorNabs>(a: T) -> T {
4079    a.vec_nabs()
4080}
4081
4082/// Vector Negative Multiply Add
4083#[inline]
4084#[target_feature(enable = "vector")]
4085#[unstable(feature = "stdarch_s390x", issue = "135681")]
4086pub unsafe fn vec_nmadd<T: sealed::VectorNmadd>(a: T, b: T, c: T) -> T {
4087    a.vec_nmadd(b, c)
4088}
4089
4090/// Vector Negative Multiply Subtract
4091#[inline]
4092#[target_feature(enable = "vector")]
4093#[unstable(feature = "stdarch_s390x", issue = "135681")]
4094pub unsafe fn vec_nmsub<T: sealed::VectorNmsub>(a: T, b: T, c: T) -> T {
4095    a.vec_nmsub(b, c)
4096}
4097
4098/// Vector Square Root
4099#[inline]
4100#[target_feature(enable = "vector")]
4101#[unstable(feature = "stdarch_s390x", issue = "135681")]
4102pub unsafe fn vec_sqrt<T: sealed::VectorSqrt>(a: T) -> T {
4103    a.vec_sqrt()
4104}
4105
4106/// Vector Splat
4107#[inline]
4108#[target_feature(enable = "vector")]
4109#[unstable(feature = "stdarch_s390x", issue = "135681")]
4110pub unsafe fn vec_splat<T: sealed::VectorSplat, const IMM: u32>(a: T) -> T {
4111    a.vec_splat::<IMM>()
4112}
4113
4114/// Vector Splats
4115#[inline]
4116#[target_feature(enable = "vector")]
4117#[unstable(feature = "stdarch_s390x", issue = "135681")]
4118pub unsafe fn vec_splats<T: sealed::VectorSplats<U>, U>(a: T) -> U {
4119    a.vec_splats()
4120}
4121
4122/// Vector AND
4123#[inline]
4124#[target_feature(enable = "vector")]
4125#[unstable(feature = "stdarch_s390x", issue = "135681")]
4126pub unsafe fn vec_and<T: sealed::VectorAnd<U>, U>(a: T, b: U) -> T::Result {
4127    a.vec_and(b)
4128}
4129
4130/// Vector OR
4131#[inline]
4132#[target_feature(enable = "vector")]
4133#[unstable(feature = "stdarch_s390x", issue = "135681")]
4134pub unsafe fn vec_or<T: sealed::VectorOr<U>, U>(a: T, b: U) -> T::Result {
4135    a.vec_or(b)
4136}
4137
4138/// Vector XOR
4139#[inline]
4140#[target_feature(enable = "vector")]
4141#[unstable(feature = "stdarch_s390x", issue = "135681")]
4142pub unsafe fn vec_xor<T: sealed::VectorXor<U>, U>(a: T, b: U) -> T::Result {
4143    a.vec_xor(b)
4144}
4145
4146/// Vector NOR
4147#[inline]
4148#[target_feature(enable = "vector")]
4149#[unstable(feature = "stdarch_s390x", issue = "135681")]
4150pub unsafe fn vec_nor<T: sealed::VectorNor<U>, U>(a: T, b: U) -> T::Result {
4151    a.vec_nor(b)
4152}
4153
4154/// Vector NAND
4155#[inline]
4156#[target_feature(enable = "vector")]
4157#[unstable(feature = "stdarch_s390x", issue = "135681")]
4158pub unsafe fn vec_nand<T: sealed::VectorNand<U>, U>(a: T, b: U) -> T::Result {
4159    a.vec_nand(b)
4160}
4161
4162/// Vector XNOR
4163#[inline]
4164#[target_feature(enable = "vector")]
4165#[unstable(feature = "stdarch_s390x", issue = "135681")]
4166pub unsafe fn vec_eqv<T: sealed::VectorEqv<U>, U>(a: T, b: U) -> T::Result {
4167    a.vec_eqv(b)
4168}
4169
4170/// Vector ANDC
4171#[inline]
4172#[target_feature(enable = "vector")]
4173#[unstable(feature = "stdarch_s390x", issue = "135681")]
4174pub unsafe fn vec_andc<T: sealed::VectorAndc<U>, U>(a: T, b: U) -> T::Result {
4175    a.vec_andc(b)
4176}
4177
4178/// Vector OR with Complement
4179///
4180/// ## Purpose
4181/// Performs a bitwise OR of the first vector with the bitwise-complemented second vector.
4182///
4183/// ## Result value
4184/// r is the bitwise OR of a and the bitwise complement of b.
4185#[inline]
4186#[target_feature(enable = "vector")]
4187#[unstable(feature = "stdarch_s390x", issue = "135681")]
4188pub unsafe fn vec_orc<T: sealed::VectorOrc<U>, U>(a: T, b: U) -> T::Result {
4189    a.vec_orc(b)
4190}
4191
4192/// Vector Floor
4193#[inline]
4194#[target_feature(enable = "vector")]
4195#[unstable(feature = "stdarch_s390x", issue = "135681")]
4196pub unsafe fn vec_floor<T: sealed::VectorFloor>(a: T) -> T {
4197    a.vec_floor()
4198}
4199
4200/// Vector Ceil
4201#[inline]
4202#[target_feature(enable = "vector")]
4203#[unstable(feature = "stdarch_s390x", issue = "135681")]
4204pub unsafe fn vec_ceil<T: sealed::VectorCeil>(a: T) -> T {
4205    a.vec_ceil()
4206}
4207
4208/// Vector Truncate
4209///
4210/// Returns a vector containing the truncated values of the corresponding elements of the given vector.
4211/// Each element of the result contains the value of the corresponding element of a, truncated to an integral value.
4212#[inline]
4213#[target_feature(enable = "vector")]
4214#[unstable(feature = "stdarch_s390x", issue = "135681")]
4215pub unsafe fn vec_trunc<T: sealed::VectorTrunc>(a: T) -> T {
4216    a.vec_trunc()
4217}
4218
4219/// Vector Round
4220///
4221/// Returns a vector containing the rounded values to the nearest representable floating-point integer,
4222/// using IEEE round-to-nearest rounding, of the corresponding elements of the given vector
4223#[inline]
4224#[target_feature(enable = "vector")]
4225#[unstable(feature = "stdarch_s390x", issue = "135681")]
4226pub unsafe fn vec_round<T: sealed::VectorRound>(a: T) -> T {
4227    a.vec_round()
4228}
4229
4230/// Vector Round to Current
4231///
4232/// Returns a vector by using the current rounding mode to round every
4233/// floating-point element in the given vector to integer.
4234#[inline]
4235#[target_feature(enable = "vector")]
4236#[unstable(feature = "stdarch_s390x", issue = "135681")]
4237pub unsafe fn vec_roundc<T: sealed::VectorRoundc>(a: T) -> T {
4238    a.vec_roundc()
4239}
4240
4241/// Vector Round toward Negative Infinity
4242///
4243/// Returns a vector containing the largest representable floating-point integral values less
4244/// than or equal to the values of the corresponding elements of the given vector.
4245#[inline]
4246#[target_feature(enable = "vector")]
4247#[unstable(feature = "stdarch_s390x", issue = "135681")]
4248pub unsafe fn vec_roundm<T: sealed::VectorFloor>(a: T) -> T {
4249    // the IBM docs note
4250    //
4251    // > vec_roundm provides the same functionality as vec_floor, except that vec_roundz would not trigger the IEEE-inexact exception.
4252    //
4253    // but in practice `vec_floor` also does not trigger that exception, so both are equivalent
4254    a.vec_floor()
4255}
4256
4257/// Vector Round toward Positive Infinity
4258///
4259/// Returns a vector containing the smallest representable floating-point integral values greater
4260/// than or equal to the values of the corresponding elements of the given vector.
4261#[inline]
4262#[target_feature(enable = "vector")]
4263#[unstable(feature = "stdarch_s390x", issue = "135681")]
4264pub unsafe fn vec_roundp<T: sealed::VectorCeil>(a: T) -> T {
4265    // the IBM docs note
4266    //
4267    // > vec_roundp provides the same functionality as vec_ceil, except that vec_roundz would not trigger the IEEE-inexact exception.
4268    //
4269    // but in practice `vec_ceil` also does not trigger that exception, so both are equivalent
4270    a.vec_ceil()
4271}
4272
4273/// Vector Round toward Zero
4274///
4275/// Returns a vector containing the truncated values of the corresponding elements of the given vector.
4276/// Each element of the result contains the value of the corresponding element of a, truncated to an integral value.
4277#[inline]
4278#[target_feature(enable = "vector")]
4279#[unstable(feature = "stdarch_s390x", issue = "135681")]
4280pub unsafe fn vec_roundz<T: sealed::VectorTrunc>(a: T) -> T {
4281    // the IBM docs note
4282    //
4283    // > vec_roundz provides the same functionality as vec_trunc, except that vec_roundz would not trigger the IEEE-inexact exception.
4284    //
4285    // but in practice `vec_trunc` also does not trigger that exception, so both are equivalent
4286    a.vec_trunc()
4287}
4288
4289/// Vector Round to Integer
4290///
4291/// Returns a vector by using the current rounding mode to round every floating-point element in the given vector to integer.
4292#[inline]
4293#[target_feature(enable = "vector")]
4294#[unstable(feature = "stdarch_s390x", issue = "135681")]
4295pub unsafe fn vec_rint<T: sealed::VectorRint>(a: T) -> T {
4296    a.vec_rint()
4297}
4298
4299/// Vector Average
4300#[inline]
4301#[target_feature(enable = "vector")]
4302#[unstable(feature = "stdarch_s390x", issue = "135681")]
4303pub unsafe fn vec_avg<T: sealed::VectorAvg<U>, U>(a: T, b: U) -> T::Result {
4304    a.vec_avg(b)
4305}
4306
4307/// Vector Shift Left
4308#[inline]
4309#[target_feature(enable = "vector")]
4310#[unstable(feature = "stdarch_s390x", issue = "135681")]
4311pub unsafe fn vec_sl<T: sealed::VectorSl<U>, U>(a: T, b: U) -> T::Result {
4312    a.vec_sl(b)
4313}
4314
4315/// Vector Shift Right
4316#[inline]
4317#[target_feature(enable = "vector")]
4318#[unstable(feature = "stdarch_s390x", issue = "135681")]
4319pub unsafe fn vec_sr<T: sealed::VectorSr<U>, U>(a: T, b: U) -> T::Result {
4320    a.vec_sr(b)
4321}
4322
4323/// Vector Shift Right Algebraic
4324#[inline]
4325#[target_feature(enable = "vector")]
4326#[unstable(feature = "stdarch_s390x", issue = "135681")]
4327pub unsafe fn vec_sra<T: sealed::VectorSra<U>, U>(a: T, b: U) -> T::Result {
4328    a.vec_sra(b)
4329}
4330
4331/// Vector Shift Left by Byte
4332#[inline]
4333#[target_feature(enable = "vector")]
4334#[unstable(feature = "stdarch_s390x", issue = "135681")]
4335pub unsafe fn vec_slb<T: sealed::VectorSlb<U>, U>(a: T, b: U) -> T::Result {
4336    a.vec_slb(b)
4337}
4338
4339/// Vector Shift Right by Byte
4340#[inline]
4341#[target_feature(enable = "vector")]
4342#[unstable(feature = "stdarch_s390x", issue = "135681")]
4343pub unsafe fn vec_srb<T: sealed::VectorSrb<U>, U>(a: T, b: U) -> T::Result {
4344    a.vec_srb(b)
4345}
4346
4347/// Vector Shift Right Algebraic by Byte
4348#[inline]
4349#[target_feature(enable = "vector")]
4350#[unstable(feature = "stdarch_s390x", issue = "135681")]
4351pub unsafe fn vec_srab<T: sealed::VectorSrab<U>, U>(a: T, b: U) -> T::Result {
4352    a.vec_srab(b)
4353}
4354
4355/// Vector Element Rotate Left
4356#[inline]
4357#[target_feature(enable = "vector")]
4358#[unstable(feature = "stdarch_s390x", issue = "135681")]
4359pub unsafe fn vec_rl<T: sealed::VectorRl<U>, U>(a: T, b: U) -> T::Result {
4360    a.vec_rl(b)
4361}
4362
4363/// Vector Shift Left
4364///
4365/// Performs a left shift for a vector by a given number of bits. Each element of the result is obtained by shifting the corresponding
4366/// element of a left by the number of bits specified by the last 3 bits of every byte of b. The bits that are shifted out are replaced by zeros.
4367#[inline]
4368#[target_feature(enable = "vector")]
4369#[unstable(feature = "stdarch_s390x", issue = "135681")]
4370pub unsafe fn vec_sll<T>(a: T, b: vector_unsigned_char) -> T
4371where
4372    T: sealed::VectorSll<vector_unsigned_char, Result = T>,
4373{
4374    a.vec_sll(b)
4375}
4376
4377/// Vector Shift Right
4378///
4379/// Performs a right shift for a vector by a given number of bits. Each element of the result is obtained by shifting the corresponding
4380/// element of a right by the number of bits specified by the last 3 bits of every byte of b. The bits that are shifted out are replaced by zeros.
4381#[inline]
4382#[target_feature(enable = "vector")]
4383#[unstable(feature = "stdarch_s390x", issue = "135681")]
4384pub unsafe fn vec_srl<T>(a: T, b: vector_unsigned_char) -> T
4385where
4386    T: sealed::VectorSrl<vector_unsigned_char, Result = T>,
4387{
4388    a.vec_srl(b)
4389}
4390
4391/// Vector Shift Right Arithmetic
4392///
4393/// Performs an algebraic right shift for a vector by a given number of bits. Each element of the result is obtained by shifting the corresponding
4394/// element of a right by the number of bits specified by the last 3 bits of every byte of b. The bits that are shifted out are replaced by copies of
4395/// the most significant bit of the element of a.
4396#[inline]
4397#[target_feature(enable = "vector")]
4398#[unstable(feature = "stdarch_s390x", issue = "135681")]
4399pub unsafe fn vec_sral<T>(a: T, b: vector_unsigned_char) -> T
4400where
4401    T: sealed::VectorSral<vector_unsigned_char, Result = T>,
4402{
4403    a.vec_sral(b)
4404}
4405
4406/// Vector Element Rotate Left Immediate
4407///
4408/// Rotates each element of a vector left by a given number of bits. Each element of the result is obtained by rotating the corresponding element
4409/// of a left by the number of bits specified by b, modulo the number of bits in the element.
4410#[inline]
4411#[target_feature(enable = "vector")]
4412#[unstable(feature = "stdarch_s390x", issue = "135681")]
4413pub unsafe fn vec_rli<T: sealed::VectorRli>(a: T, bits: core::ffi::c_ulong) -> T {
4414    a.vec_rli(bits)
4415}
4416
4417/// Vector Reverse Elements
4418///
4419/// Returns a vector with the elements of the input vector in reversed order.
4420#[inline]
4421#[target_feature(enable = "vector")]
4422#[unstable(feature = "stdarch_s390x", issue = "135681")]
4423pub unsafe fn vec_reve<T: sealed::VectorReve>(a: T) -> T {
4424    a.vec_reve()
4425}
4426
4427/// Vector Byte Reverse
4428///
4429/// Returns a vector where each vector element contains the corresponding byte-reversed vector element of the input vector.
4430#[inline]
4431#[target_feature(enable = "vector")]
4432#[unstable(feature = "stdarch_s390x", issue = "135681")]
4433pub unsafe fn vec_revb<T: sealed::VectorRevb>(a: T) -> T {
4434    a.vec_revb()
4435}
4436
4437/// Vector Merge High
4438///
4439/// Merges the most significant ("high") halves of two vectors.
4440#[inline]
4441#[target_feature(enable = "vector")]
4442#[unstable(feature = "stdarch_s390x", issue = "135681")]
4443pub unsafe fn vec_mergeh<T: sealed::VectorMergeh>(a: T, b: T) -> T {
4444    a.vec_mergeh(b)
4445}
4446
4447/// Vector Merge Low
4448///
4449/// Merges the least significant ("low") halves of two vectors.
4450#[inline]
4451#[target_feature(enable = "vector")]
4452#[unstable(feature = "stdarch_s390x", issue = "135681")]
4453pub unsafe fn vec_mergel<T: sealed::VectorMergel>(a: T, b: T) -> T {
4454    a.vec_mergel(b)
4455}
4456
4457/// Vector Pack
4458#[inline]
4459#[target_feature(enable = "vector")]
4460#[unstable(feature = "stdarch_s390x", issue = "135681")]
4461pub unsafe fn vec_pack<T: sealed::VectorPack<U>, U>(a: T, b: U) -> T::Result {
4462    a.vec_pack(b)
4463}
4464
4465/// Vector Pack Saturated
4466#[inline]
4467#[target_feature(enable = "vector")]
4468#[unstable(feature = "stdarch_s390x", issue = "135681")]
4469pub unsafe fn vec_packs<T: sealed::VectorPacks<U>, U>(a: T, b: U) -> T::Result {
4470    a.vec_packs(b)
4471}
4472
4473/// Vector Pack Saturated Condition Code
4474#[inline]
4475#[target_feature(enable = "vector")]
4476#[unstable(feature = "stdarch_s390x", issue = "135681")]
4477pub unsafe fn vec_packs_cc<T: sealed::VectorPacksCC>(a: T, b: T) -> (T::Result, i32) {
4478    a.vec_packs_cc(b)
4479}
4480
4481/// Vector Pack Saturated Unsigned
4482#[inline]
4483#[target_feature(enable = "vector")]
4484#[unstable(feature = "stdarch_s390x", issue = "135681")]
4485pub unsafe fn vec_packsu<T: sealed::VectorPacksu<U>, U>(a: T, b: U) -> T::Result {
4486    a.vec_packsu(b)
4487}
4488
4489/// Vector Pack Saturated Unsigned Condition Code
4490#[inline]
4491#[target_feature(enable = "vector")]
4492#[unstable(feature = "stdarch_s390x", issue = "135681")]
4493pub unsafe fn vec_packsu_cc<T: sealed::VectorPacksuCC>(a: T, b: T) -> (T::Result, i32) {
4494    a.vec_packsu_cc(b)
4495}
4496
4497/// Vector Unpack High
4498#[inline]
4499#[target_feature(enable = "vector")]
4500#[unstable(feature = "stdarch_s390x", issue = "135681")]
4501pub unsafe fn vec_unpackh<T: sealed::VectorUnpackh>(a: T) -> <T as sealed::VectorUnpackh>::Result {
4502    a.vec_unpackh()
4503}
4504
4505/// Vector Unpack Low
4506#[inline]
4507#[target_feature(enable = "vector")]
4508#[unstable(feature = "stdarch_s390x", issue = "135681")]
4509pub unsafe fn vec_unpackl<T: sealed::VectorUnpackl>(a: T) -> <T as sealed::VectorUnpackl>::Result {
4510    a.vec_unpackl()
4511}
4512
4513/// Vector Generate Byte Mask
4514///
4515/// Generates byte masks for elements in the return vector. For each bit in a, if the bit is one, all bit positions
4516/// in the corresponding byte element of d are set to ones. Otherwise, if the bit is zero, the corresponding byte element is set to zero.
4517#[inline]
4518#[target_feature(enable = "vector")]
4519#[unstable(feature = "stdarch_s390x", issue = "135681")]
4520#[cfg_attr(test, assert_instr(vgbm, MASK = 0x00FF))]
4521pub unsafe fn vec_genmask<const MASK: u16>() -> vector_unsigned_char {
4522    vector_unsigned_char(const { genmask::<MASK>() })
4523}
4524
4525/// Vector Generate Mask (Byte)
4526#[inline]
4527#[target_feature(enable = "vector")]
4528#[unstable(feature = "stdarch_s390x", issue = "135681")]
4529#[cfg_attr(test, assert_instr(vrepib, L = 3, H = 5))]
4530pub unsafe fn vec_genmasks_8<const L: u8, const H: u8>() -> vector_unsigned_char {
4531    vector_unsigned_char(const { [genmasks(u8::BITS, L, H) as u8; 16] })
4532}
4533
4534/// Vector Generate Mask (Halfword)
4535#[inline]
4536#[target_feature(enable = "vector")]
4537#[unstable(feature = "stdarch_s390x", issue = "135681")]
4538#[cfg_attr(test, assert_instr(vrepih, L = 3, H = 5))]
4539pub unsafe fn vec_genmasks_16<const L: u8, const H: u8>() -> vector_unsigned_short {
4540    vector_unsigned_short(const { [genmasks(u16::BITS, L, H) as u16; 8] })
4541}
4542
4543/// Vector Generate Mask (Word)
4544#[inline]
4545#[target_feature(enable = "vector")]
4546#[unstable(feature = "stdarch_s390x", issue = "135681")]
4547#[cfg_attr(test, assert_instr(vgmf, L = 3, H = 5))]
4548pub unsafe fn vec_genmasks_32<const L: u8, const H: u8>() -> vector_unsigned_int {
4549    vector_unsigned_int(const { [genmasks(u32::BITS, L, H) as u32; 4] })
4550}
4551
4552/// Vector Generate Mask (Doubleword)
4553#[inline]
4554#[target_feature(enable = "vector")]
4555#[unstable(feature = "stdarch_s390x", issue = "135681")]
4556#[cfg_attr(test, assert_instr(vgmg, L = 3, H = 5))]
4557pub unsafe fn vec_genmasks_64<const L: u8, const H: u8>() -> vector_unsigned_long_long {
4558    vector_unsigned_long_long(const { [genmasks(u64::BITS, L, H); 2] })
4559}
4560
4561/// Vector Permute
4562///
4563/// Returns a vector that contains some elements of two vectors, in the order specified by a third vector.
4564/// Each byte of the result is selected by using the least significant 5 bits of the corresponding byte of c as an index into the concatenated bytes of a and b.
4565/// Note: The vector generate mask built-in function [`vec_genmask`] could help generate the mask c.
4566#[inline]
4567#[target_feature(enable = "vector")]
4568#[unstable(feature = "stdarch_s390x", issue = "135681")]
4569pub unsafe fn vec_perm<T: sealed::VectorPerm>(a: T, b: T, c: vector_unsigned_char) -> T {
4570    a.vec_perm(b, c)
4571}
4572
4573/// Vector Sum Across Quadword
4574///
4575/// Returns a vector containing the results of performing a sum across all the elements in each of the quadword of vector a,
4576/// and the rightmost word or doubleword element of the b. The result is an unsigned 128-bit integer.
4577#[inline]
4578#[target_feature(enable = "vector")]
4579#[unstable(feature = "stdarch_s390x", issue = "135681")]
4580pub unsafe fn vec_sum_u128<T: sealed::VectorSumU128>(a: T, b: T) -> vector_unsigned_char {
4581    a.vec_sum_u128(b)
4582}
4583
4584/// Vector Sum Across Doubleword
4585///
4586/// Returns a vector containing the results of performing a sum across all the elements in each of the doubleword of vector a,
4587/// and the rightmost sub-element of the corresponding doubleword of b.
4588#[inline]
4589#[target_feature(enable = "vector")]
4590#[unstable(feature = "stdarch_s390x", issue = "135681")]
4591pub unsafe fn vec_sum2<T: sealed::VectorSum2>(a: T, b: T) -> vector_unsigned_long_long {
4592    a.vec_sum2(b)
4593}
4594
4595/// Vector Sum Across Word
4596///
4597/// Returns a vector containing the results of performing a sum across all the elements in each of the word of vector a,
4598/// and the rightmost sub-element of the corresponding word of b.
4599#[inline]
4600#[target_feature(enable = "vector")]
4601#[unstable(feature = "stdarch_s390x", issue = "135681")]
4602pub unsafe fn vec_sum4<T: sealed::VectorSum4>(a: T, b: T) -> vector_unsigned_int {
4603    a.vec_sum4(b)
4604}
4605
4606/// Vector Addition unsigned 128-bits
4607///
4608/// Adds unsigned quadword values.
4609///
4610/// This function operates on the vectors as 128-bit unsigned integers. It returns low 128 bits of a + b.
4611#[inline]
4612#[target_feature(enable = "vector")]
4613#[unstable(feature = "stdarch_s390x", issue = "135681")]
4614#[cfg_attr(test, assert_instr(vaq))]
4615pub unsafe fn vec_add_u128(
4616    a: vector_unsigned_char,
4617    b: vector_unsigned_char,
4618) -> vector_unsigned_char {
4619    let a: u128 = transmute(a);
4620    let b: u128 = transmute(b);
4621    transmute(a.wrapping_add(b))
4622}
4623
4624/// Vector Subtract unsigned 128-bits
4625///
4626/// Subtracts unsigned quadword values.
4627///
4628/// This function operates on the vectors as 128-bit unsigned integers. It returns low 128 bits of a - b.
4629#[inline]
4630#[target_feature(enable = "vector")]
4631#[unstable(feature = "stdarch_s390x", issue = "135681")]
4632#[cfg_attr(test, assert_instr(vsq))]
4633pub unsafe fn vec_sub_u128(
4634    a: vector_unsigned_char,
4635    b: vector_unsigned_char,
4636) -> vector_unsigned_char {
4637    let a: u128 = transmute(a);
4638    let b: u128 = transmute(b);
4639
4640    transmute(a.wrapping_sub(b))
4641}
4642
4643/// Vector Subtract Carryout
4644///
4645/// Returns a vector containing the borrow produced by subtracting each of corresponding elements of b from a.
4646///
4647/// On each resulting element, the value is 0 if a borrow occurred, or 1 if no borrow occurred.
4648#[inline]
4649#[target_feature(enable = "vector")]
4650#[unstable(feature = "stdarch_s390x", issue = "135681")]
4651pub unsafe fn vec_subc<T: sealed::VectorSubc<U>, U>(a: T, b: U) -> T::Result {
4652    a.vec_subc(b)
4653}
4654
4655/// Vector Subtract Carryout unsigned 128-bits
4656///
4657/// Gets the carry bit of the 128-bit subtraction of two quadword values.
4658/// This function operates on the vectors as 128-bit unsigned integers. It returns a vector containing the borrow produced by subtracting b from a, as unsigned 128-bits integers.
4659/// If no borrow occurred, the bit 127 of d is 1; otherwise it is set to 0. All other bits of d are 0.
4660#[inline]
4661#[target_feature(enable = "vector")]
4662#[unstable(feature = "stdarch_s390x", issue = "135681")]
4663#[cfg_attr(test, assert_instr(vscbiq))]
4664pub unsafe fn vec_subc_u128(
4665    a: vector_unsigned_char,
4666    b: vector_unsigned_char,
4667) -> vector_unsigned_char {
4668    // FIXME(llvm) sadly this does not work https://github.com/llvm/llvm-project/issues/129608
4669    // let a: u128 = transmute(a);
4670    // let b: u128 = transmute(b);
4671    // transmute(!a.overflowing_sub(b).1 as u128)
4672    transmute(vscbiq(transmute(a), transmute(b)))
4673}
4674
4675/// Vector Add Compute Carryout unsigned 128-bits
4676#[inline]
4677#[target_feature(enable = "vector")]
4678#[unstable(feature = "stdarch_s390x", issue = "135681")]
4679#[cfg_attr(test, assert_instr(vaccq))]
4680pub unsafe fn vec_addc_u128(
4681    a: vector_unsigned_char,
4682    b: vector_unsigned_char,
4683) -> vector_unsigned_char {
4684    let a: u128 = transmute(a);
4685    let b: u128 = transmute(b);
4686    transmute(a.overflowing_add(b).1 as u128)
4687}
4688
4689/// Vector Add With Carry unsigned 128-bits
4690#[inline]
4691#[target_feature(enable = "vector")]
4692#[unstable(feature = "stdarch_s390x", issue = "135681")]
4693#[cfg_attr(test, assert_instr(vacq))]
4694pub unsafe fn vec_adde_u128(
4695    a: vector_unsigned_char,
4696    b: vector_unsigned_char,
4697    c: vector_unsigned_char,
4698) -> vector_unsigned_char {
4699    let a: u128 = transmute(a);
4700    let b: u128 = transmute(b);
4701    let c: u128 = transmute(c);
4702    // FIXME(llvm) sadly this does not work
4703    //     let (d, _carry) = a.carrying_add(b, c & 1 != 0);
4704    //     transmute(d)
4705    transmute(vacq(a, b, c))
4706}
4707
4708/// Vector Add With Carry Compute Carry unsigned 128-bits
4709#[inline]
4710#[target_feature(enable = "vector")]
4711#[unstable(feature = "stdarch_s390x", issue = "135681")]
4712#[cfg_attr(test, assert_instr(vacccq))]
4713pub unsafe fn vec_addec_u128(
4714    a: vector_unsigned_char,
4715    b: vector_unsigned_char,
4716    c: vector_unsigned_char,
4717) -> vector_unsigned_char {
4718    let a: u128 = transmute(a);
4719    let b: u128 = transmute(b);
4720    let c: u128 = transmute(c);
4721    let (_d, carry) = a.carrying_add(b, c & 1 != 0);
4722    transmute(carry as u128)
4723}
4724
4725/// Vector Subtract with Carryout
4726///
4727/// Subtracts unsigned quadword values with carry bit from a previous operation.
4728///
4729/// This function operates on the vectors as 128-bit unsigned integers. It returns a vector containing the result of subtracting of b from a,
4730/// and the carryout bit from a previous operation.
4731///
4732/// Note: Only the borrow indication bit (127-bit) of c is used, and the other bits are ignored.
4733#[inline]
4734#[target_feature(enable = "vector")]
4735#[unstable(feature = "stdarch_s390x", issue = "135681")]
4736#[cfg_attr(test, assert_instr(vsbiq))]
4737pub unsafe fn vec_sube_u128(
4738    a: vector_unsigned_char,
4739    b: vector_unsigned_char,
4740    c: vector_unsigned_char,
4741) -> vector_unsigned_char {
4742    transmute(vsbiq(transmute(a), transmute(b), transmute(c)))
4743}
4744
4745/// Vector Subtract with Carryout, Carryout
4746///
4747/// Gets the carry bit of the 128-bit subtraction of two quadword values with carry bit from the previous operation.
4748///
4749/// It returns a vector containing the carryout produced from the result of subtracting of b from a,
4750/// and the carryout bit from a previous operation. If no borrow occurred, the 127-bit of d is 1, otherwise 0.
4751/// All other bits of d are 0.
4752///
4753/// Note: Only the borrow indication bit (127-bit) of c is used, and the other bits are ignored.
4754#[inline]
4755#[target_feature(enable = "vector")]
4756#[unstable(feature = "stdarch_s390x", issue = "135681")]
4757#[cfg_attr(test, assert_instr(vsbcbiq))]
4758pub unsafe fn vec_subec_u128(
4759    a: vector_unsigned_char,
4760    b: vector_unsigned_char,
4761    c: vector_unsigned_char,
4762) -> vector_unsigned_char {
4763    transmute(vsbcbiq(transmute(a), transmute(b), transmute(c)))
4764}
4765
4766/// Vector Splat Signed Byte
4767#[inline]
4768#[target_feature(enable = "vector")]
4769#[unstable(feature = "stdarch_s390x", issue = "135681")]
4770#[cfg_attr(test, assert_instr(vrepib, IMM = 42))]
4771pub unsafe fn vec_splat_s8<const IMM: i8>() -> vector_signed_char {
4772    vector_signed_char([IMM; 16])
4773}
4774
4775/// Vector Splat Signed Halfword
4776#[inline]
4777#[target_feature(enable = "vector")]
4778#[unstable(feature = "stdarch_s390x", issue = "135681")]
4779#[cfg_attr(test, assert_instr(vrepih, IMM = 42))]
4780pub unsafe fn vec_splat_s16<const IMM: i16>() -> vector_signed_short {
4781    vector_signed_short([IMM; 8])
4782}
4783
4784/// Vector Splat Signed Word
4785#[inline]
4786#[target_feature(enable = "vector")]
4787#[unstable(feature = "stdarch_s390x", issue = "135681")]
4788#[cfg_attr(test, assert_instr(vrepif, IMM = 42))]
4789pub unsafe fn vec_splat_s32<const IMM: i16>() -> vector_signed_int {
4790    vector_signed_int([IMM as i32; 4])
4791}
4792
4793/// Vector Splat Signed Doubleword
4794#[inline]
4795#[target_feature(enable = "vector")]
4796#[unstable(feature = "stdarch_s390x", issue = "135681")]
4797#[cfg_attr(test, assert_instr(vrepig, IMM = 42))]
4798pub unsafe fn vec_splat_s64<const IMM: i16>() -> vector_signed_long_long {
4799    vector_signed_long_long([IMM as i64; 2])
4800}
4801
4802/// Vector Splat Unsigned Byte
4803#[inline]
4804#[target_feature(enable = "vector")]
4805#[unstable(feature = "stdarch_s390x", issue = "135681")]
4806#[cfg_attr(test, assert_instr(vrepib, IMM = 42))]
4807pub unsafe fn vec_splat_u8<const IMM: u8>() -> vector_unsigned_char {
4808    vector_unsigned_char([IMM; 16])
4809}
4810
4811/// Vector Splat Unsigned Halfword
4812#[inline]
4813#[target_feature(enable = "vector")]
4814#[unstable(feature = "stdarch_s390x", issue = "135681")]
4815#[cfg_attr(test, assert_instr(vrepih, IMM = 42))]
4816pub unsafe fn vec_splat_u16<const IMM: i16>() -> vector_unsigned_short {
4817    vector_unsigned_short([IMM as u16; 8])
4818}
4819
4820/// Vector Splat Unsigned Word
4821#[inline]
4822#[target_feature(enable = "vector")]
4823#[unstable(feature = "stdarch_s390x", issue = "135681")]
4824#[cfg_attr(test, assert_instr(vrepif, IMM = 42))]
4825pub unsafe fn vec_splat_u32<const IMM: i16>() -> vector_unsigned_int {
4826    vector_unsigned_int([IMM as u32; 4])
4827}
4828
4829/// Vector Splat Unsigned Doubleword
4830#[inline]
4831#[target_feature(enable = "vector")]
4832#[unstable(feature = "stdarch_s390x", issue = "135681")]
4833#[cfg_attr(test, assert_instr(vrepig, IMM = 42))]
4834pub unsafe fn vec_splat_u64<const IMM: i16>() -> vector_unsigned_long_long {
4835    vector_unsigned_long_long([IMM as u64; 2])
4836}
4837
4838macro_rules! vec_find_any {
4839    ($($Trait:ident $fun:ident $doc:literal)*) => {
4840        $(
4841            #[inline]
4842            #[target_feature(enable = "vector")]
4843            #[unstable(feature = "stdarch_s390x", issue = "135681")]
4844            #[doc = $doc]
4845            pub unsafe fn $fun<T: sealed::$Trait<U>, U>(a: T, b: U) -> T::Result {
4846                a.$fun(b)
4847            }
4848        )*
4849    }
4850}
4851
4852vec_find_any! {
4853    VectorFindAnyEq vec_find_any_eq "Vector Find Any Element Equal with Condition Code"
4854    VectorFindAnyNe vec_find_any_ne "Vector Find Any Element Not Equal with Condition Code"
4855    VectorFindAnyEqIdx vec_find_any_eq_idx "Vector Find Any Element Equal Index with Condition Code"
4856    VectorFindAnyNeIdx vec_find_any_ne_idx "Vector Find Any Element Not Equal Index with Condition Code"
4857    VectorFindAnyEqOrZeroIdx vec_find_any_eq_or_0_idx "Vector Find Any Element Equal or Zero Index with Condition Code"
4858    VectorFindAnyNeOrZeroIdx vec_find_any_ne_or_0_idx "Vector Find Any Element Not Equal or Zero Index with Condition Code"
4859}
4860
4861macro_rules! vec_find_any_cc {
4862    ($($Trait:ident $fun:ident $doc:literal)*) => {
4863        $(
4864            #[inline]
4865            #[target_feature(enable = "vector")]
4866            #[unstable(feature = "stdarch_s390x", issue = "135681")]
4867            #[doc = $doc]
4868            pub unsafe fn $fun<T: sealed::$Trait<U>, U>(a: T, b: U) -> (T::Result, i32) {
4869                a.$fun(b)
4870            }
4871        )*
4872    }
4873}
4874
4875vec_find_any_cc! {
4876    VectorFindAnyEqCC vec_find_any_eq_cc "Vector Find Any Element Equal with Condition Code"
4877    VectorFindAnyNeCC vec_find_any_ne_cc "Vector Find Any Element Not Equal with Condition Code"
4878    VectorFindAnyEqIdxCC vec_find_any_eq_idx_cc "Vector Find Any Element Equal Index with Condition Code"
4879    VectorFindAnyNeIdxCC vec_find_any_ne_idx_cc "Vector Find Any Element Not Equal Index with Condition Code"
4880    VectorFindAnyEqOrZeroIdxCC vec_find_any_eq_or_0_idx_cc "Vector Find Any Element Equal or Zero Index with Condition Code"
4881    VectorFindAnyNeOrZeroIdxCC vec_find_any_ne_or_0_idx_cc "Vector Find Any Element Not Equal or Zero Index with Condition Code"
4882}
4883
4884/// Vector Load
4885#[inline]
4886#[target_feature(enable = "vector")]
4887#[unstable(feature = "stdarch_s390x", issue = "135681")]
4888pub unsafe fn vec_xl<T: sealed::VectorLoad>(offset: isize, ptr: *const T::ElementType) -> T {
4889    T::vec_xl(offset, ptr)
4890}
4891
4892/// Vector Load Pair
4893#[inline]
4894#[target_feature(enable = "vector")]
4895#[unstable(feature = "stdarch_s390x", issue = "135681")]
4896pub unsafe fn vec_load_pair<T: sealed::VectorLoadPair>(a: T::ElementType, b: T::ElementType) -> T {
4897    T::vec_load_pair(a, b)
4898}
4899
4900/// Vector Load to Block Boundary
4901#[inline]
4902#[target_feature(enable = "vector")]
4903#[unstable(feature = "stdarch_s390x", issue = "135681")]
4904pub unsafe fn vec_load_bndry<T: sealed::VectorLoad, const BLOCK_BOUNDARY: u16>(
4905    ptr: *const T::ElementType,
4906) -> MaybeUninit<T> {
4907    T::vec_load_bndry::<BLOCK_BOUNDARY>(ptr)
4908}
4909
4910/// Vector Store
4911#[inline]
4912#[target_feature(enable = "vector")]
4913#[unstable(feature = "stdarch_s390x", issue = "135681")]
4914pub unsafe fn vec_xst<T: sealed::VectorStore>(vector: T, offset: isize, ptr: *mut T::ElementType) {
4915    vector.vec_xst(offset, ptr)
4916}
4917
4918/// Vector Load with Length
4919#[inline]
4920#[target_feature(enable = "vector")]
4921#[unstable(feature = "stdarch_s390x", issue = "135681")]
4922pub unsafe fn vec_load_len<T: sealed::VectorLoad>(
4923    ptr: *const T::ElementType,
4924    byte_count: u32,
4925) -> T {
4926    T::vec_load_len(ptr, byte_count)
4927}
4928
4929/// Vector Store with Length
4930#[inline]
4931#[target_feature(enable = "vector")]
4932#[unstable(feature = "stdarch_s390x", issue = "135681")]
4933pub unsafe fn vec_store_len<T: sealed::VectorStore>(
4934    vector: T,
4935    ptr: *mut T::ElementType,
4936    byte_count: u32,
4937) {
4938    vector.vec_store_len(ptr, byte_count)
4939}
4940
4941/// Vector Load Rightmost with Length
4942#[inline]
4943#[target_feature(enable = "vector-packed-decimal")]
4944#[unstable(feature = "stdarch_s390x", issue = "135681")]
4945#[cfg_attr(test, assert_instr(vlrlr))]
4946pub unsafe fn vec_load_len_r(ptr: *const u8, byte_count: u32) -> vector_unsigned_char {
4947    vlrl(byte_count, ptr)
4948}
4949
4950/// Vector Store Rightmost with Length
4951#[inline]
4952#[target_feature(enable = "vector-packed-decimal")]
4953#[unstable(feature = "stdarch_s390x", issue = "135681")]
4954#[cfg_attr(test, assert_instr(vstrlr))]
4955pub unsafe fn vec_store_len_r(vector: vector_unsigned_char, ptr: *mut u8, byte_count: u32) {
4956    vstrl(vector, byte_count, ptr)
4957}
4958
4959/// Vector Multiply Add
4960#[inline]
4961#[target_feature(enable = "vector-packed-decimal")]
4962#[unstable(feature = "stdarch_s390x", issue = "135681")]
4963pub unsafe fn vec_madd<T: sealed::VectorMadd>(a: T, b: T, c: T) -> T {
4964    a.vec_madd(b, c)
4965}
4966
4967/// Vector Multiply Add
4968#[inline]
4969#[target_feature(enable = "vector-packed-decimal")]
4970#[unstable(feature = "stdarch_s390x", issue = "135681")]
4971pub unsafe fn vec_msub<T: sealed::VectorMadd>(a: T, b: T, c: T) -> T {
4972    a.vec_msub(b, c)
4973}
4974
4975/// Vector Multiply and Add Even
4976#[inline]
4977#[target_feature(enable = "vector-packed-decimal")]
4978#[unstable(feature = "stdarch_s390x", issue = "135681")]
4979pub unsafe fn vec_meadd<T: sealed::VectorMeadd>(a: T, b: T, c: T::Result) -> T::Result {
4980    a.vec_meadd(b, c)
4981}
4982
4983/// Vector Multiply and Add Odd
4984#[inline]
4985#[target_feature(enable = "vector-packed-decimal")]
4986#[unstable(feature = "stdarch_s390x", issue = "135681")]
4987pub unsafe fn vec_moadd<T: sealed::VectorMoadd>(a: T, b: T, c: T::Result) -> T::Result {
4988    a.vec_moadd(b, c)
4989}
4990
4991/// Vector Multiply and Add High
4992#[inline]
4993#[target_feature(enable = "vector-packed-decimal")]
4994#[unstable(feature = "stdarch_s390x", issue = "135681")]
4995pub unsafe fn vec_mhadd<T: sealed::VectorMhadd>(a: T, b: T, c: T::Result) -> T::Result {
4996    a.vec_mhadd(b, c)
4997}
4998
4999/// Vector Multiply and Add Low
5000#[inline]
5001#[target_feature(enable = "vector-packed-decimal")]
5002#[unstable(feature = "stdarch_s390x", issue = "135681")]
5003pub unsafe fn vec_mladd<T: sealed::VectorMladd>(a: T, b: T, c: T::Result) -> T::Result {
5004    a.vec_mladd(b, c)
5005}
5006
5007/// Vector Checksum
5008#[inline]
5009#[target_feature(enable = "vector")]
5010#[unstable(feature = "stdarch_s390x", issue = "135681")]
5011#[cfg_attr(test, assert_instr(vcksm))]
5012pub unsafe fn vec_checksum(a: vector_unsigned_int, b: vector_unsigned_int) -> vector_unsigned_int {
5013    vcksm(a, b)
5014}
5015
5016/// Vector Multiply Even
5017#[inline]
5018#[target_feature(enable = "vector")]
5019#[unstable(feature = "stdarch_s390x", issue = "135681")]
5020pub unsafe fn vec_mule<T: sealed::VectorMule<U>, U>(a: T, b: T) -> U {
5021    a.vec_mule(b)
5022}
5023
5024/// Vector Multiply Odd
5025#[inline]
5026#[target_feature(enable = "vector")]
5027#[unstable(feature = "stdarch_s390x", issue = "135681")]
5028pub unsafe fn vec_mulo<T: sealed::VectorMulo<U>, U>(a: T, b: T) -> U {
5029    a.vec_mulo(b)
5030}
5031
5032/// Vector Multiply High
5033#[inline]
5034#[target_feature(enable = "vector")]
5035#[unstable(feature = "stdarch_s390x", issue = "135681")]
5036pub unsafe fn vec_mulh<T: sealed::VectorMulh<U>, U>(a: T, b: T) -> U {
5037    a.vec_mulh(b)
5038}
5039
5040/// Vector Galois Field Multiply Sum
5041#[inline]
5042#[target_feature(enable = "vector")]
5043#[unstable(feature = "stdarch_s390x", issue = "135681")]
5044pub unsafe fn vec_gfmsum<T: sealed::VectorGfmsum<U>, U>(a: T, b: T) -> U {
5045    a.vec_gfmsum(b)
5046}
5047
5048/// Vector Galois Field Multiply Sum
5049#[inline]
5050#[target_feature(enable = "vector")]
5051#[unstable(feature = "stdarch_s390x", issue = "135681")]
5052pub unsafe fn vec_gfmsum_accum<T: sealed::VectorGfmsumAccum>(
5053    a: T,
5054    b: T,
5055    c: T::Result,
5056) -> T::Result {
5057    a.vec_gfmsum_accum(b, c)
5058}
5059
5060/// Vector Galois Field Multiply Sum 128-bits
5061#[inline]
5062#[target_feature(enable = "vector")]
5063#[unstable(feature = "stdarch_s390x", issue = "135681")]
5064#[cfg_attr(test, assert_instr(vgfmg))]
5065pub unsafe fn vec_gfmsum_128(
5066    a: vector_unsigned_long_long,
5067    b: vector_unsigned_long_long,
5068) -> vector_unsigned_char {
5069    transmute(vgfmg(a, b))
5070}
5071
5072/// Vector Galois Field Multiply Sum and Accumulate 128-bits
5073#[inline]
5074#[target_feature(enable = "vector")]
5075#[unstable(feature = "stdarch_s390x", issue = "135681")]
5076#[cfg_attr(test, assert_instr(vgfmag))]
5077pub unsafe fn vec_gfmsum_accum_128(
5078    a: vector_unsigned_long_long,
5079    b: vector_unsigned_long_long,
5080    c: vector_unsigned_char,
5081) -> vector_unsigned_char {
5082    transmute(vgfmag(a, b, transmute(c)))
5083}
5084
5085/// Vector Bit Permute
5086#[inline]
5087#[target_feature(enable = "vector-enhancements-1")]
5088#[unstable(feature = "stdarch_s390x", issue = "135681")]
5089#[cfg_attr(test, assert_instr(vbperm))]
5090pub unsafe fn vec_bperm_u128(
5091    a: vector_unsigned_char,
5092    b: vector_unsigned_char,
5093) -> vector_unsigned_long_long {
5094    vbperm(a, b)
5095}
5096
5097/// Vector Gather Element
5098#[inline]
5099#[target_feature(enable = "vector")]
5100#[unstable(feature = "stdarch_s390x", issue = "135681")]
5101pub unsafe fn vec_gather_element<T: sealed::VectorGatherElement, const D: u32>(
5102    a: T,
5103    b: T::Offset,
5104    c: *const T::Element,
5105) -> T {
5106    a.vec_gather_element::<D>(b, c)
5107}
5108
5109/// Vector Select
5110#[inline]
5111#[target_feature(enable = "vector")]
5112#[unstable(feature = "stdarch_s390x", issue = "135681")]
5113pub unsafe fn vec_sel<T: sealed::VectorSel<U>, U>(a: T, b: T, c: U) -> T {
5114    a.vec_sel(b, c)
5115}
5116
5117#[unstable(feature = "stdarch_s390x", issue = "135681")]
5118pub const __VEC_CLASS_FP_ZERO_P: u32 = 1 << 11;
5119#[unstable(feature = "stdarch_s390x", issue = "135681")]
5120pub const __VEC_CLASS_FP_ZERO_N: u32 = 1 << 10;
5121#[unstable(feature = "stdarch_s390x", issue = "135681")]
5122pub const __VEC_CLASS_FP_ZERO: u32 = __VEC_CLASS_FP_ZERO_P | __VEC_CLASS_FP_ZERO_N;
5123#[unstable(feature = "stdarch_s390x", issue = "135681")]
5124pub const __VEC_CLASS_FP_NORMAL_P: u32 = 1 << 9;
5125#[unstable(feature = "stdarch_s390x", issue = "135681")]
5126pub const __VEC_CLASS_FP_NORMAL_N: u32 = 1 << 8;
5127#[unstable(feature = "stdarch_s390x", issue = "135681")]
5128pub const __VEC_CLASS_FP_NORMAL: u32 = __VEC_CLASS_FP_NORMAL_P | __VEC_CLASS_FP_NORMAL_N;
5129#[unstable(feature = "stdarch_s390x", issue = "135681")]
5130pub const __VEC_CLASS_FP_SUBNORMAL_P: u32 = 1 << 7;
5131#[unstable(feature = "stdarch_s390x", issue = "135681")]
5132pub const __VEC_CLASS_FP_SUBNORMAL_N: u32 = 1 << 6;
5133#[unstable(feature = "stdarch_s390x", issue = "135681")]
5134pub const __VEC_CLASS_FP_SUBNORMAL: u32 = __VEC_CLASS_FP_SUBNORMAL_P | __VEC_CLASS_FP_SUBNORMAL_N;
5135#[unstable(feature = "stdarch_s390x", issue = "135681")]
5136pub const __VEC_CLASS_FP_INFINITY_P: u32 = 1 << 5;
5137#[unstable(feature = "stdarch_s390x", issue = "135681")]
5138pub const __VEC_CLASS_FP_INFINITY_N: u32 = 1 << 4;
5139#[unstable(feature = "stdarch_s390x", issue = "135681")]
5140pub const __VEC_CLASS_FP_INFINITY: u32 = __VEC_CLASS_FP_INFINITY_P | __VEC_CLASS_FP_INFINITY_N;
5141#[unstable(feature = "stdarch_s390x", issue = "135681")]
5142pub const __VEC_CLASS_FP_QNAN_P: u32 = 1 << 3;
5143#[unstable(feature = "stdarch_s390x", issue = "135681")]
5144pub const __VEC_CLASS_FP_QNAN_N: u32 = 1 << 2;
5145#[unstable(feature = "stdarch_s390x", issue = "135681")]
5146pub const __VEC_CLASS_FP_QNAN: u32 = __VEC_CLASS_FP_QNAN_P | __VEC_CLASS_FP_QNAN_N;
5147#[unstable(feature = "stdarch_s390x", issue = "135681")]
5148pub const __VEC_CLASS_FP_SNAN_P: u32 = 1 << 1;
5149#[unstable(feature = "stdarch_s390x", issue = "135681")]
5150pub const __VEC_CLASS_FP_SNAN_N: u32 = 1 << 0;
5151#[unstable(feature = "stdarch_s390x", issue = "135681")]
5152pub const __VEC_CLASS_FP_SNAN: u32 = __VEC_CLASS_FP_SNAN_P | __VEC_CLASS_FP_SNAN_N;
5153#[unstable(feature = "stdarch_s390x", issue = "135681")]
5154pub const __VEC_CLASS_FP_NAN: u32 = __VEC_CLASS_FP_QNAN | __VEC_CLASS_FP_SNAN;
5155#[unstable(feature = "stdarch_s390x", issue = "135681")]
5156pub const __VEC_CLASS_FP_NOT_NORMAL: u32 =
5157    __VEC_CLASS_FP_NAN | __VEC_CLASS_FP_SUBNORMAL | __VEC_CLASS_FP_ZERO | __VEC_CLASS_FP_INFINITY;
5158
5159/// Vector Floating-Point Test Data Class
5160///
5161/// You can use the `__VEC_CLASS_FP_*` constants as the argument for this operand
5162#[inline]
5163#[target_feature(enable = "vector")]
5164#[unstable(feature = "stdarch_s390x", issue = "135681")]
5165pub unsafe fn vec_fp_test_data_class<T: sealed::VectorFpTestDataClass, const CLASS: u32>(
5166    a: T,
5167    c: *mut i32,
5168) -> T::Result {
5169    let (x, y) = a.vec_fp_test_data_class::<CLASS>();
5170    c.write(y);
5171    x
5172}
5173
5174/// All Elements Not a Number
5175#[inline]
5176#[target_feature(enable = "vector")]
5177#[unstable(feature = "stdarch_s390x", issue = "135681")]
5178pub unsafe fn vec_all_nan<T: sealed::VectorFpTestDataClass>(a: T) -> i32 {
5179    i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 == 0)
5180}
5181
5182/// All Elements Numeric
5183#[inline]
5184#[target_feature(enable = "vector")]
5185#[unstable(feature = "stdarch_s390x", issue = "135681")]
5186pub unsafe fn vec_all_numeric<T: sealed::VectorFpTestDataClass>(a: T) -> i32 {
5187    i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 == 3)
5188}
5189
5190/// Any Elements Not a Number
5191#[inline]
5192#[target_feature(enable = "vector")]
5193#[unstable(feature = "stdarch_s390x", issue = "135681")]
5194pub unsafe fn vec_any_nan<T: sealed::VectorFpTestDataClass>(a: T) -> i32 {
5195    i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 != 3)
5196}
5197
5198/// Any Elements Numeric
5199#[inline]
5200#[target_feature(enable = "vector")]
5201#[unstable(feature = "stdarch_s390x", issue = "135681")]
5202pub unsafe fn vec_any_numeric<T: sealed::VectorFpTestDataClass>(a: T) -> i32 {
5203    i32::from(a.vec_fp_test_data_class::<__VEC_CLASS_FP_NAN>().1 != 0)
5204}
5205
5206/// Vector Test under Mask
5207#[inline]
5208#[target_feature(enable = "vector")]
5209#[unstable(feature = "stdarch_s390x", issue = "135681")]
5210pub unsafe fn vec_test_mask<T: sealed::VectorTestMask>(a: T, b: T::Mask) -> i32 {
5211    // I can't find much information about this, but this might just be a check for whether the
5212    // bitwise and of a and b is non-zero?
5213    a.vec_test_mask(b)
5214}
5215
5216/// Vector Search String
5217#[inline]
5218#[target_feature(enable = "vector")]
5219#[unstable(feature = "stdarch_s390x", issue = "135681")]
5220pub unsafe fn vec_search_string_cc<T: sealed::VectorSearchString>(
5221    a: T,
5222    b: T,
5223    c: vector_unsigned_char,
5224) -> (vector_unsigned_char, i32) {
5225    a.vec_search_string_cc(b, c)
5226}
5227
5228/// Vector Search String Until Zero
5229#[inline]
5230#[target_feature(enable = "vector")]
5231#[unstable(feature = "stdarch_s390x", issue = "135681")]
5232pub unsafe fn vec_search_string_until_zero_cc<T: sealed::VectorSearchString>(
5233    a: T,
5234    b: T,
5235    c: vector_unsigned_char,
5236) -> (vector_unsigned_char, i32) {
5237    a.vec_search_string_until_zero_cc(b, c)
5238}
5239
5240/// Vector Convert from float (even elements) to double
5241#[inline]
5242#[target_feature(enable = "vector-enhancements-1")]
5243#[unstable(feature = "stdarch_s390x", issue = "135681")]
5244// FIXME: this emits `vflls` where `vldeb` is expected
5245// #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vldeb))]
5246pub unsafe fn vec_doublee(a: vector_float) -> vector_double {
5247    let even = simd_shuffle::<_, _, f32x2>(a, a, const { u32x2::from_array([0, 2]) });
5248    simd_as(even)
5249}
5250
5251/// Vector Convert from double to float (even elements)
5252#[inline]
5253#[target_feature(enable = "vector-enhancements-1")]
5254#[unstable(feature = "stdarch_s390x", issue = "135681")]
5255// FIXME: the C version uses a shuffle mask with poison; we can't do that
5256// #[cfg_attr(all(test, target_feature = "vector-enhancements-1"), assert_instr(vledb))]
5257pub unsafe fn vec_floate(a: vector_double) -> vector_float {
5258    let truncated: f32x2 = simd_as(a);
5259    simd_shuffle(
5260        truncated,
5261        truncated,
5262        const { u32x4::from_array([0, 0, 1, 1]) },
5263    )
5264}
5265
5266/// Vector Convert from int to float
5267#[inline]
5268#[target_feature(enable = "vector")]
5269#[unstable(feature = "stdarch_s390x", issue = "135681")]
5270pub unsafe fn vec_float(a: impl sealed::VectorFloat) -> vector_float {
5271    a.vec_float()
5272}
5273
5274/// Vector Convert from long long to double
5275#[inline]
5276#[target_feature(enable = "vector")]
5277#[unstable(feature = "stdarch_s390x", issue = "135681")]
5278pub unsafe fn vec_double(a: impl sealed::VectorDouble) -> vector_double {
5279    a.vec_double()
5280}
5281
5282/// Vector Sign Extend to Doubleword
5283#[inline]
5284#[target_feature(enable = "vector")]
5285#[unstable(feature = "stdarch_s390x", issue = "135681")]
5286pub unsafe fn vec_extend_s64(a: impl sealed::VectorExtendSigned64) -> vector_signed_long_long {
5287    a.vec_extend_s64()
5288}
5289
5290/// Vector Convert floating point to signed
5291#[inline]
5292#[target_feature(enable = "vector")]
5293#[unstable(feature = "stdarch_s390x", issue = "135681")]
5294pub unsafe fn vec_signed<T: sealed::VectorSigned>(a: T) -> T::Result {
5295    a.vec_signed()
5296}
5297
5298/// Vector Convert floating point to unsigned
5299#[inline]
5300#[target_feature(enable = "vector")]
5301#[unstable(feature = "stdarch_s390x", issue = "135681")]
5302pub unsafe fn vec_unsigned<T: sealed::VectorUnsigned>(a: T) -> T::Result {
5303    a.vec_unsigned()
5304}
5305
5306/// Vector Copy Until Zero
5307#[inline]
5308#[target_feature(enable = "vector")]
5309#[unstable(feature = "stdarch_s390x", issue = "135681")]
5310pub unsafe fn vec_cp_until_zero<T: sealed::VectorCopyUntilZero>(a: T) -> T {
5311    a.vec_cp_until_zero()
5312}
5313
5314/// Vector Copy Until Zero
5315#[inline]
5316#[target_feature(enable = "vector")]
5317#[unstable(feature = "stdarch_s390x", issue = "135681")]
5318pub unsafe fn vec_cp_until_zero_cc<T: sealed::VectorCopyUntilZeroCC>(a: T) -> (T, i32) {
5319    a.vec_cp_until_zero_cc()
5320}
5321
5322/// Vector Multiply Sum Logical
5323#[inline]
5324#[target_feature(enable = "vector-enhancements-1")]
5325#[unstable(feature = "stdarch_s390x", issue = "135681")]
5326#[cfg_attr(
5327    all(test, target_feature = "vector-enhancements-1"),
5328    assert_instr(vmslg, D = 4)
5329)]
5330pub unsafe fn vec_msum_u128<const D: u32>(
5331    a: vector_unsigned_long_long,
5332    b: vector_unsigned_long_long,
5333    c: vector_unsigned_char,
5334) -> vector_unsigned_char {
5335    const {
5336        if !matches!(D, 0 | 4 | 8 | 12) {
5337            panic!("D needs to be one of 0, 4, 8, 12");
5338        }
5339    };
5340    transmute(vmslg(a, b, transmute(c), D))
5341}
5342
5343/// Vector Shift Left Double by Byte
5344#[inline]
5345#[target_feature(enable = "vector")]
5346#[unstable(feature = "stdarch_s390x", issue = "135681")]
5347pub unsafe fn vec_sld<T: sealed::VectorSld, const C: u32>(a: T, b: T) -> T {
5348    static_assert_uimm_bits!(C, 4);
5349    a.vec_sld::<C>(b)
5350}
5351
5352/// Vector Shift Left Double by Word
5353#[inline]
5354#[target_feature(enable = "vector")]
5355#[unstable(feature = "stdarch_s390x", issue = "135681")]
5356pub unsafe fn vec_sldw<T: sealed::VectorSld, const C: u32>(a: T, b: T) -> T {
5357    static_assert_uimm_bits!(C, 2);
5358    a.vec_sldw::<C>(b)
5359}
5360
5361/// Vector Shift Left Double by Bit
5362#[inline]
5363#[target_feature(enable = "vector-enhancements-2")]
5364#[unstable(feature = "stdarch_s390x", issue = "135681")]
5365pub unsafe fn vec_sldb<T: sealed::VectorSld, const C: u32>(a: T, b: T) -> T {
5366    static_assert_uimm_bits!(C, 3);
5367    a.vec_sldb::<C>(b)
5368}
5369
5370/// Vector Shift Right Double by Bit
5371#[inline]
5372#[target_feature(enable = "vector-enhancements-2")]
5373#[unstable(feature = "stdarch_s390x", issue = "135681")]
5374pub unsafe fn vec_srdb<T: sealed::VectorSrdb, const C: u32>(a: T, b: T) -> T {
5375    static_assert_uimm_bits!(C, 3);
5376    a.vec_srdb::<C>(b)
5377}
5378
5379/// Vector Compare Ranges
5380#[inline]
5381#[target_feature(enable = "vector")]
5382#[unstable(feature = "stdarch_s390x", issue = "135681")]
5383pub unsafe fn vec_cmprg<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5384    a.vstrc::<{ FindImm::Eq as u32 }>(b, c)
5385}
5386
5387/// Vector Compare Not in Ranges
5388#[inline]
5389#[target_feature(enable = "vector")]
5390#[unstable(feature = "stdarch_s390x", issue = "135681")]
5391pub unsafe fn vec_cmpnrg<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5392    a.vstrc::<{ FindImm::Ne as u32 }>(b, c)
5393}
5394
5395/// Vector Compare Ranges Index
5396#[inline]
5397#[target_feature(enable = "vector")]
5398#[unstable(feature = "stdarch_s390x", issue = "135681")]
5399pub unsafe fn vec_cmprg_idx<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5400    a.vstrc::<{ FindImm::EqIdx as u32 }>(b, c)
5401}
5402
5403/// Vector Compare Not in Ranges Index
5404#[inline]
5405#[target_feature(enable = "vector")]
5406#[unstable(feature = "stdarch_s390x", issue = "135681")]
5407pub unsafe fn vec_cmpnrg_idx<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5408    a.vstrc::<{ FindImm::NeIdx as u32 }>(b, c)
5409}
5410
5411/// Vector Compare Ranges with Condition Code
5412#[inline]
5413#[target_feature(enable = "vector")]
5414#[unstable(feature = "stdarch_s390x", issue = "135681")]
5415pub unsafe fn vec_cmprg_cc<T: sealed::VectorCompareRange>(
5416    a: T,
5417    b: T,
5418    c: T,
5419    d: *mut i32,
5420) -> T::Result {
5421    let (x, y) = a.vstrcs::<{ FindImm::Eq as u32 }>(b, c);
5422    d.write(y);
5423    x
5424}
5425
5426/// Vector Compare Not in Ranges with Condition Code
5427#[inline]
5428#[target_feature(enable = "vector")]
5429#[unstable(feature = "stdarch_s390x", issue = "135681")]
5430pub unsafe fn vec_cmpnrg_cc<T: sealed::VectorCompareRange>(
5431    a: T,
5432    b: T,
5433    c: T,
5434    d: *mut i32,
5435) -> T::Result {
5436    let (x, y) = a.vstrcs::<{ FindImm::Ne as u32 }>(b, c);
5437    d.write(y);
5438    x
5439}
5440
5441/// Vector Compare Ranges Index with Condition Code
5442#[inline]
5443#[target_feature(enable = "vector")]
5444#[unstable(feature = "stdarch_s390x", issue = "135681")]
5445pub unsafe fn vec_cmprg_idx_cc<T: sealed::VectorCompareRange>(
5446    a: T,
5447    b: T,
5448    c: T,
5449    d: *mut i32,
5450) -> T::Result {
5451    let (x, y) = a.vstrcs::<{ FindImm::EqIdx as u32 }>(b, c);
5452    d.write(y);
5453    x
5454}
5455
5456/// Vector Compare Not in Ranges Index with Condition Code
5457#[inline]
5458#[target_feature(enable = "vector")]
5459#[unstable(feature = "stdarch_s390x", issue = "135681")]
5460pub unsafe fn vec_cmpnrg_idx_cc<T: sealed::VectorCompareRange>(
5461    a: T,
5462    b: T,
5463    c: T,
5464    d: *mut i32,
5465) -> T::Result {
5466    let (x, y) = a.vstrcs::<{ FindImm::NeIdx as u32 }>(b, c);
5467    d.write(y);
5468    x
5469}
5470
5471/// Vector Compare Ranges or Zero Index
5472#[inline]
5473#[target_feature(enable = "vector")]
5474#[unstable(feature = "stdarch_s390x", issue = "135681")]
5475pub unsafe fn vec_cmprg_or_0_idx<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5476    a.vstrcz::<{ FindImm::EqIdx as u32 }>(b, c)
5477}
5478
5479/// Vector Compare Not in Ranges or Zero Index
5480#[inline]
5481#[target_feature(enable = "vector")]
5482#[unstable(feature = "stdarch_s390x", issue = "135681")]
5483pub unsafe fn vec_cmpnrg_or_0_idx<T: sealed::VectorCompareRange>(a: T, b: T, c: T) -> T::Result {
5484    a.vstrcz::<{ FindImm::NeIdx as u32 }>(b, c)
5485}
5486
5487/// Vector Compare Ranges or Zero Index with Condition Code
5488#[inline]
5489#[target_feature(enable = "vector")]
5490#[unstable(feature = "stdarch_s390x", issue = "135681")]
5491pub unsafe fn vec_cmprg_or_0_idx_cc<T: sealed::VectorCompareRange>(
5492    a: T,
5493    b: T,
5494    c: T,
5495    d: *mut i32,
5496) -> T::Result {
5497    let (x, y) = a.vstrczs::<{ FindImm::EqIdx as u32 }>(b, c);
5498    d.write(y);
5499    x
5500}
5501
5502/// Vector Compare Not in Ranges or Zero Index with Condition Code
5503#[inline]
5504#[target_feature(enable = "vector")]
5505#[unstable(feature = "stdarch_s390x", issue = "135681")]
5506pub unsafe fn vec_cmpnrg_or_0_idx_cc<T: sealed::VectorCompareRange>(
5507    a: T,
5508    b: T,
5509    c: T,
5510    d: *mut i32,
5511) -> T::Result {
5512    let (x, y) = a.vstrczs::<{ FindImm::NeIdx as u32 }>(b, c);
5513    d.write(y);
5514    x
5515}
5516
5517/// Vector Compare Equal
5518#[inline]
5519#[target_feature(enable = "vector")]
5520#[unstable(feature = "stdarch_s390x", issue = "135681")]
5521pub unsafe fn vec_cmpeq<T: sealed::VectorEquality>(a: T, b: T) -> T::Result {
5522    a.vec_cmpeq(b)
5523}
5524
5525/// Vector Compare Not Equal
5526#[inline]
5527#[target_feature(enable = "vector")]
5528#[unstable(feature = "stdarch_s390x", issue = "135681")]
5529pub unsafe fn vec_cmpne<T: sealed::VectorEquality>(a: T, b: T) -> T::Result {
5530    a.vec_cmpne(b)
5531}
5532
5533/// Vector Compare Greater Than
5534#[inline]
5535#[target_feature(enable = "vector")]
5536#[unstable(feature = "stdarch_s390x", issue = "135681")]
5537pub unsafe fn vec_cmpgt<T: sealed::VectorComparePredicate>(a: T, b: T) -> T::Result {
5538    a.vec_cmpgt(b)
5539}
5540
5541/// Vector Compare Greater Than or Equal
5542#[inline]
5543#[target_feature(enable = "vector")]
5544#[unstable(feature = "stdarch_s390x", issue = "135681")]
5545pub unsafe fn vec_cmpge<T: sealed::VectorComparePredicate>(a: T, b: T) -> T::Result {
5546    a.vec_cmpge(b)
5547}
5548
5549/// Vector Compare Less
5550#[inline]
5551#[target_feature(enable = "vector")]
5552#[unstable(feature = "stdarch_s390x", issue = "135681")]
5553pub unsafe fn vec_cmplt<T: sealed::VectorComparePredicate>(a: T, b: T) -> T::Result {
5554    a.vec_cmplt(b)
5555}
5556
5557/// Vector Compare Less Than or Equal
5558#[inline]
5559#[target_feature(enable = "vector")]
5560#[unstable(feature = "stdarch_s390x", issue = "135681")]
5561pub unsafe fn vec_cmple<T: sealed::VectorComparePredicate>(a: T, b: T) -> T::Result {
5562    a.vec_cmple(b)
5563}
5564
5565/// Vector Compare Equal Index
5566#[inline]
5567#[target_feature(enable = "vector")]
5568#[unstable(feature = "stdarch_s390x", issue = "135681")]
5569pub unsafe fn vec_cmpeq_idx<T: sealed::VectorEqualityIdx>(a: T, b: T) -> T::Result {
5570    a.vec_cmpeq_idx(b)
5571}
5572/// Vector Compare Not Equal Index
5573#[inline]
5574#[target_feature(enable = "vector")]
5575#[unstable(feature = "stdarch_s390x", issue = "135681")]
5576pub unsafe fn vec_cmpne_idx<T: sealed::VectorEqualityIdx>(a: T, b: T) -> T::Result {
5577    a.vec_cmpne_idx(b)
5578}
5579/// Vector Compare Equal Index with Condition Code
5580#[inline]
5581#[target_feature(enable = "vector")]
5582#[unstable(feature = "stdarch_s390x", issue = "135681")]
5583pub unsafe fn vec_cmpeq_idx_cc<T: sealed::VectorEqualityIdx>(a: T, b: T) -> (T::Result, i32) {
5584    a.vec_cmpeq_idx_cc(b)
5585}
5586/// Vector Compare Not Equal Index with Condition Code
5587#[inline]
5588#[target_feature(enable = "vector")]
5589#[unstable(feature = "stdarch_s390x", issue = "135681")]
5590pub unsafe fn vec_cmpne_idx_cc<T: sealed::VectorEqualityIdx>(a: T, b: T) -> (T::Result, i32) {
5591    a.vec_cmpne_idx_cc(b)
5592}
5593/// Vector Compare Equal or Zero Index
5594#[inline]
5595#[target_feature(enable = "vector")]
5596#[unstable(feature = "stdarch_s390x", issue = "135681")]
5597pub unsafe fn vec_cmpeq_or_0_idx<T: sealed::VectorEqualityIdx>(a: T, b: T) -> T::Result {
5598    a.vec_cmpeq_or_0_idx(b)
5599}
5600/// Vector Compare Not Equal or Zero Index
5601#[inline]
5602#[target_feature(enable = "vector")]
5603#[unstable(feature = "stdarch_s390x", issue = "135681")]
5604pub unsafe fn vec_cmpne_or_0_idx<T: sealed::VectorEqualityIdx>(a: T, b: T) -> T::Result {
5605    a.vec_cmpne_or_0_idx(b)
5606}
5607/// Vector Compare Equal or Zero Index with Condition Code
5608#[inline]
5609#[target_feature(enable = "vector")]
5610#[unstable(feature = "stdarch_s390x", issue = "135681")]
5611pub unsafe fn vec_cmpeq_or_0_idx_cc<T: sealed::VectorEqualityIdx>(a: T, b: T) -> (T::Result, i32) {
5612    a.vec_cmpeq_or_0_idx_cc(b)
5613}
5614/// Vector Compare Not Equal or Zero Index with Condition Code
5615#[inline]
5616#[target_feature(enable = "vector")]
5617#[unstable(feature = "stdarch_s390x", issue = "135681")]
5618pub unsafe fn vec_cmpne_or_0_idx_cc<T: sealed::VectorEqualityIdx>(a: T, b: T) -> (T::Result, i32) {
5619    a.vec_cmpne_or_0_idx_cc(b)
5620}
5621
5622/// All Elements Equal
5623#[inline]
5624#[target_feature(enable = "vector")]
5625#[unstable(feature = "stdarch_s390x", issue = "135681")]
5626pub unsafe fn vec_all_eq<T: sealed::VectorEquality>(a: T, b: T) -> i32 {
5627    simd_reduce_all(vec_cmpeq(a, b)) as i32 as i32
5628}
5629
5630/// All Elements Not Equal
5631#[inline]
5632#[target_feature(enable = "vector")]
5633#[unstable(feature = "stdarch_s390x", issue = "135681")]
5634pub unsafe fn vec_all_ne<T: sealed::VectorEquality>(a: T, b: T) -> i32 {
5635    simd_reduce_all(vec_cmpne(a, b)) as i32
5636}
5637
5638/// Any Element Equal
5639#[inline]
5640#[target_feature(enable = "vector")]
5641#[unstable(feature = "stdarch_s390x", issue = "135681")]
5642pub unsafe fn vec_any_eq<T: sealed::VectorEquality>(a: T, b: T) -> i32 {
5643    simd_reduce_any(vec_cmpeq(a, b)) as i32
5644}
5645
5646/// Any Element Not Equal
5647#[inline]
5648#[target_feature(enable = "vector")]
5649#[unstable(feature = "stdarch_s390x", issue = "135681")]
5650pub unsafe fn vec_any_ne<T: sealed::VectorEquality>(a: T, b: T) -> i32 {
5651    simd_reduce_any(vec_cmpne(a, b)) as i32
5652}
5653
5654/// All Elements Less Than
5655#[inline]
5656#[target_feature(enable = "vector")]
5657#[unstable(feature = "stdarch_s390x", issue = "135681")]
5658pub unsafe fn vec_all_lt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5659    a.vec_all_lt(b)
5660}
5661
5662/// All Elements Less Than or Equal
5663#[inline]
5664#[target_feature(enable = "vector")]
5665#[unstable(feature = "stdarch_s390x", issue = "135681")]
5666pub unsafe fn vec_all_le<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5667    a.vec_all_le(b)
5668}
5669
5670/// All Elements Greater Than
5671#[inline]
5672#[target_feature(enable = "vector")]
5673#[unstable(feature = "stdarch_s390x", issue = "135681")]
5674pub unsafe fn vec_all_gt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5675    a.vec_all_gt(b)
5676}
5677
5678/// All Elements Greater Than or Equal
5679#[inline]
5680#[target_feature(enable = "vector")]
5681#[unstable(feature = "stdarch_s390x", issue = "135681")]
5682pub unsafe fn vec_all_ge<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5683    a.vec_all_ge(b)
5684}
5685
5686/// All Elements Not Less Than
5687#[inline]
5688#[target_feature(enable = "vector")]
5689#[unstable(feature = "stdarch_s390x", issue = "135681")]
5690pub unsafe fn vec_all_nlt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5691    vec_all_ge(a, b)
5692}
5693
5694/// All Elements Not Less Than or Equal
5695#[inline]
5696#[target_feature(enable = "vector")]
5697#[unstable(feature = "stdarch_s390x", issue = "135681")]
5698pub unsafe fn vec_all_nle<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5699    vec_all_gt(a, b)
5700}
5701
5702/// All Elements Not Greater Than
5703#[inline]
5704#[target_feature(enable = "vector")]
5705#[unstable(feature = "stdarch_s390x", issue = "135681")]
5706pub unsafe fn vec_all_ngt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5707    vec_all_le(a, b)
5708}
5709
5710/// All Elements Not Greater Than or Equal
5711#[inline]
5712#[target_feature(enable = "vector")]
5713#[unstable(feature = "stdarch_s390x", issue = "135681")]
5714pub unsafe fn vec_all_nge<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5715    vec_all_lt(a, b)
5716}
5717
5718/// Any Elements Less Than
5719#[inline]
5720#[target_feature(enable = "vector")]
5721#[unstable(feature = "stdarch_s390x", issue = "135681")]
5722pub unsafe fn vec_any_lt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5723    !vec_all_ge(a, b)
5724}
5725
5726/// Any Elements Less Than or Equal
5727#[inline]
5728#[target_feature(enable = "vector")]
5729#[unstable(feature = "stdarch_s390x", issue = "135681")]
5730pub unsafe fn vec_any_le<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5731    !vec_all_gt(a, b)
5732}
5733
5734/// Any Elements Greater Than
5735#[inline]
5736#[target_feature(enable = "vector")]
5737#[unstable(feature = "stdarch_s390x", issue = "135681")]
5738pub unsafe fn vec_any_gt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5739    !vec_all_le(a, b)
5740}
5741
5742/// Any Elements Greater Than or Equal
5743#[inline]
5744#[target_feature(enable = "vector")]
5745#[unstable(feature = "stdarch_s390x", issue = "135681")]
5746pub unsafe fn vec_any_ge<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5747    !vec_all_lt(a, b)
5748}
5749
5750/// Any Elements Not Less Than
5751#[inline]
5752#[target_feature(enable = "vector")]
5753#[unstable(feature = "stdarch_s390x", issue = "135681")]
5754pub unsafe fn vec_any_nlt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5755    vec_any_ge(a, b)
5756}
5757
5758/// Any Elements Not Less Than or Equal
5759#[inline]
5760#[target_feature(enable = "vector")]
5761#[unstable(feature = "stdarch_s390x", issue = "135681")]
5762pub unsafe fn vec_any_nle<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5763    vec_any_gt(a, b)
5764}
5765
5766/// Any Elements Not Greater Than
5767#[inline]
5768#[target_feature(enable = "vector")]
5769#[unstable(feature = "stdarch_s390x", issue = "135681")]
5770pub unsafe fn vec_any_ngt<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5771    vec_any_le(a, b)
5772}
5773
5774/// Any Elements Not Greater Than or Equal
5775#[inline]
5776#[target_feature(enable = "vector")]
5777#[unstable(feature = "stdarch_s390x", issue = "135681")]
5778pub unsafe fn vec_any_nge<T: sealed::VectorCompare>(a: T, b: T) -> i32 {
5779    vec_any_lt(a, b)
5780}
5781
5782/// Vector Extract
5783#[inline]
5784#[target_feature(enable = "vector")]
5785#[unstable(feature = "stdarch_s390x", issue = "135681")]
5786pub unsafe fn vec_extract<T: sealed::VectorExtract>(a: T, b: i32) -> T::ElementType {
5787    T::vec_extract(a, b)
5788}
5789
5790/// Vector Insert
5791#[inline]
5792#[target_feature(enable = "vector")]
5793#[unstable(feature = "stdarch_s390x", issue = "135681")]
5794pub unsafe fn vec_insert<T: sealed::VectorInsert>(a: T::ElementType, b: T, c: i32) -> T {
5795    T::vec_insert(a, b, c)
5796}
5797
5798/// Vector Insert and Zero
5799#[inline]
5800#[target_feature(enable = "vector")]
5801#[unstable(feature = "stdarch_s390x", issue = "135681")]
5802pub unsafe fn vec_insert_and_zero<T: sealed::VectorInsertAndZero>(a: *const T::ElementType) -> T {
5803    T::vec_insert_and_zero(a)
5804}
5805
5806/// Vector Promote
5807#[inline]
5808#[target_feature(enable = "vector")]
5809#[unstable(feature = "stdarch_s390x", issue = "135681")]
5810pub unsafe fn vec_promote<T: sealed::VectorPromote>(a: T::ElementType, b: i32) -> MaybeUninit<T> {
5811    T::vec_promote(a, b)
5812}
5813
5814#[cfg(test)]
5815mod tests {
5816    use super::*;
5817
5818    use std::mem::transmute;
5819
5820    use crate::core_arch::simd::*;
5821    use stdarch_test::simd_test;
5822
5823    #[test]
5824    fn reverse_mask() {
5825        assert_eq!(ShuffleMask::<4>::reverse().0, [3, 2, 1, 0]);
5826    }
5827
5828    #[test]
5829    fn mergel_mask() {
5830        assert_eq!(ShuffleMask::<4>::merge_low().0, [2, 6, 3, 7]);
5831    }
5832
5833    #[test]
5834    fn mergeh_mask() {
5835        assert_eq!(ShuffleMask::<4>::merge_high().0, [0, 4, 1, 5]);
5836    }
5837
5838    #[test]
5839    fn pack_mask() {
5840        assert_eq!(ShuffleMask::<4>::pack().0, [1, 3, 5, 7]);
5841    }
5842
5843    #[test]
5844    fn test_vec_mask() {
5845        assert_eq!(
5846            genmask::<0x00FF>(),
5847            [
5848                0, 0, 0, 0, 0, 0, 0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF
5849            ]
5850        );
5851    }
5852
5853    #[test]
5854    fn test_genmasks() {
5855        assert_eq!(genmasks(u8::BITS, 3, 5), 28);
5856        assert_eq!(genmasks(u8::BITS, 3, 7), 31);
5857
5858        // If a or b is greater than 8, the operation is performed as if the value gets modulo by 8.
5859        assert_eq!(genmasks(u8::BITS, 3 + 8, 7 + 8), 31);
5860        // If a is greater than b, the operation is perform as if b equals 7.
5861        assert_eq!(genmasks(u8::BITS, 5, 4), genmasks(u8::BITS, 5, 7));
5862
5863        assert_eq!(
5864            genmasks(u16::BITS, 4, 12) as u16,
5865            u16::from_be_bytes([15, -8i8 as u8])
5866        );
5867        assert_eq!(
5868            genmasks(u32::BITS, 4, 29) as u32,
5869            u32::from_be_bytes([15, 0xFF, 0xFF, -4i8 as u8])
5870        );
5871    }
5872
5873    macro_rules! test_vec_1 {
5874        { $name: ident, $fn:ident, f32x4, [$($a:expr),+], ~[$($d:expr),+] } => {
5875            #[simd_test(enable = "vector")]
5876            unsafe fn $name() {
5877                let a: vector_float = transmute(f32x4::new($($a),+));
5878
5879                let d: vector_float = transmute(f32x4::new($($d),+));
5880                let r = transmute(vec_cmple(vec_abs(vec_sub($fn(a), d)), vec_splats(f32::EPSILON)));
5881                let e = m32x4::new(true, true, true, true);
5882                assert_eq!(e, r);
5883            }
5884        };
5885        { $name: ident, $fn:ident, $ty: ident, [$($a:expr),+], [$($d:expr),+] } => {
5886            test_vec_1! { $name, $fn, $ty -> $ty, [$($a),+], [$($d),+] }
5887        };
5888        { $name: ident, $fn:ident, $ty: ident -> $ty_out: ident, [$($a:expr),+], [$($d:expr),+] } => {
5889            #[simd_test(enable = "vector")]
5890            unsafe fn $name() {
5891                let a: s_t_l!($ty) = transmute($ty::new($($a),+));
5892
5893                let d = $ty_out::new($($d),+);
5894                let r : $ty_out = transmute($fn(a));
5895                assert_eq!(d, r);
5896            }
5897        }
5898    }
5899
5900    macro_rules! test_vec_2 {
5901        { $name: ident, $fn:ident, $ty: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
5902            test_vec_2! { $name, $fn, $ty -> $ty, [$($a),+], [$($b),+], [$($d),+] }
5903        };
5904        { $name: ident, $fn:ident, $ty: ident -> $ty_out: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
5905            test_vec_2! { $name, $fn, $ty, $ty -> $ty, [$($a),+], [$($b),+], [$($d),+] }
5906         };
5907        { $name: ident, $fn:ident, $ty1: ident, $ty2: ident -> $ty_out: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
5908            #[simd_test(enable = "vector")]
5909            unsafe fn $name() {
5910                let a: s_t_l!($ty1) = transmute($ty1::new($($a),+));
5911                let b: s_t_l!($ty2) = transmute($ty2::new($($b),+));
5912
5913                let d = $ty_out::new($($d),+);
5914                let r : $ty_out = transmute($fn(a, b));
5915                assert_eq!(d, r);
5916            }
5917         };
5918         { $name: ident, $fn:ident, $ty: ident -> $ty_out: ident, [$($a:expr),+], [$($b:expr),+], $d:expr } => {
5919            #[simd_test(enable = "vector")]
5920            unsafe fn $name() {
5921                let a: s_t_l!($ty) = transmute($ty::new($($a),+));
5922                let b: s_t_l!($ty) = transmute($ty::new($($b),+));
5923
5924                let r : $ty_out = transmute($fn(a, b));
5925                assert_eq!($d, r);
5926            }
5927         }
5928   }
5929
5930    #[simd_test(enable = "vector")]
5931    unsafe fn vec_add_i32x4_i32x4() {
5932        let x = i32x4::new(1, 2, 3, 4);
5933        let y = i32x4::new(4, 3, 2, 1);
5934        let x: vector_signed_int = transmute(x);
5935        let y: vector_signed_int = transmute(y);
5936        let z = vec_add(x, y);
5937        assert_eq!(i32x4::splat(5), transmute(z));
5938    }
5939
5940    macro_rules! test_vec_sub {
5941        { $name: ident, $ty: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
5942            test_vec_2! {$name, vec_sub, $ty, [$($a),+], [$($b),+], [$($d),+] }
5943        }
5944    }
5945
5946    test_vec_sub! { test_vec_sub_f32x4, f32x4,
5947    [-1.0, 0.0, 1.0, 2.0],
5948    [2.0, 1.0, -1.0, -2.0],
5949    [-3.0, -1.0, 2.0, 4.0] }
5950
5951    test_vec_sub! { test_vec_sub_f64x2, f64x2,
5952    [-1.0, 0.0],
5953    [2.0, 1.0],
5954    [-3.0, -1.0] }
5955
5956    test_vec_sub! { test_vec_sub_i64x2, i64x2,
5957    [-1, 0],
5958    [2, 1],
5959    [-3, -1] }
5960
5961    test_vec_sub! { test_vec_sub_u64x2, u64x2,
5962    [0, 1],
5963    [1, 0],
5964    [u64::MAX, 1] }
5965
5966    test_vec_sub! { test_vec_sub_i32x4, i32x4,
5967    [-1, 0, 1, 2],
5968    [2, 1, -1, -2],
5969    [-3, -1, 2, 4] }
5970
5971    test_vec_sub! { test_vec_sub_u32x4, u32x4,
5972    [0, 0, 1, 2],
5973    [2, 1, 0, 0],
5974    [4294967294, 4294967295, 1, 2] }
5975
5976    test_vec_sub! { test_vec_sub_i16x8, i16x8,
5977    [-1, 0, 1, 2, -1, 0, 1, 2],
5978    [2, 1, -1, -2, 2, 1, -1, -2],
5979    [-3, -1, 2, 4, -3, -1, 2, 4] }
5980
5981    test_vec_sub! { test_vec_sub_u16x8, u16x8,
5982    [0, 0, 1, 2, 0, 0, 1, 2],
5983    [2, 1, 0, 0, 2, 1, 0, 0],
5984    [65534, 65535, 1, 2, 65534, 65535, 1, 2] }
5985
5986    test_vec_sub! { test_vec_sub_i8x16, i8x16,
5987    [-1, 0, 1, 2, -1, 0, 1, 2, -1, 0, 1, 2, -1, 0, 1, 2],
5988    [2, 1, -1, -2, 2, 1, -1, -2, 2, 1, -1, -2, 2, 1, -1, -2],
5989    [-3, -1, 2, 4, -3, -1, 2, 4, -3, -1, 2, 4, -3, -1, 2, 4] }
5990
5991    test_vec_sub! { test_vec_sub_u8x16, u8x16,
5992    [0, 0, 1, 2, 0, 0, 1, 2, 0, 0, 1, 2, 0, 0, 1, 2],
5993    [2, 1, 0, 0, 2, 1, 0, 0, 2, 1, 0, 0, 2, 1, 0, 0],
5994    [254, 255, 1, 2, 254, 255, 1, 2, 254, 255, 1, 2, 254, 255, 1, 2] }
5995
5996    macro_rules! test_vec_mul {
5997        { $name: ident, $ty: ident, [$($a:expr),+], [$($b:expr),+], [$($d:expr),+] } => {
5998            test_vec_2! {$name, vec_mul, $ty, [$($a),+], [$($b),+], [$($d),+] }
5999        }
6000    }
6001
6002    test_vec_mul! { test_vec_mul_f32x4, f32x4,
6003    [-1.0, 0.0, 1.0, 2.0],
6004    [2.0, 1.0, -1.0, -2.0],
6005    [-2.0, 0.0, -1.0, -4.0] }
6006
6007    test_vec_mul! { test_vec_mul_f64x2, f64x2,
6008    [-1.0, 0.0],
6009    [2.0, 1.0],
6010    [-2.0, 0.0] }
6011
6012    test_vec_mul! { test_vec_mul_i64x2, i64x2,
6013    [i64::MAX, -4],
6014    [2, 3],
6015    [i64::MAX.wrapping_mul(2), -12] }
6016
6017    test_vec_mul! { test_vec_mul_u64x2, u64x2,
6018    [u64::MAX, 4],
6019    [2, 3],
6020    [u64::MAX.wrapping_mul(2), 12] }
6021
6022    test_vec_mul! { test_vec_mul_i32x4, i32x4,
6023    [-1, 0, 1, 2],
6024    [2, 1, -1, -2],
6025    [-2, 0, -1, -4] }
6026
6027    test_vec_mul! { test_vec_mul_u32x4, u32x4,
6028    [0, u32::MAX - 1, 1, 2],
6029    [5, 6, 7, 8],
6030    [0, 4294967284, 7, 16] }
6031
6032    test_vec_mul! { test_vec_mul_i16x8, i16x8,
6033    [-1, 0, 1, 2, -1, 0, 1, 2],
6034    [2, 1, -1, -2, 2, 1, -1, -2],
6035    [-2, 0, -1, -4, -2, 0, -1, -4] }
6036
6037    test_vec_mul! { test_vec_mul_u16x8, u16x8,
6038    [0, u16::MAX - 1, 1, 2, 3, 4, 5, 6],
6039    [5, 6, 7, 8, 9, 8, 7, 6],
6040    [0, 65524, 7, 16, 27, 32, 35, 36] }
6041
6042    test_vec_mul! { test_vec_mul_i8x16, i8x16,
6043    [-1, 0, 1, 2, -1, 0, 1, 2, -1, 0, 1, 2, -1, 0, 1, 2],
6044    [2, 1, -1, -2, 2, 1, -1, -2, 2, 1, -1, -2, 2, 1, -1, -2],
6045    [-2, 0, -1, -4, -2, 0, -1, -4, -2, 0, -1, -4, -2, 0, -1, -4] }
6046
6047    test_vec_mul! { test_vec_mul_u8x16, u8x16,
6048    [0, u8::MAX - 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 8, 7, 6, 5, 4],
6049    [5, 6, 7, 8, 9, 8, 7, 6, 5, 4, 0, u8::MAX, 1, 2, 3, 4],
6050    [0, 244, 7, 16, 27, 32, 35, 36, 35, 32, 0, 248, 7, 12, 15, 16] }
6051
6052    macro_rules! test_vec_abs {
6053        { $name: ident, $ty: ident, $a: expr, $d: expr } => {
6054            #[simd_test(enable = "vector")]
6055            unsafe fn $name() {
6056                let a: s_t_l!($ty) = vec_splats($a);
6057                let a: s_t_l!($ty) = vec_abs(a);
6058                let d = $ty::splat($d);
6059                assert_eq!(d, transmute(a));
6060            }
6061        }
6062    }
6063
6064    test_vec_abs! { test_vec_abs_i8, i8x16, -42i8, 42i8 }
6065    test_vec_abs! { test_vec_abs_i16, i16x8, -42i16, 42i16 }
6066    test_vec_abs! { test_vec_abs_i32, i32x4, -42i32, 42i32 }
6067    test_vec_abs! { test_vec_abs_i64, i64x2, -42i64, 42i64 }
6068    test_vec_abs! { test_vec_abs_f32, f32x4, -42f32, 42f32 }
6069    test_vec_abs! { test_vec_abs_f64, f64x2, -42f64, 42f64 }
6070
6071    test_vec_1! { test_vec_nabs, vec_nabs, f32x4,
6072    [core::f32::consts::PI, 1.0, 0.0, -1.0],
6073    [-core::f32::consts::PI, -1.0, 0.0, -1.0] }
6074
6075    test_vec_2! { test_vec_andc, vec_andc, i32x4,
6076    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6077    [0b00110011, 0b11110011, 0b00001100, 0b10000000],
6078    [0b11001100, 0b00001100, 0b11000000, 0b01001100] }
6079
6080    test_vec_2! { test_vec_and, vec_and, i32x4,
6081    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6082    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6083    [0b00000000, 0b11000000, 0b00001100, 0b00000000] }
6084
6085    test_vec_2! { test_vec_nand, vec_nand, i32x4,
6086    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6087    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6088    [!0b00000000, !0b11000000, !0b00001100, !0b00000000] }
6089
6090    test_vec_2! { test_vec_orc, vec_orc, u32x4,
6091    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6092    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6093    [0b11001100 | !0b00110011, 0b11001100 | !0b11110011, 0b11001100 | !0b00001100, 0b11001100 | !0b00000000] }
6094
6095    test_vec_2! { test_vec_or, vec_or, i32x4,
6096    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6097    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6098    [0b11111111, 0b11111111, 0b11001100, 0b11001100] }
6099
6100    test_vec_2! { test_vec_nor, vec_nor, i32x4,
6101    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6102    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6103    [!0b11111111, !0b11111111, !0b11001100, !0b11001100] }
6104
6105    test_vec_2! { test_vec_xor, vec_xor, i32x4,
6106    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6107    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6108    [0b11111111, 0b00111111, 0b11000000, 0b11001100] }
6109
6110    test_vec_2! { test_vec_eqv, vec_eqv, i32x4,
6111    [0b11001100, 0b11001100, 0b11001100, 0b11001100],
6112    [0b00110011, 0b11110011, 0b00001100, 0b00000000],
6113    [!0b11111111, !0b00111111, !0b11000000, !0b11001100] }
6114
6115    test_vec_1! { test_vec_floor_f32, vec_floor, f32x4,
6116        [1.1, 1.9, -0.5, -0.9],
6117        [1.0, 1.0, -1.0, -1.0]
6118    }
6119
6120    test_vec_1! { test_vec_floor_f64_1, vec_floor, f64x2,
6121        [1.1, 1.9],
6122        [1.0, 1.0]
6123    }
6124    test_vec_1! { test_vec_floor_f64_2, vec_floor, f64x2,
6125        [-0.5, -0.9],
6126        [-1.0, -1.0]
6127    }
6128
6129    test_vec_1! { test_vec_ceil_f32, vec_ceil, f32x4,
6130        [0.1, 0.5, 0.6, 0.9],
6131        [1.0, 1.0, 1.0, 1.0]
6132    }
6133    test_vec_1! { test_vec_ceil_f64_1, vec_ceil, f64x2,
6134        [0.1, 0.5],
6135        [1.0, 1.0]
6136    }
6137    test_vec_1! { test_vec_ceil_f64_2, vec_ceil, f64x2,
6138        [0.6, 0.9],
6139        [1.0, 1.0]
6140    }
6141
6142    test_vec_1! { test_vec_round_f32, vec_round, f32x4,
6143        [0.1, 0.5, 0.6, 0.9],
6144        [0.0, 0.0, 1.0, 1.0]
6145    }
6146
6147    test_vec_1! { test_vec_round_f32_even_odd, vec_round, f32x4,
6148        [0.5, 1.5, 2.5, 3.5],
6149        [0.0, 2.0, 2.0, 4.0]
6150    }
6151
6152    test_vec_1! { test_vec_round_f64_1, vec_round, f64x2,
6153        [0.1, 0.5],
6154        [0.0, 0.0]
6155    }
6156    test_vec_1! { test_vec_round_f64_2, vec_round, f64x2,
6157        [0.6, 0.9],
6158        [1.0, 1.0]
6159    }
6160
6161    test_vec_1! { test_vec_roundc_f32, vec_roundc, f32x4,
6162        [0.1, 0.5, 0.6, 0.9],
6163        [0.0, 0.0, 1.0, 1.0]
6164    }
6165
6166    test_vec_1! { test_vec_roundc_f32_even_odd, vec_roundc, f32x4,
6167        [0.5, 1.5, 2.5, 3.5],
6168        [0.0, 2.0, 2.0, 4.0]
6169    }
6170
6171    test_vec_1! { test_vec_roundc_f64_1, vec_roundc, f64x2,
6172        [0.1, 0.5],
6173        [0.0, 0.0]
6174    }
6175    test_vec_1! { test_vec_roundc_f64_2, vec_roundc, f64x2,
6176        [0.6, 0.9],
6177        [1.0, 1.0]
6178    }
6179
6180    test_vec_1! { test_vec_rint_f32, vec_rint, f32x4,
6181        [0.1, 0.5, 0.6, 0.9],
6182        [0.0, 0.0, 1.0, 1.0]
6183    }
6184
6185    test_vec_1! { test_vec_rint_f32_even_odd, vec_rint, f32x4,
6186        [0.5, 1.5, 2.5, 3.5],
6187        [0.0, 2.0, 2.0, 4.0]
6188    }
6189
6190    test_vec_1! { test_vec_rint_f64_1, vec_rint, f64x2,
6191        [0.1, 0.5],
6192        [0.0, 0.0]
6193    }
6194    test_vec_1! { test_vec_rint_f64_2, vec_rint, f64x2,
6195        [0.6, 0.9],
6196        [1.0, 1.0]
6197    }
6198
6199    test_vec_2! { test_vec_sll, vec_sll, i32x4, u8x16 -> i32x4,
6200    [1, 1, 1, 1],
6201    [0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 8],
6202    [1 << 2, 1 << 3, 1 << 4, 1] }
6203
6204    test_vec_2! { test_vec_srl, vec_srl, i32x4, u8x16 -> i32x4,
6205    [0b1000, 0b1000, 0b1000, 0b1000],
6206    [0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 16],
6207    [4, 2, 1, 8] }
6208
6209    test_vec_2! { test_vec_sral_pos, vec_sral, u32x4, u8x16 -> i32x4,
6210    [0b1000, 0b1000, 0b1000, 0b1000],
6211    [0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 16],
6212    [4, 2, 1, 8] }
6213
6214    test_vec_2! { test_vec_sral_neg, vec_sral, i32x4, u8x16 -> i32x4,
6215    [-8, -8, -8, -8],
6216    [0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 16],
6217    [-4, -2, -1, -8] }
6218
6219    test_vec_1! { test_vec_reve_f32, vec_reve, f32x4,
6220        [0.1, 0.5, 0.6, 0.9],
6221        [0.9, 0.6, 0.5, 0.1]
6222    }
6223
6224    test_vec_1! { test_vec_revb_u32, vec_revb, u32x4,
6225        [0xAABBCCDD, 0xEEFF0011, 0x22334455, 0x66778899],
6226        [0xDDCCBBAA, 0x1100FFEE, 0x55443322, 0x99887766]
6227    }
6228
6229    test_vec_2! { test_vec_mergeh_u32, vec_mergeh, u32x4,
6230        [0xAAAAAAAA, 0xBBBBBBBB, 0xCCCCCCCC, 0xDDDDDDDD],
6231        [0x00000000, 0x11111111, 0x22222222, 0x33333333],
6232        [0xAAAAAAAA, 0x00000000, 0xBBBBBBBB, 0x11111111]
6233    }
6234
6235    test_vec_2! { test_vec_mergel_u32, vec_mergel, u32x4,
6236        [0xAAAAAAAA, 0xBBBBBBBB, 0xCCCCCCCC, 0xDDDDDDDD],
6237        [0x00000000, 0x11111111, 0x22222222, 0x33333333],
6238        [0xCCCCCCCC, 0x22222222, 0xDDDDDDDD, 0x33333333]
6239    }
6240
6241    macro_rules! test_vec_perm {
6242        {$name:ident,
6243         $shorttype:ident, $longtype:ident,
6244         [$($a:expr),+], [$($b:expr),+], [$($c:expr),+], [$($d:expr),+]} => {
6245            #[simd_test(enable = "vector")]
6246            unsafe fn $name() {
6247                let a: $longtype = transmute($shorttype::new($($a),+));
6248                let b: $longtype = transmute($shorttype::new($($b),+));
6249                let c: vector_unsigned_char = transmute(u8x16::new($($c),+));
6250                let d = $shorttype::new($($d),+);
6251
6252                let r: $shorttype = transmute(vec_perm(a, b, c));
6253                assert_eq!(d, r);
6254            }
6255        }
6256    }
6257
6258    test_vec_perm! {test_vec_perm_u8x16,
6259    u8x16, vector_unsigned_char,
6260    [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
6261    [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115],
6262    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6263     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6264    [0, 1, 100, 101, 2, 3, 102, 103, 4, 5, 104, 105, 6, 7, 106, 107]}
6265    test_vec_perm! {test_vec_perm_i8x16,
6266    i8x16, vector_signed_char,
6267    [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
6268    [100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115],
6269    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6270     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6271    [0, 1, 100, 101, 2, 3, 102, 103, 4, 5, 104, 105, 6, 7, 106, 107]}
6272
6273    test_vec_perm! {test_vec_perm_m8x16,
6274    m8x16, vector_bool_char,
6275    [false, false, false, false, false, false, false, false, false, false, false, false, false, false, false, false],
6276    [true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true],
6277    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6278     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6279    [false, false, true, true, false, false, true, true, false, false, true, true, false, false, true, true]}
6280    test_vec_perm! {test_vec_perm_u16x8,
6281    u16x8, vector_unsigned_short,
6282    [0, 1, 2, 3, 4, 5, 6, 7],
6283    [10, 11, 12, 13, 14, 15, 16, 17],
6284    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6285     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6286    [0, 10, 1, 11, 2, 12, 3, 13]}
6287    test_vec_perm! {test_vec_perm_i16x8,
6288    i16x8, vector_signed_short,
6289    [0, 1, 2, 3, 4, 5, 6, 7],
6290    [10, 11, 12, 13, 14, 15, 16, 17],
6291    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6292     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6293    [0, 10, 1, 11, 2, 12, 3, 13]}
6294    test_vec_perm! {test_vec_perm_m16x8,
6295    m16x8, vector_bool_short,
6296    [false, false, false, false, false, false, false, false],
6297    [true, true, true, true, true, true, true, true],
6298    [0x00, 0x01, 0x10, 0x11, 0x02, 0x03, 0x12, 0x13,
6299     0x04, 0x05, 0x14, 0x15, 0x06, 0x07, 0x16, 0x17],
6300    [false, true, false, true, false, true, false, true]}
6301
6302    test_vec_perm! {test_vec_perm_u32x4,
6303    u32x4, vector_unsigned_int,
6304    [0, 1, 2, 3],
6305    [10, 11, 12, 13],
6306    [0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
6307     0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17],
6308    [0, 10, 1, 11]}
6309    test_vec_perm! {test_vec_perm_i32x4,
6310    i32x4, vector_signed_int,
6311    [0, 1, 2, 3],
6312    [10, 11, 12, 13],
6313    [0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
6314     0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17],
6315    [0, 10, 1, 11]}
6316    test_vec_perm! {test_vec_perm_m32x4,
6317    m32x4, vector_bool_int,
6318    [false, false, false, false],
6319    [true, true, true, true],
6320    [0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
6321     0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17],
6322    [false, true, false, true]}
6323    test_vec_perm! {test_vec_perm_f32x4,
6324    f32x4, vector_float,
6325    [0.0, 1.0, 2.0, 3.0],
6326    [1.0, 1.1, 1.2, 1.3],
6327    [0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13,
6328     0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17],
6329    [0.0, 1.0, 1.0, 1.1]}
6330
6331    test_vec_1! { test_vec_sqrt, vec_sqrt, f32x4,
6332    [core::f32::consts::PI, 1.0, 25.0, 2.0],
6333    [core::f32::consts::PI.sqrt(), 1.0, 5.0, core::f32::consts::SQRT_2] }
6334
6335    test_vec_2! { test_vec_find_any_eq, vec_find_any_eq, i32x4, i32x4 -> u32x4,
6336        [1, -2, 3, -4],
6337        [-5, 3, -7, 8],
6338        [0, 0, 0xFFFFFFFF, 0]
6339    }
6340
6341    test_vec_2! { test_vec_find_any_ne, vec_find_any_ne, i32x4, i32x4 -> u32x4,
6342        [1, -2, 3, -4],
6343        [-5, 3, -7, 8],
6344        [0xFFFFFFFF, 0xFFFFFFFF, 0, 0xFFFFFFFF]
6345    }
6346
6347    test_vec_2! { test_vec_find_any_eq_idx_1, vec_find_any_eq_idx, i32x4, i32x4 -> u32x4,
6348        [1, 2, 3, 4],
6349        [5, 3, 7, 8],
6350        [0, 8, 0, 0]
6351    }
6352    test_vec_2! { test_vec_find_any_eq_idx_2, vec_find_any_eq_idx, i32x4, i32x4 -> u32x4,
6353        [1, 2, 3, 4],
6354        [5, 6, 7, 8],
6355        [0, 16, 0, 0]
6356    }
6357
6358    test_vec_2! { test_vec_find_any_ne_idx_1, vec_find_any_ne_idx, i32x4, i32x4 -> u32x4,
6359        [1, 2, 3, 4],
6360        [1, 5, 3, 4],
6361        [0, 4, 0, 0]
6362    }
6363    test_vec_2! { test_vec_find_any_ne_idx_2, vec_find_any_ne_idx, i32x4, i32x4 -> u32x4,
6364        [1, 2, 3, 4],
6365        [1, 2, 3, 4],
6366        [0, 16, 0, 0]
6367    }
6368
6369    test_vec_2! { test_vec_find_any_eq_or_0_idx_1, vec_find_any_eq_or_0_idx, i32x4, i32x4 -> u32x4,
6370        [1, 2, 0, 4],
6371        [5, 6, 7, 8],
6372        [0, 8, 0, 0]
6373    }
6374    test_vec_2! { test_vec_find_any_ne_or_0_idx_1, vec_find_any_ne_or_0_idx, i32x4, i32x4 -> u32x4,
6375        [1, 2, 0, 4],
6376        [1, 2, 3, 4],
6377        [0, 8, 0, 0]
6378    }
6379
6380    #[simd_test(enable = "vector")]
6381    fn test_vec_find_any_eq_cc() {
6382        let a = vector_unsigned_int([1, 2, 3, 4]);
6383        let b = vector_unsigned_int([5, 3, 7, 8]);
6384
6385        let (d, c) = unsafe { vec_find_any_eq_cc(a, b) };
6386        assert_eq!(c, 1);
6387        assert_eq!(d.as_array(), &[0, 0, -1, 0]);
6388
6389        let a = vector_unsigned_int([1, 2, 3, 4]);
6390        let b = vector_unsigned_int([5, 6, 7, 8]);
6391        let (d, c) = unsafe { vec_find_any_eq_cc(a, b) };
6392        assert_eq!(c, 3);
6393        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6394    }
6395
6396    #[simd_test(enable = "vector")]
6397    fn test_vec_find_any_ne_cc() {
6398        let a = vector_unsigned_int([1, 2, 3, 4]);
6399        let b = vector_unsigned_int([5, 3, 7, 8]);
6400
6401        let (d, c) = unsafe { vec_find_any_ne_cc(a, b) };
6402        assert_eq!(c, 1);
6403        assert_eq!(d.as_array(), &[-1, -1, 0, -1]);
6404
6405        let a = vector_unsigned_int([1, 2, 3, 4]);
6406        let b = vector_unsigned_int([1, 2, 3, 4]);
6407        let (d, c) = unsafe { vec_find_any_ne_cc(a, b) };
6408        assert_eq!(c, 3);
6409        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6410    }
6411
6412    #[simd_test(enable = "vector")]
6413    fn test_vec_find_any_eq_idx_cc() {
6414        let a = vector_unsigned_int([1, 2, 3, 4]);
6415        let b = vector_unsigned_int([5, 3, 7, 8]);
6416
6417        let (d, c) = unsafe { vec_find_any_eq_idx_cc(a, b) };
6418        assert_eq!(c, 1);
6419        assert_eq!(d.as_array(), &[0, 8, 0, 0]);
6420
6421        let a = vector_unsigned_int([1, 2, 3, 4]);
6422        let b = vector_unsigned_int([5, 6, 7, 8]);
6423        let (d, c) = unsafe { vec_find_any_eq_idx_cc(a, b) };
6424        assert_eq!(c, 3);
6425        assert_eq!(d.as_array(), &[0, 16, 0, 0]);
6426    }
6427
6428    #[simd_test(enable = "vector")]
6429    fn test_vec_find_any_ne_idx_cc() {
6430        let a = vector_unsigned_int([5, 2, 3, 4]);
6431        let b = vector_unsigned_int([5, 3, 7, 8]);
6432
6433        let (d, c) = unsafe { vec_find_any_ne_idx_cc(a, b) };
6434        assert_eq!(c, 1);
6435        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
6436
6437        let a = vector_unsigned_int([1, 2, 3, 4]);
6438        let b = vector_unsigned_int([1, 2, 3, 4]);
6439        let (d, c) = unsafe { vec_find_any_ne_idx_cc(a, b) };
6440        assert_eq!(c, 3);
6441        assert_eq!(d.as_array(), &[0, 16, 0, 0]);
6442    }
6443
6444    #[simd_test(enable = "vector")]
6445    fn test_vec_find_any_eq_or_0_idx_cc() {
6446        // if no element of a matches any element of b with an equal value, and there is at least one element from a with a value of 0
6447        let a = vector_unsigned_int([0, 1, 2, 3]);
6448        let b = vector_unsigned_int([4, 5, 6, 7]);
6449        let (d, c) = unsafe { vec_find_any_eq_or_0_idx_cc(a, b) };
6450        assert_eq!(c, 0);
6451        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6452
6453        // if at least one element of a matches any element of b with an equal value, and no elements of a with a value of 0
6454        let a = vector_unsigned_int([1, 2, 3, 4]);
6455        let b = vector_unsigned_int([5, 2, 3, 4]);
6456        let (d, c) = unsafe { vec_find_any_eq_or_0_idx_cc(a, b) };
6457        assert_eq!(c, 1);
6458        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
6459
6460        // if at least one element of a matches any element of b with an equal value, and there is at least one element from a has a value of 0
6461        let a = vector_unsigned_int([1, 2, 3, 0]);
6462        let b = vector_unsigned_int([1, 2, 3, 4]);
6463        let (d, c) = unsafe { vec_find_any_eq_or_0_idx_cc(a, b) };
6464        assert_eq!(c, 2);
6465        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6466
6467        // if no element of a matches any element of b with an equal value, and there is no element from a with a value of 0.
6468        let a = vector_unsigned_int([1, 2, 3, 4]);
6469        let b = vector_unsigned_int([5, 6, 7, 8]);
6470        let (d, c) = unsafe { vec_find_any_eq_or_0_idx_cc(a, b) };
6471        assert_eq!(c, 3);
6472        assert_eq!(d.as_array(), &[0, 16, 0, 0]);
6473    }
6474
6475    #[simd_test(enable = "vector")]
6476    fn test_vec_find_any_ne_or_0_idx_cc() {
6477        // if no element of a matches any element of b with a not equal value, and there is at least one element from a with a value of 0.
6478        let a = vector_unsigned_int([0, 1, 2, 3]);
6479        let b = vector_unsigned_int([4, 1, 2, 3]);
6480        let (d, c) = unsafe { vec_find_any_ne_or_0_idx_cc(a, b) };
6481        assert_eq!(c, 0);
6482        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6483
6484        // if at least one element of a matches any element of b with a not equal value, and no elements of a with a value of 0.
6485        let a = vector_unsigned_int([4, 2, 3, 4]);
6486        let b = vector_unsigned_int([4, 5, 6, 7]);
6487        let (d, c) = unsafe { vec_find_any_ne_or_0_idx_cc(a, b) };
6488        assert_eq!(c, 1);
6489        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
6490
6491        // if at least one element of a matches any element of b with a not equal value, and there is at least one element from a has a value of 0.
6492        let a = vector_unsigned_int([1, 0, 1, 1]);
6493        let b = vector_unsigned_int([4, 5, 6, 7]);
6494        let (d, c) = unsafe { vec_find_any_ne_or_0_idx_cc(a, b) };
6495        assert_eq!(c, 2);
6496        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
6497
6498        // if no element of a matches any element of b with a not equal value, and there is no element from a with a value of 0.
6499        let a = vector_unsigned_int([4, 4, 4, 4]);
6500        let b = vector_unsigned_int([4, 5, 6, 7]);
6501        let (d, c) = unsafe { vec_find_any_ne_or_0_idx_cc(a, b) };
6502        assert_eq!(c, 3);
6503        assert_eq!(d.as_array(), &[0, 16, 0, 0]);
6504    }
6505
6506    #[simd_test(enable = "vector")]
6507    fn test_vector_load() {
6508        let expected = [0xAAAA_AAAA, 0xBBBB_BBBB, 0xCCCC_CCCC, 0xDDDD_DDDD];
6509
6510        let source: [u32; 8] = [
6511            0xAAAA_AAAA,
6512            0xBBBB_BBBB,
6513            0xCCCC_CCCC,
6514            0xDDDD_DDDD,
6515            0,
6516            0,
6517            0,
6518            0,
6519        ];
6520        assert_eq!(
6521            unsafe { vec_xl::<vector_unsigned_int>(0, source.as_ptr()) }.as_array(),
6522            &expected
6523        );
6524
6525        // offset is in bytes
6526        let source: [u32; 8] = [
6527            0x0000_AAAA,
6528            0xAAAA_BBBB,
6529            0xBBBB_CCCC,
6530            0xCCCC_DDDD,
6531            0xDDDD_0000,
6532            0,
6533            0,
6534            0,
6535        ];
6536        assert_eq!(
6537            unsafe { vec_xl::<vector_unsigned_int>(2, source.as_ptr()) }.as_array(),
6538            &expected
6539        );
6540    }
6541
6542    #[simd_test(enable = "vector")]
6543    fn test_vector_store() {
6544        let vec = vector_unsigned_int([0xAAAA_AAAA, 0xBBBB_BBBB, 0xCCCC_CCCC, 0xDDDD_DDDD]);
6545
6546        let mut dest = [0u32; 8];
6547        unsafe { vec_xst(vec, 0, dest.as_mut_ptr()) };
6548        assert_eq!(
6549            dest,
6550            [
6551                0xAAAA_AAAA,
6552                0xBBBB_BBBB,
6553                0xCCCC_CCCC,
6554                0xDDDD_DDDD,
6555                0,
6556                0,
6557                0,
6558                0
6559            ]
6560        );
6561
6562        // offset is in bytes
6563        let mut dest = [0u32; 8];
6564        unsafe { vec_xst(vec, 2, dest.as_mut_ptr()) };
6565        assert_eq!(
6566            dest,
6567            [
6568                0x0000_AAAA,
6569                0xAAAA_BBBB,
6570                0xBBBB_CCCC,
6571                0xCCCC_DDDD,
6572                0xDDDD_0000,
6573                0,
6574                0,
6575                0,
6576            ]
6577        );
6578    }
6579
6580    #[simd_test(enable = "vector")]
6581    fn test_vector_lcbb() {
6582        #[repr(align(64))]
6583        struct Align64<T>(T);
6584
6585        static ARRAY: Align64<[u8; 128]> = Align64([0; 128]);
6586
6587        assert_eq!(unsafe { __lcbb::<64>(ARRAY.0[64..].as_ptr()) }, 16);
6588        assert_eq!(unsafe { __lcbb::<64>(ARRAY.0[63..].as_ptr()) }, 1);
6589        assert_eq!(unsafe { __lcbb::<64>(ARRAY.0[56..].as_ptr()) }, 8);
6590        assert_eq!(unsafe { __lcbb::<64>(ARRAY.0[48..].as_ptr()) }, 16);
6591    }
6592
6593    test_vec_2! { test_vec_pack, vec_pack, i16x8, i16x8 -> i8x16,
6594        [0, 1, -1, 42, 32767, -32768, 30000, -30000],
6595        [32767, -32768, 12345, -12345, 0, 1, -1, 42],
6596        [0, 1, -1, 42, -1, 0, 48, -48, -1, 0, 57, -57, 0, 1, -1, 42]
6597    }
6598
6599    test_vec_2! { test_vec_packs, vec_packs, i16x8, i16x8 -> i8x16,
6600        [0, 1, -1, 42, 32767, -32768, 30000, -30000],
6601        [32767, -32768, 12345, -12345, 0, 1, -1, 42],
6602        [0, 1, -1, 42, 127, -128, 127, -128, 127, -128, 127, -128, 0, 1, -1, 42]
6603    }
6604
6605    test_vec_2! { test_vec_packsu_signed, vec_packsu, i16x8, i16x8 -> u8x16,
6606        [0, 1, -1, 42, 32767, -32768, 30000, -30000],
6607        [32767, -32768, 12345, -12345, 0, 1, -1, 42],
6608        [0, 1, 0, 42, 255, 0, 255, 0, 255, 0, 255, 0, 0, 1, 0, 42]
6609    }
6610
6611    test_vec_2! { test_vec_packsu_unsigned, vec_packsu, u16x8, u16x8 -> u8x16,
6612        [65535, 32768, 1234, 5678, 16, 8, 4, 2],
6613        [30000, 25000, 20000, 15000, 31, 63, 127, 255],
6614        [255, 255, 255, 255, 16, 8, 4, 2, 255, 255, 255, 255, 31, 63, 127, 255]
6615    }
6616
6617    test_vec_2! { test_vec_rl, vec_rl, u32x4,
6618        [0x12345678, 0x9ABCDEF0, 0x0F0F0F0F, 0x12345678],
6619        [4, 8, 12, 68],
6620        [0x23456781, 0xBCDEF09A, 0xF0F0F0F0, 0x23456781]
6621    }
6622
6623    test_vec_1! { test_vec_unpackh_i, vec_unpackh, i16x8 -> i32x4,
6624        [0x1234, -2, 0x0F0F, -32768, 0, 0, 0, 0],
6625        [0x1234, -2, 0x0F0F, -32768]
6626    }
6627
6628    test_vec_1! { test_vec_unpackh_u, vec_unpackh, u16x8 -> u32x4,
6629        [0x1234, 0xFFFF, 0x0F0F, 0x8000, 0, 0, 0, 0],
6630        [0x1234, 0xFFFF, 0x0F0F, 0x8000]
6631    }
6632
6633    test_vec_1! { test_vec_unpackl_i, vec_unpackl, i16x8 -> i32x4,
6634        [0, 0, 0, 0, 0x1234, -2, 0x0F0F, -32768],
6635        [0x1234, -2, 0x0F0F, -32768]
6636    }
6637
6638    test_vec_1! { test_vec_unpackl_u, vec_unpackl, u16x8 -> u32x4,
6639        [0, 0, 0, 0, 0x1234, 0xFFFF, 0x0F0F, 0x8000],
6640        [0x1234, 0xFFFF, 0x0F0F, 0x8000]
6641    }
6642
6643    test_vec_2! { test_vec_avg, vec_avg, u32x4,
6644        [2, 1, u32::MAX, 0],
6645        [4, 2, 2, 0],
6646        [3, (1u32 + 2).div_ceil(2), (u32::MAX as u64 + 2u64).div_ceil(2) as u32, 0]
6647    }
6648
6649    test_vec_2! { test_vec_checksum, vec_checksum, u32x4,
6650        [1, 2, 3, u32::MAX],
6651        [5, 6, 7, 8],
6652        [0, 12, 0, 0]
6653    }
6654
6655    test_vec_2! { test_vec_add_u128, vec_add_u128, u8x16,
6656        [0x01, 0x05, 0x0F, 0x1A, 0x2F, 0x3F, 0x50, 0x65,
6657                              0x7A, 0x8F, 0x9A, 0xAD, 0xB0, 0xC3, 0xD5, 0xE8],
6658        [0xF0, 0xEF, 0xC3, 0xB1, 0x92, 0x71, 0x5A, 0x43,
6659                              0x3B, 0x29, 0x13, 0x04, 0xD7, 0xA1, 0x8C, 0x76],
6660        [0xF1, 0xF4, 0xD2, 0xCB, 0xC1, 0xB0, 0xAA, 0xA8, 0xB5, 0xB8, 0xAD, 0xB2, 0x88, 0x65, 0x62, 0x5E]
6661    }
6662
6663    #[simd_test(enable = "vector")]
6664    fn test_vec_addc_u128() {
6665        unsafe {
6666            let a = u128::MAX;
6667            let b = 1u128;
6668
6669            let d: u128 = transmute(vec_addc_u128(transmute(a), transmute(b)));
6670            assert!(a.checked_add(b).is_none());
6671            assert_eq!(d, 1);
6672
6673            let a = 1u128;
6674            let b = 1u128;
6675
6676            let d: u128 = transmute(vec_addc_u128(transmute(a), transmute(b)));
6677            assert!(a.checked_add(b).is_some());
6678            assert_eq!(d, 0);
6679        }
6680    }
6681
6682    #[simd_test(enable = "vector")]
6683    fn test_vec_subc_u128() {
6684        unsafe {
6685            let a = 0u128;
6686            let b = 1u128;
6687
6688            let d: u128 = transmute(vec_subc_u128(transmute(a), transmute(b)));
6689            assert!(a.checked_sub(b).is_none());
6690            assert_eq!(d, 0);
6691
6692            let a = 1u128;
6693            let b = 1u128;
6694
6695            let d: u128 = transmute(vec_subc_u128(transmute(a), transmute(b)));
6696            assert!(a.checked_sub(b).is_some());
6697            assert_eq!(d, 1);
6698        }
6699    }
6700
6701    test_vec_2! { test_vec_mule_u, vec_mule, u16x8, u16x8 -> u32x4,
6702        [0xFFFF, 0, 2, 0, 2, 0, 1, 0],
6703        [0xFFFF, 0, 4, 0, 0xFFFF, 0, 2, 0],
6704        [0xFFFE_0001, 8, 0x0001_FFFE, 2]
6705    }
6706
6707    test_vec_2! { test_vec_mule_i, vec_mule, i16x8, i16x8 -> i32x4,
6708        [i16::MIN, 0, -2, 0, 2, 0, 1, 0],
6709        [i16::MIN, 0, 4, 0, i16::MAX, 0, 2, 0],
6710        [0x4000_0000, -8, 0xFFFE, 2]
6711    }
6712
6713    test_vec_2! { test_vec_mulo_u, vec_mulo, u16x8, u16x8 -> u32x4,
6714        [0, 0xFFFF, 0, 2, 0, 2, 0, 1],
6715        [0, 0xFFFF, 0, 4, 0, 0xFFFF, 0, 2],
6716        [0xFFFE_0001, 8, 0x0001_FFFE, 2]
6717    }
6718
6719    test_vec_2! { test_vec_mulo_i, vec_mulo, i16x8, i16x8 -> i32x4,
6720        [0, i16::MIN, 0, -2, 0, 2, 0, 1],
6721        [0, i16::MIN, 0, 4, 0, i16::MAX, 0, 2],
6722        [0x4000_0000, -8, 0xFFFE, 2]
6723    }
6724
6725    test_vec_2! { test_vec_mulh_u, vec_mulh, u32x4, u32x4 -> u32x4,
6726        [u32::MAX, 2, 2, 1],
6727        [u32::MAX, 4, u32::MAX, 2],
6728        [u32::MAX - 1, 0, 1, 0]
6729    }
6730
6731    test_vec_2! { test_vec_mulh_i, vec_mulh, i32x4, i32x4 -> i32x4,
6732        [i32::MIN, -2, 2, 1],
6733        [i32::MIN, 4, i32::MAX, 2],
6734        [0x4000_0000, -1, 0, 0]
6735    }
6736
6737    test_vec_2! { test_vec_gfmsum_1, vec_gfmsum, u16x8, u16x8 -> u32x4,
6738        [0x1234, 0x5678, 0x9ABC, 0xDEF0, 0x1357, 0x2468, 0xACE0, 0xBDF0],
6739        [0xFFFF, 0x0001, 0x8000, 0x7FFF, 0xAAAA, 0x5555, 0x1234, 0x5678],
6740        [0xE13A794, 0x68764A50, 0x94AA3E, 0x2C93F300]
6741    }
6742
6743    test_vec_2! { test_vec_gfmsum_2, vec_gfmsum, u16x8, u16x8 -> u32x4,
6744        [0x0000, 0xFFFF, 0xAAAA, 0x5555, 0x1234, 0x5678, 0x9ABC, 0xDEF0],
6745        [0xFFFF, 0x0000, 0x5555, 0xAAAA, 0x0001, 0x8000, 0x7FFF, 0x1357],
6746        [0, 0, 0x2B3C1234, 0x3781D244]
6747    }
6748
6749    #[simd_test(enable = "vector")]
6750    fn test_vec_gfmsum_128() {
6751        let a = vector_unsigned_long_long([1, 2]);
6752        let b = vector_unsigned_long_long([3, 4]);
6753
6754        let d: u128 = unsafe { transmute(vec_gfmsum_128(a, b)) };
6755        assert_eq!(d, 11);
6756
6757        let a = vector_unsigned_long_long([0x0101010101010101, 0x0202020202020202]);
6758        let b = vector_unsigned_long_long([0x0404040404040404, 0x0505050505050505]);
6759
6760        let d: u128 = unsafe { transmute(vec_gfmsum_128(a, b)) };
6761        assert_eq!(d, 0xE000E000E000E000E000E000E000E);
6762    }
6763
6764    #[simd_test(enable = "vector-enhancements-1")]
6765    fn test_vec_bperm_u128() {
6766        let a = vector_unsigned_char([65, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]);
6767        let b = vector_unsigned_char([
6768            0, 0, 0, 0, 1, 1, 1, 1, 128, 128, 128, 128, 255, 255, 255, 255,
6769        ]);
6770        let d = unsafe { vec_bperm_u128(a, b) };
6771        assert_eq!(d.as_array(), &[0xF00, 0]);
6772    }
6773
6774    #[simd_test(enable = "vector")]
6775    fn test_vec_sel() {
6776        let a = vector_signed_int([1, 2, 3, 4]);
6777        let b = vector_signed_int([5, 6, 7, 8]);
6778
6779        let e = vector_unsigned_int([9, 10, 11, 12]);
6780        let f = vector_unsigned_int([9, 9, 11, 11]);
6781
6782        let c: vector_bool_int = unsafe { simd_eq(e, f) };
6783        assert_eq!(c.as_array(), &[!0, 0, !0, 0]);
6784        let d: vector_signed_int = unsafe { vec_sel(a, b, c) };
6785        assert_eq!(d.as_array(), &[5, 2, 7, 4]);
6786    }
6787
6788    #[simd_test(enable = "vector")]
6789    fn test_vec_gather_element() {
6790        let a1: [u32; 10] = [10, 11, 12, 13, 14, 15, 16, 17, 18, 19];
6791        let a2: [u32; 10] = [20, 21, 22, 23, 24, 25, 26, 27, 28, 29];
6792
6793        let v1 = vector_unsigned_int([1, 2, 3, 4]);
6794        let v2 = vector_unsigned_int([1, 2, 3, 4]);
6795
6796        let sizeof_int = core::mem::size_of::<u32>() as u32;
6797        let v3 = vector_unsigned_int([
6798            5 * sizeof_int,
6799            8 * sizeof_int,
6800            9 * sizeof_int,
6801            6 * sizeof_int,
6802        ]);
6803
6804        unsafe {
6805            let d1 = vec_gather_element::<_, 0>(v1, v3, a1.as_ptr());
6806            assert_eq!(d1.as_array(), &[15, 2, 3, 4]);
6807            let d2 = vec_gather_element::<_, 0>(v2, v3, a2.as_ptr());
6808            assert_eq!(d2.as_array(), &[25, 2, 3, 4]);
6809        }
6810    }
6811
6812    #[simd_test(enable = "vector")]
6813    fn test_vec_fp_test_data_class() {
6814        let mut cc = 42;
6815
6816        let v1 = vector_double([0.0, f64::NAN]);
6817        let v2 = vector_double([f64::INFINITY, 1.0]);
6818        let v3 = vector_double([1.0, 2.0]);
6819
6820        unsafe {
6821            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_ZERO>(v1, &mut cc);
6822            assert_eq!(cc, 1);
6823            assert_eq!(d.as_array(), &[!0, 0]);
6824
6825            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_NAN>(v1, &mut cc);
6826            assert_eq!(cc, 1);
6827            assert_eq!(d.as_array(), &[0, !0]);
6828
6829            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_INFINITY>(v2, &mut cc);
6830            assert_eq!(cc, 1);
6831            assert_eq!(d.as_array(), &[!0, 0]);
6832
6833            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_INFINITY_N>(v2, &mut cc);
6834            assert_eq!(cc, 3);
6835            assert_eq!(d.as_array(), &[0, 0]);
6836
6837            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_NORMAL>(v2, &mut cc);
6838            assert_eq!(cc, 1);
6839            assert_eq!(d.as_array(), &[0, !0]);
6840
6841            let d = vec_fp_test_data_class::<_, __VEC_CLASS_FP_NORMAL>(v3, &mut cc);
6842            assert_eq!(cc, 0);
6843            assert_eq!(d.as_array(), &[!0, !0]);
6844        }
6845    }
6846
6847    #[simd_test(enable = "vector")]
6848    fn test_vec_fp_any_all_nan_numeric() {
6849        unsafe {
6850            assert_eq!(
6851                vec_all_nan(vector_double([f64::NAN, f64::NAN])),
6852                i32::from(true)
6853            );
6854            assert_eq!(
6855                vec_all_nan(vector_double([f64::NAN, 1.0])),
6856                i32::from(false)
6857            );
6858            assert_eq!(vec_all_nan(vector_double([0.0, 1.0])), i32::from(false));
6859
6860            assert_eq!(
6861                vec_any_nan(vector_double([f64::NAN, f64::NAN])),
6862                i32::from(true)
6863            );
6864            assert_eq!(vec_any_nan(vector_double([f64::NAN, 1.0])), i32::from(true));
6865            assert_eq!(vec_any_nan(vector_double([0.0, 1.0])), i32::from(false));
6866
6867            assert_eq!(
6868                vec_all_numeric(vector_double([f64::NAN, f64::NAN])),
6869                i32::from(false)
6870            );
6871            assert_eq!(
6872                vec_all_numeric(vector_double([f64::NAN, 1.0])),
6873                i32::from(false)
6874            );
6875            assert_eq!(vec_all_numeric(vector_double([0.0, 1.0])), i32::from(true));
6876
6877            assert_eq!(
6878                vec_any_numeric(vector_double([f64::NAN, f64::NAN])),
6879                i32::from(false)
6880            );
6881            assert_eq!(
6882                vec_any_numeric(vector_double([f64::NAN, 1.0])),
6883                i32::from(true)
6884            );
6885            assert_eq!(vec_any_numeric(vector_double([0.0, 1.0])), i32::from(true));
6886
6887            // "numeric" means "not NaN". infinities are numeric
6888            assert_eq!(
6889                vec_all_numeric(vector_double([f64::INFINITY, f64::NEG_INFINITY])),
6890                i32::from(true)
6891            );
6892            assert_eq!(
6893                vec_any_numeric(vector_double([f64::INFINITY, f64::NEG_INFINITY])),
6894                i32::from(true)
6895            );
6896        }
6897    }
6898
6899    #[simd_test(enable = "vector")]
6900    fn test_vec_test_mask() {
6901        unsafe {
6902            let v = vector_unsigned_long_long([0xFF00FF00FF00FF00; 2]);
6903            let m = vector_unsigned_long_long([0x0000FF000000FF00; 2]);
6904            assert_eq!(vec_test_mask(v, m), 3);
6905
6906            let v = vector_unsigned_long_long([u64::MAX; 2]);
6907            let m = vector_unsigned_long_long([0; 2]);
6908            assert_eq!(vec_test_mask(v, m), 0);
6909
6910            let v = vector_unsigned_long_long([0; 2]);
6911            let m = vector_unsigned_long_long([u64::MAX; 2]);
6912            assert_eq!(vec_test_mask(v, m), 0);
6913
6914            let v = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA; 2]);
6915            let m = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA; 2]);
6916            assert_eq!(vec_test_mask(v, m), 3);
6917        }
6918    }
6919
6920    #[simd_test(enable = "vector-enhancements-2")]
6921    fn test_vec_search_string_cc() {
6922        unsafe {
6923            let b = vector_unsigned_char(*b"ABCD------------");
6924            let c = vector_unsigned_char([4; 16]);
6925
6926            let haystack = vector_unsigned_char(*b"__ABCD__________");
6927            let (result, d) = vec_search_string_cc(haystack, b, c);
6928            assert_eq!(result.as_array()[7], 2);
6929            assert_eq!(d, 2);
6930
6931            let haystack = vector_unsigned_char(*b"___ABCD_________");
6932            let (result, d) = vec_search_string_cc(haystack, b, c);
6933            assert_eq!(result.as_array()[7], 3);
6934            assert_eq!(d, 2);
6935
6936            let haystack = vector_unsigned_char(*b"________________");
6937            let (result, d) = vec_search_string_cc(haystack, b, c);
6938            assert_eq!(result.as_array()[7], 16);
6939            assert_eq!(d, 0);
6940
6941            let haystack = vector_unsigned_char(*b"______\0_________");
6942            let (result, d) = vec_search_string_cc(haystack, b, c);
6943            assert_eq!(result.as_array()[7], 16);
6944            assert_eq!(d, 0);
6945
6946            let haystack = vector_unsigned_char(*b"______\0__ABCD___");
6947            let (result, d) = vec_search_string_cc(haystack, b, c);
6948            assert_eq!(result.as_array()[7], 9);
6949            assert_eq!(d, 2);
6950        }
6951    }
6952
6953    #[simd_test(enable = "vector-enhancements-2")]
6954    fn test_vec_search_string_until_zero_cc() {
6955        unsafe {
6956            let b = vector_unsigned_char(*b"ABCD\0\0\0\0\0\0\0\0\0\0\0\0");
6957            let c = vector_unsigned_char([16; 16]);
6958
6959            let haystack = vector_unsigned_char(*b"__ABCD__________");
6960            let (result, d) = vec_search_string_until_zero_cc(haystack, b, c);
6961            assert_eq!(result.as_array()[7], 2);
6962            assert_eq!(d, 2);
6963
6964            let haystack = vector_unsigned_char(*b"___ABCD_________");
6965            let (result, d) = vec_search_string_until_zero_cc(haystack, b, c);
6966            assert_eq!(result.as_array()[7], 3);
6967            assert_eq!(d, 2);
6968
6969            let haystack = vector_unsigned_char(*b"________________");
6970            let (result, d) = vec_search_string_until_zero_cc(haystack, b, c);
6971            assert_eq!(result.as_array()[7], 16);
6972            assert_eq!(d, 0);
6973
6974            let haystack = vector_unsigned_char(*b"______\0_________");
6975            let (result, d) = vec_search_string_until_zero_cc(haystack, b, c);
6976            assert_eq!(result.as_array()[7], 16);
6977            assert_eq!(d, 1);
6978
6979            let haystack = vector_unsigned_char(*b"______\0__ABCD___");
6980            let (result, d) = vec_search_string_until_zero_cc(haystack, b, c);
6981            assert_eq!(result.as_array()[7], 16);
6982            assert_eq!(d, 1);
6983        }
6984    }
6985
6986    #[simd_test(enable = "vector")]
6987    fn test_vec_doublee() {
6988        unsafe {
6989            let v = vector_float([1.0, 2.0, 3.0, 4.0]);
6990            assert_eq!(vec_doublee(v).as_array(), &[1.0, 3.0]);
6991
6992            let v = vector_float([f32::NAN, 2.0, f32::INFINITY, 4.0]);
6993            let d = vec_doublee(v);
6994            assert!(d.as_array()[0].is_nan());
6995            assert_eq!(d.as_array()[1], f64::INFINITY);
6996        }
6997    }
6998
6999    #[simd_test(enable = "vector")]
7000    fn test_vec_floate() {
7001        // NOTE: indices 1 and 3 can have an arbitrary value. With the C version
7002        // these are poison values, our version initializes the memory but its
7003        // value still should not be relied upon by application code.
7004        unsafe {
7005            let v = vector_double([1.0, 2.0]);
7006            let d = vec_floate(v);
7007            assert_eq!(d.as_array()[0], 1.0);
7008            assert_eq!(d.as_array()[2], 2.0);
7009
7010            let v = vector_double([f64::NAN, f64::INFINITY]);
7011            let d = vec_floate(v);
7012            assert!(d.as_array()[0].is_nan());
7013            assert_eq!(d.as_array()[2], f32::INFINITY);
7014
7015            let v = vector_double([f64::MIN, f64::MAX]);
7016            let d = vec_floate(v);
7017            assert_eq!(d.as_array()[0], f64::MIN as f32);
7018            assert_eq!(d.as_array()[2], f64::MAX as f32);
7019        }
7020    }
7021
7022    #[simd_test(enable = "vector")]
7023    fn test_vec_extend_s64() {
7024        unsafe {
7025            let v = vector_signed_char([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]);
7026            assert_eq!(vec_extend_s64(v).as_array(), &[7, 15]);
7027
7028            let v = vector_signed_short([0, 1, 2, 3, 4, 5, 6, 7]);
7029            assert_eq!(vec_extend_s64(v).as_array(), &[3, 7]);
7030
7031            let v = vector_signed_int([0, 1, 2, 3]);
7032            assert_eq!(vec_extend_s64(v).as_array(), &[1, 3]);
7033        }
7034    }
7035
7036    #[simd_test(enable = "vector")]
7037    fn test_vec_signed() {
7038        unsafe {
7039            let v = vector_float([1.0, 2.5, -2.5, -0.0]);
7040            assert_eq!(vec_signed(v).as_array(), &[1, 2, -2, 0]);
7041
7042            let v = vector_double([2.5, -2.5]);
7043            assert_eq!(vec_signed(v).as_array(), &[2, -2]);
7044        }
7045    }
7046
7047    #[simd_test(enable = "vector")]
7048    fn test_vec_unsigned() {
7049        // NOTE: converting a negative floating point value is UB!
7050        unsafe {
7051            let v = vector_float([1.0, 2.5, 3.5, 0.0]);
7052            assert_eq!(vec_unsigned(v).as_array(), &[1, 2, 3, 0]);
7053
7054            let v = vector_double([2.5, 3.5]);
7055            assert_eq!(vec_unsigned(v).as_array(), &[2, 3]);
7056        }
7057    }
7058
7059    #[simd_test(enable = "vector")]
7060    fn test_vec_cp_until_zero() {
7061        unsafe {
7062            let v = vector_signed_int([1, 2, 3, 4]);
7063            let d = vec_cp_until_zero(v);
7064            assert_eq!(d.as_array(), &[1, 2, 3, 4]);
7065
7066            let v = vector_signed_int([1, 2, 0, 4]);
7067            let d = vec_cp_until_zero(v);
7068            assert_eq!(d.as_array(), &[1, 2, 0, 0]);
7069        }
7070    }
7071
7072    #[simd_test(enable = "vector")]
7073    fn test_vec_cp_until_zero_cc() {
7074        unsafe {
7075            let v = vector_signed_int([1, 2, 3, 4]);
7076            let (d, cc) = vec_cp_until_zero_cc(v);
7077            assert_eq!(d.as_array(), &[1, 2, 3, 4]);
7078            assert_eq!(cc, 3);
7079
7080            let v = vector_signed_int([1, 2, 0, 4]);
7081            let (d, cc) = vec_cp_until_zero_cc(v);
7082            assert_eq!(d.as_array(), &[1, 2, 0, 0]);
7083            assert_eq!(cc, 0);
7084        }
7085    }
7086
7087    #[simd_test(enable = "vector-enhancements-1")]
7088    fn test_vec_msum_u128() {
7089        let a = vector_unsigned_long_long([1, 2]);
7090        let b = vector_unsigned_long_long([3, 4]);
7091
7092        unsafe {
7093            let c: vector_unsigned_char = transmute(100u128);
7094
7095            let d: u128 = transmute(vec_msum_u128::<0>(a, b, c));
7096            assert_eq!(d, (1 * 3) + (2 * 4) + 100);
7097
7098            let d: u128 = transmute(vec_msum_u128::<4>(a, b, c));
7099            assert_eq!(d, (1 * 3) + (2 * 4) * 2 + 100);
7100
7101            let d: u128 = transmute(vec_msum_u128::<8>(a, b, c));
7102            assert_eq!(d, (1 * 3) * 2 + (2 * 4) + 100);
7103
7104            let d: u128 = transmute(vec_msum_u128::<12>(a, b, c));
7105            assert_eq!(d, (1 * 3) * 2 + (2 * 4) * 2 + 100);
7106        }
7107    }
7108
7109    #[simd_test(enable = "vector")]
7110    fn test_vec_sld() {
7111        let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]);
7112        let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
7113
7114        unsafe {
7115            let d = vec_sld::<_, 4>(a, b);
7116            assert_eq!(d.as_array(), &[0xAAAAAAAAAAAAAAAA, 0xAAAAAAAABBBBBBBB]);
7117        }
7118    }
7119
7120    #[simd_test(enable = "vector")]
7121    fn test_vec_sldw() {
7122        let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]);
7123        let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
7124
7125        unsafe {
7126            let d = vec_sldw::<_, 1>(a, b);
7127            assert_eq!(d.as_array(), &[0xAAAAAAAAAAAAAAAA, 0xAAAAAAAABBBBBBBB]);
7128        }
7129    }
7130
7131    #[simd_test(enable = "vector-enhancements-2")]
7132    fn test_vec_sldb() {
7133        let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]);
7134        let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
7135
7136        unsafe {
7137            let d = vec_sldb::<_, 4>(a, b);
7138            assert_eq!(d.as_array(), &[0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAB]);
7139        }
7140    }
7141
7142    #[simd_test(enable = "vector-enhancements-2")]
7143    fn test_vec_srdb() {
7144        let a = vector_unsigned_long_long([0xAAAAAAAAAAAAAAAA, 0xAAAAAAAAAAAAAAAA]);
7145        let b = vector_unsigned_long_long([0xBBBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
7146
7147        unsafe {
7148            let d = vec_srdb::<_, 4>(a, b);
7149            assert_eq!(d.as_array(), &[0xABBBBBBBBBBBBBBB, 0xBBBBBBBBBBBBBBBB]);
7150        }
7151    }
7152
7153    const GT: u32 = 0x20000000;
7154    const LT: u32 = 0x40000000;
7155    const EQ: u32 = 0x80000000;
7156
7157    #[simd_test(enable = "vector")]
7158    fn test_vec_cmprg() {
7159        let a = vector_unsigned_int([11, 22, 33, 44]);
7160        let b = vector_unsigned_int([10, 20, 30, 40]);
7161
7162        let c = vector_unsigned_int([GT, LT, GT, LT]);
7163        let d = unsafe { vec_cmprg(a, b, c) };
7164        assert_eq!(d.as_array(), &[!0, 0, !0, 0]);
7165
7166        let c = vector_unsigned_int([GT, LT, 0, 0]);
7167        let d = unsafe { vec_cmprg(a, b, c) };
7168        assert_eq!(d.as_array(), &[!0, 0, 0, 0]);
7169
7170        let a = vector_unsigned_int([11, 22, 33, 30]);
7171        let b = vector_unsigned_int([10, 20, 30, 30]);
7172
7173        let c = vector_unsigned_int([GT, LT, EQ, EQ]);
7174        let d = unsafe { vec_cmprg(a, b, c) };
7175        assert_eq!(d.as_array(), &[!0, 0, 0, !0]);
7176    }
7177
7178    #[simd_test(enable = "vector")]
7179    fn test_vec_cmpnrg() {
7180        let a = vector_unsigned_int([11, 22, 33, 44]);
7181        let b = vector_unsigned_int([10, 20, 30, 40]);
7182
7183        let c = vector_unsigned_int([GT, LT, GT, LT]);
7184        let d = unsafe { vec_cmpnrg(a, b, c) };
7185        assert_eq!(d.as_array(), &[0, !0, 0, !0]);
7186
7187        let c = vector_unsigned_int([GT, LT, 0, 0]);
7188        let d = unsafe { vec_cmpnrg(a, b, c) };
7189        assert_eq!(d.as_array(), &[0, !0, !0, !0]);
7190
7191        let a = vector_unsigned_int([11, 22, 33, 30]);
7192        let b = vector_unsigned_int([10, 20, 30, 30]);
7193
7194        let c = vector_unsigned_int([GT, LT, EQ, EQ]);
7195        let d = unsafe { vec_cmpnrg(a, b, c) };
7196        assert_eq!(d.as_array(), &[0, !0, !0, 0]);
7197    }
7198
7199    #[simd_test(enable = "vector")]
7200    fn test_vec_cmprg_idx() {
7201        let a = vector_unsigned_int([1, 11, 22, 33]);
7202        let b = vector_unsigned_int([10, 20, 30, 40]);
7203
7204        let c = vector_unsigned_int([GT, LT, GT, LT]);
7205        let d = unsafe { vec_cmprg_idx(a, b, c) };
7206        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
7207    }
7208
7209    #[simd_test(enable = "vector")]
7210    fn test_vec_cmpnrg_idx() {
7211        let a = vector_unsigned_int([1, 11, 22, 33]);
7212        let b = vector_unsigned_int([10, 20, 30, 40]);
7213
7214        let c = vector_unsigned_int([GT, LT, GT, LT]);
7215        let d = unsafe { vec_cmpnrg_idx(a, b, c) };
7216        assert_eq!(d.as_array(), &[0, 0, 0, 0]);
7217    }
7218
7219    #[simd_test(enable = "vector")]
7220    fn test_vec_cmprg_or_0_idx() {
7221        let a = vector_unsigned_int([1, 0, 22, 33]);
7222        let b = vector_unsigned_int([10, 20, 30, 40]);
7223
7224        let c = vector_unsigned_int([GT, LT, GT, LT]);
7225        let d = unsafe { vec_cmprg_or_0_idx(a, b, c) };
7226        assert_eq!(d.as_array(), &[0, 4, 0, 0]);
7227    }
7228
7229    #[simd_test(enable = "vector")]
7230    fn test_vec_cmpnrg_or_0_idx() {
7231        let a = vector_unsigned_int([11, 33, 0, 22]);
7232        let b = vector_unsigned_int([10, 20, 30, 40]);
7233
7234        let c = vector_unsigned_int([GT, LT, GT, LT]);
7235        let d = unsafe { vec_cmpnrg_or_0_idx(a, b, c) };
7236        assert_eq!(d.as_array(), &[0, 8, 0, 0]);
7237    }
7238
7239    test_vec_2! { test_vec_cmpgt, vec_cmpgt, f32x4, f32x4 -> i32x4,
7240        [1.0, f32::NAN, f32::NAN, 3.14],
7241        [2.0, f32::NAN, 5.0, 2.0],
7242        [0, 0, 0, !0]
7243    }
7244
7245    test_vec_2! { test_vec_cmpge, vec_cmpge, f32x4, f32x4 -> i32x4,
7246        [1.0, f32::NAN, f32::NAN, 3.14],
7247        [1.0, f32::NAN, 5.0, 2.0],
7248        [!0, 0, 0, !0]
7249    }
7250
7251    test_vec_2! { test_vec_cmplt, vec_cmplt, f32x4, f32x4 -> i32x4,
7252        [1.0, f32::NAN, f32::NAN, 2.0],
7253        [2.0, f32::NAN, 5.0, 2.0],
7254        [!0, 0, 0, 0]
7255    }
7256
7257    test_vec_2! { test_vec_cmple, vec_cmple, f32x4, f32x4 -> i32x4,
7258        [1.0, f32::NAN, f32::NAN, 2.0],
7259        [1.0, f32::NAN, 5.0, 3.14],
7260        [!0, 0, 0, !0]
7261    }
7262
7263    test_vec_2! { test_vec_cmpeq, vec_cmpeq, f32x4, f32x4 -> i32x4,
7264        [1.0, f32::NAN, f32::NAN, 2.0],
7265        [1.0, f32::NAN, 5.0, 3.14],
7266        [!0, 0, 0, 0]
7267    }
7268
7269    test_vec_2! { test_vec_cmpne, vec_cmpne, f32x4, f32x4 -> i32x4,
7270        [1.0, f32::NAN, f32::NAN, 2.0],
7271        [1.0, f32::NAN, 5.0, 3.14],
7272        [0, !0, !0, !0]
7273    }
7274
7275    #[simd_test(enable = "vector")]
7276    fn test_vec_meadd() {
7277        let a = vector_unsigned_short([1, 0, 2, 0, 3, 0, 4, 0]);
7278        let b = vector_unsigned_short([5, 0, 6, 0, 7, 0, 8, 0]);
7279        let c = vector_unsigned_int([2, 2, 2, 2]);
7280
7281        let d = unsafe { vec_meadd(a, b, c) };
7282        assert_eq!(d.as_array(), &[7, 14, 23, 34]);
7283
7284        let a = vector_signed_short([1, 0, 2, 0, 3, 0, 4, 0]);
7285        let b = vector_signed_short([5, 0, 6, 0, 7, 0, 8, 0]);
7286        let c = vector_signed_int([2, -2, 2, -2]);
7287
7288        let d = unsafe { vec_meadd(a, b, c) };
7289        assert_eq!(d.as_array(), &[7, 10, 23, 30]);
7290    }
7291
7292    #[simd_test(enable = "vector")]
7293    fn test_vec_moadd() {
7294        let a = vector_unsigned_short([0, 1, 0, 2, 0, 3, 0, 4]);
7295        let b = vector_unsigned_short([0, 5, 0, 6, 0, 7, 0, 8]);
7296        let c = vector_unsigned_int([2, 2, 2, 2]);
7297
7298        let d = unsafe { vec_moadd(a, b, c) };
7299        assert_eq!(d.as_array(), &[7, 14, 23, 34]);
7300
7301        let a = vector_signed_short([0, 1, 0, 2, 0, 3, 0, 4]);
7302        let b = vector_signed_short([0, 5, 0, 6, 0, 7, 0, 8]);
7303        let c = vector_signed_int([2, -2, 2, -2]);
7304
7305        let d = unsafe { vec_moadd(a, b, c) };
7306        assert_eq!(d.as_array(), &[7, 10, 23, 30]);
7307    }
7308
7309    #[simd_test(enable = "vector")]
7310    fn test_vec_mhadd() {
7311        let a = vector_unsigned_int([1, 2, 3, 4]);
7312        let b = vector_unsigned_int([5, 6, 7, 8]);
7313        let c = vector_unsigned_int([u32::MAX; 4]);
7314
7315        let d = unsafe { vec_mhadd(a, b, c) };
7316        assert_eq!(d.as_array(), &[1, 1, 1, 1]);
7317
7318        let a = vector_signed_int([-1, -2, -3, -4]);
7319        let b = vector_signed_int([5, 6, 7, 8]);
7320        let c = vector_signed_int([i32::MIN; 4]);
7321
7322        let d = unsafe { vec_mhadd(a, b, c) };
7323        assert_eq!(d.as_array(), &[-1, -1, -1, -1]);
7324    }
7325
7326    #[simd_test(enable = "vector")]
7327    fn test_vec_mladd() {
7328        let a = vector_unsigned_int([1, 2, 3, 4]);
7329        let b = vector_unsigned_int([5, 6, 7, 8]);
7330        let c = vector_unsigned_int([2, 2, 2, 2]);
7331
7332        let d = unsafe { vec_mladd(a, b, c) };
7333        assert_eq!(d.as_array(), &[7, 14, 23, 34]);
7334
7335        let a = vector_signed_int([-1, -2, -3, -4]);
7336        let b = vector_signed_int([5, 6, 7, 8]);
7337        let c = vector_signed_int([2, 2, 2, 2]);
7338
7339        let d = unsafe { vec_mladd(a, b, c) };
7340        assert_eq!(d.as_array(), &[-3, -10, -19, -30]);
7341    }
7342
7343    #[simd_test(enable = "vector")]
7344    fn test_vec_extract() {
7345        let v = vector_unsigned_int([1, 2, 3, 4]);
7346
7347        assert_eq!(unsafe { vec_extract(v, 1) }, 2);
7348        assert_eq!(unsafe { vec_extract(v, 4 + 2) }, 3);
7349    }
7350
7351    #[simd_test(enable = "vector")]
7352    fn test_vec_insert() {
7353        let mut v = vector_unsigned_int([1, 2, 3, 4]);
7354
7355        v = unsafe { vec_insert(42, v, 1) };
7356        assert_eq!(v.as_array(), &[1, 42, 3, 4]);
7357
7358        v = unsafe { vec_insert(64, v, 6) };
7359        assert_eq!(v.as_array(), &[1, 42, 64, 4]);
7360    }
7361
7362    #[simd_test(enable = "vector")]
7363    fn test_vec_promote() {
7364        let v: vector_unsigned_int = unsafe { vec_promote(42, 1).assume_init() };
7365        assert_eq!(v.as_array(), &[0, 42, 0, 0]);
7366    }
7367
7368    #[simd_test(enable = "vector")]
7369    fn test_vec_insert_and_zero() {
7370        let v = unsafe { vec_insert_and_zero::<vector_unsigned_int>(&42u32) };
7371        assert_eq!(v.as_array(), vector_unsigned_int([0, 42, 0, 0]).as_array());
7372    }
7373}