portable_atomic/imp/
mod.rs

1// SPDX-License-Identifier: Apache-2.0 OR MIT
2
3// -----------------------------------------------------------------------------
4// Lock-free implementations
5
6#[cfg(not(any(
7    all(
8        portable_atomic_no_atomic_load_store,
9        not(all(target_arch = "bpf", not(feature = "critical-section"))),
10    ),
11    target_arch = "avr",
12    target_arch = "msp430",
13)))]
14#[cfg_attr(
15    portable_atomic_no_cfg_target_has_atomic,
16    cfg(not(all(
17        any(
18            target_arch = "riscv32",
19            target_arch = "riscv64",
20            feature = "critical-section",
21            portable_atomic_unsafe_assume_single_core,
22        ),
23        portable_atomic_no_atomic_cas,
24    )))
25)]
26#[cfg_attr(
27    not(portable_atomic_no_cfg_target_has_atomic),
28    cfg(not(all(
29        any(
30            target_arch = "riscv32",
31            target_arch = "riscv64",
32            feature = "critical-section",
33            portable_atomic_unsafe_assume_single_core,
34        ),
35        not(target_has_atomic = "ptr"),
36    )))
37)]
38mod core_atomic;
39
40// AVR
41#[cfg(target_arch = "avr")]
42#[cfg(not(portable_atomic_no_asm))]
43#[cfg(not(feature = "critical-section"))]
44mod avr;
45
46// MSP430
47#[cfg(target_arch = "msp430")]
48pub(crate) mod msp430;
49
50// RISC-V without A-extension
51#[cfg(any(test, not(feature = "critical-section")))]
52#[cfg_attr(
53    portable_atomic_no_cfg_target_has_atomic,
54    cfg(any(
55        all(test, not(any(miri, portable_atomic_sanitize_thread))),
56        portable_atomic_no_atomic_cas,
57    ))
58)]
59#[cfg_attr(
60    not(portable_atomic_no_cfg_target_has_atomic),
61    cfg(any(
62        all(test, not(any(miri, portable_atomic_sanitize_thread))),
63        not(target_has_atomic = "ptr"),
64    ))
65)]
66#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
67mod riscv;
68
69// x86-specific optimizations
70// Miri and Sanitizer do not support inline assembly.
71#[cfg(all(
72    any(target_arch = "x86", target_arch = "x86_64"),
73    not(any(miri, portable_atomic_sanitize_thread)),
74    any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
75))]
76mod x86;
77
78// 64-bit atomic implementations on 32-bit architectures
79#[cfg(any(target_arch = "arm", target_arch = "riscv32"))]
80mod atomic64;
81
82// 128-bit atomic implementations on 64-bit architectures
83#[cfg(any(
84    target_arch = "aarch64",
85    target_arch = "arm64ec",
86    target_arch = "powerpc64",
87    target_arch = "riscv64",
88    target_arch = "s390x",
89    target_arch = "x86_64",
90))]
91mod atomic128;
92
93// -----------------------------------------------------------------------------
94// Lock-based fallback implementations
95
96#[cfg(feature = "fallback")]
97#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
98#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
99#[cfg(any(
100    test,
101    not(any(
102        all(
103            target_arch = "aarch64",
104            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
105        ),
106        all(target_arch = "arm64ec", not(portable_atomic_no_asm)),
107        all(
108            target_arch = "x86_64",
109            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
110            any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
111        ),
112        all(
113            target_arch = "riscv64",
114            not(portable_atomic_no_asm),
115            any(
116                target_feature = "experimental-zacas",
117                portable_atomic_target_feature = "experimental-zacas",
118            ),
119        ),
120        all(
121            target_arch = "powerpc64",
122            portable_atomic_unstable_asm_experimental_arch,
123            any(
124                target_feature = "quadword-atomics",
125                portable_atomic_target_feature = "quadword-atomics",
126            ),
127        ),
128        all(target_arch = "s390x", not(portable_atomic_no_asm)),
129    ))
130))]
131mod fallback;
132
133// -----------------------------------------------------------------------------
134// Critical section based fallback implementations
135
136// On AVR, we always use critical section based fallback implementation.
137// AVR can be safely assumed to be single-core, so this is sound.
138// https://github.com/llvm/llvm-project/blob/llvmorg-19.1.0/llvm/lib/Target/AVR/AVRExpandPseudoInsts.cpp#L1074
139// MSP430 as well.
140#[cfg(any(
141    all(test, target_os = "none"),
142    portable_atomic_unsafe_assume_single_core,
143    feature = "critical-section",
144    target_arch = "avr",
145    target_arch = "msp430",
146))]
147#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(any(test, portable_atomic_no_atomic_cas)))]
148#[cfg_attr(
149    not(portable_atomic_no_cfg_target_has_atomic),
150    cfg(any(test, not(target_has_atomic = "ptr")))
151)]
152#[cfg(any(
153    target_arch = "arm",
154    target_arch = "avr",
155    target_arch = "msp430",
156    target_arch = "riscv32",
157    target_arch = "riscv64",
158    target_arch = "xtensa",
159    feature = "critical-section",
160))]
161mod interrupt;
162
163// -----------------------------------------------------------------------------
164// Atomic float implementations
165
166#[cfg(feature = "float")]
167#[allow(clippy::float_arithmetic)]
168pub(crate) mod float;
169
170// -----------------------------------------------------------------------------
171
172// has CAS | (has core atomic & !(avr | msp430 | critical section)) => core atomic
173#[cfg(not(any(
174    portable_atomic_no_atomic_load_store,
175    target_arch = "avr",
176    target_arch = "msp430",
177)))]
178#[cfg_attr(
179    portable_atomic_no_cfg_target_has_atomic,
180    cfg(not(all(
181        any(
182            target_arch = "riscv32",
183            target_arch = "riscv64",
184            feature = "critical-section",
185            portable_atomic_unsafe_assume_single_core,
186        ),
187        portable_atomic_no_atomic_cas,
188    )))
189)]
190#[cfg_attr(
191    not(portable_atomic_no_cfg_target_has_atomic),
192    cfg(not(all(
193        any(
194            target_arch = "riscv32",
195            target_arch = "riscv64",
196            feature = "critical-section",
197            portable_atomic_unsafe_assume_single_core,
198        ),
199        not(target_has_atomic = "ptr"),
200    )))
201)]
202items! {
203    pub(crate) use self::core_atomic::{
204        AtomicI16, AtomicI32, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU32, AtomicU8,
205        AtomicUsize,
206    };
207    #[cfg_attr(
208        portable_atomic_no_cfg_target_has_atomic,
209        cfg(any(
210            not(portable_atomic_no_atomic_64),
211            not(any(target_pointer_width = "16", target_pointer_width = "32")),
212        ))
213    )]
214    #[cfg_attr(
215        not(portable_atomic_no_cfg_target_has_atomic),
216        cfg(any(
217            target_has_atomic = "64",
218            not(any(target_pointer_width = "16", target_pointer_width = "32")),
219        ))
220    )]
221    pub(crate) use self::core_atomic::{AtomicI64, AtomicU64};
222}
223// bpf & !(critical section) => core atomic
224#[cfg(all(
225    target_arch = "bpf",
226    portable_atomic_no_atomic_load_store,
227    not(feature = "critical-section"),
228))]
229pub(crate) use self::core_atomic::{AtomicI64, AtomicIsize, AtomicPtr, AtomicU64, AtomicUsize};
230
231// RISC-V without A-extension & !(assume single core | critical section)
232#[cfg(not(any(portable_atomic_unsafe_assume_single_core, feature = "critical-section")))]
233#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_cas))]
234#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "ptr")))]
235#[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))]
236items! {
237    pub(crate) use self::riscv::{
238        AtomicI16, AtomicI32, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU32, AtomicU8,
239        AtomicUsize,
240    };
241    #[cfg(target_arch = "riscv64")]
242    pub(crate) use self::riscv::{AtomicI64, AtomicU64};
243}
244
245// no core atomic CAS & (assume single core | critical section) => critical section based fallback
246#[cfg(any(
247    portable_atomic_unsafe_assume_single_core,
248    feature = "critical-section",
249    target_arch = "avr",
250    target_arch = "msp430",
251))]
252#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_cas))]
253#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "ptr")))]
254items! {
255    pub(crate) use self::interrupt::{
256        AtomicI16, AtomicI8, AtomicIsize, AtomicPtr, AtomicU16, AtomicU8, AtomicUsize,
257    };
258    #[cfg(any(not(target_pointer_width = "16"), feature = "fallback"))]
259    pub(crate) use self::interrupt::{AtomicI32, AtomicU32};
260    #[cfg(any(
261        not(any(target_pointer_width = "16", target_pointer_width = "32")),
262        feature = "fallback",
263    ))]
264    pub(crate) use self::interrupt::{AtomicI64, AtomicU64};
265    #[cfg(feature = "fallback")]
266    pub(crate) use self::interrupt::{AtomicI128, AtomicU128};
267}
268
269// no core (64-bit | 128-bit) atomic & has CAS => use lock-base fallback
270#[cfg(feature = "fallback")]
271#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(not(portable_atomic_no_atomic_cas)))]
272#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(target_has_atomic = "ptr"))]
273items! {
274    #[cfg(not(any(
275        all(
276            target_arch = "arm",
277            not(any(miri, portable_atomic_sanitize_thread)),
278            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
279            any(target_os = "linux", target_os = "android"),
280            not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
281            not(portable_atomic_no_outline_atomics),
282        ),
283        all(
284            target_arch = "riscv32",
285            not(any(miri, portable_atomic_sanitize_thread)),
286            not(portable_atomic_no_asm),
287            any(
288                target_feature = "experimental-zacas",
289                portable_atomic_target_feature = "experimental-zacas",
290                all(
291                    feature = "fallback",
292                    not(portable_atomic_no_outline_atomics),
293                    any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
294                    any(target_os = "linux", target_os = "android"),
295                ),
296            ),
297        ),
298    )))]
299    #[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_64))]
300    #[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "64")))]
301    pub(crate) use self::fallback::{AtomicI64, AtomicU64};
302    #[cfg(not(any(
303        all(
304            target_arch = "aarch64",
305            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
306        ),
307        all(target_arch = "arm64ec", not(portable_atomic_no_asm)),
308        all(
309            target_arch = "x86_64",
310            not(all(
311                any(miri, portable_atomic_sanitize_thread),
312                portable_atomic_no_cmpxchg16b_intrinsic,
313            )),
314            any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
315            any(
316                target_feature = "cmpxchg16b",
317                portable_atomic_target_feature = "cmpxchg16b",
318                all(
319                    feature = "fallback",
320                    not(portable_atomic_no_outline_atomics),
321                    not(any(target_env = "sgx", miri)),
322                ),
323            ),
324        ),
325        all(
326            target_arch = "riscv64",
327            not(portable_atomic_no_asm),
328            any(
329                target_feature = "experimental-zacas",
330                portable_atomic_target_feature = "experimental-zacas",
331                all(
332                    feature = "fallback",
333                    not(portable_atomic_no_outline_atomics),
334                    any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
335                    any(target_os = "linux", target_os = "android"),
336                    not(any(miri, portable_atomic_sanitize_thread)),
337                ),
338            ),
339        ),
340        all(
341            target_arch = "powerpc64",
342            portable_atomic_unstable_asm_experimental_arch,
343            any(
344                target_feature = "quadword-atomics",
345                portable_atomic_target_feature = "quadword-atomics",
346                all(
347                    feature = "fallback",
348                    not(portable_atomic_no_outline_atomics),
349                    any(
350                        all(
351                            target_os = "linux",
352                            any(
353                                all(
354                                    target_env = "gnu",
355                                    any(target_endian = "little", not(target_feature = "crt-static")),
356                                ),
357                                all(
358                                    any(target_env = "musl", target_env = "ohos", target_env = "uclibc"),
359                                    not(target_feature = "crt-static"),
360                                ),
361                                portable_atomic_outline_atomics,
362                            ),
363                        ),
364                        target_os = "android",
365                        target_os = "freebsd",
366                        target_os = "openbsd",
367                    ),
368                    not(any(miri, portable_atomic_sanitize_thread)),
369                ),
370            ),
371        ),
372        all(target_arch = "s390x", not(portable_atomic_no_asm)),
373    )))]
374    pub(crate) use self::fallback::{AtomicI128, AtomicU128};
375}
376
377// 64-bit atomics (platform-specific)
378// pre-v6 Arm Linux
379#[cfg(feature = "fallback")]
380#[cfg(all(
381    target_arch = "arm",
382    not(any(miri, portable_atomic_sanitize_thread)),
383    any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
384    any(target_os = "linux", target_os = "android"),
385    not(any(target_feature = "v6", portable_atomic_target_feature = "v6")),
386    not(portable_atomic_no_outline_atomics),
387))]
388#[cfg_attr(portable_atomic_no_cfg_target_has_atomic, cfg(portable_atomic_no_atomic_64))]
389#[cfg_attr(not(portable_atomic_no_cfg_target_has_atomic), cfg(not(target_has_atomic = "64")))]
390pub(crate) use self::atomic64::arm_linux::{AtomicI64, AtomicU64};
391// riscv32 & (zacas | outline-atomics)
392#[cfg(all(
393    target_arch = "riscv32",
394    not(any(miri, portable_atomic_sanitize_thread)),
395    not(portable_atomic_no_asm),
396    any(
397        target_feature = "experimental-zacas",
398        portable_atomic_target_feature = "experimental-zacas",
399        all(
400            feature = "fallback",
401            not(portable_atomic_no_outline_atomics),
402            any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
403            any(target_os = "linux", target_os = "android"),
404        ),
405    ),
406))]
407pub(crate) use self::atomic64::riscv32::{AtomicI64, AtomicU64};
408
409// 128-bit atomics (platform-specific)
410// AArch64
411#[cfg(any(
412    all(target_arch = "aarch64", any(not(portable_atomic_no_asm), portable_atomic_unstable_asm)),
413    all(target_arch = "arm64ec", not(portable_atomic_no_asm))
414))]
415pub(crate) use self::atomic128::aarch64::{AtomicI128, AtomicU128};
416// x86_64 & (cmpxchg16b | outline-atomics)
417#[cfg(all(
418    target_arch = "x86_64",
419    not(all(any(miri, portable_atomic_sanitize_thread), portable_atomic_no_cmpxchg16b_intrinsic)),
420    any(not(portable_atomic_no_asm), portable_atomic_unstable_asm),
421    any(
422        target_feature = "cmpxchg16b",
423        portable_atomic_target_feature = "cmpxchg16b",
424        all(
425            feature = "fallback",
426            not(portable_atomic_no_outline_atomics),
427            not(any(target_env = "sgx", miri)),
428        ),
429    ),
430))]
431pub(crate) use self::atomic128::x86_64::{AtomicI128, AtomicU128};
432// riscv64 & (zacas | outline-atomics)
433#[cfg(all(
434    target_arch = "riscv64",
435    not(portable_atomic_no_asm),
436    any(
437        target_feature = "experimental-zacas",
438        portable_atomic_target_feature = "experimental-zacas",
439        all(
440            feature = "fallback",
441            not(portable_atomic_no_outline_atomics),
442            any(test, portable_atomic_outline_atomics), // TODO(riscv): currently disabled by default
443            any(target_os = "linux", target_os = "android"),
444            not(any(miri, portable_atomic_sanitize_thread)),
445        ),
446    ),
447))]
448pub(crate) use self::atomic128::riscv64::{AtomicI128, AtomicU128};
449// powerpc64 & (pwr8 | outline-atomics)
450#[cfg(all(
451    target_arch = "powerpc64",
452    portable_atomic_unstable_asm_experimental_arch,
453    any(
454        target_feature = "quadword-atomics",
455        portable_atomic_target_feature = "quadword-atomics",
456        all(
457            feature = "fallback",
458            not(portable_atomic_no_outline_atomics),
459            any(
460                all(
461                    target_os = "linux",
462                    any(
463                        all(
464                            target_env = "gnu",
465                            any(target_endian = "little", not(target_feature = "crt-static")),
466                        ),
467                        all(
468                            any(target_env = "musl", target_env = "ohos", target_env = "uclibc"),
469                            not(target_feature = "crt-static"),
470                        ),
471                        portable_atomic_outline_atomics,
472                    ),
473                ),
474                target_os = "android",
475                target_os = "freebsd",
476                target_os = "openbsd",
477            ),
478            not(any(miri, portable_atomic_sanitize_thread)),
479        ),
480    ),
481))]
482pub(crate) use self::atomic128::powerpc64::{AtomicI128, AtomicU128};
483// s390x
484#[cfg(all(target_arch = "s390x", not(portable_atomic_no_asm)))]
485pub(crate) use self::atomic128::s390x::{AtomicI128, AtomicU128};
portable_atomic/imp/mod.rs

portable_atomic/imp/
mod.rs
