mirror of
https://review.haiku-os.org/buildtools
synced 2025-02-15 02:07:42 +01:00
92b3138b83
Updated dependencies: * GMP 6.2.1 * ISL 0.24 * MPL 1.2.1 * MPFR 4.1.0 The dependencies were pulled in by running the ./contrib/download_prerequisites script and then manually removing the symbolic links and archives, and renaming the directories (i.e mv isl-0.24 to isl)
1142 lines
38 KiB
D
1142 lines
38 KiB
D
/**
|
|
* The core.internal.atomic module comtains the low-level atomic features available in hardware.
|
|
* This module may be a routing layer for compiler intrinsics.
|
|
*
|
|
* Copyright: Copyright Manu Evans 2019.
|
|
* License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost License 1.0)
|
|
* Authors: Sean Kelly, Alex Rønne Petersen, Manu Evans
|
|
* Source: $(DRUNTIMESRC core/internal/_atomic.d)
|
|
*/
|
|
|
|
module core.internal.atomic;
|
|
|
|
import core.atomic : MemoryOrder, has128BitCAS;
|
|
|
|
version (DigitalMars)
|
|
{
|
|
private
|
|
{
|
|
enum : int
|
|
{
|
|
AX, BX, CX, DX, DI, SI, R8, R9
|
|
}
|
|
|
|
immutable string[4][8] registerNames = [
|
|
[ "AL", "AX", "EAX", "RAX" ],
|
|
[ "BL", "BX", "EBX", "RBX" ],
|
|
[ "CL", "CX", "ECX", "RCX" ],
|
|
[ "DL", "DX", "EDX", "RDX" ],
|
|
[ "DIL", "DI", "EDI", "RDI" ],
|
|
[ "SIL", "SI", "ESI", "RSI" ],
|
|
[ "R8B", "R8W", "R8D", "R8" ],
|
|
[ "R9B", "R9W", "R9D", "R9" ],
|
|
];
|
|
|
|
template RegIndex(T)
|
|
{
|
|
static if (T.sizeof == 1)
|
|
enum RegIndex = 0;
|
|
else static if (T.sizeof == 2)
|
|
enum RegIndex = 1;
|
|
else static if (T.sizeof == 4)
|
|
enum RegIndex = 2;
|
|
else static if (T.sizeof == 8)
|
|
enum RegIndex = 3;
|
|
else
|
|
static assert(false, "Invalid type");
|
|
}
|
|
|
|
enum SizedReg(int reg, T = size_t) = registerNames[reg][RegIndex!T];
|
|
}
|
|
|
|
inout(T) atomicLoad(MemoryOrder order = MemoryOrder.seq, T)(inout(T)* src) pure nothrow @nogc @trusted
|
|
if (CanCAS!T)
|
|
{
|
|
static assert(order != MemoryOrder.rel, "invalid MemoryOrder for atomicLoad()");
|
|
|
|
static if (T.sizeof == size_t.sizeof * 2)
|
|
{
|
|
version (D_InlineAsm_X86)
|
|
{
|
|
asm pure nothrow @nogc @trusted
|
|
{
|
|
push EDI;
|
|
push EBX;
|
|
mov EBX, 0;
|
|
mov ECX, 0;
|
|
mov EAX, 0;
|
|
mov EDX, 0;
|
|
mov EDI, src;
|
|
lock; cmpxchg8b [EDI];
|
|
pop EBX;
|
|
pop EDI;
|
|
}
|
|
}
|
|
else version (D_InlineAsm_X86_64)
|
|
{
|
|
version (Windows)
|
|
{
|
|
static if (RegisterReturn!T)
|
|
{
|
|
enum SrcPtr = SizedReg!CX;
|
|
enum RetPtr = null;
|
|
}
|
|
else
|
|
{
|
|
enum SrcPtr = SizedReg!DX;
|
|
enum RetPtr = SizedReg!CX;
|
|
}
|
|
|
|
mixin (simpleFormat(q{
|
|
asm pure nothrow @nogc @trusted
|
|
{
|
|
naked;
|
|
push RBX;
|
|
mov R8, %0;
|
|
?1 mov R9, %1;
|
|
mov RBX, 0;
|
|
mov RCX, 0;
|
|
mov RAX, 0;
|
|
mov RDX, 0;
|
|
lock; cmpxchg16b [R8];
|
|
?1 mov [R9], RAX;
|
|
?1 mov 8[R9], RDX;
|
|
pop RBX;
|
|
ret;
|
|
}
|
|
}, [SrcPtr, RetPtr]));
|
|
}
|
|
else
|
|
{
|
|
asm pure nothrow @nogc @trusted
|
|
{
|
|
naked;
|
|
push RBX;
|
|
mov RBX, 0;
|
|
mov RCX, 0;
|
|
mov RAX, 0;
|
|
mov RDX, 0;
|
|
lock; cmpxchg16b [RDI];
|
|
pop RBX;
|
|
ret;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else static if (needsLoadBarrier!order)
|
|
{
|
|
version (D_InlineAsm_X86)
|
|
{
|
|
enum SrcReg = SizedReg!CX;
|
|
enum ZeroReg = SizedReg!(DX, T);
|
|
enum ResReg = SizedReg!(AX, T);
|
|
|
|
mixin (simpleFormat(q{
|
|
asm pure nothrow @nogc @trusted
|
|
{
|
|
mov %1, 0;
|
|
mov %2, 0;
|
|
mov %0, src;
|
|
lock; cmpxchg [%0], %1;
|
|
}
|
|
}, [SrcReg, ZeroReg, ResReg]));
|
|
}
|
|
else version (D_InlineAsm_X86_64)
|
|
{
|
|
version (Windows)
|
|
enum SrcReg = SizedReg!CX;
|
|
else
|
|
enum SrcReg = SizedReg!DI;
|
|
enum ZeroReg = SizedReg!(DX, T);
|
|
enum ResReg = SizedReg!(AX, T);
|
|
|
|
mixin (simpleFormat(q{
|
|
asm pure nothrow @nogc @trusted
|
|
{
|
|
naked;
|
|
mov %1, 0;
|
|
mov %2, 0;
|
|
lock; cmpxchg [%0], %1;
|
|
ret;
|
|
}
|
|
}, [SrcReg, ZeroReg, ResReg]));
|
|
}
|
|
}
|
|
else
|
|
return *src;
|
|
}
|
|
|
|
void atomicStore(MemoryOrder order = MemoryOrder.seq, T)(T* dest, T value) pure nothrow @nogc @trusted
|
|
if (CanCAS!T)
|
|
{
|
|
static assert(order != MemoryOrder.acq, "Invalid MemoryOrder for atomicStore()");
|
|
|
|
static if (T.sizeof == size_t.sizeof * 2)
|
|
{
|
|
version (D_InlineAsm_X86)
|
|
{
|
|
asm pure nothrow @nogc @trusted
|
|
{
|
|
push EDI;
|
|
push EBX;
|
|
lea EDI, value;
|
|
mov EBX, [EDI];
|
|
mov ECX, 4[EDI];
|
|
mov EDI, dest;
|
|
mov EAX, [EDI];
|
|
mov EDX, 4[EDI];
|
|
L1: lock; cmpxchg8b [EDI];
|
|
jne L1;
|
|
pop EBX;
|
|
pop EDI;
|
|
}
|
|
}
|
|
else version (D_InlineAsm_X86_64)
|
|
{
|
|
version (Windows)
|
|
{
|
|
asm pure nothrow @nogc @trusted
|
|
{
|
|
naked;
|
|
push RBX;
|
|
mov R8, RDX;
|
|
mov RAX, [RDX];
|
|
mov RDX, 8[RDX];
|
|
mov RBX, [RCX];
|
|
mov RCX, 8[RCX];
|
|
L1: lock; cmpxchg16b [R8];
|
|
jne L1;
|
|
pop RBX;
|
|
ret;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
asm pure nothrow @nogc @trusted
|
|
{
|
|
naked;
|
|
push RBX;
|
|
mov RBX, RDI;
|
|
mov RCX, RSI;
|
|
mov RDI, RDX;
|
|
mov RAX, [RDX];
|
|
mov RDX, 8[RDX];
|
|
L1: lock; cmpxchg16b [RDI];
|
|
jne L1;
|
|
pop RBX;
|
|
ret;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else static if (needsStoreBarrier!order)
|
|
atomicExchange!(order, false)(dest, value);
|
|
else
|
|
*dest = value;
|
|
}
|
|
|
|
T atomicFetchAdd(MemoryOrder order = MemoryOrder.seq, bool result = true, T)(T* dest, T value) pure nothrow @nogc @trusted
|
|
if (is(T : ulong))
|
|
{
|
|
version (D_InlineAsm_X86)
|
|
{
|
|
static assert(T.sizeof <= 4, "64bit atomicFetchAdd not supported on 32bit target." );
|
|
|
|
enum DestReg = SizedReg!DX;
|
|
enum ValReg = SizedReg!(AX, T);
|
|
|
|
mixin (simpleFormat(q{
|
|
asm pure nothrow @nogc @trusted
|
|
{
|
|
mov %1, value;
|
|
mov %0, dest;
|
|
lock; xadd[%0], %1;
|
|
}
|
|
}, [DestReg, ValReg]));
|
|
}
|
|
else version (D_InlineAsm_X86_64)
|
|
{
|
|
version (Windows)
|
|
{
|
|
enum DestReg = SizedReg!DX;
|
|
enum ValReg = SizedReg!(CX, T);
|
|
}
|
|
else
|
|
{
|
|
enum DestReg = SizedReg!SI;
|
|
enum ValReg = SizedReg!(DI, T);
|
|
}
|
|
enum ResReg = result ? SizedReg!(AX, T) : null;
|
|
|
|
mixin (simpleFormat(q{
|
|
asm pure nothrow @nogc @trusted
|
|
{
|
|
naked;
|
|
lock; xadd[%0], %1;
|
|
?2 mov %2, %1;
|
|
ret;
|
|
}
|
|
}, [DestReg, ValReg, ResReg]));
|
|
}
|
|
else
|
|
static assert (false, "Unsupported architecture.");
|
|
}
|
|
|
|
T atomicFetchSub(MemoryOrder order = MemoryOrder.seq, bool result = true, T)(T* dest, T value) pure nothrow @nogc @trusted
|
|
if (is(T : ulong))
|
|
{
|
|
return atomicFetchAdd(dest, cast(T)-cast(IntOrLong!T)value);
|
|
}
|
|
|
|
T atomicExchange(MemoryOrder order = MemoryOrder.seq, bool result = true, T)(T* dest, T value) pure nothrow @nogc @trusted
|
|
if (CanCAS!T)
|
|
{
|
|
version (D_InlineAsm_X86)
|
|
{
|
|
static assert(T.sizeof <= 4, "64bit atomicExchange not supported on 32bit target." );
|
|
|
|
enum DestReg = SizedReg!CX;
|
|
enum ValReg = SizedReg!(AX, T);
|
|
|
|
mixin (simpleFormat(q{
|
|
asm pure nothrow @nogc @trusted
|
|
{
|
|
mov %1, value;
|
|
mov %0, dest;
|
|
xchg [%0], %1;
|
|
}
|
|
}, [DestReg, ValReg]));
|
|
}
|
|
else version (D_InlineAsm_X86_64)
|
|
{
|
|
version (Windows)
|
|
{
|
|
enum DestReg = SizedReg!DX;
|
|
enum ValReg = SizedReg!(CX, T);
|
|
}
|
|
else
|
|
{
|
|
enum DestReg = SizedReg!SI;
|
|
enum ValReg = SizedReg!(DI, T);
|
|
}
|
|
enum ResReg = result ? SizedReg!(AX, T) : null;
|
|
|
|
mixin (simpleFormat(q{
|
|
asm pure nothrow @nogc @trusted
|
|
{
|
|
naked;
|
|
xchg [%0], %1;
|
|
?2 mov %2, %1;
|
|
ret;
|
|
}
|
|
}, [DestReg, ValReg, ResReg]));
|
|
}
|
|
else
|
|
static assert (false, "Unsupported architecture.");
|
|
}
|
|
|
|
alias atomicCompareExchangeWeak = atomicCompareExchangeStrong;
|
|
|
|
bool atomicCompareExchangeStrong(MemoryOrder succ = MemoryOrder.seq, MemoryOrder fail = MemoryOrder.seq, T)(T* dest, T* compare, T value) pure nothrow @nogc @trusted
|
|
if (CanCAS!T)
|
|
{
|
|
version (D_InlineAsm_X86)
|
|
{
|
|
static if (T.sizeof <= 4)
|
|
{
|
|
enum DestAddr = SizedReg!CX;
|
|
enum CmpAddr = SizedReg!DI;
|
|
enum Val = SizedReg!(DX, T);
|
|
enum Cmp = SizedReg!(AX, T);
|
|
|
|
mixin (simpleFormat(q{
|
|
asm pure nothrow @nogc @trusted
|
|
{
|
|
push %1;
|
|
mov %2, value;
|
|
mov %1, compare;
|
|
mov %3, [%1];
|
|
mov %0, dest;
|
|
lock; cmpxchg [%0], %2;
|
|
mov [%1], %3;
|
|
setz AL;
|
|
pop %1;
|
|
}
|
|
}, [DestAddr, CmpAddr, Val, Cmp]));
|
|
}
|
|
else static if (T.sizeof == 8)
|
|
{
|
|
asm pure nothrow @nogc @trusted
|
|
{
|
|
push EDI;
|
|
push EBX;
|
|
lea EDI, value;
|
|
mov EBX, [EDI];
|
|
mov ECX, 4[EDI];
|
|
mov EDI, compare;
|
|
mov EAX, [EDI];
|
|
mov EDX, 4[EDI];
|
|
mov EDI, dest;
|
|
lock; cmpxchg8b [EDI];
|
|
mov EDI, compare;
|
|
mov [EDI], EAX;
|
|
mov 4[EDI], EDX;
|
|
setz AL;
|
|
pop EBX;
|
|
pop EDI;
|
|
}
|
|
}
|
|
else
|
|
static assert(T.sizeof <= 8, "128bit atomicCompareExchangeStrong not supported on 32bit target." );
|
|
}
|
|
else version (D_InlineAsm_X86_64)
|
|
{
|
|
static if (T.sizeof <= 8)
|
|
{
|
|
version (Windows)
|
|
{
|
|
enum DestAddr = SizedReg!R8;
|
|
enum CmpAddr = SizedReg!DX;
|
|
enum Val = SizedReg!(CX, T);
|
|
}
|
|
else
|
|
{
|
|
enum DestAddr = SizedReg!DX;
|
|
enum CmpAddr = SizedReg!SI;
|
|
enum Val = SizedReg!(DI, T);
|
|
}
|
|
enum Res = SizedReg!(AX, T);
|
|
|
|
mixin (simpleFormat(q{
|
|
asm pure nothrow @nogc @trusted
|
|
{
|
|
naked;
|
|
mov %3, [%1];
|
|
lock; cmpxchg [%0], %2;
|
|
jne compare_fail;
|
|
mov AL, 1;
|
|
ret;
|
|
compare_fail:
|
|
mov [%1], %3;
|
|
xor AL, AL;
|
|
ret;
|
|
}
|
|
}, [DestAddr, CmpAddr, Val, Res]));
|
|
}
|
|
else
|
|
{
|
|
version (Windows)
|
|
{
|
|
asm pure nothrow @nogc @trusted
|
|
{
|
|
naked;
|
|
push RBX;
|
|
mov R9, RDX;
|
|
mov RAX, [RDX];
|
|
mov RDX, 8[RDX];
|
|
mov RBX, [RCX];
|
|
mov RCX, 8[RCX];
|
|
lock; cmpxchg16b [R8];
|
|
pop RBX;
|
|
jne compare_fail;
|
|
mov AL, 1;
|
|
ret;
|
|
compare_fail:
|
|
mov [R9], RAX;
|
|
mov 8[R9], RDX;
|
|
xor AL, AL;
|
|
ret;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
asm pure nothrow @nogc @trusted
|
|
{
|
|
naked;
|
|
push RBX;
|
|
mov R8, RCX;
|
|
mov R9, RDX;
|
|
mov RAX, [RDX];
|
|
mov RDX, 8[RDX];
|
|
mov RBX, RDI;
|
|
mov RCX, RSI;
|
|
lock; cmpxchg16b [R8];
|
|
pop RBX;
|
|
jne compare_fail;
|
|
mov AL, 1;
|
|
ret;
|
|
compare_fail:
|
|
mov [R9], RAX;
|
|
mov 8[R9], RDX;
|
|
xor AL, AL;
|
|
ret;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else
|
|
static assert (false, "Unsupported architecture.");
|
|
}
|
|
|
|
alias atomicCompareExchangeWeakNoResult = atomicCompareExchangeStrongNoResult;
|
|
|
|
bool atomicCompareExchangeStrongNoResult(MemoryOrder succ = MemoryOrder.seq, MemoryOrder fail = MemoryOrder.seq, T)(T* dest, const T compare, T value) pure nothrow @nogc @trusted
|
|
if (CanCAS!T)
|
|
{
|
|
version (D_InlineAsm_X86)
|
|
{
|
|
static if (T.sizeof <= 4)
|
|
{
|
|
enum DestAddr = SizedReg!CX;
|
|
enum Cmp = SizedReg!(AX, T);
|
|
enum Val = SizedReg!(DX, T);
|
|
|
|
mixin (simpleFormat(q{
|
|
asm pure nothrow @nogc @trusted
|
|
{
|
|
mov %2, value;
|
|
mov %1, compare;
|
|
mov %0, dest;
|
|
lock; cmpxchg [%0], %2;
|
|
setz AL;
|
|
}
|
|
}, [DestAddr, Cmp, Val]));
|
|
}
|
|
else static if (T.sizeof == 8)
|
|
{
|
|
asm pure nothrow @nogc @trusted
|
|
{
|
|
push EDI;
|
|
push EBX;
|
|
lea EDI, value;
|
|
mov EBX, [EDI];
|
|
mov ECX, 4[EDI];
|
|
lea EDI, compare;
|
|
mov EAX, [EDI];
|
|
mov EDX, 4[EDI];
|
|
mov EDI, dest;
|
|
lock; cmpxchg8b [EDI];
|
|
setz AL;
|
|
pop EBX;
|
|
pop EDI;
|
|
}
|
|
}
|
|
else
|
|
static assert(T.sizeof <= 8, "128bit atomicCompareExchangeStrong not supported on 32bit target." );
|
|
}
|
|
else version (D_InlineAsm_X86_64)
|
|
{
|
|
static if (T.sizeof <= 8)
|
|
{
|
|
version (Windows)
|
|
{
|
|
enum DestAddr = SizedReg!R8;
|
|
enum Cmp = SizedReg!(DX, T);
|
|
enum Val = SizedReg!(CX, T);
|
|
}
|
|
else
|
|
{
|
|
enum DestAddr = SizedReg!DX;
|
|
enum Cmp = SizedReg!(SI, T);
|
|
enum Val = SizedReg!(DI, T);
|
|
}
|
|
enum AXReg = SizedReg!(AX, T);
|
|
|
|
mixin (simpleFormat(q{
|
|
asm pure nothrow @nogc @trusted
|
|
{
|
|
naked;
|
|
mov %3, %1;
|
|
lock; cmpxchg [%0], %2;
|
|
setz AL;
|
|
ret;
|
|
}
|
|
}, [DestAddr, Cmp, Val, AXReg]));
|
|
}
|
|
else
|
|
{
|
|
version (Windows)
|
|
{
|
|
asm pure nothrow @nogc @trusted
|
|
{
|
|
naked;
|
|
push RBX;
|
|
mov RAX, [RDX];
|
|
mov RDX, 8[RDX];
|
|
mov RBX, [RCX];
|
|
mov RCX, 8[RCX];
|
|
lock; cmpxchg16b [R8];
|
|
setz AL;
|
|
pop RBX;
|
|
ret;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
asm pure nothrow @nogc @trusted
|
|
{
|
|
naked;
|
|
push RBX;
|
|
mov RAX, RDX;
|
|
mov RDX, RCX;
|
|
mov RBX, RDI;
|
|
mov RCX, RSI;
|
|
lock; cmpxchg16b [R8];
|
|
setz AL;
|
|
pop RBX;
|
|
ret;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
else
|
|
static assert (false, "Unsupported architecture.");
|
|
}
|
|
|
|
void atomicFence(MemoryOrder order = MemoryOrder.seq)() pure nothrow @nogc @trusted
|
|
{
|
|
// TODO: `mfence` should only be required for seq_cst operations, but this depends on
|
|
// the compiler's backend knowledge to not reorder code inappropriately,
|
|
// so we'll apply it conservatively.
|
|
static if (order != MemoryOrder.raw)
|
|
{
|
|
version (D_InlineAsm_X86)
|
|
{
|
|
import core.cpuid;
|
|
|
|
// TODO: review this implementation; it seems way overly complicated
|
|
asm pure nothrow @nogc @trusted
|
|
{
|
|
naked;
|
|
|
|
call sse2;
|
|
test AL, AL;
|
|
jne Lcpuid;
|
|
|
|
// Fast path: We have SSE2, so just use mfence.
|
|
mfence;
|
|
jmp Lend;
|
|
|
|
Lcpuid:
|
|
|
|
// Slow path: We use cpuid to serialize. This is
|
|
// significantly slower than mfence, but is the
|
|
// only serialization facility we have available
|
|
// on older non-SSE2 chips.
|
|
push EBX;
|
|
|
|
mov EAX, 0;
|
|
cpuid;
|
|
|
|
pop EBX;
|
|
|
|
Lend:
|
|
|
|
ret;
|
|
}
|
|
}
|
|
else version (D_InlineAsm_X86_64)
|
|
{
|
|
asm pure nothrow @nogc @trusted
|
|
{
|
|
naked;
|
|
mfence;
|
|
ret;
|
|
}
|
|
}
|
|
else
|
|
static assert (false, "Unsupported architecture.");
|
|
}
|
|
}
|
|
|
|
void pause() pure nothrow @nogc @trusted
|
|
{
|
|
version (D_InlineAsm_X86)
|
|
{
|
|
asm pure nothrow @nogc @trusted
|
|
{
|
|
naked;
|
|
rep; nop;
|
|
ret;
|
|
}
|
|
}
|
|
else version (D_InlineAsm_X86_64)
|
|
{
|
|
asm pure nothrow @nogc @trusted
|
|
{
|
|
naked;
|
|
// pause; // TODO: DMD should add this opcode to its inline asm
|
|
rep; nop;
|
|
ret;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
// ARM should `yield`
|
|
// other architectures? otherwise some sort of nop...
|
|
}
|
|
}
|
|
}
|
|
else version (GNU)
|
|
{
|
|
import gcc.builtins;
|
|
import gcc.config;
|
|
|
|
inout(T) atomicLoad(MemoryOrder order = MemoryOrder.seq, T)(inout(T)* src) pure nothrow @nogc @trusted
|
|
if (CanCAS!T)
|
|
{
|
|
static assert(order != MemoryOrder.rel, "invalid MemoryOrder for atomicLoad()");
|
|
|
|
static if (GNU_Have_Atomics || GNU_Have_LibAtomic)
|
|
{
|
|
static if (T.sizeof == ubyte.sizeof)
|
|
{
|
|
ubyte value = __atomic_load_1(cast(shared)src, order);
|
|
return *cast(typeof(return)*)&value;
|
|
}
|
|
else static if (T.sizeof == ushort.sizeof)
|
|
{
|
|
ushort value = __atomic_load_2(cast(shared)src, order);
|
|
return *cast(typeof(return)*)&value;
|
|
}
|
|
else static if (T.sizeof == uint.sizeof)
|
|
{
|
|
uint value = __atomic_load_4(cast(shared)src, order);
|
|
return *cast(typeof(return)*)&value;
|
|
}
|
|
else static if (T.sizeof == ulong.sizeof && GNU_Have_64Bit_Atomics)
|
|
{
|
|
ulong value = __atomic_load_8(cast(shared)src, order);
|
|
return *cast(typeof(return)*)&value;
|
|
}
|
|
else static if (GNU_Have_LibAtomic)
|
|
{
|
|
T value;
|
|
__atomic_load(T.sizeof, cast(shared)src, &value, order);
|
|
return *cast(typeof(return)*)&value;
|
|
}
|
|
else
|
|
static assert(0, "Invalid template type specified.");
|
|
}
|
|
else
|
|
{
|
|
getAtomicMutex.lock();
|
|
scope(exit) getAtomicMutex.unlock();
|
|
return *cast(typeof(return)*)&src;
|
|
}
|
|
}
|
|
|
|
void atomicStore(MemoryOrder order = MemoryOrder.seq, T)(T* dest, T value) pure nothrow @nogc @trusted
|
|
if (CanCAS!T)
|
|
{
|
|
static assert(order != MemoryOrder.acq, "Invalid MemoryOrder for atomicStore()");
|
|
|
|
static if (GNU_Have_Atomics || GNU_Have_LibAtomic)
|
|
{
|
|
static if (T.sizeof == ubyte.sizeof)
|
|
__atomic_store_1(cast(shared)dest, *cast(ubyte*)&value, order);
|
|
else static if (T.sizeof == ushort.sizeof)
|
|
__atomic_store_2(cast(shared)dest, *cast(ushort*)&value, order);
|
|
else static if (T.sizeof == uint.sizeof)
|
|
__atomic_store_4(cast(shared)dest, *cast(uint*)&value, order);
|
|
else static if (T.sizeof == ulong.sizeof && GNU_Have_64Bit_Atomics)
|
|
__atomic_store_8(cast(shared)dest, *cast(ulong*)&value, order);
|
|
else static if (GNU_Have_LibAtomic)
|
|
__atomic_store(T.sizeof, cast(shared)dest, cast(void*)&value, order);
|
|
else
|
|
static assert(0, "Invalid template type specified.");
|
|
}
|
|
else
|
|
{
|
|
getAtomicMutex.lock();
|
|
*dest = value;
|
|
getAtomicMutex.unlock();
|
|
}
|
|
}
|
|
|
|
T atomicFetchAdd(MemoryOrder order = MemoryOrder.seq, bool result = true, T)(T* dest, T value) pure nothrow @nogc @trusted
|
|
if (is(T : ulong))
|
|
{
|
|
static if (GNU_Have_Atomics || GNU_Have_LibAtomic)
|
|
{
|
|
static if (T.sizeof == ubyte.sizeof)
|
|
return __atomic_fetch_add_1(cast(shared)dest, value, order);
|
|
else static if (T.sizeof == ushort.sizeof)
|
|
return __atomic_fetch_add_2(cast(shared)dest, value, order);
|
|
else static if (T.sizeof == uint.sizeof)
|
|
return __atomic_fetch_add_4(cast(shared)dest, value, order);
|
|
else static if (T.sizeof == ulong.sizeof && GNU_Have_64Bit_Atomics)
|
|
return __atomic_fetch_add_8(cast(shared)dest, value, order);
|
|
else static if (GNU_Have_LibAtomic)
|
|
return __atomic_fetch_add(T.sizeof, cast(shared)dest, cast(void*)&value, order);
|
|
else
|
|
static assert(0, "Invalid template type specified.");
|
|
}
|
|
else
|
|
{
|
|
getAtomicMutex.lock();
|
|
scope(exit) getAtomicMutex.unlock();
|
|
T tmp = *dest;
|
|
*dest += value;
|
|
return tmp;
|
|
}
|
|
}
|
|
|
|
T atomicFetchSub(MemoryOrder order = MemoryOrder.seq, bool result = true, T)(T* dest, T value) pure nothrow @nogc @trusted
|
|
if (is(T : ulong))
|
|
{
|
|
static if (GNU_Have_Atomics || GNU_Have_LibAtomic)
|
|
{
|
|
static if (T.sizeof == ubyte.sizeof)
|
|
return __atomic_fetch_sub_1(cast(shared)dest, value, order);
|
|
else static if (T.sizeof == ushort.sizeof)
|
|
return __atomic_fetch_sub_2(cast(shared)dest, value, order);
|
|
else static if (T.sizeof == uint.sizeof)
|
|
return __atomic_fetch_sub_4(cast(shared)dest, value, order);
|
|
else static if (T.sizeof == ulong.sizeof && GNU_Have_64Bit_Atomics)
|
|
return __atomic_fetch_sub_8(cast(shared)dest, value, order);
|
|
else static if (GNU_Have_LibAtomic)
|
|
return __atomic_fetch_sub(T.sizeof, cast(shared)dest, cast(void*)&value, order);
|
|
else
|
|
static assert(0, "Invalid template type specified.");
|
|
}
|
|
else
|
|
{
|
|
getAtomicMutex.lock();
|
|
scope(exit) getAtomicMutex.unlock();
|
|
T tmp = *dest;
|
|
*dest -= value;
|
|
return tmp;
|
|
}
|
|
}
|
|
|
|
T atomicExchange(MemoryOrder order = MemoryOrder.seq, bool result = true, T)(T* dest, T value) pure nothrow @nogc @trusted
|
|
if (is(T : ulong) || is(T == class) || is(T == interface) || is(T U : U*))
|
|
{
|
|
static if (GNU_Have_Atomics || GNU_Have_LibAtomic)
|
|
{
|
|
static if (T.sizeof == byte.sizeof)
|
|
{
|
|
ubyte res = __atomic_exchange_1(cast(shared)dest, *cast(ubyte*)&value, order);
|
|
return *cast(typeof(return)*)&res;
|
|
}
|
|
else static if (T.sizeof == short.sizeof)
|
|
{
|
|
ushort res = __atomic_exchange_2(cast(shared)dest, *cast(ushort*)&value, order);
|
|
return *cast(typeof(return)*)&res;
|
|
}
|
|
else static if (T.sizeof == int.sizeof)
|
|
{
|
|
uint res = __atomic_exchange_4(cast(shared)dest, *cast(uint*)&value, order);
|
|
return *cast(typeof(return)*)&res;
|
|
}
|
|
else static if (T.sizeof == long.sizeof && GNU_Have_64Bit_Atomics)
|
|
{
|
|
ulong res = __atomic_exchange_8(cast(shared)dest, *cast(ulong*)&value, order);
|
|
return *cast(typeof(return)*)&res;
|
|
}
|
|
else static if (GNU_Have_LibAtomic)
|
|
{
|
|
T res = void;
|
|
__atomic_exchange(T.sizeof, cast(shared)dest, cast(void*)&value, &res, order);
|
|
return res;
|
|
}
|
|
else
|
|
static assert(0, "Invalid template type specified.");
|
|
}
|
|
else
|
|
{
|
|
getAtomicMutex.lock();
|
|
scope(exit) getAtomicMutex.unlock();
|
|
|
|
T res = *dest;
|
|
*dest = value;
|
|
return res;
|
|
}
|
|
}
|
|
|
|
bool atomicCompareExchangeWeak(MemoryOrder succ = MemoryOrder.seq, MemoryOrder fail = MemoryOrder.seq, T)(T* dest, T* compare, T value) pure nothrow @nogc @trusted
|
|
if (CanCAS!T)
|
|
{
|
|
return atomicCompareExchangeImpl!(succ, fail, true)(dest, compare, value);
|
|
}
|
|
|
|
bool atomicCompareExchangeStrong(MemoryOrder succ = MemoryOrder.seq, MemoryOrder fail = MemoryOrder.seq, T)(T* dest, T* compare, T value) pure nothrow @nogc @trusted
|
|
if (CanCAS!T)
|
|
{
|
|
return atomicCompareExchangeImpl!(succ, fail, false)(dest, compare, value);
|
|
}
|
|
|
|
bool atomicCompareExchangeStrongNoResult(MemoryOrder succ = MemoryOrder.seq, MemoryOrder fail = MemoryOrder.seq, T)(T* dest, const T compare, T value) pure nothrow @nogc @trusted
|
|
if (CanCAS!T)
|
|
{
|
|
return atomicCompareExchangeImpl!(succ, fail, false)(dest, cast(T*)&compare, value);
|
|
}
|
|
|
|
bool atomicCompareExchangeWeakNoResult(MemoryOrder succ = MemoryOrder.seq, MemoryOrder fail = MemoryOrder.seq, T)(T* dest, const T compare, T value) pure nothrow @nogc @trusted
|
|
if (CanCAS!T)
|
|
{
|
|
return atomicCompareExchangeImpl!(succ, fail, true)(dest, cast(T*)&compare, value);
|
|
}
|
|
|
|
private bool atomicCompareExchangeImpl(MemoryOrder succ = MemoryOrder.seq, MemoryOrder fail = MemoryOrder.seq, bool weak, T)(T* dest, T* compare, T value) pure nothrow @nogc @trusted
|
|
if (CanCAS!T)
|
|
{
|
|
bool res = void;
|
|
|
|
static if (GNU_Have_Atomics || GNU_Have_LibAtomic)
|
|
{
|
|
static if (T.sizeof == byte.sizeof)
|
|
res = __atomic_compare_exchange_1(cast(shared)dest, compare, *cast(ubyte*)&value,
|
|
weak, succ, fail);
|
|
else static if (T.sizeof == short.sizeof)
|
|
res = __atomic_compare_exchange_2(cast(shared)dest, compare, *cast(ushort*)&value,
|
|
weak, succ, fail);
|
|
else static if (T.sizeof == int.sizeof)
|
|
res = __atomic_compare_exchange_4(cast(shared)dest, compare, *cast(uint*)&value,
|
|
weak, succ, fail);
|
|
else static if (T.sizeof == long.sizeof && GNU_Have_64Bit_Atomics)
|
|
res = __atomic_compare_exchange_8(cast(shared)dest, compare, *cast(ulong*)&value,
|
|
weak, succ, fail);
|
|
else static if (GNU_Have_LibAtomic)
|
|
res = __atomic_compare_exchange(T.sizeof, cast(shared)dest, compare, cast(void*)&value,
|
|
succ, fail);
|
|
else
|
|
static assert(0, "Invalid template type specified.");
|
|
}
|
|
else
|
|
{
|
|
static if (T.sizeof == byte.sizeof)
|
|
alias U = byte;
|
|
else static if (T.sizeof == short.sizeof)
|
|
alias U = short;
|
|
else static if (T.sizeof == int.sizeof)
|
|
alias U = int;
|
|
else static if (T.sizeof == long.sizeof)
|
|
alias U = long;
|
|
else
|
|
static assert(0, "Invalid template type specified.");
|
|
|
|
getAtomicMutex.lock();
|
|
scope(exit) getAtomicMutex.unlock();
|
|
|
|
if (*cast(U*)dest == *cast(U*)&compare)
|
|
{
|
|
*dest = value;
|
|
res = true;
|
|
}
|
|
else
|
|
{
|
|
*compare = *dest;
|
|
res = false;
|
|
}
|
|
}
|
|
|
|
return res;
|
|
}
|
|
|
|
void atomicFence(MemoryOrder order = MemoryOrder.seq)() pure nothrow @nogc @trusted
|
|
{
|
|
static if (GNU_Have_Atomics || GNU_Have_LibAtomic)
|
|
__atomic_thread_fence(order);
|
|
else
|
|
{
|
|
getAtomicMutex.lock();
|
|
getAtomicMutex.unlock();
|
|
}
|
|
}
|
|
|
|
void pause() pure nothrow @nogc @trusted
|
|
{
|
|
version (X86)
|
|
{
|
|
__builtin_ia32_pause();
|
|
}
|
|
else version (X86_64)
|
|
{
|
|
__builtin_ia32_pause();
|
|
}
|
|
else
|
|
{
|
|
// Other architectures? Some sort of nop or barrier.
|
|
}
|
|
}
|
|
|
|
static if (!GNU_Have_Atomics && !GNU_Have_LibAtomic)
|
|
{
|
|
// Use system mutex for atomics, faking the purity of the functions so
|
|
// that they can be used in pure/nothrow/@safe code.
|
|
extern (C) private pure @trusted @nogc nothrow
|
|
{
|
|
static if (GNU_Thread_Model == ThreadModel.Posix)
|
|
{
|
|
import core.sys.posix.pthread;
|
|
alias atomicMutexHandle = pthread_mutex_t;
|
|
|
|
pragma(mangle, "pthread_mutex_init") int fakePureMutexInit(pthread_mutex_t*, pthread_mutexattr_t*);
|
|
pragma(mangle, "pthread_mutex_lock") int fakePureMutexLock(pthread_mutex_t*);
|
|
pragma(mangle, "pthread_mutex_unlock") int fakePureMutexUnlock(pthread_mutex_t*);
|
|
}
|
|
else static if (GNU_Thread_Model == ThreadModel.Win32)
|
|
{
|
|
import core.sys.windows.winbase;
|
|
alias atomicMutexHandle = CRITICAL_SECTION;
|
|
|
|
pragma(mangle, "InitializeCriticalSection") int fakePureMutexInit(CRITICAL_SECTION*);
|
|
pragma(mangle, "EnterCriticalSection") void fakePureMutexLock(CRITICAL_SECTION*);
|
|
pragma(mangle, "LeaveCriticalSection") int fakePureMutexUnlock(CRITICAL_SECTION*);
|
|
}
|
|
else
|
|
{
|
|
alias atomicMutexHandle = int;
|
|
}
|
|
}
|
|
|
|
// Implements lock/unlock operations.
|
|
private struct AtomicMutex
|
|
{
|
|
int lock() pure @trusted @nogc nothrow
|
|
{
|
|
static if (GNU_Thread_Model == ThreadModel.Posix)
|
|
{
|
|
if (!_inited)
|
|
{
|
|
fakePureMutexInit(&_handle, null);
|
|
_inited = true;
|
|
}
|
|
return fakePureMutexLock(&_handle);
|
|
}
|
|
else
|
|
{
|
|
static if (GNU_Thread_Model == ThreadModel.Win32)
|
|
{
|
|
if (!_inited)
|
|
{
|
|
fakePureMutexInit(&_handle);
|
|
_inited = true;
|
|
}
|
|
fakePureMutexLock(&_handle);
|
|
}
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
int unlock() pure @trusted @nogc nothrow
|
|
{
|
|
static if (GNU_Thread_Model == ThreadModel.Posix)
|
|
return fakePureMutexUnlock(&_handle);
|
|
else
|
|
{
|
|
static if (GNU_Thread_Model == ThreadModel.Win32)
|
|
fakePureMutexUnlock(&_handle);
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
private:
|
|
atomicMutexHandle _handle;
|
|
bool _inited;
|
|
}
|
|
|
|
// Internal static mutex reference.
|
|
private AtomicMutex* _getAtomicMutex() @trusted @nogc nothrow
|
|
{
|
|
__gshared static AtomicMutex mutex;
|
|
return &mutex;
|
|
}
|
|
|
|
// Pure alias for _getAtomicMutex.
|
|
pragma(mangle, _getAtomicMutex.mangleof)
|
|
private AtomicMutex* getAtomicMutex() pure @trusted @nogc nothrow @property;
|
|
}
|
|
}
|
|
|
|
private:
|
|
|
|
version (Windows)
|
|
{
|
|
enum RegisterReturn(T) = is(T : U[], U) || is(T : R delegate(A), R, A...);
|
|
}
|
|
|
|
enum CanCAS(T) = is(T : ulong) ||
|
|
is(T == class) ||
|
|
is(T == interface) ||
|
|
is(T : U*, U) ||
|
|
is(T : U[], U) ||
|
|
is(T : R delegate(A), R, A...) ||
|
|
(is(T == struct) && __traits(isPOD, T) &&
|
|
(T.sizeof <= size_t.sizeof*2 || // no more than 2 words
|
|
(T.sizeof == 16 && has128BitCAS)) && // or supports 128-bit CAS
|
|
(T.sizeof & (T.sizeof - 1)) == 0 // is power of 2
|
|
);
|
|
|
|
template IntOrLong(T)
|
|
{
|
|
static if (T.sizeof > 4)
|
|
alias IntOrLong = long;
|
|
else
|
|
alias IntOrLong = int;
|
|
}
|
|
|
|
// NOTE: x86 loads implicitly have acquire semantics so a memory
|
|
// barrier is only necessary on releases.
|
|
template needsLoadBarrier( MemoryOrder ms )
|
|
{
|
|
enum bool needsLoadBarrier = ms == MemoryOrder.seq;
|
|
}
|
|
|
|
|
|
// NOTE: x86 stores implicitly have release semantics so a memory
|
|
// barrier is only necessary on acquires.
|
|
template needsStoreBarrier( MemoryOrder ms )
|
|
{
|
|
enum bool needsStoreBarrier = ms == MemoryOrder.seq;
|
|
}
|
|
|
|
// this is a helper to build asm blocks
|
|
string simpleFormat(string format, scope string[] args)
|
|
{
|
|
string result;
|
|
outer: while (format.length)
|
|
{
|
|
foreach (i; 0 .. format.length)
|
|
{
|
|
if (format[i] == '%' || format[i] == '?')
|
|
{
|
|
bool isQ = format[i] == '?';
|
|
result ~= format[0 .. i++];
|
|
assert (i < format.length, "Invalid format string");
|
|
if (format[i] == '%' || format[i] == '?')
|
|
{
|
|
assert(!isQ, "Invalid format string");
|
|
result ~= format[i++];
|
|
}
|
|
else
|
|
{
|
|
int index = 0;
|
|
assert (format[i] >= '0' && format[i] <= '9', "Invalid format string");
|
|
while (i < format.length && format[i] >= '0' && format[i] <= '9')
|
|
index = index * 10 + (ubyte(format[i++]) - ubyte('0'));
|
|
if (!isQ)
|
|
result ~= args[index];
|
|
else if (!args[index])
|
|
{
|
|
size_t j = i;
|
|
for (; j < format.length;)
|
|
{
|
|
if (format[j++] == '\n')
|
|
break;
|
|
}
|
|
i = j;
|
|
}
|
|
}
|
|
format = format[i .. $];
|
|
continue outer;
|
|
}
|
|
}
|
|
result ~= format;
|
|
break;
|
|
}
|
|
return result;
|
|
}
|