|
|
@ -1,5 +1,6 @@
|
|
|
|
/*
|
|
|
|
/*
|
|
|
|
Copyright (c) 2018-2019, tevador <tevador@gmail.com>
|
|
|
|
Copyright (c) 2018-2019, tevador <tevador@gmail.com>
|
|
|
|
|
|
|
|
Copyright (c) 2019, SChernykh <https://github.com/SChernykh>
|
|
|
|
|
|
|
|
|
|
|
|
All rights reserved.
|
|
|
|
All rights reserved.
|
|
|
|
|
|
|
|
|
|
|
@ -32,13 +33,15 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
|
|
|
|
|
|
|
|
namespace ARMV8A {
|
|
|
|
namespace ARMV8A {
|
|
|
|
|
|
|
|
|
|
|
|
constexpr uint32_t B = 0x14000000;
|
|
|
|
constexpr uint32_t B = 0x14000000;
|
|
|
|
constexpr uint32_t EOR = 0xCA000000;
|
|
|
|
constexpr uint32_t EOR = 0xCA000000;
|
|
|
|
constexpr uint32_t EOR32 = 0x4A000000;
|
|
|
|
constexpr uint32_t EOR32 = 0x4A000000;
|
|
|
|
constexpr uint32_t ADD = 0x8B000000;
|
|
|
|
constexpr uint32_t ADD = 0x8B000000;
|
|
|
|
constexpr uint32_t MOVZ = 0xD2800000;
|
|
|
|
constexpr uint32_t MOVZ = 0xD2800000;
|
|
|
|
constexpr uint32_t MOVN = 0x92800000;
|
|
|
|
constexpr uint32_t MOVN = 0x92800000;
|
|
|
|
constexpr uint32_t MOVK = 0xF2800000;
|
|
|
|
constexpr uint32_t MOVK = 0xF2800000;
|
|
|
|
|
|
|
|
constexpr uint32_t ADD_IMM_LO = 0x91000000;
|
|
|
|
|
|
|
|
constexpr uint32_t ADD_IMM_HI = 0x91400000;
|
|
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
@ -49,6 +52,8 @@ static const size_t PrologueSize = ((uint8_t*)randomx_program_aarch64_vm_instruc
|
|
|
|
|
|
|
|
|
|
|
|
constexpr uint32_t IntRegMap[8] = { 4, 5, 6, 7, 12, 13, 14, 15 };
|
|
|
|
constexpr uint32_t IntRegMap[8] = { 4, 5, 6, 7, 12, 13, 14, 15 };
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
template<typename T> static constexpr size_t Log2(T value) { return (value > 1) ? (Log2(value / 2) + 1) : 0; }
|
|
|
|
|
|
|
|
|
|
|
|
JitCompilerA64::JitCompilerA64()
|
|
|
|
JitCompilerA64::JitCompilerA64()
|
|
|
|
: code((uint8_t*) allocMemoryPages(CodeSize))
|
|
|
|
: code((uint8_t*) allocMemoryPages(CodeSize))
|
|
|
|
{
|
|
|
|
{
|
|
|
@ -101,6 +106,70 @@ size_t JitCompilerA64::getCodeSize()
|
|
|
|
return CodeSize;
|
|
|
|
return CodeSize;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
void JitCompilerA64::emitMovImmediate(uint32_t dst, uint32_t imm, uint8_t* code, uint32_t& codePos)
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
uint32_t k = codePos;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if (imm < (1 << 16))
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
// movz tmp_reg, imm32 (16 low bits)
|
|
|
|
|
|
|
|
emit32(ARMV8A::MOVZ | dst | (imm << 5), code, k);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
else
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
if (static_cast<int32_t>(imm) < 0)
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
// movn tmp_reg, ~imm32 (16 high bits)
|
|
|
|
|
|
|
|
emit32(ARMV8A::MOVN | dst | (1 << 21) | ((~imm >> 16) << 5), code, k);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
else
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
// movz tmp_reg, imm32 (16 high bits)
|
|
|
|
|
|
|
|
emit32(ARMV8A::MOVZ | dst | (1 << 21) | ((imm >> 16) << 5), code, k);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// movk tmp_reg, imm32 (16 low bits)
|
|
|
|
|
|
|
|
emit32(ARMV8A::MOVK | dst | ((imm & 0xFFFF) << 5), code, k);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
codePos = k;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
void JitCompilerA64::emitAddImmediate(uint32_t dst, uint32_t src, uint32_t imm, uint8_t* code, uint32_t& codePos)
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
uint32_t k = codePos;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if (imm < (1 << 24))
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
const uint32_t imm_lo = imm & ((1 << 12) - 1);
|
|
|
|
|
|
|
|
const uint32_t imm_hi = imm >> 12;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if (imm_lo && imm_hi)
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
emit32(ARMV8A::ADD_IMM_LO | dst | (src << 5) | (imm_lo << 10), code, k);
|
|
|
|
|
|
|
|
emit32(ARMV8A::ADD_IMM_HI | dst | (dst << 5) | (imm_hi << 10), code, k);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
else if (imm_lo)
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
emit32(ARMV8A::ADD_IMM_LO | dst | (src << 5) | (imm_lo << 10), code, k);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
else
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
emit32(ARMV8A::ADD_IMM_HI | dst | (src << 5) | (imm_hi << 10), code, k);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
else
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
constexpr uint32_t tmp_reg = 21;
|
|
|
|
|
|
|
|
emitMovImmediate(tmp_reg, imm, code, k);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// add dst, src, tmp_reg
|
|
|
|
|
|
|
|
emit32(ARMV8A::ADD | dst | (src << 5) | (tmp_reg << 16), code, k);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
codePos = k;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
void JitCompilerA64::h_IADD_RS(Instruction& instr, int i, uint32_t& codePos)
|
|
|
|
void JitCompilerA64::h_IADD_RS(Instruction& instr, int i, uint32_t& codePos)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
uint32_t k = codePos;
|
|
|
|
uint32_t k = codePos;
|
|
|
@ -113,23 +182,46 @@ void JitCompilerA64::h_IADD_RS(Instruction& instr, int i, uint32_t& codePos)
|
|
|
|
emit32(ARMV8A::ADD | dst | (dst << 5) | (shift << 10) | (src << 16), code, k);
|
|
|
|
emit32(ARMV8A::ADD | dst | (dst << 5) | (shift << 10) | (src << 16), code, k);
|
|
|
|
|
|
|
|
|
|
|
|
if (instr.dst == RegisterNeedsDisplacement)
|
|
|
|
if (instr.dst == RegisterNeedsDisplacement)
|
|
|
|
|
|
|
|
emitAddImmediate(dst, dst, instr.getImm32(), code, k);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
codePos = k;
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
void JitCompilerA64::h_IADD_M(Instruction& instr, int i, uint32_t& codePos)
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
uint32_t k = codePos;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
const uint32_t src = IntRegMap[instr.src];
|
|
|
|
|
|
|
|
const uint32_t dst = IntRegMap[instr.dst];
|
|
|
|
|
|
|
|
uint32_t imm = instr.getImm32();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
constexpr uint32_t tmp_reg = 21;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if (src != dst)
|
|
|
|
{
|
|
|
|
{
|
|
|
|
const uint32_t imm32 = instr.getImm32();
|
|
|
|
imm &= instr.getModMem() ? (RANDOMX_SCRATCHPAD_L1 - 1) : (RANDOMX_SCRATCHPAD_L2 - 1);
|
|
|
|
if (static_cast<int32_t>(imm32) < 0)
|
|
|
|
emitAddImmediate(tmp_reg, src, imm, code, k);
|
|
|
|
{
|
|
|
|
|
|
|
|
// movn x21, ~imm32 (16 high bits)
|
|
|
|
constexpr uint32_t t = 0x927d0000 | tmp_reg | (tmp_reg << 5);
|
|
|
|
emit32(ARMV8A::MOVN | 21 | (1 << 21) | ((~imm32 >> 16) << 5), code, k);
|
|
|
|
constexpr uint32_t andInstrL1 = t | ((Log2(RANDOMX_SCRATCHPAD_L1) - 4) << 10);
|
|
|
|
}
|
|
|
|
constexpr uint32_t andInstrL2 = t | ((Log2(RANDOMX_SCRATCHPAD_L2) - 4) << 10);
|
|
|
|
else
|
|
|
|
|
|
|
|
{
|
|
|
|
emit32(instr.getModMem() ? andInstrL1 : andInstrL2, code, k);
|
|
|
|
// movz x21, imm32 (16 high bits)
|
|
|
|
|
|
|
|
emit32(ARMV8A::MOVZ | 21 | (1 << 21) | ((imm32 >> 16) << 5), code, k);
|
|
|
|
// ldr tmp_reg, [x2, tmp_reg]
|
|
|
|
}
|
|
|
|
emit32(0xf8606840 | tmp_reg | (tmp_reg << 16), code, k);
|
|
|
|
// movk x21, imm32 (16 low bits)
|
|
|
|
|
|
|
|
emit32(ARMV8A::MOVK | 21 | ((imm32 & 0xFFFF) << 5), code, k);
|
|
|
|
|
|
|
|
// add dst, x21
|
|
|
|
|
|
|
|
emit32(ARMV8A::ADD | dst | (dst << 5) | (21 << 16), code, k);
|
|
|
|
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
else
|
|
|
|
|
|
|
|
{
|
|
|
|
|
|
|
|
imm = (imm & ScratchpadL3Mask) >> 3;
|
|
|
|
|
|
|
|
emitMovImmediate(tmp_reg, imm, code, k);
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// ldr tmp_reg, [x2, tmp_reg, lsl 3]
|
|
|
|
|
|
|
|
emit32(0xf8607840 | tmp_reg | (tmp_reg << 16), code, k);
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// add dst, dst, tmp_reg
|
|
|
|
|
|
|
|
emit32(ARMV8A::ADD | dst | (dst << 5) | (tmp_reg << 16), code, k);
|
|
|
|
|
|
|
|
|
|
|
|
codePos = k;
|
|
|
|
codePos = k;
|
|
|
|
}
|
|
|
|
}
|
|
|
|