|
|
|
|
@ -34,36 +34,207 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
|
#include "vm_compiled_light.hpp"
|
|
|
|
|
#include "blake2/blake2.h"
|
|
|
|
|
#include "jit_compiler_x86_static.hpp"
|
|
|
|
|
#include "assembly_generator_x86.hpp"
|
|
|
|
|
#include <cassert>
|
|
|
|
|
|
|
|
|
|
const RandomX_Configuration RandomX_DefaultConfig = {
|
|
|
|
|
/* ArgonMemory */ 262144,
|
|
|
|
|
/* ArgonIterations */ 3,
|
|
|
|
|
/* ArgonLanes */ 1,
|
|
|
|
|
/* ArgonSalt */ "RandomX\x03",
|
|
|
|
|
/* ProgramSize */ 256,
|
|
|
|
|
/* ProgramIterations */ 2048,
|
|
|
|
|
/* ProgramCount */ 8,
|
|
|
|
|
/* codeShhPrefetchTweaked */ {},
|
|
|
|
|
};
|
|
|
|
|
RandomX_ConfigurationWownero::RandomX_ConfigurationWownero()
|
|
|
|
|
{
|
|
|
|
|
ArgonSalt = "RandomWOW\x01";
|
|
|
|
|
ProgramIterations = 1024;
|
|
|
|
|
ProgramCount = 16;
|
|
|
|
|
ScratchpadL2_Size = 131072;
|
|
|
|
|
ScratchpadL3_Size = 1048576;
|
|
|
|
|
|
|
|
|
|
RANDOMX_FREQ_IROR_R = 10;
|
|
|
|
|
RANDOMX_FREQ_IROL_R = 0;
|
|
|
|
|
RANDOMX_FREQ_FSWAP_R = 8;
|
|
|
|
|
RANDOMX_FREQ_FADD_R = 20;
|
|
|
|
|
RANDOMX_FREQ_FSUB_R = 20;
|
|
|
|
|
RANDOMX_FREQ_FMUL_R = 20;
|
|
|
|
|
|
|
|
|
|
fillAes4Rx4_Key[0] = rx_set_int_vec_i128(0xcf359e95, 0x141f82b7, 0x7ffbe4a6, 0xf890465d);
|
|
|
|
|
fillAes4Rx4_Key[1] = rx_set_int_vec_i128(0x6741ffdc, 0xbd5c5ac3, 0xfee8278a, 0x6a55c450);
|
|
|
|
|
fillAes4Rx4_Key[2] = rx_set_int_vec_i128(0x3d324aac, 0xa7279ad2, 0xd524fde4, 0x114c47a4);
|
|
|
|
|
fillAes4Rx4_Key[3] = rx_set_int_vec_i128(0x76f6db08, 0x42d3dbd9, 0x99a9aeff, 0x810c3a2a);
|
|
|
|
|
fillAes4Rx4_Key[4] = fillAes4Rx4_Key[0];
|
|
|
|
|
fillAes4Rx4_Key[5] = fillAes4Rx4_Key[1];
|
|
|
|
|
fillAes4Rx4_Key[6] = fillAes4Rx4_Key[2];
|
|
|
|
|
fillAes4Rx4_Key[7] = fillAes4Rx4_Key[3];
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
RandomX_Configuration RandomX_CurrentConfig = RandomX_DefaultConfig;
|
|
|
|
|
RandomX_ConfigurationLoki::RandomX_ConfigurationLoki()
|
|
|
|
|
{
|
|
|
|
|
ArgonIterations = 4;
|
|
|
|
|
ArgonLanes = 2;
|
|
|
|
|
ArgonSalt = "RandomXL\x12";
|
|
|
|
|
ProgramSize = 320;
|
|
|
|
|
ProgramCount = 7;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void RandomX_Configuration::initCode()
|
|
|
|
|
RandomX_ConfigurationBase::RandomX_ConfigurationBase()
|
|
|
|
|
: ArgonMemory(262144)
|
|
|
|
|
, ArgonIterations(3)
|
|
|
|
|
, ArgonLanes(1)
|
|
|
|
|
, ArgonSalt("RandomX\x03")
|
|
|
|
|
, CacheAccesses(8)
|
|
|
|
|
, SuperscalarLatency(170)
|
|
|
|
|
, DatasetBaseSize(2147483648)
|
|
|
|
|
, DatasetExtraSize(33554368)
|
|
|
|
|
, ScratchpadL1_Size(16384)
|
|
|
|
|
, ScratchpadL2_Size(262144)
|
|
|
|
|
, ScratchpadL3_Size(2097152)
|
|
|
|
|
, ProgramSize(256)
|
|
|
|
|
, ProgramIterations(2048)
|
|
|
|
|
, ProgramCount(8)
|
|
|
|
|
, JumpBits(8)
|
|
|
|
|
, JumpOffset(8)
|
|
|
|
|
, RANDOMX_FREQ_IADD_RS(25)
|
|
|
|
|
, RANDOMX_FREQ_IADD_M(7)
|
|
|
|
|
, RANDOMX_FREQ_ISUB_R(16)
|
|
|
|
|
, RANDOMX_FREQ_ISUB_M(7)
|
|
|
|
|
, RANDOMX_FREQ_IMUL_R(16)
|
|
|
|
|
, RANDOMX_FREQ_IMUL_M(4)
|
|
|
|
|
, RANDOMX_FREQ_IMULH_R(4)
|
|
|
|
|
, RANDOMX_FREQ_IMULH_M(1)
|
|
|
|
|
, RANDOMX_FREQ_ISMULH_R(4)
|
|
|
|
|
, RANDOMX_FREQ_ISMULH_M(1)
|
|
|
|
|
, RANDOMX_FREQ_IMUL_RCP(8)
|
|
|
|
|
, RANDOMX_FREQ_INEG_R(2)
|
|
|
|
|
, RANDOMX_FREQ_IXOR_R(15)
|
|
|
|
|
, RANDOMX_FREQ_IXOR_M(5)
|
|
|
|
|
, RANDOMX_FREQ_IROR_R(8)
|
|
|
|
|
, RANDOMX_FREQ_IROL_R(2)
|
|
|
|
|
, RANDOMX_FREQ_ISWAP_R(4)
|
|
|
|
|
, RANDOMX_FREQ_FSWAP_R(4)
|
|
|
|
|
, RANDOMX_FREQ_FADD_R(16)
|
|
|
|
|
, RANDOMX_FREQ_FADD_M(5)
|
|
|
|
|
, RANDOMX_FREQ_FSUB_R(16)
|
|
|
|
|
, RANDOMX_FREQ_FSUB_M(5)
|
|
|
|
|
, RANDOMX_FREQ_FSCAL_R(6)
|
|
|
|
|
, RANDOMX_FREQ_FMUL_R(32)
|
|
|
|
|
, RANDOMX_FREQ_FDIV_M(4)
|
|
|
|
|
, RANDOMX_FREQ_FSQRT_R(6)
|
|
|
|
|
, RANDOMX_FREQ_CBRANCH(16)
|
|
|
|
|
, RANDOMX_FREQ_CFROUND(1)
|
|
|
|
|
, RANDOMX_FREQ_ISTORE(16)
|
|
|
|
|
, RANDOMX_FREQ_NOP(0)
|
|
|
|
|
{
|
|
|
|
|
const uint8_t* a = (const uint8_t*) &randomx_sshash_prefetch;
|
|
|
|
|
const uint8_t* b = (const uint8_t*) &randomx_sshash_end;
|
|
|
|
|
memcpy(codeShhPrefetchTweaked, a, b - a);
|
|
|
|
|
*(uint32_t*)(codeShhPrefetchTweaked + 3) = ArgonMemory * 16 - 1;
|
|
|
|
|
fillAes4Rx4_Key[0] = rx_set_int_vec_i128(0x99e5d23f, 0x2f546d2b, 0xd1833ddb, 0x6421aadd);
|
|
|
|
|
fillAes4Rx4_Key[1] = rx_set_int_vec_i128(0xa5dfcde5, 0x06f79d53, 0xb6913f55, 0xb20e3450);
|
|
|
|
|
fillAes4Rx4_Key[2] = rx_set_int_vec_i128(0x171c02bf, 0x0aa4679f, 0x515e7baf, 0x5c3ed904);
|
|
|
|
|
fillAes4Rx4_Key[3] = rx_set_int_vec_i128(0xd8ded291, 0xcd673785, 0xe78f5d08, 0x85623763);
|
|
|
|
|
fillAes4Rx4_Key[4] = rx_set_int_vec_i128(0x229effb4, 0x3d518b6d, 0xe3d6a7a6, 0xb5826f73);
|
|
|
|
|
fillAes4Rx4_Key[5] = rx_set_int_vec_i128(0xb272b7d2, 0xe9024d4e, 0x9c10b3d9, 0xc7566bf3);
|
|
|
|
|
fillAes4Rx4_Key[6] = rx_set_int_vec_i128(0xf63befa7, 0x2ba9660a, 0xf765a38b, 0xf273c9e7);
|
|
|
|
|
fillAes4Rx4_Key[7] = rx_set_int_vec_i128(0xc0b0762d, 0x0c06d1fd, 0x915839de, 0x7a7cd609);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
extern "C" {
|
|
|
|
|
void RandomX_ConfigurationBase::Apply()
|
|
|
|
|
{
|
|
|
|
|
#if defined(_M_X64) || defined(__x86_64__)
|
|
|
|
|
{
|
|
|
|
|
const uint8_t* a = (const uint8_t*)&randomx_sshash_prefetch;
|
|
|
|
|
const uint8_t* b = (const uint8_t*)&randomx_sshash_end;
|
|
|
|
|
memcpy(codeShhPrefetchTweaked, a, b - a);
|
|
|
|
|
*(uint32_t*)(codeShhPrefetchTweaked + 3) = ArgonMemory * 16 - 1;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const uint32_t DatasetBaseMask = DatasetBaseSize - RANDOMX_DATASET_ITEM_SIZE;
|
|
|
|
|
{
|
|
|
|
|
const uint8_t* a = (const uint8_t*)&randomx_program_read_dataset;
|
|
|
|
|
const uint8_t* b = (const uint8_t*)&randomx_program_read_dataset_sshash_init;
|
|
|
|
|
memcpy(codeReadDatasetTweaked, a, b - a);
|
|
|
|
|
*(uint32_t*)(codeReadDatasetTweaked + 7) = DatasetBaseMask;
|
|
|
|
|
*(uint32_t*)(codeReadDatasetTweaked + 23) = DatasetBaseMask;
|
|
|
|
|
}
|
|
|
|
|
{
|
|
|
|
|
const uint8_t* a = (const uint8_t*)&randomx_program_read_dataset_sshash_init;
|
|
|
|
|
const uint8_t* b = (const uint8_t*)&randomx_program_read_dataset_sshash_fin;
|
|
|
|
|
memcpy(codeReadDatasetLightSshInitTweaked, a, b - a);
|
|
|
|
|
*(uint32_t*)(codeReadDatasetLightSshInitTweaked + 59) = DatasetBaseMask;
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
CacheLineAlignMask_Calculated = (DatasetBaseSize - 1) & ~(RANDOMX_DATASET_ITEM_SIZE - 1);
|
|
|
|
|
DatasetExtraItems_Calculated = DatasetExtraSize / RANDOMX_DATASET_ITEM_SIZE;
|
|
|
|
|
|
|
|
|
|
ScratchpadL1Mask_Calculated = (ScratchpadL1_Size / sizeof(uint64_t) - 1) * 8;
|
|
|
|
|
ScratchpadL1Mask16_Calculated = (ScratchpadL1_Size / sizeof(uint64_t) / 2 - 1) * 16;
|
|
|
|
|
ScratchpadL2Mask_Calculated = (ScratchpadL2_Size / sizeof(uint64_t) - 1) * 8;
|
|
|
|
|
ScratchpadL2Mask16_Calculated = (ScratchpadL2_Size / sizeof(uint64_t) / 2 - 1) * 16;
|
|
|
|
|
ScratchpadL3Mask_Calculated = (((ScratchpadL3_Size / sizeof(uint64_t)) - 1) * 8);
|
|
|
|
|
ScratchpadL3Mask64_Calculated = ((ScratchpadL3_Size / sizeof(uint64_t)) / 8 - 1) * 64;
|
|
|
|
|
|
|
|
|
|
#if defined(_M_X64) || defined(__x86_64__)
|
|
|
|
|
{
|
|
|
|
|
const uint8_t* a = (const uint8_t*)&randomx_program_loop_load;
|
|
|
|
|
const uint8_t* b = (const uint8_t*)&randomx_program_start;
|
|
|
|
|
memcpy(codeLoopLoadTweaked, a, b - a);
|
|
|
|
|
*(uint32_t*)(codeLoopLoadTweaked + 4) = ScratchpadL3Mask64_Calculated;
|
|
|
|
|
*(uint32_t*)(codeLoopLoadTweaked + 50) = ScratchpadL3Mask64_Calculated;
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
ConditionMask_Calculated = (1 << JumpBits) - 1;
|
|
|
|
|
|
|
|
|
|
constexpr int CEIL_NULL = 0;
|
|
|
|
|
int k = 0;
|
|
|
|
|
|
|
|
|
|
void randomx_apply_config(const RandomX_Configuration* config) {
|
|
|
|
|
RandomX_CurrentConfig = *config;
|
|
|
|
|
RandomX_CurrentConfig.initCode();
|
|
|
|
|
#if defined(_M_X64) || defined(__x86_64__)
|
|
|
|
|
#define JIT_HANDLE(x, prev) randomx::JitCompilerX86::engine[k] = &randomx::JitCompilerX86::h_##x
|
|
|
|
|
#else
|
|
|
|
|
#define JIT_HANDLE(x, prev)
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#define INST_HANDLE(x, prev) \
|
|
|
|
|
CEIL_##x = CEIL_##prev + RANDOMX_FREQ_##x; \
|
|
|
|
|
for (int i = 0; i < RANDOMX_FREQ_##x; ++i, ++k) \
|
|
|
|
|
{ \
|
|
|
|
|
randomx::AssemblyGeneratorX86::engine[k] = &randomx::AssemblyGeneratorX86::h_##x; \
|
|
|
|
|
JIT_HANDLE(x, prev); \
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
INST_HANDLE(IADD_RS, NULL);
|
|
|
|
|
INST_HANDLE(IADD_M, IADD_RS);
|
|
|
|
|
INST_HANDLE(ISUB_R, IADD_M);
|
|
|
|
|
INST_HANDLE(ISUB_M, ISUB_R);
|
|
|
|
|
INST_HANDLE(IMUL_R, ISUB_M);
|
|
|
|
|
INST_HANDLE(IMUL_M, IMUL_R);
|
|
|
|
|
INST_HANDLE(IMULH_R, IMUL_M);
|
|
|
|
|
INST_HANDLE(IMULH_M, IMULH_R);
|
|
|
|
|
INST_HANDLE(ISMULH_R, IMULH_M);
|
|
|
|
|
INST_HANDLE(ISMULH_M, ISMULH_R);
|
|
|
|
|
INST_HANDLE(IMUL_RCP, ISMULH_M);
|
|
|
|
|
INST_HANDLE(INEG_R, IMUL_RCP);
|
|
|
|
|
INST_HANDLE(IXOR_R, INEG_R);
|
|
|
|
|
INST_HANDLE(IXOR_M, IXOR_R);
|
|
|
|
|
INST_HANDLE(IROR_R, IXOR_M);
|
|
|
|
|
INST_HANDLE(IROL_R, IROR_R);
|
|
|
|
|
INST_HANDLE(ISWAP_R, IROL_R);
|
|
|
|
|
INST_HANDLE(FSWAP_R, ISWAP_R);
|
|
|
|
|
INST_HANDLE(FADD_R, FSWAP_R);
|
|
|
|
|
INST_HANDLE(FADD_M, FADD_R);
|
|
|
|
|
INST_HANDLE(FSUB_R, FADD_M);
|
|
|
|
|
INST_HANDLE(FSUB_M, FSUB_R);
|
|
|
|
|
INST_HANDLE(FSCAL_R, FSUB_M);
|
|
|
|
|
INST_HANDLE(FMUL_R, FSCAL_R);
|
|
|
|
|
INST_HANDLE(FDIV_M, FMUL_R);
|
|
|
|
|
INST_HANDLE(FSQRT_R, FDIV_M);
|
|
|
|
|
INST_HANDLE(CBRANCH, FSQRT_R);
|
|
|
|
|
INST_HANDLE(CFROUND, CBRANCH);
|
|
|
|
|
INST_HANDLE(ISTORE, CFROUND);
|
|
|
|
|
INST_HANDLE(NOP, ISTORE);
|
|
|
|
|
#undef INST_HANDLE
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
RandomX_ConfigurationMonero RandomX_MoneroConfig;
|
|
|
|
|
RandomX_ConfigurationWownero RandomX_WowneroConfig;
|
|
|
|
|
RandomX_ConfigurationLoki RandomX_LokiConfig;
|
|
|
|
|
|
|
|
|
|
RandomX_ConfigurationBase RandomX_CurrentConfig;
|
|
|
|
|
|
|
|
|
|
extern "C" {
|
|
|
|
|
|
|
|
|
|
randomx_cache *randomx_alloc_cache(randomx_flags flags) {
|
|
|
|
|
randomx_cache *cache;
|
|
|
|
|
|
|
|
|
|
@ -75,7 +246,7 @@ extern "C" {
|
|
|
|
|
cache->jit = nullptr;
|
|
|
|
|
cache->initialize = &randomx::initCache;
|
|
|
|
|
cache->datasetInit = &randomx::initDataset;
|
|
|
|
|
cache->memory = (uint8_t*)randomx::DefaultAllocator::allocMemory(RandomX_CurrentConfig.ArgonMemory * randomx::ArgonBlockSize);
|
|
|
|
|
cache->memory = (uint8_t*)randomx::DefaultAllocator::allocMemory(RANDOMX_CACHE_MAX_SIZE);
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
case RANDOMX_FLAG_JIT:
|
|
|
|
|
@ -83,7 +254,7 @@ extern "C" {
|
|
|
|
|
cache->jit = new randomx::JitCompiler();
|
|
|
|
|
cache->initialize = &randomx::initCacheCompile;
|
|
|
|
|
cache->datasetInit = cache->jit->getDatasetInitFunc();
|
|
|
|
|
cache->memory = (uint8_t*)randomx::DefaultAllocator::allocMemory(RandomX_CurrentConfig.ArgonMemory * randomx::ArgonBlockSize);
|
|
|
|
|
cache->memory = (uint8_t*)randomx::DefaultAllocator::allocMemory(RANDOMX_CACHE_MAX_SIZE);
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
case RANDOMX_FLAG_LARGE_PAGES:
|
|
|
|
|
@ -91,7 +262,7 @@ extern "C" {
|
|
|
|
|
cache->jit = nullptr;
|
|
|
|
|
cache->initialize = &randomx::initCache;
|
|
|
|
|
cache->datasetInit = &randomx::initDataset;
|
|
|
|
|
cache->memory = (uint8_t*)randomx::LargePageAllocator::allocMemory(RandomX_CurrentConfig.ArgonMemory * randomx::ArgonBlockSize);
|
|
|
|
|
cache->memory = (uint8_t*)randomx::LargePageAllocator::allocMemory(RANDOMX_CACHE_MAX_SIZE);
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
case RANDOMX_FLAG_JIT | RANDOMX_FLAG_LARGE_PAGES:
|
|
|
|
|
@ -99,7 +270,7 @@ extern "C" {
|
|
|
|
|
cache->jit = new randomx::JitCompiler();
|
|
|
|
|
cache->initialize = &randomx::initCacheCompile;
|
|
|
|
|
cache->datasetInit = cache->jit->getDatasetInitFunc();
|
|
|
|
|
cache->memory = (uint8_t*)randomx::LargePageAllocator::allocMemory(RandomX_CurrentConfig.ArgonMemory * randomx::ArgonBlockSize);
|
|
|
|
|
cache->memory = (uint8_t*)randomx::LargePageAllocator::allocMemory(RANDOMX_CACHE_MAX_SIZE);
|
|
|
|
|
break;
|
|
|
|
|
|
|
|
|
|
default:
|
|
|
|
|
@ -135,11 +306,11 @@ extern "C" {
|
|
|
|
|
dataset = new randomx_dataset();
|
|
|
|
|
if (flags & RANDOMX_FLAG_LARGE_PAGES) {
|
|
|
|
|
dataset->dealloc = &randomx::deallocDataset<randomx::LargePageAllocator>;
|
|
|
|
|
dataset->memory = (uint8_t*)randomx::LargePageAllocator::allocMemory(randomx::DatasetSize);
|
|
|
|
|
dataset->memory = (uint8_t*)randomx::LargePageAllocator::allocMemory(RANDOMX_DATASET_MAX_SIZE);
|
|
|
|
|
}
|
|
|
|
|
else {
|
|
|
|
|
dataset->dealloc = &randomx::deallocDataset<randomx::DefaultAllocator>;
|
|
|
|
|
dataset->memory = (uint8_t*)randomx::DefaultAllocator::allocMemory(randomx::DatasetSize);
|
|
|
|
|
dataset->memory = (uint8_t*)randomx::DefaultAllocator::allocMemory(RANDOMX_DATASET_MAX_SIZE);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
catch (std::exception &ex) {
|
|
|
|
|
@ -152,7 +323,7 @@ extern "C" {
|
|
|
|
|
return dataset;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
constexpr unsigned long DatasetItemCount = randomx::DatasetSize / RANDOMX_DATASET_ITEM_SIZE;
|
|
|
|
|
#define DatasetItemCount ((RandomX_CurrentConfig.DatasetBaseSize + RandomX_CurrentConfig.DatasetExtraSize) / RANDOMX_DATASET_ITEM_SIZE)
|
|
|
|
|
|
|
|
|
|
unsigned long randomx_dataset_item_count() {
|
|
|
|
|
return DatasetItemCount;
|
|
|
|
|
|
SChernykh
5 years ago