diff --git a/src/coreclr/jit/emit.h b/src/coreclr/jit/emit.h index 4b7af2f73e31dc..9ac5ed4439e05b 100644 --- a/src/coreclr/jit/emit.h +++ b/src/coreclr/jit/emit.h @@ -1121,6 +1121,28 @@ class emitter idAddr()->_idReg4 = reg; assert(reg == idAddr()->_idReg4); } + bool idHasReg3() const + { + switch (idInsFmt()) + { + case IF_RWR_RRD_RRD: + case IF_RWR_RRD_RRD_CNS: + case IF_RWR_RRD_RRD_RRD: + return true; + default: + return false; + } + } + bool idHasReg4() const + { + switch (idInsFmt()) + { + case IF_RWR_RRD_RRD_RRD: + return true; + default: + return false; + } + } #endif // defined(TARGET_XARCH) #ifdef TARGET_ARMARCH insOpts idInsOpt() const diff --git a/src/coreclr/jit/emitxarch.cpp b/src/coreclr/jit/emitxarch.cpp index 20aa8f4ba4eaad..c3c632c02eda01 100644 --- a/src/coreclr/jit/emitxarch.cpp +++ b/src/coreclr/jit/emitxarch.cpp @@ -217,10 +217,10 @@ bool emitter::IsEvexEncodedInstruction(instruction ins) const case INS_phminposuw: case INS_mpsadbw: case INS_pclmulqdq: - case INS_aesdec: - case INS_aesdeclast: case INS_aesenc: case INS_aesenclast: + case INS_aesdec: + case INS_aesdeclast: case INS_aesimc: case INS_aeskeygenassist: case INS_vzeroupper: @@ -260,21 +260,30 @@ bool emitter::IsEvexEncodedInstruction(instruction ins) const case INS_prefetcht2: case INS_sfence: // Might need new INS_*suffix* instructions for these. - case INS_por: // INS_pord, INS_porq. - case INS_pxor: // INS_pxord, INS_pxorq - case INS_movdqa: // INS_movdqa32, INS_movdqa64. - case INS_movdqu: // INS_movdqu8, INS_movdqu16, INS_movdqu32, INS_movdqu64. - case INS_pand: // INS_pandd, INS_pandq. - case INS_pandn: // INS_pandnd, INS_pandnq. - case INS_vextractf128: // INS_vextractf32x4, INS_vextractf64x2. - case INS_vextracti128: // INS_vextracti32x4, INS_vextracti64x2. - case INS_vinsertf128: // INS_vinsertf32x4, INS_vinsertf64x2. - case INS_vinserti128: // INS_vinserti32x4, INS_vinserti64x2. case INS_vbroadcastf128: // INS_vbroadcastf32x4, INS_vbroadcastf64x2. case INS_vbroadcasti128: // INS_vbroadcasti32x4, INS_vbroadcasti64x2. - { - return false; - } + + // TODO-XARCH-AVX512 these need to be encoded with the proper individual EVEX instructions (movdqu8, + // movdqu16 etc) + // For implementation speed, I have set it up so the standing instruction will default to the 32-bit operand + // type + // i.e., movdqu => movdqu32 etc + // Since we are not using k registers yet, this will have no impact on correctness but will affect things + // once + // k registers are used (as that is the point of the "break out operand type" of these instructions) + // case INS_movdqa: // INS_movdqa32, INS_movdqa64. + // case INS_movdqu: // INS_movdqu8, INS_movdqu16, INS_movdqu32, INS_movdqu64. + // case INS_pand: // INS_pandd, INS_pandq. + // case INS_pandn: // INS_pandnd, INS_pandnq. + // case INS_por: // INS_pord, INS_porq. + // case INS_pxor: // INS_pxord, INS_pxorq + // case INS_vextractf128: // INS_vextractf32x4, INS_vextractf64x2. + // case INS_vextracti128: // INS_vextracti32x4, INS_vextracti64x2. + // case INS_vinsertf128: // INS_vinsertf32x4, INS_vinsertf64x2. + // case INS_vinserti128: // INS_vinserti32x4, INS_vinserti64x2. + { + return false; + } default: { break; @@ -761,9 +770,11 @@ bool emitter::Is4ByteSSEInstruction(instruction ins) // Return Value: // true if this instruction requires a VEX or EVEX prefix. // -bool emitter::TakesSimdPrefix(instruction ins) const +bool emitter::TakesSimdPrefix(const instrDesc* id) const { - return TakesEvexPrefix(ins) || TakesVexPrefix(ins); + instruction ins = id->idIns(); + + return TakesEvexPrefix(id) || TakesVexPrefix(ins); } //------------------------------------------------------------------------ @@ -785,13 +796,15 @@ bool emitter::TakesSimdPrefix(instruction ins) const // Return Value: // true if this instruction requires a EVEX prefix. // -bool emitter::TakesEvexPrefix(instruction ins) const +bool emitter::TakesEvexPrefix(const instrDesc* id) const { if (!emitComp->DoJitStressEvexEncoding()) { return false; } + instruction ins = id->idIns(); + // TODO-XArch-AVX512: Revisit 'HasKMaskRegisterDest()' check once KMask support is added. return IsEvexEncodedInstruction(ins) && !HasKMaskRegisterDest(ins); } @@ -1059,6 +1072,20 @@ bool emitter::TakesRexWPrefix(instruction ins, emitAttr attr) #endif //! TARGET_AMD64 } +//------------------------------------------------------------------------ +// IsHighSIMDReg: Checks if a register is strictly an EVEX encoded high SIMD +// registers (mm16-mm31). +// +// Arguments: +// reg -- register to check +// +// Return Value: +// true if the register is strictly an EVEX encoded high SIMD register +bool emitter::IsHighSIMDReg(regNumber reg) const +{ + return false; +} + // Returns true if using this register will require a REX.* prefix. // Since XMM registers overlap with YMM registers, this routine // can also be used to know whether a YMM register if the @@ -1125,6 +1152,23 @@ bool IsXMMReg(regNumber reg) #endif // !TARGET_AMD64 } +//------------------------------------------------------------------------ +// HighAwareRegEncoding: For EVEX encoded high SIMD registers (mm16-mm31), +// get a register encoding for bits 0-4, where the 5th bit is encoded via +// EVEX.R', EVEX.R, or EVEX.X. +// +// Arguments: +// reg -- register to encode +// +// Return Value: +// bits 0-4 of register encoding +// +unsigned HighAwareRegEncoding(regNumber reg) +{ + static_assert((REG_XMM0 & 0x7) == 0, "bad XMMBASE"); + return (unsigned)(reg & 0xF); +} + // Returns bits to be encoded in instruction for the given register. unsigned RegEncoding(regNumber reg) { @@ -1135,11 +1179,13 @@ unsigned RegEncoding(regNumber reg) // Utility routines that abstract the logic of adding REX.W, REX.R, REX.X, REX.B and REX prefixes // SSE2: separate 1-byte prefix gets added before opcode. // AVX: specific bits within VEX prefix need to be set in bit-inverted form. -emitter::code_t emitter::AddRexWPrefix(instruction ins, code_t code) +emitter::code_t emitter::AddRexWPrefix(const instrDesc* id, code_t code) { + instruction ins = id->idIns(); + if (UseEvexEncoding() && IsEvexEncodedInstruction(ins)) { - if (TakesEvexPrefix(ins) && codeEvexMigrationCheck(code)) // TODO-XArch-AVX512: Remove codeEvexMigrationCheck(). + if (TakesEvexPrefix(id) && codeEvexMigrationCheck(code)) // TODO-XArch-AVX512: Remove codeEvexMigrationCheck(). { // W-bit is available in 4-byte EVEX prefix that starts with byte 62. assert(hasEvexPrefix(code)); @@ -1169,11 +1215,13 @@ emitter::code_t emitter::AddRexWPrefix(instruction ins, code_t code) #ifdef TARGET_AMD64 -emitter::code_t emitter::AddRexRPrefix(instruction ins, code_t code) +emitter::code_t emitter::AddRexRPrefix(const instrDesc* id, code_t code) { + instruction ins = id->idIns(); + if (UseEvexEncoding() && IsEvexEncodedInstruction(ins)) { - if (TakesEvexPrefix(ins) && codeEvexMigrationCheck(code)) // TODO-XArch-AVX512: Remove codeEvexMigrationCheck(). + if (TakesEvexPrefix(id) && codeEvexMigrationCheck(code)) // TODO-XArch-AVX512: Remove codeEvexMigrationCheck(). { // R-bit is available in 4-byte EVEX prefix that starts with byte 62. assert(hasEvexPrefix(code)); @@ -1197,11 +1245,13 @@ emitter::code_t emitter::AddRexRPrefix(instruction ins, code_t code) return code | 0x4400000000ULL; } -emitter::code_t emitter::AddRexXPrefix(instruction ins, code_t code) +emitter::code_t emitter::AddRexXPrefix(const instrDesc* id, code_t code) { + instruction ins = id->idIns(); + if (UseEvexEncoding() && IsEvexEncodedInstruction(ins)) { - if (TakesEvexPrefix(ins)) + if (TakesEvexPrefix(id)) { // X-bit is available in 4-byte EVEX prefix that starts with byte 62. assert(hasEvexPrefix(code)); @@ -1224,11 +1274,13 @@ emitter::code_t emitter::AddRexXPrefix(instruction ins, code_t code) return code | 0x4200000000ULL; } -emitter::code_t emitter::AddRexBPrefix(instruction ins, code_t code) +emitter::code_t emitter::AddRexBPrefix(const instrDesc* id, code_t code) { + instruction ins = id->idIns(); + if (UseEvexEncoding() && IsEvexEncodedInstruction(ins)) { - if (TakesEvexPrefix(ins) && codeEvexMigrationCheck(code)) // TODO-XArch-AVX512: Remove codeEvexMigrationCheck(). + if (TakesEvexPrefix(id) && codeEvexMigrationCheck(code)) // TODO-XArch-AVX512: Remove codeEvexMigrationCheck(). { // B-bit is available in 4-byte EVEX prefix that starts with byte 62. assert(hasEvexPrefix(code)); @@ -1260,6 +1312,39 @@ emitter::code_t emitter::AddRexPrefix(instruction ins, code_t code) return code | 0x4000000000ULL; } + +//------------------------------------------------------------------------ +// AddEvexVPrimePrefix: Add the EVEX.V' bit to the EVEX prefix. EVEX.V' +// is encoded in inverted form. +// +// Arguments: +// code -- register to encode +// +// Return Value: +// code with EVEX.V' set in verted form. +// +emitter::code_t emitter::AddEvexVPrimePrefix(code_t code) +{ + assert(UseEvexEncoding() && hasEvexPrefix(code)); + return emitter::code_t(code & 0xFFFFFFF7FFFFFFFFULL); +} + +//------------------------------------------------------------------------ +// AddEvexRPrimePrefix: Add the EVEX.R' bit to the EVEX prefix. EVEX.R' +// is encoded in inverted form. +// +// Arguments: +// code -- register to encode +// +// Return Value: +// code with EVEX.R' set in verted form. +// +emitter::code_t emitter::AddEvexRPrimePrefix(code_t code) +{ + assert(UseEvexEncoding() && hasEvexPrefix(code)); + return emitter::code_t(code & 0xFFEFFFFFFFFFFFFFULL); +} + #endif // TARGET_AMD64 bool isPrefix(BYTE b) @@ -1962,15 +2047,18 @@ unsigned emitter::emitGetVexPrefixSize(instruction ins, emitAttr attr) // Returns: // Updated size. // -unsigned emitter::emitGetAdjustedSizeEvexAware(instruction ins, emitAttr attr, code_t code) +// TODO-XARCH-AVX512 come back and check whether we can id `id` directly (no need) +// to pass emitAttr size +unsigned emitter::emitGetAdjustedSizeEvexAware(const instrDesc* id, emitAttr attr, code_t code) { - unsigned adjustedSize = 0; + unsigned adjustedSize = 0; + instruction ins = id->idIns(); // TODO-XArch-AVX512: Remove redundant code and possiblly collapse EVEX and VEX into a single pathway // IsEvexEncodedInstruction(ins) is `true` for AVX/SSE instructions also which needs to be VEX encoded unless // explicitly // asked for EVEX. - if (IsEvexEncodedInstruction(ins) && TakesEvexPrefix(ins)) + if (IsEvexEncodedInstruction(ins) && TakesEvexPrefix(id)) { // EVEX prefix encodes some bytes of the opcode and as a result, overall size of the instruction reduces. // Therefore, to estimate the size adding EVEX prefix size and size of instruction opcode bytes will always @@ -2669,10 +2757,12 @@ bool emitter::EncodedBySSE38orSSE3A(instruction ins) * part of an opcode. */ -inline unsigned emitter::insEncodeReg012(instruction ins, regNumber reg, emitAttr size, code_t* code) +inline unsigned emitter::insEncodeReg012(const instrDesc* id, regNumber reg, emitAttr size, code_t* code) { assert(reg < REG_STK); + instruction ins = id->idIns(); + #ifdef TARGET_AMD64 // Either code is not NULL or reg is not an extended reg. // If reg is an extended reg, instruction needs to be prefixed with 'REX' @@ -2681,7 +2771,14 @@ inline unsigned emitter::insEncodeReg012(instruction ins, regNumber reg, emitAtt if (IsExtendedReg(reg)) { - *code = AddRexBPrefix(ins, *code); // REX.B + if (IsHighSIMDReg(reg)) + { + *code = AddRexXPrefix(id, *code); // EVEX.X + } + if (reg & 0x8) + { + *code = AddRexBPrefix(id, *code); // REX.B + } } else if ((EA_SIZE(size) == EA_1BYTE) && (reg > REG_RBX) && (code != nullptr)) { @@ -2703,10 +2800,12 @@ inline unsigned emitter::insEncodeReg012(instruction ins, regNumber reg, emitAtt * part of an opcode. */ -inline unsigned emitter::insEncodeReg345(instruction ins, regNumber reg, emitAttr size, code_t* code) +inline unsigned emitter::insEncodeReg345(const instrDesc* id, regNumber reg, emitAttr size, code_t* code) { assert(reg < REG_STK); + instruction ins = id->idIns(); + #ifdef TARGET_AMD64 // Either code is not NULL or reg is not an extended reg. // If reg is an extended reg, instruction needs to be prefixed with 'REX' @@ -2715,7 +2814,14 @@ inline unsigned emitter::insEncodeReg345(instruction ins, regNumber reg, emitAtt if (IsExtendedReg(reg)) { - *code = AddRexRPrefix(ins, *code); // REX.R + if (IsHighSIMDReg(reg)) + { + *code = AddEvexRPrimePrefix(*code); // EVEX.R' + } + if (reg & 0x8) + { + *code = AddRexRPrefix(id, *code); // REX.R + } } else if ((EA_SIZE(size) == EA_1BYTE) && (reg > REG_RBX) && (code != nullptr)) { @@ -2736,8 +2842,10 @@ inline unsigned emitter::insEncodeReg345(instruction ins, regNumber reg, emitAtt * Returns modified SIMD opcode with the specified register encoded in bits 3-6 of * byte 2 of VEX and EVEX prefix. */ -inline emitter::code_t emitter::insEncodeReg3456(instruction ins, regNumber reg, emitAttr size, code_t code) +inline emitter::code_t emitter::insEncodeReg3456(const instrDesc* id, regNumber reg, emitAttr size, code_t code) { + instruction ins = id->idIns(); + assert(reg < REG_STK); assert(IsVexOrEvexEncodedInstruction(ins)); assert(hasVexOrEvexPrefix(code)); @@ -2755,10 +2863,20 @@ inline emitter::code_t emitter::insEncodeReg3456(instruction ins, regNumber reg, assert(regBits <= 0xF); if (UseEvexEncoding() && IsEvexEncodedInstruction(ins)) { - if (TakesEvexPrefix(ins) && codeEvexMigrationCheck(code)) + if (TakesEvexPrefix(id) && codeEvexMigrationCheck(code)) { - assert(hasEvexPrefix(code) && TakesEvexPrefix(ins)); + assert(hasEvexPrefix(code) && TakesEvexPrefix(id)); + // TODO-XARCH-AVX512 I don't like that we redefine regBits on the EVEX case. + // Rather see these paths cleaned up. + regBits = HighAwareRegEncoding(reg); +#if defined(TARGET_AMD64) + if (IsHighSIMDReg(reg)) + { + // Have to set the EVEX V' bit + code = AddEvexVPrimePrefix(code); + } +#endif // Shift count = 5-bytes of opcode + 0-2 bits for EVEX regBits <<= 43; return code ^ regBits; @@ -2766,6 +2884,10 @@ inline emitter::code_t emitter::insEncodeReg3456(instruction ins, regNumber reg, } if (UseVEXEncoding() && IsVexEncodedInstruction(ins)) { + + // Both prefix encodes register operand in 1's complement form + assert(regBits <= 0xF); + if (TakesVexPrefix(ins)) { assert(hasVexPrefix(code)); @@ -2785,8 +2907,10 @@ inline emitter::code_t emitter::insEncodeReg3456(instruction ins, regNumber reg, * Used exclusively to generate the REX.X bit and truncate the register. */ -inline unsigned emitter::insEncodeRegSIB(instruction ins, regNumber reg, code_t* code) +inline unsigned emitter::insEncodeRegSIB(const instrDesc* id, regNumber reg, code_t* code) { + instruction ins = id->idIns(); + assert(reg < REG_STK); #ifdef TARGET_AMD64 @@ -2797,7 +2921,14 @@ inline unsigned emitter::insEncodeRegSIB(instruction ins, regNumber reg, code_t* if (IsExtendedReg(reg)) { - *code = AddRexXPrefix(ins, *code); // REX.X + if (IsHighSIMDReg(reg)) + { + *code = AddEvexVPrimePrefix(*code); // EVEX.X + } + if (reg & 0x8) + { + *code = AddRexXPrefix(id, *code); // REX.B + } } unsigned regBits = RegEncoding(reg); #else // !TARGET_AMD64 @@ -2813,7 +2944,7 @@ inline unsigned emitter::insEncodeRegSIB(instruction ins, regNumber reg, code_t* * Returns the "[r/m]" opcode with the mod/RM field set to register. */ -inline emitter::code_t emitter::insEncodeMRreg(instruction ins, code_t code) +inline emitter::code_t emitter::insEncodeMRreg(const instrDesc* id, code_t code) { // If Byte 4 (which is 0xFF00) is 0, that's where the RM encoding goes. // Otherwise, it will be placed after the 4 byte encoding. @@ -2831,7 +2962,7 @@ inline emitter::code_t emitter::insEncodeMRreg(instruction ins, code_t code) * Returns the given "[r/m]" opcode with the mod/RM field set to register. */ -inline emitter::code_t emitter::insEncodeRMreg(instruction ins, code_t code) +inline emitter::code_t emitter::insEncodeRMreg(const instrDesc* id, code_t code) { // If Byte 4 (which is 0xFF00) is 0, that's where the RM encoding goes. // Otherwise, it will be placed after the 4 byte encoding. @@ -2849,11 +2980,11 @@ inline emitter::code_t emitter::insEncodeRMreg(instruction ins, code_t code) * the given register. */ -inline emitter::code_t emitter::insEncodeMRreg(instruction ins, regNumber reg, emitAttr size, code_t code) +inline emitter::code_t emitter::insEncodeMRreg(const instrDesc* id, regNumber reg, emitAttr size, code_t code) { assert((code & 0xC000) == 0); code |= 0xC000; - unsigned regcode = insEncodeReg012(ins, reg, size, &code) << 8; + unsigned regcode = insEncodeReg012(id, reg, size, &code) << 8; code |= regcode; return code; } @@ -2864,11 +2995,11 @@ inline emitter::code_t emitter::insEncodeMRreg(instruction ins, regNumber reg, e * the given register. */ -inline emitter::code_t emitter::insEncodeMIreg(instruction ins, regNumber reg, emitAttr size, code_t code) +inline emitter::code_t emitter::insEncodeMIreg(const instrDesc* id, regNumber reg, emitAttr size, code_t code) { assert((code & 0xC000) == 0); code |= 0xC000; - unsigned regcode = insEncodeReg012(ins, reg, size, &code) << 8; + unsigned regcode = insEncodeReg012(id, reg, size, &code) << 8; code |= regcode; return code; } @@ -2889,13 +3020,13 @@ inline bool insNeedsRRIb(instruction ins) * Returns the "reg,reg,imm8" opcode with both the reg's set to the * the given register. */ -inline emitter::code_t emitter::insEncodeRRIb(instruction ins, regNumber reg, emitAttr size) +inline emitter::code_t emitter::insEncodeRRIb(const instrDesc* id, regNumber reg, emitAttr size) { assert(size == EA_4BYTE); // All we handle for now. - assert(insNeedsRRIb(ins)); + assert(insNeedsRRIb(id->idIns())); // If this list gets longer, use a switch, or a table lookup. code_t code = 0x69c0; - unsigned regcode = insEncodeReg012(ins, reg, size, &code); + unsigned regcode = insEncodeReg012(id, reg, size, &code); // We use the same register as source and destination. (Could have another version that does both regs...) code |= regcode; code |= (regcode << 3); @@ -2908,10 +3039,10 @@ inline emitter::code_t emitter::insEncodeRRIb(instruction ins, regNumber reg, em * nibble of the opcode */ -inline emitter::code_t emitter::insEncodeOpreg(instruction ins, regNumber reg, emitAttr size) +inline emitter::code_t emitter::insEncodeOpreg(const instrDesc* id, regNumber reg, emitAttr size) { - code_t code = insCodeRR(ins); - unsigned regcode = insEncodeReg012(ins, reg, size, &code); + code_t code = insCodeRR(id->idIns()); + unsigned regcode = insEncodeReg012(id, reg, size, &code); code |= regcode; return code; } @@ -3082,7 +3213,7 @@ inline UNATIVE_OFFSET emitter::emitInsSizeRR(instrDesc* id, code_t code) instruction ins = id->idIns(); emitAttr attr = id->idOpSize(); - UNATIVE_OFFSET sz = emitGetAdjustedSizeEvexAware(ins, attr, code); + UNATIVE_OFFSET sz = emitGetAdjustedSizeEvexAware(id, attr, code); bool includeRexPrefixSize = true; // REX prefix @@ -3141,8 +3272,10 @@ inline UNATIVE_OFFSET emitter::emitInsSizeRR(instrDesc* id, code_t code, int val return valSize + emitInsSizeRR(id, code); } -inline UNATIVE_OFFSET emitter::emitInsSizeRR(instruction ins, regNumber reg1, regNumber reg2, emitAttr attr) +inline UNATIVE_OFFSET emitter::emitInsSizeRR(instrDesc* id, regNumber reg1, regNumber reg2, emitAttr attr) { + instruction ins = id->idIns(); + emitAttr size = EA_SIZE(attr); // If Byte 4 (which is 0xFF00) is zero, that's where the RM encoding goes. @@ -3150,7 +3283,7 @@ inline UNATIVE_OFFSET emitter::emitInsSizeRR(instruction ins, regNumber reg1, re // This would probably be better expressed as a different format or something? code_t code = insCodeRM(ins); - UNATIVE_OFFSET sz = emitGetAdjustedSizeEvexAware(ins, size, insCodeRM(ins)); + UNATIVE_OFFSET sz = emitGetAdjustedSizeEvexAware(id, size, insCodeRM(ins)); bool includeRexPrefixSize = true; // REX prefix @@ -3170,7 +3303,7 @@ inline UNATIVE_OFFSET emitter::emitInsSizeRR(instruction ins, regNumber reg1, re } else { - sz += emitInsSize(insEncodeRMreg(ins, code), includeRexPrefixSize); + sz += emitInsSize(insEncodeRMreg(id, code), includeRexPrefixSize); } return sz; @@ -3299,7 +3432,7 @@ inline UNATIVE_OFFSET emitter::emitInsSizeSVCalcDisp(instrDesc* id, code_t code, assert(emitComp->lvaTempsHaveLargerOffsetThanVars()); // Check whether we can use compressed displacement if EVEX. - if (TakesEvexPrefix(id->idIns())) + if (TakesEvexPrefix(id)) { bool compressedFitsInByte = false; TryEvexCompressDisp8Byte(id, ssize_t(offs), &compressedFitsInByte); @@ -3343,7 +3476,7 @@ inline UNATIVE_OFFSET emitter::emitInsSizeSVCalcDisp(instrDesc* id, code_t code, #endif // !FEATURE_FIXED_OUT_ARGS bool useSmallEncoding = false; - if (TakesEvexPrefix(id->idIns())) + if (TakesEvexPrefix(id)) { TryEvexCompressDisp8Byte(id, ssize_t(offs), &useSmallEncoding); } @@ -3372,7 +3505,7 @@ inline UNATIVE_OFFSET emitter::emitInsSizeSV(instrDesc* id, code_t code, int var assert(id->idIns() != INS_invalid); instruction ins = id->idIns(); emitAttr attrSize = id->idOpSize(); - UNATIVE_OFFSET prefix = emitGetAdjustedSizeEvexAware(ins, attrSize, code); + UNATIVE_OFFSET prefix = emitGetAdjustedSizeEvexAware(id, attrSize, code); // REX prefix if (TakesRexWPrefix(ins, attrSize) || IsExtendedReg(id->idReg1(), attrSize) || @@ -3390,7 +3523,7 @@ inline UNATIVE_OFFSET emitter::emitInsSizeSV(instrDesc* id, code_t code, int var instruction ins = id->idIns(); emitAttr attrSize = id->idOpSize(); UNATIVE_OFFSET valSize = EA_SIZE_IN_BYTES(attrSize); - UNATIVE_OFFSET prefix = emitGetAdjustedSizeEvexAware(ins, attrSize, code); + UNATIVE_OFFSET prefix = emitGetAdjustedSizeEvexAware(id, attrSize, code); bool valInByte = ((signed char)val == val) && (ins != INS_mov) && (ins != INS_test); #ifdef TARGET_AMD64 @@ -3496,7 +3629,7 @@ UNATIVE_OFFSET emitter::emitInsSizeAM(instrDesc* id, code_t code) } else { - if (TakesEvexPrefix(ins)) + if (TakesEvexPrefix(id)) { dsp = TryEvexCompressDisp8Byte(id, dsp, &dspInByte); } @@ -3523,7 +3656,7 @@ UNATIVE_OFFSET emitter::emitInsSizeAM(instrDesc* id, code_t code) size = 2; } - size += emitGetAdjustedSizeEvexAware(ins, attrSize, code); + size += emitGetAdjustedSizeEvexAware(id, attrSize, code); if (hasRexPrefix(code)) { @@ -3741,7 +3874,7 @@ inline UNATIVE_OFFSET emitter::emitInsSizeCV(instrDesc* id, code_t code) // can be reached via RIP-relative addressing. UNATIVE_OFFSET size = sizeof(INT32); - size += emitGetAdjustedSizeEvexAware(ins, attrSize, code); + size += emitGetAdjustedSizeEvexAware(id, attrSize, code); bool includeRexPrefixSize = true; @@ -3983,7 +4116,7 @@ void emitter::emitIns(instruction ins, emitAttr attr) insFormat fmt = IF_NONE; - sz += emitGetAdjustedSizeEvexAware(ins, attr, code); + sz += emitGetAdjustedSizeEvexAware(id, attr, code); if (TakesRexWPrefix(ins, attr)) { sz += emitGetRexPrefixSize(ins); @@ -5105,7 +5238,7 @@ void emitter::emitIns_R(instruction ins, emitAttr attr, regNumber reg) /* We expect this to always be a 'big' opcode */ - assert(insEncodeMRreg(ins, reg, attr, insCodeMR(ins)) & 0x00FF0000); + assert(insEncodeMRreg(id, reg, attr, insCodeMR(ins)) & 0x00FF0000); size = attr; @@ -5125,7 +5258,7 @@ void emitter::emitIns_R(instruction ins, emitAttr attr, regNumber reg) id->idReg1(reg); // Vex bytes - sz += emitGetAdjustedSizeEvexAware(ins, attr, insEncodeMRreg(ins, reg, attr, insCodeMR(ins))); + sz += emitGetAdjustedSizeEvexAware(id, attr, insEncodeMRreg(id, reg, attr, insCodeMR(ins))); // REX byte if (IsExtendedReg(reg, attr) || TakesRexWPrefix(ins, attr)) @@ -5267,7 +5400,12 @@ void emitter::emitIns_R_I(instruction ins, break; } - sz += emitGetAdjustedSizeEvexAware(ins, attr, insCodeMI(ins)); + id = emitNewInstrSC(attr, val); + id->idIns(ins); + id->idInsFmt(fmt); + id->idReg1(reg); + + sz += emitGetAdjustedSizeEvexAware(id, attr, insCodeMI(ins)); // Do we need a REX prefix for AMD64? We need one if we are using any extended register (REX.R), or if we have a // 64-bit sized operand (REX.W). Note that IMUL in our encoding is special, with a "built-in", implicit, target @@ -5277,10 +5415,6 @@ void emitter::emitIns_R_I(instruction ins, sz += emitGetRexPrefixSize(ins); } - id = emitNewInstrSC(attr, val); - id->idIns(ins); - id->idInsFmt(fmt); - id->idReg1(reg); id->idCodeSize(sz); #ifdef DEBUG @@ -5859,13 +5993,13 @@ void emitter::emitIns_Mov(instruction ins, emitAttr attr, regNumber dstReg, regN return; } - UNATIVE_OFFSET sz = emitInsSizeRR(ins, dstReg, srcReg, attr); - instrDesc* id = emitNewInstrSmall(attr); id->idIns(ins); id->idInsFmt(fmt); id->idReg1(dstReg); id->idReg2(srcReg); + + UNATIVE_OFFSET sz = emitInsSizeRR(id, dstReg, srcReg, attr); id->idCodeSize(sz); dispIns(id); @@ -5890,8 +6024,6 @@ void emitter::emitIns_R_R(instruction ins, emitAttr attr, regNumber reg1, regNum assert(size <= EA_32BYTE); noway_assert(emitVerifyEncodable(ins, size, reg1, reg2)); - UNATIVE_OFFSET sz = emitInsSizeRR(ins, reg1, reg2, attr); - /* Special case: "XCHG" uses a different format */ insFormat fmt = (ins == INS_xchg) ? IF_RRW_RRW : emitInsModeFormat(ins, IF_RRD_RRD); @@ -5900,6 +6032,8 @@ void emitter::emitIns_R_R(instruction ins, emitAttr attr, regNumber reg1, regNum id->idInsFmt(fmt); id->idReg1(reg1); id->idReg2(reg2); + + UNATIVE_OFFSET sz = emitInsSizeRR(id, reg1, reg2, attr); id->idCodeSize(sz); dispIns(id); @@ -8970,7 +9104,7 @@ void emitter::emitIns_Call(EmitCallType callType, { // Tailcall with addressing mode/register needs to be rex.w // prefixed to be recognized as part of epilog by unwinder. - code = AddRexWPrefix(ins, code); + code = AddRexWPrefix(id, code); } sz = emitInsSizeAM(id, code); @@ -11329,13 +11463,13 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) { // tail call with addressing mode (or through register) needs rex.w // prefix to be recognized by unwinder as part of epilog. - code = AddRexWPrefix(ins, code); + code = AddRexWPrefix(id, code); } // Special case: call via a register if (id->idIsCallRegPtr()) { - code = insEncodeMRreg(ins, reg, EA_PTRSIZE, code); + code = insEncodeMRreg(id, reg, EA_PTRSIZE, code); dst += emitOutputSimdPrefixIfNeeded(ins, dst, code); dst += emitOutputWord(dst, code); goto DONE; @@ -11349,14 +11483,14 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) // Compute the REX prefix if it exists if (IsExtendedReg(reg, EA_PTRSIZE)) { - insEncodeReg012(ins, reg, EA_PTRSIZE, &code); + insEncodeReg012(id, reg, EA_PTRSIZE, &code); // TODO-Cleanup: stop casting RegEncoding() back to a regNumber. reg = (regNumber)RegEncoding(reg); } if (IsExtendedReg(rgx, EA_PTRSIZE)) { - insEncodeRegSIB(ins, rgx, &code); + insEncodeRegSIB(id, rgx, &code); // TODO-Cleanup: stop casting RegEncoding() back to a regNumber. rgx = (regNumber)RegEncoding(rgx); } @@ -11402,7 +11536,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) { case IF_RWR_ARD: - assert(code == (insCodeRM(ins) | (insEncodeReg345(ins, REG_EAX, EA_PTRSIZE, NULL) << 8))); + assert(code == (insCodeRM(ins) | (insEncodeReg345(id, REG_EAX, EA_PTRSIZE, NULL) << 8))); code &= ~((code_t)0xFFFFFFFF); code |= 0xA0; @@ -11411,7 +11545,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) case IF_AWR_RRD: - assert(code == (insCodeMR(ins) | (insEncodeReg345(ins, REG_EAX, EA_PTRSIZE, NULL) << 8))); + assert(code == (insCodeMR(ins) | (insEncodeReg345(id, REG_EAX, EA_PTRSIZE, NULL) << 8))); code &= ~((code_t)0xFFFFFFFF); code |= 0xA2; @@ -11428,10 +11562,10 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) // Emit SIMD prefix if required // There are some callers who already add SIMD prefix and call this routine. // Therefore, add SIMD prefix is one is not already present. - code = AddSimdPrefixIfNeededAndNotPresent(ins, code, size); + code = AddSimdPrefixIfNeededAndNotPresent(id, code, size); // For this format, moves do not support a third operand, so we only need to handle the binary ops. - if (TakesSimdPrefix(ins)) + if (TakesSimdPrefix(id)) { if (IsDstDstSrcAVXInstruction(ins)) { @@ -11456,33 +11590,33 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) } // encode source operand reg in 'vvvv' bits in 1's complement form - code = insEncodeReg3456(ins, src1, size, code); + code = insEncodeReg3456(id, src1, size, code); } else if (IsDstSrcSrcAVXInstruction(ins)) { - code = insEncodeReg3456(ins, id->idReg2(), size, code); + code = insEncodeReg3456(id, id->idReg2(), size, code); } } // Emit the REX prefix if required // TODO-XARCH-AVX512 : Update this check once all paths have EVEX support. - // Explore moving IsWEvexOpcodeExtension() logic inside TakesRexWPrefix(). Not doing so currently - // since we cannot differentiate EVEX vs VEX without 'code' until all paths have EVEX support. - if (TakesRexWPrefix(ins, size) || (codeEvexMigrationCheck(code) && IsWEvexOpcodeExtension(ins))) + // Explore moving IsWEvexOpcodeExtension() logic inside TakesRexWPrefix(). Not doind so currently + // since we cannot differentiate EVEX vs VEX without 'code' untill all paths have EVEX support. + if (TakesRexWPrefix(ins, size) || (codeEvexMigrationCheck(code) && IsWEvexOpcodeExtension(id))) { - code = AddRexWPrefix(ins, code); + code = AddRexWPrefix(id, code); } if (IsExtendedReg(reg, EA_PTRSIZE)) { - insEncodeReg012(ins, reg, EA_PTRSIZE, &code); + insEncodeReg012(id, reg, EA_PTRSIZE, &code); // TODO-Cleanup: stop casting RegEncoding() back to a regNumber. reg = (regNumber)RegEncoding(reg); } if (IsExtendedReg(rgx, EA_PTRSIZE)) { - insEncodeRegSIB(ins, rgx, &code); + insEncodeRegSIB(id, rgx, &code); // TODO-Cleanup: stop casting RegEncoding() back to a regNumber. rgx = (regNumber)RegEncoding(rgx); } @@ -11522,7 +11656,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) } } } - unsigned regcode = insEncodeReg345(ins, reg345, size, &code); + unsigned regcode = insEncodeReg345(id, reg345, size, &code); dst += emitOutputSimdPrefixIfNeeded(ins, dst, code); @@ -11652,7 +11786,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) } else { - if (TakesEvexPrefix(ins)) + if (TakesEvexPrefix(id)) { dsp = TryEvexCompressDisp8Byte(id, dsp, &dspInByte); } @@ -11887,7 +12021,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) { // Put the register in the opcode - code |= insEncodeReg012(ins, reg, EA_PTRSIZE, nullptr); + code |= insEncodeReg012(id, reg, EA_PTRSIZE, nullptr); // Is there a displacement? if (dspIsZero) @@ -11917,7 +12051,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) else { // Put the register in the opcode - code |= insEncodeReg012(ins, reg, EA_PTRSIZE, nullptr) << 8; + code |= insEncodeReg012(id, reg, EA_PTRSIZE, nullptr) << 8; // Is there a displacement? if (dspIsZero) @@ -11963,8 +12097,8 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) if (reg != REG_NA) { // The address is "[reg + {2/4/8} * rgx + icon]" - regByte = insEncodeReg012(ins, reg, EA_PTRSIZE, nullptr) | - insEncodeReg345(ins, rgx, EA_PTRSIZE, nullptr) | insSSval(mul); + regByte = insEncodeReg012(id, reg, EA_PTRSIZE, nullptr) | + insEncodeReg345(id, rgx, EA_PTRSIZE, nullptr) | insSSval(mul); if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) { @@ -12030,8 +12164,8 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) else { // The address is "[{2/4/8} * rgx + icon]" - regByte = insEncodeReg012(ins, REG_EBP, EA_PTRSIZE, nullptr) | - insEncodeReg345(ins, rgx, EA_PTRSIZE, nullptr) | insSSval(mul); + regByte = insEncodeReg012(id, REG_EBP, EA_PTRSIZE, nullptr) | + insEncodeReg345(id, rgx, EA_PTRSIZE, nullptr) | insSSval(mul); if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) { @@ -12060,7 +12194,7 @@ BYTE* emitter::emitOutputAM(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) else { // The address is "[reg+rgx+dsp]" - regByte = insEncodeReg012(ins, reg, EA_PTRSIZE, nullptr) | insEncodeReg345(ins, rgx, EA_PTRSIZE, nullptr); + regByte = insEncodeReg012(id, reg, EA_PTRSIZE, nullptr) | insEncodeReg345(id, rgx, EA_PTRSIZE, nullptr); if (EncodedBySSE38orSSE3A(ins) || (ins == INS_crc32)) { @@ -12298,16 +12432,16 @@ BYTE* emitter::emitOutputSV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) // Add VEX or EVEX prefix if required. // There are some callers who already add prefix and call this routine. // Therefore, add VEX or EVEX prefix if one is not already present. - code = AddSimdPrefixIfNeededAndNotPresent(ins, code, size); + code = AddSimdPrefixIfNeededAndNotPresent(id, code, size); // Compute the REX prefix // TODO-XARCH-AVX512 : Update this check once all paths have EVEX support. // Explore moving IsWEvexOpcodeExtension() logic inside TakesRexWPrefix(). // Not doing so currently since we cannot differentiate EVEX vs VEX without // 'code' until all paths have EVEX support. - if (TakesRexWPrefix(ins, size) || (codeEvexMigrationCheck(code) && IsWEvexOpcodeExtension(ins))) + if (TakesRexWPrefix(ins, size) || (codeEvexMigrationCheck(code) && IsWEvexOpcodeExtension(id))) { - code = AddRexWPrefix(ins, code); + code = AddRexWPrefix(id, code); } // Special case emitting AVX instructions @@ -12334,9 +12468,9 @@ BYTE* emitter::emitOutputSV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) } else { - code = insEncodeReg3456(ins, id->idReg1(), size, code); + code = insEncodeReg3456(id, id->idReg1(), size, code); } - unsigned regcode = insEncodeReg345(ins, reg345, size, &code); + unsigned regcode = insEncodeReg345(id, reg345, size, &code); dst += emitOutputSimdPrefixIfNeeded(ins, dst, code); @@ -12467,7 +12601,7 @@ BYTE* emitter::emitOutputSV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) // function, to which the remainder of the emitter logic should handle properly. // TODO-XARCH-AVX512 : embedded broadcast might change this int dspAsByte = dsp; - if (TakesEvexPrefix(ins)) + if (TakesEvexPrefix(id)) { dspAsByte = int(TryEvexCompressDisp8Byte(id, ssize_t(dsp), &dspInByte)); } @@ -12521,7 +12655,7 @@ BYTE* emitter::emitOutputSV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) // TODO-XARCH-AVX512 : working to wrap up all adjusted disp8 compression logic into the following // function, to which the remainder of the emitter logic should handle properly. // TODO-XARCH-AVX512 : embedded broadcast might change this - if (TakesEvexPrefix(ins)) + if (TakesEvexPrefix(id)) { dspAsByte = int(TryEvexCompressDisp8Byte(id, ssize_t(dsp), &dspInByte)); } @@ -12747,12 +12881,12 @@ BYTE* emitter::emitOutputCV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) // Compute VEX/EVEX prefix // Some of its callers already add EVEX/VEX prefix and then call this routine. // Therefore add EVEX/VEX prefix is not already present. - code = AddSimdPrefixIfNeededAndNotPresent(ins, code, size); + code = AddSimdPrefixIfNeededAndNotPresent(id, code, size); // Compute the REX prefix - if (TakesRexWPrefix(ins, size) || (codeEvexMigrationCheck(code) && IsWEvexOpcodeExtension(ins))) + if (TakesRexWPrefix(ins, size) || (codeEvexMigrationCheck(code) && IsWEvexOpcodeExtension(id))) { - code = AddRexWPrefix(ins, code); + code = AddRexWPrefix(id, code); } // `addc` is used for two kinds if instructions @@ -12787,7 +12921,7 @@ BYTE* emitter::emitOutputCV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) { case IF_RWR_MRD: - assert(code == (insCodeRM(ins) | (insEncodeReg345(ins, REG_EAX, EA_PTRSIZE, NULL) << 8) | 0x0500)); + assert(code == (insCodeRM(ins) | (insEncodeReg345(id, REG_EAX, EA_PTRSIZE, NULL) << 8) | 0x0500)); code &= ~((code_t)0xFFFFFFFF); code |= 0xA0; @@ -12796,7 +12930,7 @@ BYTE* emitter::emitOutputCV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) case IF_MWR_RRD: - assert(code == (insCodeMR(ins) | (insEncodeReg345(ins, REG_EAX, EA_PTRSIZE, NULL) << 8) | 0x0500)); + assert(code == (insCodeMR(ins) | (insEncodeReg345(id, REG_EAX, EA_PTRSIZE, NULL) << 8) | 0x0500)); code &= ~((code_t)0xFFFFFFFF); code |= 0xA2; @@ -12834,9 +12968,9 @@ BYTE* emitter::emitOutputCV(BYTE* dst, instrDesc* id, code_t code, CnsVal* addc) } else { - code = insEncodeReg3456(ins, id->idReg1(), size, code); + code = insEncodeReg3456(id, id->idReg1(), size, code); } - unsigned regcode = insEncodeReg345(ins, reg345, size, &code); + unsigned regcode = insEncodeReg345(id, reg345, size, &code); dst += emitOutputSimdPrefixIfNeeded(ins, dst, code); @@ -13216,13 +13350,13 @@ BYTE* emitter::emitOutputR(BYTE* dst, instrDesc* id) code |= 0x1; } - if (TakesRexWPrefix(ins, size) || (codeEvexMigrationCheck(code) && IsWEvexOpcodeExtension(ins))) + if (TakesRexWPrefix(ins, size) || (codeEvexMigrationCheck(code) && IsWEvexOpcodeExtension(id))) { - code = AddRexWPrefix(ins, code); + code = AddRexWPrefix(id, code); } // Register... - unsigned regcode = insEncodeReg012(ins, reg, size, &code); + unsigned regcode = insEncodeReg012(id, reg, size, &code); // Output the REX prefix dst += emitOutputSimdPrefixIfNeeded(ins, dst, code); @@ -13236,7 +13370,7 @@ BYTE* emitter::emitOutputR(BYTE* dst, instrDesc* id) // Output a size prefix for a 16-bit operand dst += emitOutputByte(dst, 0x66); } - dst += emitOutputByte(dst, insCodeRR(ins) | insEncodeReg012(ins, reg, size, nullptr)); + dst += emitOutputByte(dst, insCodeRR(ins) | insEncodeReg012(id, reg, size, nullptr)); } break; @@ -13246,9 +13380,9 @@ BYTE* emitter::emitOutputR(BYTE* dst, instrDesc* id) case INS_push_hide: assert(size == EA_PTRSIZE); - code = insEncodeOpreg(ins, reg, size); + code = insEncodeOpreg(id, reg, size); - assert(!TakesSimdPrefix(ins)); + assert(!TakesSimdPrefix(id)); assert(!TakesRexWPrefix(ins, size)); // Output the REX prefix @@ -13268,13 +13402,13 @@ BYTE* emitter::emitOutputR(BYTE* dst, instrDesc* id) code = insCodeRR(ins); - if (TakesRexWPrefix(ins, size) || (codeEvexMigrationCheck(code) && IsWEvexOpcodeExtension(ins))) + if (TakesRexWPrefix(ins, size) || (codeEvexMigrationCheck(code) && IsWEvexOpcodeExtension(id))) { - code = AddRexWPrefix(ins, code); + code = AddRexWPrefix(id, code); } // Register... - unsigned regcode = insEncodeReg012(ins, reg, size, &code); + unsigned regcode = insEncodeReg012(id, reg, size, &code); // Output the REX prefix dst += emitOutputSimdPrefixIfNeeded(ins, dst, code); @@ -13303,7 +13437,7 @@ BYTE* emitter::emitOutputR(BYTE* dst, instrDesc* id) assert(id->idGCref() == GCT_NONE); assert(size == EA_1BYTE); - code = insEncodeMRreg(ins, reg, EA_1BYTE, insCodeMR(ins)); + code = insEncodeMRreg(id, reg, EA_1BYTE, insCodeMR(ins)); // Output the REX prefix dst += emitOutputSimdPrefixIfNeeded(ins, dst, code); @@ -13329,7 +13463,7 @@ BYTE* emitter::emitOutputR(BYTE* dst, instrDesc* id) assert(id->idGCref() == GCT_NONE); - code = insEncodeMRreg(ins, reg, size, insCodeMR(ins)); + code = insEncodeMRreg(id, reg, size, insCodeMR(ins)); if (size != EA_1BYTE) { @@ -13343,11 +13477,11 @@ BYTE* emitter::emitOutputR(BYTE* dst, instrDesc* id) } } - code = AddSimdPrefixIfNeeded(ins, code, size); + code = AddSimdPrefixIfNeeded(id, code, size); - if (TakesRexWPrefix(ins, size) || (codeEvexMigrationCheck(code) && IsWEvexOpcodeExtension(ins))) + if (TakesRexWPrefix(ins, size) || (codeEvexMigrationCheck(code) && IsWEvexOpcodeExtension(id))) { - code = AddRexWPrefix(ins, code); + code = AddRexWPrefix(id, code); } // Output the REX prefix @@ -13434,36 +13568,36 @@ BYTE* emitter::emitOutputRR(BYTE* dst, instrDesc* id) { code = insCodeMR(ins); } - code = AddSimdPrefixIfNeeded(ins, code, size); - code = insEncodeRMreg(ins, code); + code = AddSimdPrefixIfNeeded(id, code, size); + code = insEncodeRMreg(id, code); // TODO-XARCH-AVX512 : Update this check once all paths have EVEX support. // Explore moving IsWEvexOpcodeExtension() logic to instruction table as flag. - if (TakesRexWPrefix(ins, size) || (codeEvexMigrationCheck(code) && IsWEvexOpcodeExtension(ins))) + if (TakesRexWPrefix(ins, size) || (codeEvexMigrationCheck(code) && IsWEvexOpcodeExtension(id))) { - code = AddRexWPrefix(ins, code); + code = AddRexWPrefix(id, code); } } else if ((ins == INS_movsx) || (ins == INS_movzx) || (insIsCMOV(ins))) { assert(hasCodeRM(ins) && !hasCodeMI(ins) && !hasCodeMR(ins)); code = insCodeRM(ins); - code = AddSimdPrefixIfNeeded(ins, code, size); - code = insEncodeRMreg(ins, code) | (int)(size == EA_2BYTE); + code = AddSimdPrefixIfNeeded(id, code, size); + code = insEncodeRMreg(id, code) | (int)(size == EA_2BYTE); #ifdef TARGET_AMD64 assert((size < EA_4BYTE) || (insIsCMOV(ins))); if ((size == EA_8BYTE) || (ins == INS_movsx)) { - code = AddRexWPrefix(ins, code); + code = AddRexWPrefix(id, code); } } else if (ins == INS_movsxd) { assert(hasCodeRM(ins) && !hasCodeMI(ins) && !hasCodeMR(ins)); code = insCodeRM(ins); - code = AddSimdPrefixIfNeeded(ins, code, size); - code = insEncodeRMreg(ins, code); + code = AddSimdPrefixIfNeeded(id, code, size); + code = insEncodeRMreg(id, code); #endif // TARGET_AMD64 } @@ -13473,8 +13607,8 @@ BYTE* emitter::emitOutputRR(BYTE* dst, instrDesc* id) { assert(hasCodeRM(ins) && !hasCodeMI(ins) && !hasCodeMR(ins)); code = insCodeRM(ins); - code = AddSimdPrefixIfNeeded(ins, code, size); - code = insEncodeRMreg(ins, code); + code = AddSimdPrefixIfNeeded(id, code, size); + code = insEncodeRMreg(id, code); if ((ins == INS_crc32) && (size > EA_1BYTE)) { code |= 0x0100; @@ -13487,15 +13621,15 @@ BYTE* emitter::emitOutputRR(BYTE* dst, instrDesc* id) } else if (size == EA_8BYTE) { - code = AddRexWPrefix(ins, code); + code = AddRexWPrefix(id, code); } } #endif // FEATURE_HW_INTRINSICS else { - assert(!TakesSimdPrefix(ins)); + assert(!TakesSimdPrefix(id)); code = insCodeMR(ins); - code = insEncodeMRreg(ins, code); + code = insEncodeMRreg(id, code); if (ins != INS_test) { @@ -13525,7 +13659,7 @@ BYTE* emitter::emitOutputRR(BYTE* dst, instrDesc* id) // Don't need to zero out the high bits explicitly if ((ins != INS_xor) || (reg1 != reg2)) { - code = AddRexWPrefix(ins, code); + code = AddRexWPrefix(id, code); } else { @@ -13562,10 +13696,10 @@ BYTE* emitter::emitOutputRR(BYTE* dst, instrDesc* id) } } - unsigned regCode = insEncodeReg345(ins, regFor345Bits, size, &code); - regCode |= insEncodeReg012(ins, regFor012Bits, size, &code); + unsigned regCode = insEncodeReg345(id, regFor345Bits, size, &code); + regCode |= insEncodeReg012(id, regFor012Bits, size, &code); - if (TakesSimdPrefix(ins)) + if (TakesSimdPrefix(id)) { // In case of AVX instructions that take 3 operands, we generally want to encode reg1 // as first source. In this case, reg1 is both a source and a destination. @@ -13577,12 +13711,12 @@ BYTE* emitter::emitOutputRR(BYTE* dst, instrDesc* id) if (IsDstDstSrcAVXInstruction(ins)) { // encode source/dest operand reg in 'vvvv' bits in 1's complement form - code = insEncodeReg3456(ins, reg1, size, code); + code = insEncodeReg3456(id, reg1, size, code); } else if (IsDstSrcSrcAVXInstruction(ins)) { // encode source operand reg in 'vvvv' bits in 1's complement form - code = insEncodeReg3456(ins, reg2, size, code); + code = insEncodeReg3456(id, reg2, size, code); } } @@ -13824,21 +13958,21 @@ BYTE* emitter::emitOutputRRR(BYTE* dst, instrDesc* id) emitAttr size = id->idOpSize(); code = insCodeRM(ins); - code = AddSimdPrefixIfNeeded(ins, code, size); + code = AddSimdPrefixIfNeeded(id, code, size); - code = insEncodeRMreg(ins, code); + code = insEncodeRMreg(id, code); // TODO-XARCH-AVX512 : Update this check once all paths have EVEX support. // Explore moving IsWEvexOpcodeExtension() logic to instruction table as flag. - if (TakesRexWPrefix(ins, size) || (codeEvexMigrationCheck(code) && IsWEvexOpcodeExtension(ins))) + if (TakesRexWPrefix(ins, size) || (codeEvexMigrationCheck(code) && IsWEvexOpcodeExtension(id))) { - code = AddRexWPrefix(ins, code); + code = AddRexWPrefix(id, code); } - unsigned regCode = insEncodeReg345(ins, targetReg, size, &code); - regCode |= insEncodeReg012(ins, src2, size, &code); + unsigned regCode = insEncodeReg345(id, targetReg, size, &code); + regCode |= insEncodeReg012(id, src2, size, &code); // encode source operand reg in 'vvvv' bits in 1's complement form - code = insEncodeReg3456(ins, src1, size, code); + code = insEncodeReg3456(id, src1, size, code); // Output the REX/VEX/EVEX prefix dst += emitOutputSimdPrefixIfNeeded(ins, dst, code); @@ -13932,17 +14066,17 @@ BYTE* emitter::emitOutputRI(BYTE* dst, instrDesc* id) // Get the 'base' opcode. code = insCodeMI(ins); - code = AddSimdPrefixIfNeeded(ins, code, size); - code = insEncodeMIreg(ins, reg, size, code); + code = AddSimdPrefixIfNeeded(id, code, size); + code = insEncodeMIreg(id, reg, size, code); assert(code & 0x00FF0000); - if (TakesSimdPrefix(ins)) + if (TakesSimdPrefix(id)) { // The 'vvvv' bits encode the destination register, which for this case (RI) // is the same as the source. - code = insEncodeReg3456(ins, reg, size, code); + code = insEncodeReg3456(id, reg, size, code); } - unsigned regcode = (insEncodeReg345(ins, regOpcode, size, &code) | insEncodeReg012(ins, reg, size, &code)) << 8; + unsigned regcode = (insEncodeReg345(id, regOpcode, size, &code) | insEncodeReg012(id, reg, size, &code)) << 8; // Output the REX prefix dst += emitOutputSimdPrefixIfNeeded(ins, dst, code); @@ -13970,15 +14104,15 @@ BYTE* emitter::emitOutputRI(BYTE* dst, instrDesc* id) assert(code < 0x100); code |= 0x08; // Set the 'w' bit - unsigned regcode = insEncodeReg012(ins, reg, size, &code); + unsigned regcode = insEncodeReg012(id, reg, size, &code); code |= regcode; // This is INS_mov and will not take VEX prefix assert(!TakesVexPrefix(ins)); - if (TakesRexWPrefix(ins, size) || (codeEvexMigrationCheck(code) && IsWEvexOpcodeExtension(ins))) + if (TakesRexWPrefix(ins, size) || (codeEvexMigrationCheck(code) && IsWEvexOpcodeExtension(id))) { - code = AddRexWPrefix(ins, code); + code = AddRexWPrefix(id, code); } dst += emitOutputSimdPrefixIfNeeded(ins, dst, code); @@ -14070,13 +14204,13 @@ BYTE* emitter::emitOutputRI(BYTE* dst, instrDesc* id) // r/m, immed form, but do have a dstReg,srcReg,imm8 form. if (valInByte && useSigned && insNeedsRRIb(ins)) { - code = insEncodeRRIb(ins, reg, size); + code = insEncodeRRIb(id, reg, size); } else { code = insCodeMI(ins); - code = AddSimdPrefixIfNeeded(ins, code, size); - code = insEncodeMIreg(ins, reg, size, code); + code = AddSimdPrefixIfNeeded(id, code, size); + code = insEncodeMIreg(id, reg, size, code); } } @@ -14100,7 +14234,7 @@ BYTE* emitter::emitOutputRI(BYTE* dst, instrDesc* id) /* Set the 'w' bit to get the large version */ /* and the REX.W bit to get the really large version */ - code = AddRexWPrefix(ins, code); + code = AddRexWPrefix(id, code); code |= 0x1; break; #endif @@ -14309,9 +14443,9 @@ BYTE* emitter::emitOutputIV(BYTE* dst, instrDesc* id) } else { - if (TakesRexWPrefix(ins, size) || (codeEvexMigrationCheck(code) && IsWEvexOpcodeExtension(ins))) + if (TakesRexWPrefix(ins, size) || (codeEvexMigrationCheck(code) && IsWEvexOpcodeExtension(id))) { - code = AddRexWPrefix(ins, code); + code = AddRexWPrefix(id, code); dst += emitOutputSimdPrefixIfNeeded(ins, dst, code); } @@ -14619,7 +14753,7 @@ BYTE* emitter::emitOutputLJ(insGroup* ig, BYTE* dst, instrDesc* i) idAmd->idCodeSize(sz); code = insCodeRM(ins); - code |= (insEncodeReg345(ins, id->idReg1(), EA_PTRSIZE, &code) << 8); + code |= (insEncodeReg345(id, id->idReg1(), EA_PTRSIZE, &code) << 8); dst = emitOutputAM(dst, idAmd, code, nullptr); @@ -14720,7 +14854,7 @@ ssize_t emitter::GetInputSizeInBytes(instrDesc* id) // ssize_t emitter::TryEvexCompressDisp8Byte(instrDesc* id, ssize_t dsp, bool* dspInByte) { - assert(TakesEvexPrefix(id->idIns())); + assert(TakesEvexPrefix(id)); insTupleType tt = insTupleTypeInfo(id->idIns()); assert(hasTupleTypeInfo(id->idIns())); @@ -14931,12 +15065,12 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) #ifdef TARGET_AMD64 // Support only scalar AVX instructions and hence size is hard coded to 4-byte. - code = AddSimdPrefixIfNeeded(ins, code, EA_4BYTE); + code = AddSimdPrefixIfNeeded(id, code, EA_4BYTE); if (((ins == INS_cdq) || (ins == INS_cwde)) && - (TakesRexWPrefix(ins, id->idOpSize()) || (codeEvexMigrationCheck(code) && IsWEvexOpcodeExtension(ins)))) + (TakesRexWPrefix(ins, id->idOpSize()) || (codeEvexMigrationCheck(code) && IsWEvexOpcodeExtension(id)))) { - code = AddRexWPrefix(ins, code); + code = AddRexWPrefix(id, code); } dst += emitOutputSimdPrefixIfNeeded(ins, dst, code); #endif @@ -15210,8 +15344,8 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) case IF_RRW_SHF: code = insCodeMR(ins); // Emit the VEX prefix if it exists - code = AddSimdPrefixIfNeeded(ins, code, size); - code = insEncodeMRreg(ins, id->idReg1(), size, code); + code = AddSimdPrefixIfNeeded(id, code, size); + code = insEncodeMRreg(id, id->idReg1(), size, code); // set the W bit if (size != EA_1BYTE) @@ -15220,9 +15354,9 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) } // Emit the REX prefix if it exists - if (TakesRexWPrefix(ins, size) || (codeEvexMigrationCheck(code) && IsWEvexOpcodeExtension(ins))) + if (TakesRexWPrefix(ins, size) || (codeEvexMigrationCheck(code) && IsWEvexOpcodeExtension(id))) { - code = AddRexWPrefix(ins, code); + code = AddRexWPrefix(id, code); } // Output a size prefix for a 16-bit operand @@ -15278,8 +15412,8 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) { code = insCodeMR(ins); // Emit the VEX prefix if it exists - code = AddSimdPrefixIfNeeded(ins, code, size); - code = insEncodeMRreg(ins, code); + code = AddSimdPrefixIfNeeded(id, code, size); + code = insEncodeMRreg(id, code); mReg = id->idReg1(); rReg = id->idReg2(); } @@ -15288,7 +15422,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) code = insCodeMI(ins); // Emit the VEX prefix if it exists - code = AddSimdPrefixIfNeeded(ins, code, size); + code = AddSimdPrefixIfNeeded(id, code, size); assert((code & 0xC000) == 0); code |= 0xC000; @@ -15302,19 +15436,19 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) { code = insCodeRM(ins); // Emit the VEX prefix if it exists - code = AddSimdPrefixIfNeeded(ins, code, size); - code = insEncodeRMreg(ins, code); + code = AddSimdPrefixIfNeeded(id, code, size); + code = insEncodeRMreg(id, code); mReg = id->idReg2(); rReg = id->idReg1(); } assert(code & 0x00FF0000); - if (TakesRexWPrefix(ins, size) || (codeEvexMigrationCheck(code) && IsWEvexOpcodeExtension(ins))) + if (TakesRexWPrefix(ins, size) || (codeEvexMigrationCheck(code) && IsWEvexOpcodeExtension(id))) { - code = AddRexWPrefix(ins, code); + code = AddRexWPrefix(id, code); } - if (TakesSimdPrefix(ins)) + if (TakesSimdPrefix(id)) { if (IsDstDstSrcAVXInstruction(ins)) { @@ -15324,17 +15458,17 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) // (Though we will need to handle the few ops that can have the 'vvvv' bits as destination, // e.g. pslldq, when/if we support those instructions with 2 registers.) // (see x64 manual Table 2-9. Instructions with a VEX.vvvv destination) - code = insEncodeReg3456(ins, id->idReg1(), size, code); + code = insEncodeReg3456(id, id->idReg1(), size, code); } else if (IsDstSrcSrcAVXInstruction(ins)) { // This is a "merge" move instruction. // Encode source operand reg in 'vvvv' bits in 1's complement form - code = insEncodeReg3456(ins, id->idReg2(), size, code); + code = insEncodeReg3456(id, id->idReg2(), size, code); } } - regcode = (insEncodeReg345(ins, rReg, size, &code) | insEncodeReg012(ins, mReg, size, &code)); + regcode = (insEncodeReg345(id, rReg, size, &code) | insEncodeReg012(id, mReg, size, &code)); // Output the REX prefix dst += emitOutputSimdPrefixIfNeeded(ins, dst, code); @@ -15449,8 +15583,8 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) } else { - code = AddSimdPrefixIfNeeded(ins, code, size); - regcode = (insEncodeReg345(ins, id->idReg1(), size, &code) << 8); + code = AddSimdPrefixIfNeeded(id, code, size); + regcode = (insEncodeReg345(id, id->idReg1(), size, &code) << 8); dst = emitOutputAM(dst, id, code | regcode, &cnsVal); } @@ -15477,8 +15611,8 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) } else { - code = AddSimdPrefixIfNeeded(ins, code, size); - regcode = (insEncodeReg345(ins, id->idReg1(), size, &code) << 8); + code = AddSimdPrefixIfNeeded(id, code, size); + regcode = (insEncodeReg345(id, id->idReg1(), size, &code) << 8); dst = emitOutputAM(dst, id, code | regcode); } sz = emitSizeOfInsDsc(id); @@ -15506,8 +15640,8 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) } else { - code = AddSimdPrefixIfNeeded(ins, code, size); - regcode = (insEncodeReg345(ins, id->idReg1(), size, &code) << 8); + code = AddSimdPrefixIfNeeded(id, code, size); + regcode = (insEncodeReg345(id, id->idReg1(), size, &code) << 8); dst = emitOutputAM(dst, id, code | regcode, &cnsVal); } sz = emitSizeOfInsDsc(id); @@ -15518,8 +15652,8 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) case IF_AWR_RRD: case IF_ARW_RRD: code = insCodeMR(ins); - code = AddSimdPrefixIfNeeded(ins, code, size); - regcode = (insEncodeReg345(ins, id->idReg1(), size, &code) << 8); + code = AddSimdPrefixIfNeeded(id, code, size); + regcode = (insEncodeReg345(id, id->idReg1(), size, &code) << 8); dst = emitOutputAM(dst, id, code | regcode); sz = emitSizeOfInsDsc(id); break; @@ -15527,7 +15661,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) case IF_AWR_RRD_RRD: { code = insCodeMR(ins); - code = AddSimdPrefixIfNeeded(ins, code, size); + code = AddSimdPrefixIfNeeded(id, code, size); dst = emitOutputAM(dst, id, code); sz = emitSizeOfInsDsc(id); break; @@ -15617,7 +15751,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) } else { - code = AddSimdPrefixIfNeeded(ins, code, size); + code = AddSimdPrefixIfNeeded(id, code, size); // In case of AVX instructions that take 3 operands, encode reg1 as first source. // Note that reg1 is both a source and a destination. @@ -15628,10 +15762,10 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) if (IsDstDstSrcAVXInstruction(ins)) { // encode source operand reg in 'vvvv' bits in 1's complement form - code = insEncodeReg3456(ins, id->idReg1(), size, code); + code = insEncodeReg3456(id, id->idReg1(), size, code); } - regcode = (insEncodeReg345(ins, id->idReg1(), size, &code) << 8); + regcode = (insEncodeReg345(id, id->idReg1(), size, &code) << 8); dst = emitOutputSV(dst, id, code | regcode, &cnsVal); } @@ -15652,15 +15786,15 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) } else { - code = AddSimdPrefixIfNeeded(ins, code, size); + code = AddSimdPrefixIfNeeded(id, code, size); if (IsDstDstSrcAVXInstruction(ins)) { // encode source operand reg in 'vvvv' bits in 1's complement form - code = insEncodeReg3456(ins, id->idReg1(), size, code); + code = insEncodeReg3456(id, id->idReg1(), size, code); } - regcode = (insEncodeReg345(ins, id->idReg1(), size, &code) << 8); + regcode = (insEncodeReg345(id, id->idReg1(), size, &code) << 8); dst = emitOutputSV(dst, id, code | regcode); } @@ -15673,8 +15807,8 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) assert(IsVexOrEvexEncodedInstruction(ins)); code = insCodeRM(ins); - code = AddSimdPrefixIfNeeded(ins, code, size); - code = insEncodeReg3456(ins, id->idReg2(), size, + code = AddSimdPrefixIfNeeded(id, code, size); + code = insEncodeReg3456(id, id->idReg2(), size, code); // encode source operand reg in 'vvvv' bits in 1's complement form // 4-byte AVX instructions are special cased inside emitOutputSV @@ -15685,7 +15819,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) } else { - regcode = (insEncodeReg345(ins, id->idReg1(), size, &code) << 8); + regcode = (insEncodeReg345(id, id->idReg1(), size, &code) << 8); dst = emitOutputSV(dst, id, code | regcode); } break; @@ -15699,8 +15833,8 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) emitGetInsCns(id, &cnsVal); code = insCodeRM(ins); - code = AddSimdPrefixIfNeeded(ins, code, size); - code = insEncodeReg3456(ins, id->idReg2(), size, + code = AddSimdPrefixIfNeeded(id, code, size); + code = insEncodeReg3456(id, id->idReg2(), size, code); // encode source operand reg in 'vvvv' bits in 1's complement form // 4-byte AVX instructions are special cased inside emitOutputSV @@ -15711,7 +15845,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) } else { - regcode = (insEncodeReg345(ins, id->idReg1(), size, &code) << 8); + regcode = (insEncodeReg345(id, id->idReg1(), size, &code) << 8); dst = emitOutputSV(dst, id, code | regcode, &cnsVal); } @@ -15723,7 +15857,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) case IF_SWR_RRD: case IF_SRW_RRD: code = insCodeMR(ins); - code = AddSimdPrefixIfNeeded(ins, code, size); + code = AddSimdPrefixIfNeeded(id, code, size); // In case of AVX instructions that take 3 operands, encode reg1 as first source. // Note that reg1 is both a source and a destination. @@ -15734,10 +15868,10 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) if (IsDstDstSrcAVXInstruction(ins)) { // encode source operand reg in 'vvvv' bits in 1's complement form - code = insEncodeReg3456(ins, id->idReg1(), size, code); + code = insEncodeReg3456(id, id->idReg1(), size, code); } - regcode = (insEncodeReg345(ins, id->idReg1(), size, &code) << 8); + regcode = (insEncodeReg345(id, id->idReg1(), size, &code) << 8); dst = emitOutputSV(dst, id, code | regcode); break; @@ -15771,7 +15905,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) } else { - code = AddSimdPrefixIfNeeded(ins, code, size); + code = AddSimdPrefixIfNeeded(id, code, size); // In case of AVX instructions that take 3 operands, encode reg1 as first source. // Note that reg1 is both a source and a destination. @@ -15782,10 +15916,10 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) if (IsDstDstSrcAVXInstruction(ins)) { // encode source operand reg in 'vvvv' bits in 1's complement form - code = insEncodeReg3456(ins, id->idReg1(), size, code); + code = insEncodeReg3456(id, id->idReg1(), size, code); } - regcode = (insEncodeReg345(ins, id->idReg1(), size, &code) << 8); + regcode = (insEncodeReg345(id, id->idReg1(), size, &code) << 8); dst = emitOutputCV(dst, id, code | regcode | 0x0500, &cnsVal); } @@ -15816,15 +15950,15 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) } else { - code = AddSimdPrefixIfNeeded(ins, code, size); + code = AddSimdPrefixIfNeeded(id, code, size); if (IsDstDstSrcAVXInstruction(ins)) { // encode source operand reg in 'vvvv' bits in 1's complement form - code = insEncodeReg3456(ins, id->idReg1(), size, code); + code = insEncodeReg3456(id, id->idReg1(), size, code); } - regcode = (insEncodeReg345(ins, id->idReg1(), size, &code) << 8); + regcode = (insEncodeReg345(id, id->idReg1(), size, &code) << 8); dst = emitOutputCV(dst, id, code | regcode | 0x0500); } @@ -15838,8 +15972,8 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) assert(IsVexOrEvexEncodedInstruction(ins)); code = insCodeRM(ins); - code = AddSimdPrefixIfNeeded(ins, code, size); - code = insEncodeReg3456(ins, id->idReg2(), size, + code = AddSimdPrefixIfNeeded(id, code, size); + code = insEncodeReg3456(id, id->idReg2(), size, code); // encode source operand reg in 'vvvv' bits in 1's complement form // Special case 4-byte AVX instructions @@ -15849,7 +15983,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) } else { - regcode = (insEncodeReg345(ins, id->idReg1(), size, &code) << 8); + regcode = (insEncodeReg345(id, id->idReg1(), size, &code) << 8); dst = emitOutputCV(dst, id, code | regcode | 0x0500); } sz = emitSizeOfInsDsc(id); @@ -15864,8 +15998,8 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) emitGetInsCns(id, &cnsVal); code = insCodeRM(ins); - code = AddSimdPrefixIfNeeded(ins, code, size); - code = insEncodeReg3456(ins, id->idReg2(), size, + code = AddSimdPrefixIfNeeded(id, code, size); + code = insEncodeReg3456(id, id->idReg2(), size, code); // encode source operand reg in 'vvvv' bits in 1's complement form // Special case 4-byte AVX instructions @@ -15875,7 +16009,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) } else { - regcode = (insEncodeReg345(ins, id->idReg1(), size, &code) << 8); + regcode = (insEncodeReg345(id, id->idReg1(), size, &code) << 8); dst = emitOutputCV(dst, id, code | regcode | 0x0500, &cnsVal); } sz = emitSizeOfInsDsc(id); @@ -15884,7 +16018,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) case IF_RWR_MRD_OFF: code = insCode(ins); - code = AddSimdPrefixIfNeeded(ins, code, size); + code = AddSimdPrefixIfNeeded(id, code, size); // In case of AVX instructions that take 3 operands, encode reg1 as first source. // Note that reg1 is both a source and a destination. @@ -15895,10 +16029,10 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) if (IsDstDstSrcAVXInstruction(ins)) { // encode source operand reg in 'vvvv' bits in 1's complement form - code = insEncodeReg3456(ins, id->idReg1(), size, code); + code = insEncodeReg3456(id, id->idReg1(), size, code); } - regcode = insEncodeReg012(id->idIns(), id->idReg1(), size, &code); + regcode = insEncodeReg012(id, id->idReg1(), size, &code); dst = emitOutputCV(dst, id, code | 0x30 | regcode); sz = emitSizeOfInsDsc(id); break; @@ -15907,7 +16041,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) case IF_MWR_RRD: case IF_MRW_RRD: code = insCodeMR(ins); - code = AddSimdPrefixIfNeeded(ins, code, size); + code = AddSimdPrefixIfNeeded(id, code, size); // In case of AVX instructions that take 3 operands, encode reg1 as first source. // Note that reg1 is both a source and a destination. @@ -15918,10 +16052,10 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) if (IsDstDstSrcAVXInstruction(ins)) { // encode source operand reg in 'vvvv' bits in 1's complement form - code = insEncodeReg3456(ins, id->idReg1(), size, code); + code = insEncodeReg3456(id, id->idReg1(), size, code); } - regcode = (insEncodeReg345(ins, id->idReg1(), size, &code) << 8); + regcode = (insEncodeReg345(id, id->idReg1(), size, &code) << 8); dst = emitOutputCV(dst, id, code | regcode | 0x0500); sz = emitSizeOfInsDsc(id); break; diff --git a/src/coreclr/jit/emitxarch.h b/src/coreclr/jit/emitxarch.h index 13f79bc3833cca..dc6fe3fed49238 100644 --- a/src/coreclr/jit/emitxarch.h +++ b/src/coreclr/jit/emitxarch.h @@ -44,7 +44,7 @@ UNATIVE_OFFSET emitInsSizeSV(instrDesc* id, code_t code, int var, int dsp); UNATIVE_OFFSET emitInsSizeSV(instrDesc* id, code_t code, int var, int dsp, int val); UNATIVE_OFFSET emitInsSizeRR(instrDesc* id, code_t code); UNATIVE_OFFSET emitInsSizeRR(instrDesc* id, code_t code, int val); -UNATIVE_OFFSET emitInsSizeRR(instruction ins, regNumber reg1, regNumber reg2, emitAttr attr); +UNATIVE_OFFSET emitInsSizeRR(instrDesc* id, regNumber reg1, regNumber reg2, emitAttr attr); UNATIVE_OFFSET emitInsSizeAM(instrDesc* id, code_t code); UNATIVE_OFFSET emitInsSizeAM(instrDesc* id, code_t code, int val); UNATIVE_OFFSET emitInsSizeCV(instrDesc* id, code_t code); @@ -72,18 +72,18 @@ unsigned emitGetVexPrefixSize(instruction ins, emitAttr attr); unsigned emitGetEvexPrefixSize(instruction ins); unsigned emitGetPrefixSize(code_t code, bool includeRexPrefixSize); unsigned emitGetAdjustedSize(instruction ins, emitAttr attr, code_t code); -unsigned emitGetAdjustedSizeEvexAware(instruction ins, emitAttr attr, code_t code); +unsigned emitGetAdjustedSizeEvexAware(const instrDesc* id, emitAttr attr, code_t code); -unsigned insEncodeReg012(instruction ins, regNumber reg, emitAttr size, code_t* code); -unsigned insEncodeReg345(instruction ins, regNumber reg, emitAttr size, code_t* code); -code_t insEncodeReg3456(instruction ins, regNumber reg, emitAttr size, code_t code); -unsigned insEncodeRegSIB(instruction ins, regNumber reg, code_t* code); +unsigned insEncodeReg012(const instrDesc* id, regNumber reg, emitAttr size, code_t* code); +unsigned insEncodeReg345(const instrDesc* id, regNumber reg, emitAttr size, code_t* code); +code_t insEncodeReg3456(const instrDesc* id, regNumber reg, emitAttr size, code_t code); +unsigned insEncodeRegSIB(const instrDesc* id, regNumber reg, code_t* code); -code_t insEncodeMRreg(instruction ins, code_t code); -code_t insEncodeRMreg(instruction ins, code_t code); -code_t insEncodeMRreg(instruction ins, regNumber reg, emitAttr size, code_t code); -code_t insEncodeRRIb(instruction ins, regNumber reg, emitAttr size); -code_t insEncodeOpreg(instruction ins, regNumber reg, emitAttr size); +code_t insEncodeMRreg(const instrDesc* id, code_t code); +code_t insEncodeRMreg(const instrDesc* id, code_t code); +code_t insEncodeMRreg(const instrDesc* id, regNumber reg, emitAttr size, code_t code); +code_t insEncodeRRIb(const instrDesc* id, regNumber reg, emitAttr size); +code_t insEncodeOpreg(const instrDesc* id, regNumber reg, emitAttr size); unsigned insSSval(unsigned scale); @@ -102,14 +102,17 @@ bool IsVexEncodedInstruction(instruction ins) const; bool IsEvexEncodedInstruction(instruction ins) const; bool IsVexOrEvexEncodedInstruction(instruction ins) const; -code_t insEncodeMIreg(instruction ins, regNumber reg, emitAttr size, code_t code); +code_t insEncodeMIreg(const instrDesc* id, regNumber reg, emitAttr size, code_t code); -code_t AddRexWPrefix(instruction ins, code_t code); -code_t AddRexRPrefix(instruction ins, code_t code); -code_t AddRexXPrefix(instruction ins, code_t code); -code_t AddRexBPrefix(instruction ins, code_t code); +code_t AddRexWPrefix(const instrDesc* id, code_t code); +code_t AddRexRPrefix(const instrDesc* id, code_t code); +code_t AddRexXPrefix(const instrDesc* id, code_t code); +code_t AddRexBPrefix(const instrDesc* id, code_t code); code_t AddRexPrefix(instruction ins, code_t code); +code_t AddEvexVPrimePrefix(code_t code); +code_t AddEvexRPrimePrefix(code_t code); + bool EncodedBySSE38orSSE3A(instruction ins); bool Is4ByteSSEInstruction(instruction ins); static bool IsMovInstruction(instruction ins); @@ -180,13 +183,15 @@ code_t AddVexPrefixIfNeededAndNotPresent(instruction ins, code_t code, emitAttr // Returns: // `true` if W bit needs to be set to 1. // -bool IsWEvexOpcodeExtension(instruction ins) +bool IsWEvexOpcodeExtension(const instrDesc* id) { - if (!TakesEvexPrefix(ins)) + if (!TakesEvexPrefix(id)) { return false; } + instruction ins = id->idIns(); + switch (ins) { case INS_movq: @@ -485,7 +490,7 @@ bool UseSimdEncoding() const #define EVEX_PREFIX_MASK 0xFF00000000000000ULL #define EVEX_PREFIX_CODE 0x6200000000000000ULL -bool TakesEvexPrefix(instruction ins) const; +bool TakesEvexPrefix(const instrDesc* id) const; //------------------------------------------------------------------------ // hasEvexPrefix: Returns true if the instruction encoding already @@ -513,9 +518,13 @@ code_t AddEvexPrefix(instruction ins, code_t code, emitAttr attr); // // Returns: // code with prefix added. -code_t AddSimdPrefixIfNeeded(instruction ins, code_t code, emitAttr size) +// TODO-XARCH-AVX512 come back and check whether we can id `id` directly (no need) +// to pass emitAttr size +code_t AddSimdPrefixIfNeeded(const instrDesc* id, code_t code, emitAttr size) { - if (TakesEvexPrefix(ins)) + instruction ins = id->idIns(); + + if (TakesEvexPrefix(id)) { code = AddEvexPrefix(ins, code, size); } @@ -536,11 +545,14 @@ code_t AddSimdPrefixIfNeeded(instruction ins, code_t code, emitAttr size) // size - operand size // // Returns: -// `true` if code has an Evex prefix. -// -code_t AddSimdPrefixIfNeededAndNotPresent(instruction ins, code_t code, emitAttr size) +// TRUE if code has an Evex prefix. +// TODO-XARCH-AVX512 come back and check whether we can id `id` directly (no need) +// to pass emitAttr size +code_t AddSimdPrefixIfNeededAndNotPresent(const instrDesc* id, code_t code, emitAttr size) { - if (TakesEvexPrefix(ins)) + instruction ins = id->idIns(); + + if (TakesEvexPrefix(id)) { code = !hasEvexPrefix(code) ? AddEvexPrefix(ins, code, size) : code; } @@ -551,7 +563,7 @@ code_t AddSimdPrefixIfNeededAndNotPresent(instruction ins, code_t code, emitAttr return code; } -bool TakesSimdPrefix(instruction ins) const; +bool TakesSimdPrefix(const instrDesc* id) const; //------------------------------------------------------------------------ // hasVexOrEvexPrefix: Returns true if the instruction encoding already @@ -1023,4 +1035,7 @@ inline bool HasEmbeddedBroadcast(instrDesc* id) return false; } +inline bool HasHighSIMDReg(const instrDesc* id) const; +inline bool IsHighSIMDReg(regNumber) const; + #endif // TARGET_XARCH