Enable AVX512 Additional 16 SIMD Registers

eng/pipelines/common/templates/runtimes/run-test-job.yml

@@ -536,6 +536,7 @@ jobs:
         ${{ if in(parameters.testGroup, 'jitstress-isas-avx512') }}:
           scenarios:
           - jitstress_isas_avx512_forceevex
+          - jitstress_isas_avx512_forceevex_stresshighregs
         ${{ if in(parameters.testGroup, 'jitstressregs-x86') }}:
           scenarios:
           - jitstressregs1_x86_noavx

src/coreclr/jit/codegencommon.cpp

@@ -29,6 +29,13 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
 
 #include "patchpointinfo.h"
 
+// Please see the comment for these instance variables in `compiler.h`
+#if defined(TARGET_AMD64)
+#define RBM_ALLFLOAT_USE (this->rbmAllFloat)
+#define RBM_FLT_CALLEE_TRASH_USE (this->rbmFltCalleeTrash)
+#define CNT_CALLEE_TRASH_FLOAT_USE (this->cntCalleeTrashFloat)
+#endif
+
 /*****************************************************************************/
 
 void CodeGenInterface::setFramePointerRequiredEH(bool value)
@@ -775,6 +782,16 @@ void Compiler::compChangeLife(VARSET_VALARG_TP newLife)
     }
 }
 
+#undef RBM_ALLFLOAT_USE
+#undef RBM_FLT_CALLEE_TRASH_USE
+#undef CNT_CALLEE_TRASH_FLOAT_USE
+
+#if defined(TARGET_AMD64)
+#define RBM_ALLFLOAT_USE (compiler->rbmAllFloat)
+#define RBM_FLT_CALLEE_TRASH_USE (compiler->rbmFltCalleeTrash)
+#define CNT_CALLEE_TRASH_FLOAT_USE (compiler->cntCalleeTrashFloat)
+#endif
+
 // Need an explicit instantiation.
 template void Compiler::compChangeLife<true>(VARSET_VALARG_TP newLife);
 
@@ -9411,4 +9428,4 @@ bool CodeGen::genCanOmitNormalizationForBswap16(GenTree* tree)
     }
 
     return (cast->gtCastType == TYP_USHORT) || (cast->gtCastType == TYP_SHORT);
-}
+}

src/coreclr/jit/codegenlinear.cpp

@@ -17,6 +17,13 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
 #include "emit.h"
 #include "codegen.h"
 
+// Please see the comment for these instance variables in `compiler.h`
+#if defined(TARGET_AMD64)
+#define RBM_ALLFLOAT_USE (compiler->rbmAllFloat)
+#define RBM_FLT_CALLEE_TRASH_USE (compiler->rbmFltCalleeTrash)
+#define CNT_CALLEE_TRASH_FLOAT_USE (compiler->cntCalleeTrashFloat)
+#endif
+
 //------------------------------------------------------------------------
 // genInitializeRegisterState: Initialize the register state contained in 'regSet'.
 //
@@ -2683,4 +2690,4 @@ void CodeGen::genCodeForSetcc(GenTreeCC* setcc)
     inst_SETCC(setcc->gtCondition, setcc->TypeGet(), setcc->GetRegNum());
     genProduceReg(setcc);
 }
-#endif // !TARGET_LOONGARCH64
+#endif // !TARGET_LOONGARCH64

src/coreclr/jit/codegenxarch.cpp

@@ -23,6 +23,12 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
 #include "gcinfoencoder.h"
 #include "patchpointinfo.h"
 
+#if defined(TARGET_AMD64)
+#define RBM_ALLFLOAT_USE (compiler->rbmAllFloat)
+#define RBM_FLT_CALLEE_TRASH_USE (compiler->rbmFltCalleeTrash)
+#define CNT_CALLEE_TRASH_FLOAT_USE (compiler->cntCalleeTrashFloat)
+#endif
+
 //---------------------------------------------------------------------
 // genSetGSSecurityCookie: Set the "GS" security cookie in the prolog.
 //
@@ -3524,7 +3530,7 @@ void CodeGen::genStructPutArgUnroll(GenTreePutArgStk* putArgNode)
         //       this probably needs to be changed.
 
         // Load
-        genCodeForLoadOffset(INS_movdqu, EA_8BYTE, xmmTmpReg, src, offset);
+        genCodeForLoadOffset(INS_movdqu, EA_16BYTE, xmmTmpReg, src, offset);
         // Store
         genStoreRegToStackArg(TYP_STRUCT, xmmTmpReg, offset);
 
@@ -8347,7 +8353,7 @@ void CodeGen::genStoreRegToStackArg(var_types type, regNumber srcReg, int offset
     {
         ins = INS_movdqu;
         // This should be changed!
-        attr = EA_8BYTE;
+        attr = EA_16BYTE;
         size = 16;
     }
     else
@@ -11047,4 +11053,4 @@ bool CodeGenInterface::genCodeAddrNeedsReloc(size_t addr)
 #endif // TARGET_X86
 }
 
-#endif // TARGET_XARCH
+#endif // TARGET_XARCH

src/coreclr/jit/compiler.cpp

@@ -3325,6 +3325,24 @@ void Compiler::compInitOptions(JitFlags* jitFlags)
         opts.compJitSaveFpLrWithCalleeSavedRegisters = JitConfig.JitSaveFpLrWithCalleeSavedRegisters();
     }
 #endif // defined(DEBUG) && defined(TARGET_ARM64)
+
+#if defined(TARGET_AMD64)
+    rbmAllFloat         = RBM_ALLFLOAT_INIT;
+    rbmFltCalleeTrash   = RBM_FLT_CALLEE_TRASH_INIT;
+    cntCalleeTrashFloat = CNT_CALLEE_TRASH_FLOAT_INIT;
+    actualRegCount      = ACTUAL_REG_COUNT;
+
+    if (DoJitStressEvexEncoding())
+    {
+        rbmAllFloat |= RBM_HIGHFLOAT;
+        rbmFltCalleeTrash |= RBM_HIGHFLOAT;
+        cntCalleeTrashFloat += CNT_CALLEE_TRASH_HIGHFLOAT;
+    }
+    else
+    {
+        actualRegCount -= CNT_HIGHFLOAT;
+    }
+#endif // TARGET_AMD64
 }
 
 #ifdef DEBUG
@@ -10275,4 +10293,4 @@ void Compiler::EnregisterStats::Dump(FILE* fout) const
     PRINT_STATS(m_stressLclFld, m_addrExposed);
     PRINT_STATS(m_dispatchRetBuf, m_addrExposed);
 }
-#endif // TRACK_ENREG_STATS
+#endif // TRACK_ENREG_STATS

src/coreclr/jit/compiler.h

@@ -10635,6 +10635,25 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
     GenTree* fgMorphMultiregStructArg(CallArg* arg);
 
     bool killGCRefs(GenTree* tree);
+
+#if defined(TARGET_AMD64)
+public:
+    // The following are for initializing register allocator "constants" defined in targetamd64.h
+    // that now depend upon runtime ISA information, e.g., the presence of AVX512F/VL, which increases
+    // the number of simd (xmm,ymm, and zmm) registers from 16 to 32.
+    // As only 64-bit xarch has the capability to have the additional registers, we limit the changes
+    // to TARGET_AMD64 only.
+    //
+    // Users of `targetamd64.h` need to define three macros, RBM_ALLFLOAT_USE, RBM_FLT_CALLEE_TRASH_USE,
+    // and CNT_CALLEE_TRASH_FLOAT_USE which should point to these three variables respectively.
+    // We did this to avoid polluting all `targetXXX.h` macro definitions with a compiler parameter, where only
+    // TARGET_AMD64 requires one.
+    regMaskTP rbmAllFloat;
+    regMaskTP rbmFltCalleeTrash;
+    unsigned  cntCalleeTrashFloat;
+    unsigned  actualRegCount;
+#endif // TARGET_AMD64
+
 }; // end of class Compiler
 
 //---------------------------------------------------------------------------------------------------------------------

src/coreclr/jit/emit.cpp

@@ -20,6 +20,14 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
 #include "emit.h"
 #include "codegen.h"
 
+// Please see the comment for these instance variables in `compiler.h`
+#if defined(TARGET_AMD64)
+#define RBM_ALLFLOAT_USE (emitComp->rbmAllFloat)
+#define RBM_FLT_CALLEE_TRASH_USE (emitComp->rbmFltCalleeTrash)
+#define CNT_CALLEE_TRASH_FLOAT_USE (emitComp->cntCalleeTrashFloat)
+#define ACTUAL_REG_COUNT (emitComp->actualRegCount)
+#endif
+
 /*****************************************************************************
  *
  *  Represent an emitter location.
@@ -3378,6 +3386,7 @@ emitter::instrDesc* emitter::emitNewInstrCallInd(int              argCnt,
 #endif // TARGET_XARCH
 
         /* Save the live GC registers in the unused register fields */
+        assert((gcrefRegs & RBM_CALLEE_TRASH) == 0);
         emitEncodeCallGCregs(gcrefRegs, id);
 
         return id;
@@ -3450,6 +3459,7 @@ emitter::instrDesc* emitter::emitNewInstrCallDir(int              argCnt,
         assert(!id->idIsLargeCns());
 
         /* Save the live GC registers in the unused register fields */
+        assert((gcrefRegs & RBM_CALLEE_TRASH) == 0);
         emitEncodeCallGCregs(gcrefRegs, id);
 
         return id;
@@ -9945,3 +9955,7 @@ void emitter::emitEnableGC()
     }
 }
 #endif // !defined(JIT32_GCENCODER)
+
+#undef RBM_ALLFLOAT_USE
+#undef RBM_FLT_CALLEE_TRASH_USE
+#undef CNT_CALLEE_TRASH_FLOAT_USE

src/coreclr/jit/emit.h

@@ -1121,6 +1121,28 @@ class emitter
             idAddr()->_idReg4 = reg;
             assert(reg == idAddr()->_idReg4);
         }
+        bool idHasReg3() const
+        {
+            switch (idInsFmt())
+            {
+                case IF_RWR_RRD_RRD:
+                case IF_RWR_RRD_RRD_CNS:
+                case IF_RWR_RRD_RRD_RRD:
+                    return true;
+                default:
+                    return false;
+            }
+        }
+        bool idHasReg4() const
+        {
+            switch (idInsFmt())
+            {
+                case IF_RWR_RRD_RRD_RRD:
+                    return true;
+                default:
+                    return false;
+            }
+        }
 #endif // defined(TARGET_XARCH)
 #ifdef TARGET_ARMARCH
         insOpts idInsOpt() const

src/coreclr/jit/emitinl.h

@@ -7,6 +7,13 @@
 
 #ifdef TARGET_XARCH
 
+// Please see the comment for these instance variables in `compiler.h`
+#if defined(TARGET_AMD64)
+#define RBM_ALLFLOAT_USE (emitComp->rbmAllFloat)
+#define RBM_FLT_CALLEE_TRASH_USE (emitComp->rbmFltCalleeTrash)
+#define CNT_CALLEE_TRASH_FLOAT_USE (emitComp->cntCalleeTrashFloat)
+#endif
+
 /* static */
 inline bool emitter::instrIs3opImul(instruction ins)
 {
@@ -205,17 +212,18 @@ inline ssize_t emitter::emitGetInsAmdAny(instrDesc* id)
     return id->idAddr()->iiaAddrMode.amDisp;
 }
 
+#undef RBM_ALLFLOAT_USE
+#undef RBM_FLT_CALLEE_TRASH_USE
+#undef CNT_CALLEE_TRASH_FLOAT_USE
+
 #endif // TARGET_XARCH
 
 /*****************************************************************************
  *
  *  Convert between a register mask and a smaller version for storage.
  */
-
 /*static*/ inline void emitter::emitEncodeCallGCregs(regMaskTP regmask, instrDesc* id)
 {
-    assert((regmask & RBM_CALLEE_TRASH) == 0);
-
     unsigned encodeMask;
 
 #ifdef TARGET_X86