Enable AVX512 Additional 16 SIMD Registers

eng/pipelines/common/templates/runtimes/run-test-job.yml

@@ -536,6 +536,7 @@ jobs:
         ${{ if in(parameters.testGroup, 'jitstress-isas-avx512') }}:
           scenarios:
           - jitstress_isas_avx512_forceevex
+          - jitstress_isas_avx512_forceevex_stresshighregs
         ${{ if in(parameters.testGroup, 'jitstressregs-x86') }}:
           scenarios:
           - jitstressregs1_x86_noavx

src/coreclr/jit/codegen.h

@@ -35,6 +35,17 @@ class CodeGen final : public CodeGenInterface
         GenTree* addr, bool fold, bool* revPtr, GenTree** rv1Ptr, GenTree** rv2Ptr, unsigned* mulPtr, ssize_t* cnsPtr);
 
 private:
+#if defined(TARGET_AMD64)
+    regMaskTP get_RBM_ALLFLOAT() const
+    {
+        return compiler->rbmAllFloat;
+    }
+    regMaskTP get_RBM_FLT_CALLEE_TRASH() const
+    {
+        return compiler->rbmFltCalleeTrash;
+    }
+#endif // TARGET_AMD64
+
 #if defined(TARGET_XARCH)
     // Bit masks used in negating a float or double number.
     // This is to avoid creating more than one data constant for these bitmasks when a

src/coreclr/jit/codegenxarch.cpp

@@ -3524,7 +3524,7 @@ void CodeGen::genStructPutArgUnroll(GenTreePutArgStk* putArgNode)
         //       this probably needs to be changed.
 
         // Load
-        genCodeForLoadOffset(INS_movdqu, EA_8BYTE, xmmTmpReg, src, offset);
+        genCodeForLoadOffset(INS_movdqu, EA_16BYTE, xmmTmpReg, src, offset);
         // Store
         genStoreRegToStackArg(TYP_STRUCT, xmmTmpReg, offset);
 
@@ -8347,7 +8347,7 @@ void CodeGen::genStoreRegToStackArg(var_types type, regNumber srcReg, int offset
     {
         ins = INS_movdqu;
         // This should be changed!
-        attr = EA_8BYTE;
+        attr = EA_16BYTE;
         size = 16;
     }
     else

src/coreclr/jit/compiler.cpp

@@ -3325,6 +3325,24 @@ void Compiler::compInitOptions(JitFlags* jitFlags)
         opts.compJitSaveFpLrWithCalleeSavedRegisters = JitConfig.JitSaveFpLrWithCalleeSavedRegisters();
     }
 #endif // defined(DEBUG) && defined(TARGET_ARM64)
+
+#if defined(TARGET_AMD64)
+    rbmAllFloat         = RBM_ALLFLOAT_INIT;
+    rbmFltCalleeTrash   = RBM_FLT_CALLEE_TRASH_INIT;
+    cntCalleeTrashFloat = CNT_CALLEE_TRASH_FLOAT_INIT;
+    availableRegCount   = ACTUAL_REG_COUNT;
+
+    if (DoJitStressEvexEncoding())
+    {
+        rbmAllFloat |= RBM_HIGHFLOAT;
+        rbmFltCalleeTrash |= RBM_HIGHFLOAT;
+        cntCalleeTrashFloat += CNT_CALLEE_TRASH_HIGHFLOAT;
+    }
+    else
+    {
+        availableRegCount -= CNT_HIGHFLOAT;
+    }
+#endif // TARGET_AMD64
 }
 
 #ifdef DEBUG
@@ -3528,6 +3546,37 @@ bool Compiler::compPromoteFewerStructs(unsigned lclNum)
     return rejectThisPromo;
 }
 
+//------------------------------------------------------------------------
+// dumpRegMask: display a register mask. For well-known sets of registers, display a well-known token instead of
+// a potentially large number of registers.
+//
+// Arguments:
+//   regs - The set of registers to display
+//
+void Compiler::dumpRegMask(regMaskTP regs) const
+{
+    if (regs == RBM_ALLINT)
+    {
+        printf("[allInt]");
+    }
+    else if (regs == (RBM_ALLINT & ~RBM_FPBASE))
+    {
+        printf("[allIntButFP]");
+    }
+    else if (regs == RBM_ALLFLOAT)
+    {
+        printf("[allFloat]");
+    }
+    else if (regs == RBM_ALLDOUBLE)
+    {
+        printf("[allDouble]");
+    }
+    else
+    {
+        dspRegMask(regs);
+    }
+}
+
 #endif // DEBUG
 
 void Compiler::compInitDebuggingInfo()

src/coreclr/jit/compiler.h

@@ -10361,6 +10361,8 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
 
     bool compJitHaltMethod();
 
+    void dumpRegMask(regMaskTP regs) const;
+
 #endif
 
     /*
@@ -10635,6 +10637,48 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
     GenTree* fgMorphMultiregStructArg(CallArg* arg);
 
     bool killGCRefs(GenTree* tree);
+
+#if defined(TARGET_AMD64)
+private:
+    // The following are for initializing register allocator "constants" defined in targetamd64.h
+    // that now depend upon runtime ISA information, e.g., the presence of AVX512F/VL, which increases
+    // the number of SIMD (xmm, ymm, and zmm) registers from 16 to 32.
+    // As only 64-bit xarch has the capability to have the additional registers, we limit the changes
+    // to TARGET_AMD64 only.
+    //
+    // Users of these values need to define four accessor functions:
+    //
+    //    regMaskTP get_RBM_ALLFLOAT();
+    //    regMaskTP get_RBM_FLT_CALLEE_TRASH();
+    //    unsigned get_CNT_CALLEE_TRASH_FLOAT();
+    //    unsigned get_AVAILABLE_REG_COUNT();
+    //
+    // which return the values of these variables.
+    //
+    // This was done to avoid polluting all `targetXXX.h` macro definitions with a compiler parameter, where only
+    // TARGET_AMD64 requires one.
+    //
+    regMaskTP rbmAllFloat;
+    regMaskTP rbmFltCalleeTrash;
+    unsigned  cntCalleeTrashFloat;
+    unsigned  availableRegCount;
+
+public:
+    regMaskTP get_RBM_ALLFLOAT() const
+    {
+        return rbmAllFloat;
+    }
+    regMaskTP get_RBM_FLT_CALLEE_TRASH() const
+    {
+        return rbmFltCalleeTrash;
+    }
+    unsigned get_CNT_CALLEE_TRASH_FLOAT() const
+    {
+        return cntCalleeTrashFloat;
+    }
+
+#endif // TARGET_AMD64
+
 }; // end of class Compiler
 
 //---------------------------------------------------------------------------------------------------------------------

src/coreclr/jit/emit.cpp

@@ -99,6 +99,17 @@ void emitLocation::Print(LONG compMethodID) const
 }
 #endif // DEBUG
 
+#if defined(TARGET_AMD64)
+inline regMaskTP emitter::get_RBM_FLT_CALLEE_TRASH() const
+{
+    return emitComp->rbmFltCalleeTrash;
+}
+inline unsigned emitter::get_AVAILABLE_REG_COUNT() const
+{
+    return emitComp->availableRegCount;
+}
+#endif // TARGET_AMD64
+
 /*****************************************************************************
  *
  *  Return the name of an instruction format.
@@ -3204,11 +3215,19 @@ void emitter::emitDispRegSet(regMaskTP regs)
 
     for (reg = REG_FIRST; reg < ACTUAL_REG_COUNT; reg = REG_NEXT(reg))
     {
-        if ((regs & genRegMask(reg)) == 0)
+        if (regs == RBM_NONE)
+        {
+            break;
+        }
+
+        regMaskTP curReg = genRegMask(reg);
+        if ((regs & curReg) == 0)
         {
             continue;
         }
 
+        regs -= curReg;
+
         if (sp)
         {
             printf(" ");
@@ -3378,6 +3397,7 @@ emitter::instrDesc* emitter::emitNewInstrCallInd(int              argCnt,
 #endif // TARGET_XARCH
 
         /* Save the live GC registers in the unused register fields */
+        assert((gcrefRegs & RBM_CALLEE_TRASH) == 0);
         emitEncodeCallGCregs(gcrefRegs, id);
 
         return id;
@@ -3450,6 +3470,7 @@ emitter::instrDesc* emitter::emitNewInstrCallDir(int              argCnt,
         assert(!id->idIsLargeCns());
 
         /* Save the live GC registers in the unused register fields */
+        assert((gcrefRegs & RBM_CALLEE_TRASH) == 0);
         emitEncodeCallGCregs(gcrefRegs, id);
 
         return id;

src/coreclr/jit/emit.h

@@ -1121,6 +1121,28 @@ class emitter
             idAddr()->_idReg4 = reg;
             assert(reg == idAddr()->_idReg4);
         }
+        bool idHasReg3() const
+        {
+            switch (idInsFmt())
+            {
+                case IF_RWR_RRD_RRD:
+                case IF_RWR_RRD_RRD_CNS:
+                case IF_RWR_RRD_RRD_RRD:
+                    return true;
+                default:
+                    return false;
+            }
+        }
+        bool idHasReg4() const
+        {
+            switch (idInsFmt())
+            {
+                case IF_RWR_RRD_RRD_RRD:
+                    return true;
+                default:
+                    return false;
+            }
+        }
 #endif // defined(TARGET_XARCH)
 #ifdef TARGET_ARMARCH
         insOpts idInsOpt() const
@@ -1951,6 +1973,11 @@ class emitter
     CORINFO_FIELD_HANDLE emitBlkConst(const void* cnsAddr, unsigned cnsSize, unsigned cnsAlign, var_types elemType);
 
 private:
+#if defined(TARGET_AMD64)
+    regMaskTP get_RBM_FLT_CALLEE_TRASH() const;
+    unsigned  get_AVAILABLE_REG_COUNT() const;
+#endif // TARGET_AMD64
+
     CORINFO_FIELD_HANDLE emitFltOrDblConst(double constValue, emitAttr attr);
     CORINFO_FIELD_HANDLE emitSimd8Const(simd8_t constValue);
     CORINFO_FIELD_HANDLE emitSimd16Const(simd16_t constValue);

src/coreclr/jit/emitinl.h

@@ -211,11 +211,8 @@ inline ssize_t emitter::emitGetInsAmdAny(instrDesc* id)
  *
  *  Convert between a register mask and a smaller version for storage.
  */
-
 /*static*/ inline void emitter::emitEncodeCallGCregs(regMaskTP regmask, instrDesc* id)
 {
-    assert((regmask & RBM_CALLEE_TRASH) == 0);
-
     unsigned encodeMask;
 
 #ifdef TARGET_X86