aarch64_vector.ad

//
// Copyright (c) 2020, 2024, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2020, 2024, Arm Limited. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
// under the terms of the GNU General Public License version 2 only, as
// published by the Free Software Foundation.
//
// This code is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
// version 2 for more details (a copy is included in the LICENSE file that
// accompanied this code).
//
// You should have received a copy of the GNU General Public License version
// 2 along with this work; if not, write to the Free Software Foundation,
// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
//
// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
// or visit www.oracle.com if you need additional information or have any
// questions.
//
//

// This file is automatically generated by running "m4 aarch64_vector_ad.m4". Do not edit!

// AArch64 VECTOR Architecture Description File


// 4 bit signed offset -- for predicated load/store

operand vmemA_immIOffset4() %{
  // (esize / msize) = 1
  predicate(Address::offset_ok_for_sve_immed(n->get_int(), 4,
            Matcher::scalable_vector_reg_size(T_BYTE)));
  match(ConI);
  op_cost(0);
  format %{ %}
  interface(CONST_INTER);
%}

operand vmemA_immLOffset4() %{
  // (esize / msize) = 1
  predicate(Address::offset_ok_for_sve_immed(n->get_long(), 4,
            Matcher::scalable_vector_reg_size(T_BYTE)));
  match(ConL);
  op_cost(0);
  format %{ %}
  interface(CONST_INTER);
%}

operand vmemA_indOffI4(iRegP reg, vmemA_immIOffset4 off) %{
  constraint(ALLOC_IN_RC(ptr_reg));
  match(AddP reg off);
  op_cost(0);
  format %{ "[$reg, $off]" %}
  interface(MEMORY_INTER) %{
    base($reg);
    index(0xffffffff);
    scale(0x0);
    disp($off);
  %}
%}

operand vmemA_indOffL4(iRegP reg, vmemA_immLOffset4 off) %{
  constraint(ALLOC_IN_RC(ptr_reg));
  match(AddP reg off);
  op_cost(0);
  format %{ "[$reg, $off]" %}
  interface(MEMORY_INTER) %{
    base($reg);
    index(0xffffffff);
    scale(0x0);
    disp($off);
  %}
%}

// The indOff of vmemA is valid only when the vector element (load to/store from)
// size equals to memory element (load from/store to) size.
opclass vmemA(indirect, vmemA_indOffI4, vmemA_indOffL4);

source_hpp %{
  // Assert that the given node is not a variable shift.
  bool assert_not_var_shift(const Node* n);

  Assembler::SIMD_Arrangement get_arrangement(const Node* n);
  Assembler::SIMD_RegVariant get_reg_variant(const Node* n);
%}

source %{

  typedef void (C2_MacroAssembler::* sve_mem_insn_predicate)(FloatRegister Rt, Assembler::SIMD_RegVariant T,
                                                             PRegister Pg, const Address &adr);

  // Predicated load/store, with optional ptrue to all elements of given predicate register.
  static void loadStoreA_predicated(C2_MacroAssembler* masm, bool is_store, FloatRegister reg,
                                    PRegister pg, BasicType mem_elem_bt, BasicType vector_elem_bt,
                                    int opcode, Register base, int index, int size, int disp) {
    sve_mem_insn_predicate insn;
    int mesize = type2aelembytes(mem_elem_bt);
    if (index == -1) {
      assert(size == 0, "unsupported address mode: scale size = %d", size);
      switch(mesize) {
      case 1:
        insn = is_store ? &C2_MacroAssembler::sve_st1b : &C2_MacroAssembler::sve_ld1b;
        break;
      case 2:
        insn = is_store ? &C2_MacroAssembler::sve_st1h : &C2_MacroAssembler::sve_ld1h;
        break;
      case 4:
        insn = is_store ? &C2_MacroAssembler::sve_st1w : &C2_MacroAssembler::sve_ld1w;
        break;
      case 8:
        insn = is_store ? &C2_MacroAssembler::sve_st1d : &C2_MacroAssembler::sve_ld1d;
        break;
      default:
        assert(false, "unsupported");
        ShouldNotReachHere();
      }
      int imm4 = disp / mesize / Matcher::scalable_vector_reg_size(vector_elem_bt);
      (masm->*insn)(reg, Assembler::elemType_to_regVariant(vector_elem_bt), pg, Address(base, imm4));
    } else {
      assert(false, "unimplemented");
      ShouldNotReachHere();
    }
  }

  bool Matcher::match_rule_supported_auto_vectorization(int opcode, int vlen, BasicType bt) {
    if (UseSVE == 0) {
      // These operations are not profitable to be vectorized on NEON, because no direct
      // NEON instructions support them. But the match rule support for them is profitable for
      // Vector API intrinsics.
      if ((opcode == Op_VectorCastD2X && bt == T_INT) ||
          (opcode == Op_VectorCastL2X && bt == T_FLOAT) ||
          (opcode == Op_CountLeadingZerosV && bt == T_LONG) ||
          (opcode == Op_CountTrailingZerosV && bt == T_LONG) ||
          // The implementations of Op_AddReductionVD/F in Neon are for the Vector API only.
          // They are not suitable for auto-vectorization because the result would not conform
          // to the JLS, Section Evaluation Order.
          opcode == Op_AddReductionVD || opcode == Op_AddReductionVF ||
          opcode == Op_MulReductionVD || opcode == Op_MulReductionVF ||
          opcode == Op_MulVL) {
        return false;
      }
    }
    return match_rule_supported_vector(opcode, vlen, bt);
  }

  // Identify extra cases that we might want to provide match rules for vector nodes and
  // other intrinsics guarded with vector length (vlen) and element type (bt).
  bool Matcher::match_rule_supported_vector(int opcode, int vlen, BasicType bt) {
    if (!match_rule_supported(opcode)) {
      return false;
    }

    int length_in_bytes = vlen * type2aelembytes(bt);
    if (UseSVE == 0 && length_in_bytes > FloatRegister::neon_vl) {
      return false;
    }

    // Check whether specific Op is supported.
    // Fail fast, otherwise fall through to common vector_size_supported() check.
    switch (opcode) {
      case Op_AndVMask:
      case Op_OrVMask:
      case Op_XorVMask:
      case Op_MaskAll:
      case Op_VectorMaskGen:
      case Op_LoadVectorMasked:
      case Op_StoreVectorMasked:
      case Op_StoreVectorScatter:
      case Op_StoreVectorScatterMasked:
      case Op_PopulateIndex:
      case Op_CompressM:
      case Op_CompressV:
        if (UseSVE == 0) {
          return false;
        }
        break;
      case Op_LoadVectorGather:
      case Op_LoadVectorGatherMasked:
        if (UseSVE == 0 || is_subword_type(bt)) {
          return false;
        }
        break;
      case Op_MulAddVS2VI:
        if (length_in_bytes != 16) {
          return false;
        }
        break;
      case Op_MulReductionVD:
      case Op_MulReductionVF:
      case Op_MulReductionVI:
      case Op_MulReductionVL:
        // No vector multiply reduction instructions, but we do
        // emit scalar instructions for 64/128-bit vectors.
        if (length_in_bytes != 8 && length_in_bytes != 16) {
          return false;
        }
        break;
      case Op_VectorMaskCmp:
        if (length_in_bytes < 8) {
          return false;
        }
        break;
      case Op_VectorLoadShuffle:
      case Op_VectorRearrange:
        if (vlen < 4) {
          return false;
        }
        break;
      case Op_ExpandV:
        if (UseSVE < 2 || is_subword_type(bt)) {
          return false;
        }
        break;
      case Op_VectorMaskToLong:
        if (UseSVE > 0 && vlen > 64) {
          return false;
        }
        break;
      case Op_VectorLongToMask:
        if (vlen > 64 || !VM_Version::supports_svebitperm()) {
          return false;
        }
        break;
      case Op_CompressBitsV:
      case Op_ExpandBitsV:
        if (!VM_Version::supports_svebitperm()) {
          return false;
        }
        break;
      default:
        break;
    }
    return vector_size_supported(bt, vlen);
  }

  bool Matcher::match_rule_supported_vector_masked(int opcode, int vlen, BasicType bt) {
    // Only SVE supports masked operations.
    if (UseSVE == 0) {
      return false;
    }

    // If an opcode does not support the masked version,
    // unpredicated node with VectorBlend node will be used instead.
    switch(opcode) {
      case Op_VectorRearrange:
      case Op_MulReductionVD:
      case Op_MulReductionVF:
      case Op_MulReductionVI:
      case Op_MulReductionVL:
      case Op_CompressBitsV:
      case Op_ExpandBitsV:
        return false;
      // We use Op_LoadVectorMasked to implement the predicated Op_LoadVector.
      // Hence we turn to check whether Op_LoadVectorMasked is supported. The
      // same as vector store/gather/scatter.
      case Op_LoadVector:
        opcode = Op_LoadVectorMasked;
        break;
      case Op_StoreVector:
        opcode = Op_StoreVectorMasked;
        break;
      case Op_LoadVectorGather:
        opcode = Op_LoadVectorGatherMasked;
        break;
      case Op_StoreVectorScatter:
        opcode = Op_StoreVectorScatterMasked;
        break;
      default:
        break;
    }

    return match_rule_supported_vector(opcode, vlen, bt);
  }

  bool Matcher::vector_needs_partial_operations(Node* node, const TypeVect* vt) {
    // Only SVE has partial vector operations
    if (UseSVE == 0) {
      return false;
    }

    switch(node->Opcode()) {
      case Op_VectorLoadMask:
      case Op_VectorMaskCmp:
      case Op_LoadVectorGather:
      case Op_StoreVectorScatter:
      case Op_AddReductionVF:
      case Op_AddReductionVD:
      case Op_AndReductionV:
      case Op_OrReductionV:
      case Op_XorReductionV:
      // Mask is needed for partial Op_VectorMaskFirstTrue, because when the
      // input predicate is all-false, the result should be the vector length
      // instead of the vector register size.
      case Op_VectorMaskFirstTrue:
        return true;
      case Op_MaskAll:
        return !node->in(1)->is_Con();
      case Op_LoadVector:
      case Op_StoreVector:
        // We use NEON load/store instructions if the vector length is <= 128 bits.
        return vt->length_in_bytes() > 16;
      case Op_AddReductionVI:
      case Op_AddReductionVL:
        // We may prefer using NEON instructions rather than SVE partial operations.
        return !VM_Version::use_neon_for_vector(vt->length_in_bytes());
      case Op_MinReductionV:
      case Op_MaxReductionV:
        // For BYTE/SHORT/INT/FLOAT/DOUBLE types, we may prefer using NEON
        // instructions rather than SVE partial operations.
        return vt->element_basic_type() == T_LONG ||
               !VM_Version::use_neon_for_vector(vt->length_in_bytes());
      default:
        // For other ops whose vector size is smaller than the max vector size, a
        // full-sized unpredicated operation does not impact the final vector result.
        return false;
    }
  }

  bool Matcher::vector_rearrange_requires_load_shuffle(BasicType elem_bt, int vlen) {
    return false;
  }

  // Assert that the given node is not a variable shift.
  bool assert_not_var_shift(const Node* n) {
    assert(!n->as_ShiftV()->is_var_shift(), "illegal variable shift");
    return true;
  }

  Assembler::SIMD_Arrangement get_arrangement(const Node* n) {
    BasicType bt = Matcher::vector_element_basic_type(n);
    uint length_in_bytes = Matcher::vector_length_in_bytes(n);
    return Assembler::esize2arrangement((uint)type2aelembytes(bt),
                                        /* isQ */ length_in_bytes == 16);
  }

  Assembler::SIMD_RegVariant get_reg_variant(const Node* n) {
    BasicType bt = Matcher::vector_element_basic_type(n);
    return Assembler::elemType_to_regVariant(bt);
  }
%}


// All VECTOR instructions

// ------------------------------ Vector load/store ----------------------------

// Load Vector (16 bits)
instruct loadV2(vReg dst, vmem2 mem) %{
  predicate(n->as_LoadVector()->memory_size() == 2);
  match(Set dst (LoadVector mem));
  format %{ "loadV2 $dst, $mem\t# vector (16 bits)" %}
  ins_encode( aarch64_enc_ldrvH(dst, mem) );
  ins_pipe(pipe_slow);
%}

// Store Vector (16 bits)
instruct storeV2(vReg src, vmem2 mem) %{
  predicate(n->as_StoreVector()->memory_size() == 2);
  match(Set mem (StoreVector mem src));
  format %{ "storeV2 $mem, $src\t# vector (16 bits)" %}
  ins_encode( aarch64_enc_strvH(src, mem) );
  ins_pipe(pipe_slow);
%}

// Load Vector (32 bits)
instruct loadV4(vReg dst, vmem4 mem) %{
  predicate(n->as_LoadVector()->memory_size() == 4);
  match(Set dst (LoadVector mem));
  format %{ "loadV4 $dst, $mem\t# vector (32 bits)" %}
  ins_encode( aarch64_enc_ldrvS(dst, mem) );
  ins_pipe(pipe_slow);
%}

// Store Vector (32 bits)
instruct storeV4(vReg src, vmem4 mem) %{
  predicate(n->as_StoreVector()->memory_size() == 4);
  match(Set mem (StoreVector mem src));
  format %{ "storeV4 $mem, $src\t# vector (32 bits)" %}
  ins_encode( aarch64_enc_strvS(src, mem) );
  ins_pipe(pipe_slow);
%}

// Load Vector (64 bits)
instruct loadV8(vReg dst, vmem8 mem) %{
  predicate(n->as_LoadVector()->memory_size() == 8);
  match(Set dst (LoadVector mem));
  format %{ "loadV8 $dst, $mem\t# vector (64 bits)" %}
  ins_encode( aarch64_enc_ldrvD(dst, mem) );
  ins_pipe(pipe_slow);
%}

// Store Vector (64 bits)
instruct storeV8(vReg src, vmem8 mem) %{
  predicate(n->as_StoreVector()->memory_size() == 8);
  match(Set mem (StoreVector mem src));
  format %{ "storeV8 $mem, $src\t# vector (64 bits)" %}
  ins_encode( aarch64_enc_strvD(src, mem) );
  ins_pipe(pipe_slow);
%}

// Load Vector (128 bits)
instruct loadV16(vReg dst, vmem16 mem) %{
  predicate(n->as_LoadVector()->memory_size() == 16);
  match(Set dst (LoadVector mem));
  format %{ "loadV16 $dst, $mem\t# vector (128 bits)" %}
  ins_encode( aarch64_enc_ldrvQ(dst, mem) );
  ins_pipe(pipe_slow);
%}

// Store Vector (128 bits)
instruct storeV16(vReg src, vmem16 mem) %{
  predicate(n->as_StoreVector()->memory_size() == 16);
  match(Set mem (StoreVector mem src));
  format %{ "storeV16 $mem, $src\t# vector (128 bits)" %}
  ins_encode( aarch64_enc_strvQ(src, mem) );
  ins_pipe(pipe_slow);
%}

// Load Vector (> 128 bits)
instruct loadV(vReg dst, vmemA mem) %{
  predicate(n->as_LoadVector()->memory_size() > 16);
  match(Set dst (LoadVector mem));
  format %{ "loadV $dst, $mem\t# vector (sve)" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    loadStoreA_predicated(masm, /* is_store */ false,
                          $dst$$FloatRegister, ptrue, bt, bt, $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
  %}
  ins_pipe(pipe_slow);
%}

// Store Vector (> 128 bits)
instruct storeV(vReg src, vmemA mem) %{
  predicate(n->as_StoreVector()->memory_size() > 16);
  match(Set mem (StoreVector mem src));
  format %{ "storeV $mem, $src\t# vector (sve)" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    BasicType bt = Matcher::vector_element_basic_type(this, $src);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    loadStoreA_predicated(masm, /* is_store */ true,
                          $src$$FloatRegister, ptrue, bt, bt, $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
  %}
  ins_pipe(pipe_slow);
%}

// vector load/store - predicated

instruct loadV_masked(vReg dst, vmemA mem, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst (LoadVectorMasked mem pg));
  format %{ "loadV_masked $dst, $pg, $mem" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    loadStoreA_predicated(masm, /* is_store */ false, $dst$$FloatRegister,
                          $pg$$PRegister, bt, bt, $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
  %}
  ins_pipe(pipe_slow);
%}

instruct storeV_masked(vReg src, vmemA mem, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set mem (StoreVectorMasked mem (Binary src pg)));
  format %{ "storeV_masked $mem, $pg, $src" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src);
    loadStoreA_predicated(masm, /* is_store */ true, $src$$FloatRegister,
                          $pg$$PRegister, bt, bt, $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
  %}
  ins_pipe(pipe_slow);
%}

// vector load const

instruct vloadcon(vReg dst, immI0 src) %{
  match(Set dst (VectorLoadConst src));
  format %{ "vloadcon $dst, $src\t# load/generate iota indices" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    if (UseSVE == 0) {
      uint length_in_bytes = Matcher::vector_length_in_bytes(this);
      assert(length_in_bytes <= 16, "must be");
      // The iota indices are ordered by type B/S/I/L/F/D, and the offset between two types is 16.
      int offset = exact_log2(type2aelembytes(bt)) << 4;
      if (is_floating_point_type(bt)) {
        offset += 32;
      }
      __ lea(rscratch1, ExternalAddress(StubRoutines::aarch64::vector_iota_indices() + offset));
      if (length_in_bytes == 16) {
        __ ldrq($dst$$FloatRegister, rscratch1);
      } else {
        __ ldrd($dst$$FloatRegister, rscratch1);
      }
    } else {
      Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
      __ sve_index($dst$$FloatRegister, size, 0, 1);
      if (is_floating_point_type(bt)) {
        __ sve_scvtf($dst$$FloatRegister, size, ptrue, $dst$$FloatRegister, size);
      }
    }
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector add -----------------------------------

// vector add

instruct vaddB(vReg dst, vReg src1, vReg src2) %{
  match(Set dst (AddVB src1 src2));
  format %{ "vaddB $dst, $src1, $src2" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ addv($dst$$FloatRegister, get_arrangement(this),
              $src1$$FloatRegister, $src2$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_add($dst$$FloatRegister, __ B, $src1$$FloatRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddS(vReg dst, vReg src1, vReg src2) %{
  match(Set dst (AddVS src1 src2));
  format %{ "vaddS $dst, $src1, $src2" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ addv($dst$$FloatRegister, get_arrangement(this),
              $src1$$FloatRegister, $src2$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_add($dst$$FloatRegister, __ H, $src1$$FloatRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddI(vReg dst, vReg src1, vReg src2) %{
  match(Set dst (AddVI src1 src2));
  format %{ "vaddI $dst, $src1, $src2" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ addv($dst$$FloatRegister, get_arrangement(this),
              $src1$$FloatRegister, $src2$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_add($dst$$FloatRegister, __ S, $src1$$FloatRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddL(vReg dst, vReg src1, vReg src2) %{
  match(Set dst (AddVL src1 src2));
  format %{ "vaddL $dst, $src1, $src2" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ addv($dst$$FloatRegister, get_arrangement(this),
              $src1$$FloatRegister, $src2$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_add($dst$$FloatRegister, __ D, $src1$$FloatRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddF(vReg dst, vReg src1, vReg src2) %{
  match(Set dst (AddVF src1 src2));
  format %{ "vaddF $dst, $src1, $src2" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ fadd($dst$$FloatRegister, get_arrangement(this),
              $src1$$FloatRegister, $src2$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_fadd($dst$$FloatRegister, __ S, $src1$$FloatRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddD(vReg dst, vReg src1, vReg src2) %{
  match(Set dst (AddVD src1 src2));
  format %{ "vaddD $dst, $src1, $src2" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ fadd($dst$$FloatRegister, get_arrangement(this),
              $src1$$FloatRegister, $src2$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_fadd($dst$$FloatRegister, __ D, $src1$$FloatRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector add - predicated

instruct vaddB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AddVB (Binary dst_src1 src2) pg));
  format %{ "vaddB_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_add($dst_src1$$FloatRegister, __ B, $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AddVS (Binary dst_src1 src2) pg));
  format %{ "vaddS_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_add($dst_src1$$FloatRegister, __ H, $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AddVI (Binary dst_src1 src2) pg));
  format %{ "vaddI_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_add($dst_src1$$FloatRegister, __ S, $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AddVL (Binary dst_src1 src2) pg));
  format %{ "vaddL_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_add($dst_src1$$FloatRegister, __ D, $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AddVF (Binary dst_src1 src2) pg));
  format %{ "vaddF_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_fadd($dst_src1$$FloatRegister, __ S, $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AddVD (Binary dst_src1 src2) pg));
  format %{ "vaddD_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_fadd($dst_src1$$FloatRegister, __ D, $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector add reg imm (unpredicated)

instruct vaddImmB(vReg dst_src, immBAddSubV con) %{
  predicate(UseSVE > 0);
  match(Set dst_src (AddVB dst_src (Replicate con)));
  format %{ "vaddImmB $dst_src, $dst_src, $con" %}
  ins_encode %{
    int val = (int)$con$$constant;
    if (val > 0) {
      __ sve_add($dst_src$$FloatRegister, __ B, val);
    } else {
      __ sve_sub($dst_src$$FloatRegister, __ B, -val);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddImmS(vReg dst_src, immIAddSubV con) %{
  predicate(UseSVE > 0);
  match(Set dst_src (AddVS dst_src (Replicate con)));
  format %{ "vaddImmS $dst_src, $dst_src, $con" %}
  ins_encode %{
    int val = (int)$con$$constant;
    if (val > 0) {
      __ sve_add($dst_src$$FloatRegister, __ H, val);
    } else {
      __ sve_sub($dst_src$$FloatRegister, __ H, -val);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddImmI(vReg dst_src, immIAddSubV con) %{
  predicate(UseSVE > 0);
  match(Set dst_src (AddVI dst_src (Replicate con)));
  format %{ "vaddImmI $dst_src, $dst_src, $con" %}
  ins_encode %{
    int val = (int)$con$$constant;
    if (val > 0) {
      __ sve_add($dst_src$$FloatRegister, __ S, val);
    } else {
      __ sve_sub($dst_src$$FloatRegister, __ S, -val);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddImmL(vReg dst_src, immLAddSubV con) %{
  predicate(UseSVE > 0);
  match(Set dst_src (AddVL dst_src (Replicate con)));
  format %{ "vaddImmL $dst_src, $dst_src, $con" %}
  ins_encode %{
    int val = (int)$con$$constant;
    if (val > 0) {
      __ sve_add($dst_src$$FloatRegister, __ D, val);
    } else {
      __ sve_sub($dst_src$$FloatRegister, __ D, -val);
    }
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector sub -----------------------------------

// vector sub

instruct vsubB(vReg dst, vReg src1, vReg src2) %{
  match(Set dst (SubVB src1 src2));
  format %{ "vsubB $dst, $src1, $src2" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ subv($dst$$FloatRegister, get_arrangement(this),
              $src1$$FloatRegister, $src2$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_sub($dst$$FloatRegister, __ B, $src1$$FloatRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubS(vReg dst, vReg src1, vReg src2) %{
  match(Set dst (SubVS src1 src2));
  format %{ "vsubS $dst, $src1, $src2" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ subv($dst$$FloatRegister, get_arrangement(this),
              $src1$$FloatRegister, $src2$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_sub($dst$$FloatRegister, __ H, $src1$$FloatRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubI(vReg dst, vReg src1, vReg src2) %{
  match(Set dst (SubVI src1 src2));
  format %{ "vsubI $dst, $src1, $src2" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ subv($dst$$FloatRegister, get_arrangement(this),
              $src1$$FloatRegister, $src2$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_sub($dst$$FloatRegister, __ S, $src1$$FloatRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubL(vReg dst, vReg src1, vReg src2) %{
  match(Set dst (SubVL src1 src2));
  format %{ "vsubL $dst, $src1, $src2" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ subv($dst$$FloatRegister, get_arrangement(this),
              $src1$$FloatRegister, $src2$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_sub($dst$$FloatRegister, __ D, $src1$$FloatRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubF(vReg dst, vReg src1, vReg src2) %{
  match(Set dst (SubVF src1 src2));
  format %{ "vsubF $dst, $src1, $src2" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ fsub($dst$$FloatRegister, get_arrangement(this),
              $src1$$FloatRegister, $src2$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_fsub($dst$$FloatRegister, __ S, $src1$$FloatRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubD(vReg dst, vReg src1, vReg src2) %{
  match(Set dst (SubVD src1 src2));
  format %{ "vsubD $dst, $src1, $src2" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ fsub($dst$$FloatRegister, get_arrangement(this),
              $src1$$FloatRegister, $src2$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_fsub($dst$$FloatRegister, __ D, $src1$$FloatRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector sub - predicated

instruct vsubB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (SubVB (Binary dst_src1 src2) pg));
  format %{ "vsubB_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_sub($dst_src1$$FloatRegister, __ B, $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (SubVS (Binary dst_src1 src2) pg));
  format %{ "vsubS_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_sub($dst_src1$$FloatRegister, __ H, $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (SubVI (Binary dst_src1 src2) pg));
  format %{ "vsubI_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_sub($dst_src1$$FloatRegister, __ S, $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (SubVL (Binary dst_src1 src2) pg));
  format %{ "vsubL_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_sub($dst_src1$$FloatRegister, __ D, $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (SubVF (Binary dst_src1 src2) pg));
  format %{ "vsubF_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_fsub($dst_src1$$FloatRegister, __ S, $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (SubVD (Binary dst_src1 src2) pg));
  format %{ "vsubD_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_fsub($dst_src1$$FloatRegister, __ D, $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector mul -----------------------------------

// vector mul - BYTE, CHAR, SHORT, INT

instruct vmulB_neon(vReg dst, vReg src1, vReg src2) %{
  predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst (MulVB src1 src2));
  format %{ "vmulB_neon $dst, $src1, $src2" %}
  ins_encode %{
    __ mulv($dst$$FloatRegister, get_arrangement(this),
            $src1$$FloatRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulB_sve(vReg dst_src1, vReg src2) %{
  predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst_src1 (MulVB dst_src1 src2));
  format %{ "vmulB_sve $dst_src1, $dst_src1, $src2" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    __ sve_mul($dst_src1$$FloatRegister, __ B, ptrue, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulS_neon(vReg dst, vReg src1, vReg src2) %{
  predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst (MulVS src1 src2));
  format %{ "vmulS_neon $dst, $src1, $src2" %}
  ins_encode %{
    __ mulv($dst$$FloatRegister, get_arrangement(this),
            $src1$$FloatRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulS_sve(vReg dst_src1, vReg src2) %{
  predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst_src1 (MulVS dst_src1 src2));
  format %{ "vmulS_sve $dst_src1, $dst_src1, $src2" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    __ sve_mul($dst_src1$$FloatRegister, __ H, ptrue, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulI_neon(vReg dst, vReg src1, vReg src2) %{
  predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst (MulVI src1 src2));
  format %{ "vmulI_neon $dst, $src1, $src2" %}
  ins_encode %{
    __ mulv($dst$$FloatRegister, get_arrangement(this),
            $src1$$FloatRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulI_sve(vReg dst_src1, vReg src2) %{
  predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst_src1 (MulVI dst_src1 src2));
  format %{ "vmulI_sve $dst_src1, $dst_src1, $src2" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    __ sve_mul($dst_src1$$FloatRegister, __ S, ptrue, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector mul - LONG

instruct vmulL_neon(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE == 0);
  match(Set dst (MulVL src1 src2));
  format %{ "vmulL_neon $dst, $src1, $src2\t# 2L" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    assert(length_in_bytes == 16, "must be");
    __ umov(rscratch1, $src1$$FloatRegister, __ D, 0);
    __ umov(rscratch2, $src2$$FloatRegister, __ D, 0);
    __ mul(rscratch2, rscratch2, rscratch1);
    __ mov($dst$$FloatRegister, __ D, 0, rscratch2);
    __ umov(rscratch1, $src1$$FloatRegister, __ D, 1);
    __ umov(rscratch2, $src2$$FloatRegister, __ D, 1);
    __ mul(rscratch2, rscratch2, rscratch1);
    __ mov($dst$$FloatRegister, __ D, 1, rscratch2);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulL_sve(vReg dst_src1, vReg src2) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (MulVL dst_src1 src2));
  format %{ "vmulL_sve $dst_src1, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_mul($dst_src1$$FloatRegister, __ D, ptrue, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector mul - floating-point

instruct vmulF(vReg dst, vReg src1, vReg src2) %{
  match(Set dst (MulVF src1 src2));
  format %{ "vmulF $dst, $src1, $src2" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ fmul($dst$$FloatRegister, get_arrangement(this),
              $src1$$FloatRegister, $src2$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_fmul($dst$$FloatRegister, __ S, $src1$$FloatRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulD(vReg dst, vReg src1, vReg src2) %{
  match(Set dst (MulVD src1 src2));
  format %{ "vmulD $dst, $src1, $src2" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ fmul($dst$$FloatRegister, get_arrangement(this),
              $src1$$FloatRegister, $src2$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_fmul($dst$$FloatRegister, __ D, $src1$$FloatRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector mul - predicated

instruct vmulB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (MulVB (Binary dst_src1 src2) pg));
  format %{ "vmulB_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_mul($dst_src1$$FloatRegister, __ B, $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (MulVS (Binary dst_src1 src2) pg));
  format %{ "vmulS_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_mul($dst_src1$$FloatRegister, __ H, $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (MulVI (Binary dst_src1 src2) pg));
  format %{ "vmulI_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_mul($dst_src1$$FloatRegister, __ S, $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (MulVL (Binary dst_src1 src2) pg));
  format %{ "vmulL_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_mul($dst_src1$$FloatRegister, __ D, $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (MulVF (Binary dst_src1 src2) pg));
  format %{ "vmulF_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_fmul($dst_src1$$FloatRegister, __ S, $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (MulVD (Binary dst_src1 src2) pg));
  format %{ "vmulD_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_fmul($dst_src1$$FloatRegister, __ D, $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector float div -----------------------------

// vector float div

instruct vdivF_neon(vReg dst, vReg src1, vReg src2) %{
  predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst (DivVF src1 src2));
  format %{ "vdivF_neon $dst, $src1, $src2" %}
  ins_encode %{
    __ fdiv($dst$$FloatRegister, get_arrangement(this),
            $src1$$FloatRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vdivF_sve(vReg dst_src1, vReg src2) %{
  predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst_src1 (DivVF dst_src1 src2));
  format %{ "vdivF_sve $dst_src1, $dst_src1, $src2" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    __ sve_fdiv($dst_src1$$FloatRegister, __ S, ptrue, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vdivD_neon(vReg dst, vReg src1, vReg src2) %{
  predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst (DivVD src1 src2));
  format %{ "vdivD_neon $dst, $src1, $src2" %}
  ins_encode %{
    __ fdiv($dst$$FloatRegister, get_arrangement(this),
            $src1$$FloatRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vdivD_sve(vReg dst_src1, vReg src2) %{
  predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst_src1 (DivVD dst_src1 src2));
  format %{ "vdivD_sve $dst_src1, $dst_src1, $src2" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    __ sve_fdiv($dst_src1$$FloatRegister, __ D, ptrue, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector float div - predicated

instruct vdivF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (DivVF (Binary dst_src1 src2) pg));
  format %{ "vdivF_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_fdiv($dst_src1$$FloatRegister, __ S, $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vdivD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (DivVD (Binary dst_src1 src2) pg));
  format %{ "vdivD_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_fdiv($dst_src1$$FloatRegister, __ D, $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector and -----------------------------------

// vector and

instruct vand(vReg dst, vReg src1, vReg src2) %{
  match(Set dst (AndV src1 src2));
  format %{ "vand $dst, $src1, $src2" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ andr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
              $src1$$FloatRegister, $src2$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_and($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector and - predicated

instruct vand_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AndV (Binary dst_src1 src2) pg));
  format %{ "vand_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_and($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
               $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector and reg imm (unpredicated)

instruct vandImmB(vReg dst_src, immBLog con) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE);
  match(Set dst_src (AndV dst_src (Replicate con)));
  format %{ "vandImmB $dst_src, $dst_src, $con" %}
  ins_encode %{
    __ sve_and($dst_src$$FloatRegister, __ B, (uint64_t)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct vandImmS(vReg dst_src, immSLog con) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_SHORT);
  match(Set dst_src (AndV dst_src (Replicate con)));
  format %{ "vandImmS $dst_src, $dst_src, $con" %}
  ins_encode %{
    __ sve_and($dst_src$$FloatRegister, __ H, (uint64_t)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct vandImmI(vReg dst_src, immILog con) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_INT);
  match(Set dst_src (AndV dst_src (Replicate con)));
  format %{ "vandImmI $dst_src, $dst_src, $con" %}
  ins_encode %{
    __ sve_and($dst_src$$FloatRegister, __ S, (uint64_t)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct vandImmL(vReg dst_src, immLLog con) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_LONG);
  match(Set dst_src (AndV dst_src (Replicate con)));
  format %{ "vandImmL $dst_src, $dst_src, $con" %}
  ins_encode %{
    __ sve_and($dst_src$$FloatRegister, __ D, (uint64_t)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector or ------------------------------------

// vector or

instruct vor(vReg dst, vReg src1, vReg src2) %{
  match(Set dst (OrV src1 src2));
  format %{ "vor $dst, $src1, $src2" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ orr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
              $src1$$FloatRegister, $src2$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_orr($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector or - predicated

instruct vor_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (OrV (Binary dst_src1 src2) pg));
  format %{ "vor_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_orr($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
               $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector or reg imm (unpredicated)

instruct vorImmB(vReg dst_src, immBLog con) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE);
  match(Set dst_src (OrV dst_src (Replicate con)));
  format %{ "vorImmB $dst_src, $dst_src, $con" %}
  ins_encode %{
    __ sve_orr($dst_src$$FloatRegister, __ B, (uint64_t)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct vorImmS(vReg dst_src, immSLog con) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_SHORT);
  match(Set dst_src (OrV dst_src (Replicate con)));
  format %{ "vorImmS $dst_src, $dst_src, $con" %}
  ins_encode %{
    __ sve_orr($dst_src$$FloatRegister, __ H, (uint64_t)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct vorImmI(vReg dst_src, immILog con) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_INT);
  match(Set dst_src (OrV dst_src (Replicate con)));
  format %{ "vorImmI $dst_src, $dst_src, $con" %}
  ins_encode %{
    __ sve_orr($dst_src$$FloatRegister, __ S, (uint64_t)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct vorImmL(vReg dst_src, immLLog con) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_LONG);
  match(Set dst_src (OrV dst_src (Replicate con)));
  format %{ "vorImmL $dst_src, $dst_src, $con" %}
  ins_encode %{
    __ sve_orr($dst_src$$FloatRegister, __ D, (uint64_t)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector xor -----------------------------------

// vector xor

instruct vxor(vReg dst, vReg src1, vReg src2) %{
  match(Set dst (XorV src1 src2));
  format %{ "vxor $dst, $src1, $src2" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ eor($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
              $src1$$FloatRegister, $src2$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_eor($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector xor - predicated

instruct vxor_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (XorV (Binary dst_src1 src2) pg));
  format %{ "vxor_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_eor($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
               $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector xor reg imm (unpredicated)

instruct vxorImmB(vReg dst_src, immBLog con) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE);
  match(Set dst_src (XorV dst_src (Replicate con)));
  format %{ "vxorImmB $dst_src, $dst_src, $con" %}
  ins_encode %{
    __ sve_eor($dst_src$$FloatRegister, __ B, (uint64_t)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct vxorImmS(vReg dst_src, immSLog con) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_SHORT);
  match(Set dst_src (XorV dst_src (Replicate con)));
  format %{ "vxorImmS $dst_src, $dst_src, $con" %}
  ins_encode %{
    __ sve_eor($dst_src$$FloatRegister, __ H, (uint64_t)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct vxorImmI(vReg dst_src, immILog con) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_INT);
  match(Set dst_src (XorV dst_src (Replicate con)));
  format %{ "vxorImmI $dst_src, $dst_src, $con" %}
  ins_encode %{
    __ sve_eor($dst_src$$FloatRegister, __ S, (uint64_t)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct vxorImmL(vReg dst_src, immLLog con) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_LONG);
  match(Set dst_src (XorV dst_src (Replicate con)));
  format %{ "vxorImmL $dst_src, $dst_src, $con" %}
  ins_encode %{
    __ sve_eor($dst_src$$FloatRegister, __ D, (uint64_t)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

// vector eor3 (unpredicated)

instruct veor3_neon(vReg dst, vReg src1, vReg src2, vReg src3) %{
  predicate(VM_Version::supports_sha3() &&
            VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst (XorV src1 (XorV src2 src3)));
  format %{ "veor3_neon $dst, $src1, $src2, $src3" %}
  ins_encode %{
    __ eor3($dst$$FloatRegister, __ T16B, $src1$$FloatRegister,
            $src2$$FloatRegister, $src3$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct veor3_sve(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseSVE == 2 && !VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst_src1 (XorV dst_src1 (XorV src2 src3)));
  format %{ "veor3_sve $dst_src1, $dst_src1, $src2, $src3" %}
  ins_encode %{
    __ sve_eor3($dst_src1$$FloatRegister, $src2$$FloatRegister, $src3$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector not -----------------------------------

// vector not

instruct vnotI(vReg dst, vReg src, immI_M1 m1) %{
  match(Set dst (XorV src (Replicate m1)));
  format %{ "vnotI $dst, $src" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ notr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
              $src$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_not($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vnotL(vReg dst, vReg src, immL_M1 m1) %{
  match(Set dst (XorV src (Replicate m1)));
  format %{ "vnotL $dst, $src" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ notr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
              $src$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_not($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector not - predicated

instruct vnotI_masked(vReg dst_src, immI_M1 m1, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (XorV (Binary dst_src (Replicate m1)) pg));
  format %{ "vnotI_masked $dst_src, $pg, $dst_src" %}
  ins_encode %{
    __ sve_not($dst_src$$FloatRegister, get_reg_variant(this),
               $pg$$PRegister, $dst_src$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vnotL_masked(vReg dst_src, immL_M1 m1, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (XorV (Binary dst_src (Replicate m1)) pg));
  format %{ "vnotL_masked $dst_src, $pg, $dst_src" %}
  ins_encode %{
    __ sve_not($dst_src$$FloatRegister, get_reg_variant(this),
               $pg$$PRegister, $dst_src$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector and_not -------------------------------

// vector and_not

instruct vand_notI(vReg dst, vReg src1, vReg src2, immI_M1 m1) %{
  match(Set dst (AndV src1 (XorV src2 (Replicate m1))));
  format %{ "vand_notI $dst, $src1, $src2" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ bic($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
             $src1$$FloatRegister, $src2$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_bic($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vand_notL(vReg dst, vReg src1, vReg src2, immL_M1 m1) %{
  match(Set dst (AndV src1 (XorV src2 (Replicate m1))));
  format %{ "vand_notL $dst, $src1, $src2" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ bic($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
             $src1$$FloatRegister, $src2$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_bic($dst$$FloatRegister, $src1$$FloatRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector and_not - predicated

instruct vand_notI_masked(vReg dst_src1, vReg src2, immI_M1 m1, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (Replicate m1))) pg));
  format %{ "vand_notI_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_bic($dst_src1$$FloatRegister, get_reg_variant(this),
               $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vand_notL_masked(vReg dst_src1, vReg src2, immL_M1 m1, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (Replicate m1))) pg));
  format %{ "vand_notL_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_bic($dst_src1$$FloatRegister, get_reg_variant(this),
               $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector abs -----------------------------------

// vector abs

instruct vabsB(vReg dst, vReg src) %{
  match(Set dst (AbsVB src));
  format %{ "vabsB $dst, $src" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ absr($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_abs($dst$$FloatRegister, __ B, ptrue, $src$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsS(vReg dst, vReg src) %{
  match(Set dst (AbsVS src));
  format %{ "vabsS $dst, $src" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ absr($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_abs($dst$$FloatRegister, __ H, ptrue, $src$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsI(vReg dst, vReg src) %{
  match(Set dst (AbsVI src));
  format %{ "vabsI $dst, $src" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ absr($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_abs($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsL(vReg dst, vReg src) %{
  match(Set dst (AbsVL src));
  format %{ "vabsL $dst, $src" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ absr($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_abs($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsF(vReg dst, vReg src) %{
  match(Set dst (AbsVF src));
  format %{ "vabsF $dst, $src" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ fabs($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_fabs($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsD(vReg dst, vReg src) %{
  match(Set dst (AbsVD src));
  format %{ "vabsD $dst, $src" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ fabs($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_fabs($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector abs - predicated

instruct vabsB_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (AbsVB dst_src pg));
  format %{ "vabsB_masked $dst_src, $pg, $dst_src" %}
  ins_encode %{
    __ sve_abs($dst_src$$FloatRegister, __ B, $pg$$PRegister, $dst_src$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsS_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (AbsVS dst_src pg));
  format %{ "vabsS_masked $dst_src, $pg, $dst_src" %}
  ins_encode %{
    __ sve_abs($dst_src$$FloatRegister, __ H, $pg$$PRegister, $dst_src$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsI_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (AbsVI dst_src pg));
  format %{ "vabsI_masked $dst_src, $pg, $dst_src" %}
  ins_encode %{
    __ sve_abs($dst_src$$FloatRegister, __ S, $pg$$PRegister, $dst_src$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsL_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (AbsVL dst_src pg));
  format %{ "vabsL_masked $dst_src, $pg, $dst_src" %}
  ins_encode %{
    __ sve_abs($dst_src$$FloatRegister, __ D, $pg$$PRegister, $dst_src$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsF_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (AbsVF dst_src pg));
  format %{ "vabsF_masked $dst_src, $pg, $dst_src" %}
  ins_encode %{
    __ sve_fabs($dst_src$$FloatRegister, __ S, $pg$$PRegister, $dst_src$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsD_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (AbsVD dst_src pg));
  format %{ "vabsD_masked $dst_src, $pg, $dst_src" %}
  ins_encode %{
    __ sve_fabs($dst_src$$FloatRegister, __ D, $pg$$PRegister, $dst_src$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector fabd ----------------------------------

// vector fabs diff

instruct vfabd_neon(vReg dst, vReg src1, vReg src2) %{
  predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst (AbsVF (SubVF src1 src2)));
  match(Set dst (AbsVD (SubVD src1 src2)));
  format %{ "vfabd_neon $dst, $src1, $src2" %}
  ins_encode %{
    __ fabd($dst$$FloatRegister, get_arrangement(this),
            $src1$$FloatRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vfabd_sve(vReg dst_src1, vReg src2) %{
  predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst_src1 (AbsVF (SubVF dst_src1 src2)));
  match(Set dst_src1 (AbsVD (SubVD dst_src1 src2)));
  format %{ "vfabd_sve $dst_src1, $dst_src1, $src2" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_fabd($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                ptrue, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector fabs diff - predicated

instruct vfabd_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AbsVF (SubVF (Binary dst_src1 src2) pg) pg));
  match(Set dst_src1 (AbsVD (SubVD (Binary dst_src1 src2) pg) pg));
  format %{ "vfabd_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_fabd($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector neg -----------------------------------

// vector neg

instruct vnegI(vReg dst, vReg src) %{
  match(Set dst (NegVI src));
  format %{ "vnegI $dst, $src" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ negr($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      BasicType bt = Matcher::vector_element_basic_type(this);
      __ sve_neg($dst$$FloatRegister, __ elemType_to_regVariant(bt),
                 ptrue, $src$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vnegL(vReg dst, vReg src) %{
  match(Set dst (NegVL src));
  format %{ "vnegL $dst, $src" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ negr($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_neg($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vnegF(vReg dst, vReg src) %{
  match(Set dst (NegVF src));
  format %{ "vnegF $dst, $src" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ fneg($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_fneg($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vnegD(vReg dst, vReg src) %{
  match(Set dst (NegVD src));
  format %{ "vnegD $dst, $src" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ fneg($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_fneg($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector neg - predicated

instruct vnegI_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (NegVI dst_src pg));
  format %{ "vnegI_masked $dst_src, $pg, $dst_src" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_neg($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
               $pg$$PRegister, $dst_src$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vnegL_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (NegVL dst_src pg));
  format %{ "vnegL_masked $dst_src, $pg, $dst_src" %}
  ins_encode %{
    __ sve_neg($dst_src$$FloatRegister, __ D, $pg$$PRegister, $dst_src$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vnegF_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (NegVF dst_src pg));
  format %{ "vnegF_masked $dst_src, $pg, $dst_src" %}
  ins_encode %{
    __ sve_fneg($dst_src$$FloatRegister, __ S, $pg$$PRegister, $dst_src$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vnegD_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (NegVD dst_src pg));
  format %{ "vnegD_masked $dst_src, $pg, $dst_src" %}
  ins_encode %{
    __ sve_fneg($dst_src$$FloatRegister, __ D, $pg$$PRegister, $dst_src$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector sqrt ----------------------------------

// vector sqrt

instruct vsqrtF(vReg dst, vReg src) %{
  match(Set dst (SqrtVF src));
  format %{ "vsqrtF $dst, $src" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ fsqrt($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_fsqrt($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vsqrtD(vReg dst, vReg src) %{
  match(Set dst (SqrtVD src));
  format %{ "vsqrtD $dst, $src" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ fsqrt($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_fsqrt($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector sqrt - predicated

instruct vsqrtF_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (SqrtVF dst_src pg));
  format %{ "vsqrtF_masked $dst_src, $pg, $dst_src" %}
  ins_encode %{
    __ sve_fsqrt($dst_src$$FloatRegister, __ S, $pg$$PRegister, $dst_src$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vsqrtD_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (SqrtVD dst_src pg));
  format %{ "vsqrtD_masked $dst_src, $pg, $dst_src" %}
  ins_encode %{
    __ sve_fsqrt($dst_src$$FloatRegister, __ D, $pg$$PRegister, $dst_src$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector min -----------------------------------

// vector min - LONG

instruct vminL_neon(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n) == T_LONG);
  match(Set dst (MinV src1 src2));
  effect(TEMP_DEF dst);
  format %{ "vminL_neon $dst, $src1, $src2\t# 2L" %}
  ins_encode %{
    __ cm(Assembler::GT, $dst$$FloatRegister, __ T2D, $src1$$FloatRegister, $src2$$FloatRegister);
    __ bsl($dst$$FloatRegister, __ T16B, $src2$$FloatRegister, $src1$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vminL_sve(vReg dst_src1, vReg src2) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_LONG);
  match(Set dst_src1 (MinV dst_src1 src2));
  format %{ "vminL_sve $dst_src1, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_smin($dst_src1$$FloatRegister, __ D, ptrue, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector min - B/S/I/F/D

instruct vmin_neon(vReg dst, vReg src1, vReg src2) %{
  predicate(Matcher::vector_element_basic_type(n) != T_LONG &&
            VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst (MinV src1 src2));
  format %{ "vmin_neon $dst, $src1, $src2\t# B/S/I/F/D" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    if (is_floating_point_type(bt)) {
      __ fmin($dst$$FloatRegister, get_arrangement(this),
              $src1$$FloatRegister, $src2$$FloatRegister);
    } else {
      assert(is_integral_type(bt) && bt != T_LONG, "unsupported type");
      __ minv($dst$$FloatRegister, get_arrangement(this),
              $src1$$FloatRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vmin_sve(vReg dst_src1, vReg src2) %{
  predicate(Matcher::vector_element_basic_type(n) != T_LONG &&
            !VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst_src1 (MinV dst_src1 src2));
  format %{ "vmin_sve $dst_src1, $dst_src1, $src2\t# B/S/I/F/D" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    BasicType bt = Matcher::vector_element_basic_type(this);
    if (is_floating_point_type(bt)) {
      __ sve_fmin($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                  ptrue, $src2$$FloatRegister);
    } else {
      assert(is_integral_type(bt) && bt != T_LONG, "unsupported type");
      __ sve_smin($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                  ptrue, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector min - predicated

instruct vmin_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (MinV (Binary dst_src1 src2) pg));
  format %{ "vmin_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    if (is_floating_point_type(bt)) {
      __ sve_fmin($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                  $pg$$PRegister, $src2$$FloatRegister);
    } else {
      assert(is_integral_type(bt), "unsupported type");
      __ sve_smin($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                  $pg$$PRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector max -----------------------------------

// vector max - LONG

instruct vmaxL_neon(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n) == T_LONG);
  match(Set dst (MaxV src1 src2));
  effect(TEMP_DEF dst);
  format %{ "vmaxL_neon $dst, $src1, $src2\t# 2L" %}
  ins_encode %{
    __ cm(Assembler::GT, $dst$$FloatRegister, __ T2D, $src1$$FloatRegister, $src2$$FloatRegister);
    __ bsl($dst$$FloatRegister, __ T16B, $src1$$FloatRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmaxL_sve(vReg dst_src1, vReg src2) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_LONG);
  match(Set dst_src1 (MaxV dst_src1 src2));
  format %{ "vmaxL_sve $dst_src1, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_smax($dst_src1$$FloatRegister, __ D, ptrue, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector max - B/S/I/F/D

instruct vmax_neon(vReg dst, vReg src1, vReg src2) %{
  predicate(Matcher::vector_element_basic_type(n) != T_LONG &&
            VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst (MaxV src1 src2));
  format %{ "vmax_neon $dst, $src1, $src2\t# B/S/I/F/D" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    if (is_floating_point_type(bt)) {
      __ fmax($dst$$FloatRegister, get_arrangement(this),
              $src1$$FloatRegister, $src2$$FloatRegister);
    } else {
      assert(is_integral_type(bt) && bt != T_LONG, "unsupported type");
      __ maxv($dst$$FloatRegister, get_arrangement(this),
              $src1$$FloatRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vmax_sve(vReg dst_src1, vReg src2) %{
  predicate(Matcher::vector_element_basic_type(n) != T_LONG &&
            !VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst_src1 (MaxV dst_src1 src2));
  format %{ "vmax_sve $dst_src1, $dst_src1, $src2\t# B/S/I/F/D" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    BasicType bt = Matcher::vector_element_basic_type(this);
    if (is_floating_point_type(bt)) {
      __ sve_fmax($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                  ptrue, $src2$$FloatRegister);
    } else {
      assert(is_integral_type(bt) && bt != T_LONG, "unsupported type");
      __ sve_smax($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                  ptrue, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector max - predicated

instruct vmax_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (MaxV (Binary dst_src1 src2) pg));
  format %{ "vmax_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    if (is_floating_point_type(bt)) {
      __ sve_fmax($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                  $pg$$PRegister, $src2$$FloatRegister);
    } else {
      assert(is_integral_type(bt), "unsupported type");
      __ sve_smax($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                  $pg$$PRegister, $src2$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ MLA RELATED ----------------------------------

// vector mla
// dst_src1 = dst_src1 + src2 * src3

instruct vmla(vReg dst_src1, vReg src2, vReg src3) %{
  match(Set dst_src1 (AddVB dst_src1 (MulVB src2 src3)));
  match(Set dst_src1 (AddVS dst_src1 (MulVS src2 src3)));
  match(Set dst_src1 (AddVI dst_src1 (MulVI src2 src3)));
  format %{ "vmla $dst_src1, $src2, $src3" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ mlav($dst_src1$$FloatRegister, get_arrangement(this),
              $src2$$FloatRegister, $src3$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_mla($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                 ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vmlaL(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AddVL dst_src1 (MulVL src2 src3)));
  format %{ "vmlaL $dst_src1, $src2, $src3" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_mla($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
               ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmla_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AddVB (Binary dst_src1 (MulVB src2 src3)) pg));
  match(Set dst_src1 (AddVS (Binary dst_src1 (MulVS src2 src3)) pg));
  match(Set dst_src1 (AddVI (Binary dst_src1 (MulVI src2 src3)) pg));
  match(Set dst_src1 (AddVL (Binary dst_src1 (MulVL src2 src3)) pg));
  format %{ "vmla_masked $dst_src1, $pg, $src2, $src3" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_mla($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
               $pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector fmla
// dst_src1 = src2 * src3 + dst_src1

instruct vfmla(vReg dst_src1, vReg src2, vReg src3) %{
  match(Set dst_src1 (FmaVF dst_src1 (Binary src2 src3)));
  match(Set dst_src1 (FmaVD dst_src1 (Binary src2 src3)));
  format %{ "vfmla $dst_src1, $src2, $src3" %}
  ins_encode %{
    assert(UseFMA, "Needs FMA instructions support.");
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ fmla($dst_src1$$FloatRegister, get_arrangement(this),
              $src2$$FloatRegister, $src3$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      BasicType bt = Matcher::vector_element_basic_type(this);
      __ sve_fmla($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                  ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector fmad - predicated
// dst_src1 = dst_src1 * src2 + src3

instruct vfmad_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (FmaVF (Binary dst_src1 src2) (Binary src3 pg)));
  match(Set dst_src1 (FmaVD (Binary dst_src1 src2) (Binary src3 pg)));
  format %{ "vfmad_masked $dst_src1, $pg, $src2, $src3" %}
  ins_encode %{
    assert(UseFMA, "Needs FMA instructions support.");
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_fmad($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                $pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector mls
// dst_src1 = dst_src1 - src2 * src3

instruct vmls(vReg dst_src1, vReg src2, vReg src3) %{
  match(Set dst_src1 (SubVB dst_src1 (MulVB src2 src3)));
  match(Set dst_src1 (SubVS dst_src1 (MulVS src2 src3)));
  match(Set dst_src1 (SubVI dst_src1 (MulVI src2 src3)));
  format %{ "vmls $dst_src1, $src2, $src3" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ mlsv($dst_src1$$FloatRegister, get_arrangement(this),
              $src2$$FloatRegister, $src3$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_mls($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                 ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vmlsL(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (SubVL dst_src1 (MulVL src2 src3)));
  format %{ "vmlsL $dst_src1, $src2, $src3" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_mls($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                 ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmls_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (SubVB (Binary dst_src1 (MulVB src2 src3)) pg));
  match(Set dst_src1 (SubVS (Binary dst_src1 (MulVS src2 src3)) pg));
  match(Set dst_src1 (SubVI (Binary dst_src1 (MulVI src2 src3)) pg));
  match(Set dst_src1 (SubVL (Binary dst_src1 (MulVL src2 src3)) pg));
  format %{ "vmls_masked $dst_src1, $pg, $src2, $src3" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_mls($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
               $pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector fmls

// dst_src1 = src2 * (-src3) + dst_src1
// "(-src2) * src3 + dst_src1" has been idealized to "src3 * (-src2) + dst_src1"
instruct vfmls(vReg dst_src1, vReg src2, vReg src3) %{
  match(Set dst_src1 (FmaVF dst_src1 (Binary src2 (NegVF src3))));
  match(Set dst_src1 (FmaVD dst_src1 (Binary src2 (NegVD src3))));
  format %{ "vfmls $dst_src1, $src2, $src3" %}
  ins_encode %{
    assert(UseFMA, "Needs FMA instructions support.");
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ fmls($dst_src1$$FloatRegister, get_arrangement(this),
              $src2$$FloatRegister, $src3$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      BasicType bt = Matcher::vector_element_basic_type(this);
      __ sve_fmls($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                  ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector fmsb - predicated

// dst_src1 = dst_src1 * (-src2) + src3
instruct vfmsb_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (FmaVF (Binary dst_src1 (NegVF src2)) (Binary src3 pg)));
  match(Set dst_src1 (FmaVD (Binary dst_src1 (NegVD src2)) (Binary src3 pg)));
  format %{ "vfmsb_masked $dst_src1, $pg, $src2, $src3" %}
  ins_encode %{
    assert(UseFMA, "Needs FMA instructions support.");
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_fmsb($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                $pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector fnmla (sve)

// dst_src1 = src2 * (-src3) - dst_src1
// "(-src2) * src3 - dst_src1" has been idealized to "src3 * (-src2) - dst_src1"
instruct vfnmla(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 (NegVF src3))));
  match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 (NegVD src3))));
  format %{ "vfnmla $dst_src1, $src2, $src3" %}
  ins_encode %{
    assert(UseFMA, "Needs FMA instructions support.");
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_fnmla($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                 ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector fnmad - predicated

// dst_src1 = dst_src1 * (-src2) - src3
instruct vfnmad_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (FmaVF (Binary dst_src1 (NegVF src2)) (Binary (NegVF src3) pg)));
  match(Set dst_src1 (FmaVD (Binary dst_src1 (NegVD src2)) (Binary (NegVD src3) pg)));
  format %{ "vfnmad_masked $dst_src1, $pg, $src2, $src3" %}
  ins_encode %{
    assert(UseFMA, "Needs FMA instructions support.");
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_fnmad($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                 $pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector fnmls (sve)

// dst_src1 = src2 * src3 - dst_src1
instruct vfnmls(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 src3)));
  match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 src3)));
  format %{ "vfnmls $dst_src1, $src2, $src3" %}
  ins_encode %{
    assert(UseFMA, "Needs FMA instructions support.");
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_fnmls($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                 ptrue, $src2$$FloatRegister, $src3$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector fnmsb - predicated

// dst_src1 = dst_src1 * src2 - src3
instruct vfnmsb_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (FmaVF (Binary dst_src1 src2) (Binary (NegVF src3) pg)));
  match(Set dst_src1 (FmaVD (Binary dst_src1 src2) (Binary (NegVD src3) pg)));
  format %{ "vfnmsb_masked $dst_src1, $pg, $src2, $src3" %}
  ins_encode %{
    assert(UseFMA, "Needs FMA instructions support.");
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_fnmsb($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
                 $pg$$PRegister, $src2$$FloatRegister, $src3$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// MulAddVS2VI
// Vector Multiply-Add Shorts into Integer

instruct vmuladdS2I(vReg dst, vReg src1, vReg src2, vReg tmp) %{
  predicate(Matcher::vector_length_in_bytes(n) == 16 &&
            Matcher::vector_element_basic_type(n->in(1)) == T_SHORT);
  match(Set dst (MulAddVS2VI src1 src2));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "vmuladdS2I $dst, $src1, $src2\t# KILL $tmp" %}
  ins_encode %{
    __ smullv($tmp$$FloatRegister, __ T4H, $src1$$FloatRegister, $src2$$FloatRegister);
    __ smullv($dst$$FloatRegister, __ T8H, $src1$$FloatRegister, $src2$$FloatRegister);
    __ addpv($dst$$FloatRegister, __ T4S, $tmp$$FloatRegister, $dst$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector shift ---------------------------------

// Vector right shift in AArch64 ASIMD
//
// Right shifts with vector shift count on AArch64 ASIMD are implemented
// as left shift by negative shift count.
// There are two cases for vector shift count.
//
// Case 1: The vector shift count is from replication.
//        |            |
//    LoadVector  RShiftCntV
//        |       /
//     RShiftVI
//
// Case 2: The vector shift count is from loading.
// This case isn't supported by middle-end now. But it's supported by
// panama/vectorIntrinsics(JEP 338: Vector API).
//        |            |
//    LoadVector  LoadVector
//        |       /
//     RShiftVI
//
// The negate is conducted in RShiftCntV rule for case 1, whereas it's done in
// RShiftV* rules for case 2. Because there exists an optimization opportunity
// for case 1, that is, multiple neg instructions in inner loop can be hoisted
// to outer loop and merged into one neg instruction.
//
// Note that ShiftVNode::is_var_shift() indicates whether the vector shift
// count is a variable vector(case 2) or not(a vector generated by RShiftCntV,
// i.e. case 1).

// vector shift count

instruct vshiftcntL(vReg dst, iRegIorL2I cnt) %{
  match(Set dst (LShiftCntV cnt));
  format %{ "vshiftcntL $dst, $cnt" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ dup($dst$$FloatRegister, get_arrangement(this), $cnt$$Register);
    } else {
      assert(UseSVE > 0, "must be sve");
      BasicType bt = Matcher::vector_element_basic_type(this);
      __ sve_dup($dst$$FloatRegister, __ elemType_to_regVariant(bt), $cnt$$Register);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vshiftcntR(vReg dst, iRegIorL2I cnt) %{
  match(Set dst (RShiftCntV cnt));
  format %{ "vshiftcntR $dst, $cnt" %}
  ins_encode %{
    if (UseSVE == 0) {
      uint length_in_bytes = Matcher::vector_length_in_bytes(this);
      assert(length_in_bytes <= 16, "must be");
      __ negw(rscratch1, $cnt$$Register);
      __ dup($dst$$FloatRegister, get_arrangement(this), rscratch1);
    } else {
      BasicType bt = Matcher::vector_element_basic_type(this);
      __ sve_dup($dst$$FloatRegister, __ elemType_to_regVariant(bt), $cnt$$Register);
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector shift left

instruct vlsl_neon(vReg dst, vReg src, vReg shift) %{
  predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst (LShiftVB src shift));
  match(Set dst (LShiftVS src shift));
  match(Set dst (LShiftVI src shift));
  match(Set dst (LShiftVL src shift));
  format %{ "vlsl_neon $dst, $src, $shift" %}
  ins_encode %{
    __ sshl($dst$$FloatRegister, get_arrangement(this),
            $src$$FloatRegister, $shift$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsl_sve(vReg dst_src, vReg shift) %{
  predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n)));
  match(Set dst_src (LShiftVB dst_src shift));
  match(Set dst_src (LShiftVS dst_src shift));
  match(Set dst_src (LShiftVI dst_src shift));
  match(Set dst_src (LShiftVL dst_src shift));
  format %{ "vlsl_sve $dst_src, $dst_src, $shift" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_lsl($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
               ptrue, $shift$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector shift right (arithmetic)

instruct vasr_neon(vReg dst, vReg src, vReg shift) %{
  predicate(UseSVE == 0 && !n->as_ShiftV()->is_var_shift());
  match(Set dst (RShiftVB src shift));
  match(Set dst (RShiftVS src shift));
  match(Set dst (RShiftVI src shift));
  match(Set dst (RShiftVL src shift));
  format %{ "vasr_neon $dst, $src, $shift\t# not variable shift" %}
  ins_encode %{
    __ sshl($dst$$FloatRegister, get_arrangement(this),
            $src$$FloatRegister, $shift$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vasr_neon_var(vReg dst, vReg src, vReg shift) %{
  predicate(UseSVE == 0 && n->as_ShiftV()->is_var_shift());
  match(Set dst (RShiftVB src shift));
  match(Set dst (RShiftVS src shift));
  match(Set dst (RShiftVI src shift));
  match(Set dst (RShiftVL src shift));
  effect(TEMP_DEF dst);
  format %{ "vasr_neon_var $dst, $src, $shift\t# variable shift" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    __ negr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
            $shift$$FloatRegister);
    __ sshl($dst$$FloatRegister, get_arrangement(this),
            $src$$FloatRegister, $dst$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vasr_sve(vReg dst_src, vReg shift) %{
  predicate(UseSVE > 0);
  match(Set dst_src (RShiftVB dst_src shift));
  match(Set dst_src (RShiftVS dst_src shift));
  match(Set dst_src (RShiftVI dst_src shift));
  match(Set dst_src (RShiftVL dst_src shift));
  format %{ "vasr_sve $dst_src, $dst_src, $shift" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_asr($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
               ptrue, $shift$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector shift right (logical)

instruct vlsr_neon(vReg dst, vReg src, vReg shift) %{
  predicate(UseSVE == 0 && !n->as_ShiftV()->is_var_shift());
  match(Set dst (URShiftVB src shift));
  match(Set dst (URShiftVS src shift));
  match(Set dst (URShiftVI src shift));
  match(Set dst (URShiftVL src shift));
  format %{ "vlsr_neon $dst, $src, $shift\t# not variable shift" %}
  ins_encode %{
    __ ushl($dst$$FloatRegister, get_arrangement(this),
            $src$$FloatRegister, $shift$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsr_neon_var(vReg dst, vReg src, vReg shift) %{
  predicate(UseSVE == 0 && n->as_ShiftV()->is_var_shift());
  match(Set dst (URShiftVB src shift));
  match(Set dst (URShiftVS src shift));
  match(Set dst (URShiftVI src shift));
  match(Set dst (URShiftVL src shift));
  effect(TEMP_DEF dst);
  format %{ "vlsr_neon_var $dst, $src, $shift\t# variable shift" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    __ negr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
            $shift$$FloatRegister);
    __ ushl($dst$$FloatRegister, get_arrangement(this),
            $src$$FloatRegister, $dst$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsr_sve(vReg dst_src, vReg shift) %{
  predicate(UseSVE > 0);
  match(Set dst_src (URShiftVB dst_src shift));
  match(Set dst_src (URShiftVS dst_src shift));
  match(Set dst_src (URShiftVI dst_src shift));
  match(Set dst_src (URShiftVL dst_src shift));
  format %{ "vlsr_sve $dst_src, $dst_src, $shift" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_lsr($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
               ptrue, $shift$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector shift with imm

instruct vlsl_imm(vReg dst, vReg src, immI shift) %{
  predicate(assert_not_var_shift(n));
  match(Set dst (LShiftVB src (LShiftCntV shift)));
  match(Set dst (LShiftVS src (LShiftCntV shift)));
  match(Set dst (LShiftVI src (LShiftCntV shift)));
  match(Set dst (LShiftVL src (LShiftCntV shift)));
  format %{ "vlsl_imm $dst, $src, $shift" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    int con = (int)$shift$$constant;
    if (is_subword_type(bt)) {
      // Optimize for B/S
      int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
      if (con >= esize_in_bits) {
        if (VM_Version::use_neon_for_vector(length_in_bytes)) {
          __ eor($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
                 $src$$FloatRegister, $src$$FloatRegister);
        } else {
          assert(UseSVE > 0, "must be sve");
          __ sve_eor($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
        }
        return;
      }
    }
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ shl($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister, con);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_lsl($dst$$FloatRegister, __ elemType_to_regVariant(bt), $src$$FloatRegister, con);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vasr_imm(vReg dst, vReg src, immI_positive shift) %{
  predicate(assert_not_var_shift(n));
  match(Set dst (RShiftVB src (RShiftCntV shift)));
  match(Set dst (RShiftVS src (RShiftCntV shift)));
  match(Set dst (RShiftVI src (RShiftCntV shift)));
  match(Set dst (RShiftVL src (RShiftCntV shift)));
  format %{ "vasr_imm $dst, $src, $shift" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    int con = (int)$shift$$constant;
    if (is_subword_type(bt)) {
      // Refine con for B/S
      int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
      if (con >= esize_in_bits) con = esize_in_bits - 1;
    }
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ sshr($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister, con);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_asr($dst$$FloatRegister, __ elemType_to_regVariant(bt), $src$$FloatRegister, con);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsr_imm(vReg dst, vReg src, immI_positive shift) %{
  predicate(assert_not_var_shift(n));
  match(Set dst (URShiftVB src (RShiftCntV shift)));
  match(Set dst (URShiftVS src (RShiftCntV shift)));
  match(Set dst (URShiftVI src (RShiftCntV shift)));
  match(Set dst (URShiftVL src (RShiftCntV shift)));
  format %{ "vlsr_imm $dst, $src, $shift" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    int con = (int)$shift$$constant;
    if (is_subword_type(bt)) {
      // Optimize for B/S
      int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
      if (con >= esize_in_bits) {
        if (VM_Version::use_neon_for_vector(length_in_bytes)) {
          __ eor($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
                 $src$$FloatRegister, $src$$FloatRegister);
        } else {
          assert(UseSVE > 0, "must be sve");
          __ sve_eor($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
        }
        return;
      }
    }
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ ushr($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister, con);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_lsr($dst$$FloatRegister, __ elemType_to_regVariant(bt), $src$$FloatRegister, con);
    }
  %}
  ins_pipe(pipe_slow);
%}

// shift right add with imm (vector length <= 128 bits only)

instruct vasra_imm(vReg dst, vReg src, immI_positive shift) %{
  predicate(Matcher::vector_length_in_bytes(n) <= 16);
  match(Set dst (AddVB dst (RShiftVB src (RShiftCntV shift))));
  match(Set dst (AddVS dst (RShiftVS src (RShiftCntV shift))));
  match(Set dst (AddVI dst (RShiftVI src (RShiftCntV shift))));
  match(Set dst (AddVL dst (RShiftVL src (RShiftCntV shift))));
  format %{ "vasra_imm $dst, $src, $shift" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    int con = (int)$shift$$constant;
    if (is_subword_type(bt)) {
      // Refine con for B/S
      int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
      if (con >= esize_in_bits) con = esize_in_bits - 1;
    }
    __ ssra($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister, con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsra_imm(vReg dst, vReg src, immI_positive shift) %{
  predicate(Matcher::vector_length_in_bytes(n) <= 16);
  match(Set dst (AddVB dst (URShiftVB src (RShiftCntV shift))));
  match(Set dst (AddVS dst (URShiftVS src (RShiftCntV shift))));
  match(Set dst (AddVI dst (URShiftVI src (RShiftCntV shift))));
  match(Set dst (AddVL dst (URShiftVL src (RShiftCntV shift))));
  format %{ "vlsra_imm $dst, $src, $shift" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    int con = (int)$shift$$constant;
    if (is_subword_type(bt)) { // for B/H
      int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
      if (con < esize_in_bits) {
        __ usra($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister, con);
      }
    } else { // for S/D
      assert(type2aelembytes(bt) == 4 || type2aelembytes(bt) == 8, "unsupported type");
      __ usra($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister, con);
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector shift - predicated

instruct vlsl_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (LShiftVB (Binary dst_src1 src2) pg));
  match(Set dst_src1 (LShiftVS (Binary dst_src1 src2) pg));
  match(Set dst_src1 (LShiftVI (Binary dst_src1 src2) pg));
  match(Set dst_src1 (LShiftVL (Binary dst_src1 src2) pg));
  format %{ "vlsl_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_lsl($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
               $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vasr_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (RShiftVB (Binary dst_src1 src2) pg));
  match(Set dst_src1 (RShiftVS (Binary dst_src1 src2) pg));
  match(Set dst_src1 (RShiftVI (Binary dst_src1 src2) pg));
  match(Set dst_src1 (RShiftVL (Binary dst_src1 src2) pg));
  format %{ "vasr_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_asr($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
               $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsr_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (URShiftVB (Binary dst_src1 src2) pg));
  match(Set dst_src1 (URShiftVS (Binary dst_src1 src2) pg));
  match(Set dst_src1 (URShiftVI (Binary dst_src1 src2) pg));
  match(Set dst_src1 (URShiftVL (Binary dst_src1 src2) pg));
  format %{ "vlsr_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_lsr($dst_src1$$FloatRegister, __ elemType_to_regVariant(bt),
               $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector shift with imm - predicated

instruct vlsl_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (LShiftVB (Binary dst_src (LShiftCntV shift)) pg));
  match(Set dst_src (LShiftVS (Binary dst_src (LShiftCntV shift)) pg));
  match(Set dst_src (LShiftVI (Binary dst_src (LShiftCntV shift)) pg));
  match(Set dst_src (LShiftVL (Binary dst_src (LShiftCntV shift)) pg));
  format %{ "vlsl_imm_masked $dst_src, $pg, $dst_src, $shift" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
    int con = (int)$shift$$constant;
    assert(con >= 0 && con < esize_in_bits, "invalid shift immediate");
    __ sve_lsl($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
               $pg$$PRegister, con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vasr_imm_masked(vReg dst_src, immI_positive shift, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (RShiftVB (Binary dst_src (RShiftCntV shift)) pg));
  match(Set dst_src (RShiftVS (Binary dst_src (RShiftCntV shift)) pg));
  match(Set dst_src (RShiftVI (Binary dst_src (RShiftCntV shift)) pg));
  match(Set dst_src (RShiftVL (Binary dst_src (RShiftCntV shift)) pg));
  format %{ "vasr_imm_masked $dst_src, $pg, $dst_src, $shift" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
    int con = (int)$shift$$constant;
    assert(con > 0 && con < esize_in_bits, "invalid shift immediate");
    __ sve_asr($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
               $pg$$PRegister, con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsr_imm_masked(vReg dst_src, immI_positive shift, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (URShiftVB (Binary dst_src (RShiftCntV shift)) pg));
  match(Set dst_src (URShiftVS (Binary dst_src (RShiftCntV shift)) pg));
  match(Set dst_src (URShiftVI (Binary dst_src (RShiftCntV shift)) pg));
  match(Set dst_src (URShiftVL (Binary dst_src (RShiftCntV shift)) pg));
  format %{ "vlsr_imm_masked $dst_src, $pg, $dst_src, $shift" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    int esize_in_bits = type2aelembytes(bt) * BitsPerByte;
    int con = (int)$shift$$constant;
    assert(con > 0 && con < esize_in_bits, "invalid shift immediate");
    __ sve_lsr($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
               $pg$$PRegister, con);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector reduction add -------------------------

// reduction addI

instruct reduce_addI_neon(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc, vReg tmp) %{
  predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))));
  match(Set dst (AddReductionVI isrc vsrc));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "reduce_addI_neon $dst, $isrc, $vsrc\t# KILL $tmp" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    __ neon_reduce_add_integral($dst$$Register, bt,
                                $isrc$$Register, $vsrc$$FloatRegister,
                                length_in_bytes, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_addI_sve(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc, vRegD tmp) %{
  predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))));
  match(Set dst (AddReductionVI isrc vsrc));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "reduce_addI_sve $dst, $isrc, $vsrc\t# KILL $tmp" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister,
                           ptrue, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// reduction addL

instruct reduce_addL_neon(iRegLNoSp dst, iRegL isrc, vReg vsrc, vReg tmp) %{
  predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))));
  match(Set dst (AddReductionVL isrc vsrc));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "reduce_addL_neon $dst, $isrc, $vsrc\t# KILL $tmp" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    __ neon_reduce_add_integral($dst$$Register, bt,
                                $isrc$$Register, $vsrc$$FloatRegister,
                                length_in_bytes, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_addL_sve(iRegLNoSp dst, iRegL isrc, vReg vsrc, vRegD tmp) %{
  predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))));
  match(Set dst (AddReductionVL isrc vsrc));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "reduce_addL_sve $dst, $isrc, $vsrc\t# KILL $tmp" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister,
                           ptrue, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// reduction addF

instruct reduce_non_strict_order_add2F_neon(vRegF dst, vRegF fsrc, vReg vsrc) %{
  // Non-strictly ordered floating-point add reduction for a 64-bits-long vector. This rule is
  // intended for the VectorAPI (which allows for non-strictly ordered add reduction).
  predicate(Matcher::vector_length(n->in(2)) == 2 && !n->as_Reduction()->requires_strict_order());
  match(Set dst (AddReductionVF fsrc vsrc));
  effect(TEMP_DEF dst);
  format %{ "reduce_non_strict_order_add2F_neon $dst, $fsrc, $vsrc" %}
  ins_encode %{
    __ faddp($dst$$FloatRegister, $vsrc$$FloatRegister, __ S);
    __ fadds($dst$$FloatRegister, $dst$$FloatRegister, $fsrc$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_non_strict_order_add4F_neon(vRegF dst, vRegF fsrc, vReg vsrc, vReg tmp) %{
  // Non-strictly ordered floating-point add reduction for 128-bits-long vector. This rule is
  // intended for the VectorAPI (which allows for non-strictly ordered add reduction).
  predicate(Matcher::vector_length(n->in(2)) == 4 && !n->as_Reduction()->requires_strict_order());
  match(Set dst (AddReductionVF fsrc vsrc));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "reduce_non_strict_order_add4F_neon $dst, $fsrc, $vsrc\t# KILL $tmp" %}
  ins_encode %{
    __ faddp($tmp$$FloatRegister, __ T4S, $vsrc$$FloatRegister, $vsrc$$FloatRegister);
    __ faddp($dst$$FloatRegister, $tmp$$FloatRegister, __ S);
    __ fadds($dst$$FloatRegister, $dst$$FloatRegister, $fsrc$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// This rule calculates the reduction result in strict order. Two cases will
// reach here:
// 1. Non strictly-ordered AddReductionVF when vector size > 128-bits. For example -
//    AddReductionVF generated by Vector API. For vector size > 128-bits, it is more
//    beneficial performance-wise to generate direct SVE instruction even if it is
//    strictly ordered.
// 2. Strictly-ordered AddReductionVF. For example - AddReductionVF generated by
//    auto-vectorization on SVE machine.
instruct reduce_addF_sve(vRegF dst_src1, vReg src2) %{
  predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))) ||
            n->as_Reduction()->requires_strict_order());
  match(Set dst_src1 (AddReductionVF dst_src1 src2));
  format %{ "reduce_addF_sve $dst_src1, $dst_src1, $src2" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src2);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    __ sve_fadda($dst_src1$$FloatRegister, __ S, ptrue, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// reduction addD

instruct reduce_non_strict_order_add2D_neon(vRegD dst, vRegD dsrc, vReg vsrc) %{
  // Non-strictly ordered floating-point add reduction for doubles. This rule is
  // intended for the VectorAPI (which allows for non-strictly ordered add reduction).
  predicate(!n->as_Reduction()->requires_strict_order());
  match(Set dst (AddReductionVD dsrc vsrc));
  effect(TEMP_DEF dst);
  format %{ "reduce_non_strict_order_add2D_neon $dst, $dsrc, $vsrc\t# 2D" %}
  ins_encode %{
    __ faddp($dst$$FloatRegister, $vsrc$$FloatRegister, __ D);
    __ faddd($dst$$FloatRegister, $dst$$FloatRegister, $dsrc$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// This rule calculates the reduction result in strict order. Two cases will
// reach here:
// 1. Non strictly-ordered AddReductionVD when vector size > 128-bits. For example -
//    AddReductionVD generated by Vector API. For vector size > 128-bits, it is more
//    beneficial performance-wise to generate direct SVE instruction even if it is
//    strictly ordered.
// 2. Strictly-ordered AddReductionVD. For example - AddReductionVD generated by
//    auto-vectorization on SVE machine.
instruct reduce_addD_sve(vRegD dst_src1, vReg src2) %{
  predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))) ||
            n->as_Reduction()->requires_strict_order());
  match(Set dst_src1 (AddReductionVD dst_src1 src2));
  format %{ "reduce_addD_sve $dst_src1, $dst_src1, $src2" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src2);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    __ sve_fadda($dst_src1$$FloatRegister, __ D, ptrue, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// reduction add - predicated

instruct reduce_addI_masked(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc, pRegGov pg, vRegD tmp) %{
  predicate(UseSVE > 0);
  match(Set dst (AddReductionVI (Binary isrc vsrc) pg));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "reduce_addI_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister,
                           $pg$$PRegister, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_addL_masked(iRegLNoSp dst, iRegL isrc, vReg vsrc, pRegGov pg, vRegD tmp) %{
  predicate(UseSVE > 0);
  match(Set dst (AddReductionVL (Binary isrc vsrc) pg));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "reduce_addL_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister,
                           $pg$$PRegister, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_addF_masked(vRegF dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AddReductionVF (Binary dst_src1 src2) pg));
  format %{ "reduce_addF_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_fadda($dst_src1$$FloatRegister, __ S,
                 $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_addD_masked(vRegD dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AddReductionVD (Binary dst_src1 src2) pg));
  format %{ "reduce_addD_masked $dst_src1, $pg, $dst_src1, $src2" %}
  ins_encode %{
    __ sve_fadda($dst_src1$$FloatRegister, __ D,
                 $pg$$PRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector reduction mul -------------------------

instruct reduce_mulI(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc,
                     vReg tmp1, vReg tmp2) %{
  predicate(Matcher::vector_length_in_bytes(n->in(2)) == 8 ||
            Matcher::vector_length_in_bytes(n->in(2)) == 16);
  match(Set dst (MulReductionVI isrc vsrc));
  effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2);
  format %{ "reduce_mulI $dst, $isrc, $vsrc\t# vector (64/128 bits). KILL $tmp1, $tmp2" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    __ neon_reduce_mul_integral($dst$$Register, bt, $isrc$$Register,
                                $vsrc$$FloatRegister, length_in_bytes,
                                $tmp1$$FloatRegister, $tmp2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_mulL(iRegLNoSp dst, iRegL isrc, vReg vsrc) %{
  predicate(Matcher::vector_length_in_bytes(n->in(2)) == 16);
  match(Set dst (MulReductionVL isrc vsrc));
  effect(TEMP_DEF dst);
  format %{ "reduce_mulL $dst, $isrc, $vsrc\t# 2L" %}
  ins_encode %{
    __ neon_reduce_mul_integral($dst$$Register, T_LONG, $isrc$$Register,
                                $vsrc$$FloatRegister, 16, fnoreg, fnoreg);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_mulF(vRegF dst, vRegF fsrc, vReg vsrc, vReg tmp) %{
  predicate(Matcher::vector_length_in_bytes(n->in(2)) <= 16);
  match(Set dst (MulReductionVF fsrc vsrc));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "reduce_mulF $dst, $fsrc, $vsrc\t# 2F/4F. KILL $tmp" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    __ neon_reduce_mul_fp($dst$$FloatRegister, T_FLOAT, $fsrc$$FloatRegister,
                          $vsrc$$FloatRegister, length_in_bytes, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_mulD(vRegD dst, vRegD dsrc, vReg vsrc, vReg tmp) %{
  predicate(Matcher::vector_length_in_bytes(n->in(2)) == 16);
  match(Set dst (MulReductionVD dsrc vsrc));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "reduce_mulD $dst, $dsrc, $vsrc\t# 2D. KILL $tmp" %}
  ins_encode %{
    __ neon_reduce_mul_fp($dst$$FloatRegister, T_DOUBLE, $dsrc$$FloatRegister,
                          $vsrc$$FloatRegister, 16, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector reduction and -------------------------

// reduction andI

instruct reduce_andI_neon(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc) %{
  predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n->in(2)) != T_LONG);
  match(Set dst (AndReductionV isrc vsrc));
  effect(TEMP_DEF dst);
  format %{ "reduce_andI_neon $dst, $isrc, $vsrc" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    __ neon_reduce_logical(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister, length_in_bytes);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_andI_sve(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc, vRegD tmp) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(2)) != T_LONG);
  match(Set dst (AndReductionV isrc vsrc));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "reduce_andI_sve $dst, $isrc, $vsrc\t# KILL $tmp" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister,
                           ptrue, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// reduction andL

instruct reduce_andL_neon(iRegLNoSp dst, iRegL isrc, vReg vsrc) %{
  predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
  match(Set dst (AndReductionV isrc vsrc));
  effect(TEMP_DEF dst);
  format %{ "reduce_andL_neon $dst, $isrc, $vsrc" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    __ neon_reduce_logical(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister, length_in_bytes);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_andL_sve(iRegLNoSp dst, iRegL isrc, vReg vsrc, vRegD tmp) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
  match(Set dst (AndReductionV isrc vsrc));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "reduce_andL_sve $dst, $isrc, $vsrc\t# KILL $tmp" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister,
                           ptrue, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// reduction and - predicated

instruct reduce_andI_masked(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc, pRegGov pg, vRegD tmp) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) != T_LONG);
  match(Set dst (AndReductionV (Binary isrc vsrc) pg));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "reduce_andI_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister,
                           $pg$$PRegister, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_andL_masked(iRegLNoSp dst, iRegL isrc, vReg vsrc, pRegGov pg, vRegD tmp) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_LONG);
  match(Set dst (AndReductionV (Binary isrc vsrc) pg));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "reduce_andL_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister,
                           $pg$$PRegister, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector reduction or --------------------------

// reduction orI

instruct reduce_orI_neon(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc) %{
  predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n->in(2)) != T_LONG);
  match(Set dst (OrReductionV isrc vsrc));
  effect(TEMP_DEF dst);
  format %{ "reduce_orI_neon $dst, $isrc, $vsrc" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    __ neon_reduce_logical(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister, length_in_bytes);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_orI_sve(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc, vRegD tmp) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(2)) != T_LONG);
  match(Set dst (OrReductionV isrc vsrc));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "reduce_orI_sve $dst, $isrc, $vsrc\t# KILL $tmp" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister,
                           ptrue, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// reduction orL

instruct reduce_orL_neon(iRegLNoSp dst, iRegL isrc, vReg vsrc) %{
  predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
  match(Set dst (OrReductionV isrc vsrc));
  effect(TEMP_DEF dst);
  format %{ "reduce_orL_neon $dst, $isrc, $vsrc" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    __ neon_reduce_logical(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister, length_in_bytes);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_orL_sve(iRegLNoSp dst, iRegL isrc, vReg vsrc, vRegD tmp) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
  match(Set dst (OrReductionV isrc vsrc));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "reduce_orL_sve $dst, $isrc, $vsrc\t# KILL $tmp" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister,
                           ptrue, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// reduction or - predicated

instruct reduce_orI_masked(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc, pRegGov pg, vRegD tmp) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) != T_LONG);
  match(Set dst (OrReductionV (Binary isrc vsrc) pg));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "reduce_orI_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister,
                           $pg$$PRegister, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_orL_masked(iRegLNoSp dst, iRegL isrc, vReg vsrc, pRegGov pg, vRegD tmp) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_LONG);
  match(Set dst (OrReductionV (Binary isrc vsrc) pg));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "reduce_orL_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister,
                           $pg$$PRegister, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector reduction xor -------------------------

// reduction xorI

instruct reduce_xorI_neon(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc) %{
  predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n->in(2)) != T_LONG);
  match(Set dst (XorReductionV isrc vsrc));
  effect(TEMP_DEF dst);
  format %{ "reduce_xorI_neon $dst, $isrc, $vsrc" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    __ neon_reduce_logical(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister, length_in_bytes);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_xorI_sve(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc, vRegD tmp) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(2)) != T_LONG);
  match(Set dst (XorReductionV isrc vsrc));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "reduce_xorI_sve $dst, $isrc, $vsrc\t# KILL $tmp" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister,
                           ptrue, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// reduction xorL

instruct reduce_xorL_neon(iRegLNoSp dst, iRegL isrc, vReg vsrc) %{
  predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
  match(Set dst (XorReductionV isrc vsrc));
  effect(TEMP_DEF dst);
  format %{ "reduce_xorL_neon $dst, $isrc, $vsrc" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    __ neon_reduce_logical(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister, length_in_bytes);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_xorL_sve(iRegLNoSp dst, iRegL isrc, vReg vsrc, vRegD tmp) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
  match(Set dst (XorReductionV isrc vsrc));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "reduce_xorL_sve $dst, $isrc, $vsrc\t# KILL $tmp" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister,
                           ptrue, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// reduction xor - predicated

instruct reduce_xorI_masked(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc, pRegGov pg, vRegD tmp) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) != T_LONG);
  match(Set dst (XorReductionV (Binary isrc vsrc) pg));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "reduce_xorI_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister,
                           $pg$$PRegister, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_xorL_masked(iRegLNoSp dst, iRegL isrc, vReg vsrc, pRegGov pg, vRegD tmp) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_LONG);
  match(Set dst (XorReductionV (Binary isrc vsrc) pg));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "reduce_xorL_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister,
                           $pg$$PRegister, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector reduction max -------------------------

// reduction maxI

instruct reduce_maxI_neon(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc,
                          vReg tmp, rFlagsReg cr) %{
  predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))) &&
            (Matcher::vector_element_basic_type(n->in(2)) == T_BYTE ||
             Matcher::vector_element_basic_type(n->in(2)) == T_SHORT ||
             Matcher::vector_element_basic_type(n->in(2)) == T_INT));
  match(Set dst (MaxReductionV isrc vsrc));
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  format %{ "reduce_maxI_neon $dst, $isrc, $vsrc\t# KILL $tmp, cr" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    __ neon_reduce_minmax_integral(this->ideal_Opcode(), $dst$$Register, bt,
                                   $isrc$$Register, $vsrc$$FloatRegister,
                                   length_in_bytes, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_maxI_sve(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc,
                         vRegD tmp, rFlagsReg cr) %{
  predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))) &&
            (Matcher::vector_element_basic_type(n->in(2)) == T_BYTE ||
             Matcher::vector_element_basic_type(n->in(2)) == T_SHORT ||
             Matcher::vector_element_basic_type(n->in(2)) == T_INT));
  match(Set dst (MaxReductionV isrc vsrc));
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  format %{ "reduce_maxI_sve $dst, $isrc, $vsrc\t# KILL $tmp, cr" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister,
                           ptrue, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// reduction maxL

instruct reduce_maxL_neon(iRegLNoSp dst, iRegL isrc, vReg vsrc, rFlagsReg cr) %{
  predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
  match(Set dst (MaxReductionV isrc vsrc));
  effect(TEMP_DEF dst, KILL cr);
  format %{ "reduce_maxL_neon $dst, $isrc, $vsrc\t# 2L. KILL cr" %}
  ins_encode %{
    __ neon_reduce_minmax_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
                                   $isrc$$Register, $vsrc$$FloatRegister,
                                   /* vector_length_in_bytes */ 16, fnoreg);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_maxL_sve(iRegLNoSp dst, iRegL isrc, vReg vsrc,
                         vRegD tmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
  match(Set dst (MaxReductionV isrc vsrc));
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  format %{ "reduce_maxL_sve $dst, $isrc, $vsrc\t# KILL $tmp, cr" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
                           $isrc$$Register, $vsrc$$FloatRegister,
                           ptrue, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// reduction maxF

instruct reduce_maxF(vRegF dst, vRegF fsrc, vReg vsrc) %{
  predicate(Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT);
  match(Set dst (MaxReductionV fsrc vsrc));
  effect(TEMP_DEF dst);
  format %{ "reduce_maxF $dst, $fsrc, $vsrc" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      if (length_in_bytes == 8) {
        __ fmaxp($dst$$FloatRegister, $vsrc$$FloatRegister, __ S);
      } else {
        __ fmaxv($dst$$FloatRegister, __ T4S, $vsrc$$FloatRegister);
      }
    } else {
      assert(UseSVE > 0, "must be sve");
      assert(length_in_bytes == MaxVectorSize, "invalid vector length");
      __ sve_fmaxv($dst$$FloatRegister, __ S, ptrue, $vsrc$$FloatRegister);
    }
    __ fmaxs($dst$$FloatRegister, $dst$$FloatRegister, $fsrc$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// reduction maxD

instruct reduce_maxD(vRegD dst, vRegD dsrc, vReg vsrc) %{
  predicate(Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE);
  match(Set dst (MaxReductionV dsrc vsrc));
  effect(TEMP_DEF dst);
  format %{ "reduce_maxD $dst, $dsrc, $vsrc" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ fmaxp($dst$$FloatRegister, $vsrc$$FloatRegister, __ D);
    } else {
      assert(UseSVE > 0, "must be sve");
      assert(length_in_bytes == MaxVectorSize, "invalid vector length");
      __ sve_fmaxv($dst$$FloatRegister, __ D, ptrue, $vsrc$$FloatRegister);
    }
    __ fmaxd($dst$$FloatRegister, $dst$$FloatRegister, $dsrc$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// reduction max - predicated

instruct reduce_maxI_masked(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc, pRegGov pg,
                            vRegD tmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            (Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_BYTE ||
             Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_SHORT ||
             Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_INT));
  match(Set dst (MaxReductionV (Binary isrc vsrc) pg));
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  format %{ "reduce_maxI_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp, cr" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister,
                           $pg$$PRegister, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_maxL_masked(iRegLNoSp dst, iRegL isrc, vReg vsrc, pRegGov pg,
                            vRegD tmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_LONG);
  match(Set dst (MaxReductionV (Binary isrc vsrc) pg));
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  format %{ "reduce_maxL_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp, cr" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister,
                           $pg$$PRegister, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_maxF_masked(vRegF dst, vRegF fsrc, vReg vsrc, pRegGov pg) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_FLOAT);
  match(Set dst (MaxReductionV (Binary fsrc vsrc) pg));
  effect(TEMP_DEF dst);
  format %{ "reduce_maxF_masked $dst, $fsrc, $pg, $vsrc" %}
  ins_encode %{
    __ sve_fmaxv($dst$$FloatRegister, __ S, $pg$$PRegister, $vsrc$$FloatRegister);
    __ fmaxs($dst$$FloatRegister, $dst$$FloatRegister, $fsrc$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_maxD_masked(vRegD dst, vRegD dsrc, vReg vsrc, pRegGov pg) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_DOUBLE);
  match(Set dst (MaxReductionV (Binary dsrc vsrc) pg));
  effect(TEMP_DEF dst);
  format %{ "reduce_maxD_masked $dst, $dsrc, $pg, $vsrc" %}
  ins_encode %{
    __ sve_fmaxv($dst$$FloatRegister, __ D, $pg$$PRegister, $vsrc$$FloatRegister);
    __ fmaxd($dst$$FloatRegister, $dst$$FloatRegister, $dsrc$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector reduction min -------------------------

// reduction minI

instruct reduce_minI_neon(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc,
                          vReg tmp, rFlagsReg cr) %{
  predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))) &&
            (Matcher::vector_element_basic_type(n->in(2)) == T_BYTE ||
             Matcher::vector_element_basic_type(n->in(2)) == T_SHORT ||
             Matcher::vector_element_basic_type(n->in(2)) == T_INT));
  match(Set dst (MinReductionV isrc vsrc));
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  format %{ "reduce_minI_neon $dst, $isrc, $vsrc\t# KILL $tmp, cr" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    __ neon_reduce_minmax_integral(this->ideal_Opcode(), $dst$$Register, bt,
                                   $isrc$$Register, $vsrc$$FloatRegister,
                                   length_in_bytes, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_minI_sve(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc,
                         vRegD tmp, rFlagsReg cr) %{
  predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(2))) &&
            (Matcher::vector_element_basic_type(n->in(2)) == T_BYTE ||
             Matcher::vector_element_basic_type(n->in(2)) == T_SHORT ||
             Matcher::vector_element_basic_type(n->in(2)) == T_INT));
  match(Set dst (MinReductionV isrc vsrc));
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  format %{ "reduce_minI_sve $dst, $isrc, $vsrc\t# KILL $tmp, cr" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister,
                           ptrue, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// reduction minL

instruct reduce_minL_neon(iRegLNoSp dst, iRegL isrc, vReg vsrc, rFlagsReg cr) %{
  predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
  match(Set dst (MinReductionV isrc vsrc));
  effect(TEMP_DEF dst, KILL cr);
  format %{ "reduce_minL_neon $dst, $isrc, $vsrc\t# 2L. KILL cr" %}
  ins_encode %{
    __ neon_reduce_minmax_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
                                   $isrc$$Register, $vsrc$$FloatRegister,
                                   /* vector_length_in_bytes */ 16, fnoreg);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_minL_sve(iRegLNoSp dst, iRegL isrc, vReg vsrc,
                         vRegD tmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(2)) == T_LONG);
  match(Set dst (MinReductionV isrc vsrc));
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  format %{ "reduce_minL_sve $dst, $isrc, $vsrc\t# KILL $tmp, cr" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
                           $isrc$$Register, $vsrc$$FloatRegister,
                           ptrue, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// reduction minF

instruct reduce_minF(vRegF dst, vRegF fsrc, vReg vsrc) %{
  predicate(Matcher::vector_element_basic_type(n->in(2)) == T_FLOAT);
  match(Set dst (MinReductionV fsrc vsrc));
  effect(TEMP_DEF dst);
  format %{ "reduce_minF $dst, $fsrc, $vsrc" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      if (length_in_bytes == 8) {
        __ fminp($dst$$FloatRegister, $vsrc$$FloatRegister, __ S);
      } else {
        __ fminv($dst$$FloatRegister, __ T4S, $vsrc$$FloatRegister);
      }
    } else {
      assert(UseSVE > 0, "must be sve");
      assert(length_in_bytes == MaxVectorSize, "invalid vector length");
      __ sve_fminv($dst$$FloatRegister, __ S, ptrue, $vsrc$$FloatRegister);
    }
    __ fmins($dst$$FloatRegister, $dst$$FloatRegister, $fsrc$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// reduction minD

instruct reduce_minD(vRegD dst, vRegD dsrc, vReg vsrc) %{
  predicate(Matcher::vector_element_basic_type(n->in(2)) == T_DOUBLE);
  match(Set dst (MinReductionV dsrc vsrc));
  effect(TEMP_DEF dst);
  format %{ "reduce_minD $dst, $dsrc, $vsrc" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $vsrc);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ fminp($dst$$FloatRegister, $vsrc$$FloatRegister, __ D);
    } else {
      assert(UseSVE > 0, "must be sve");
      assert(length_in_bytes == MaxVectorSize, "invalid vector length");
      __ sve_fminv($dst$$FloatRegister, __ D, ptrue, $vsrc$$FloatRegister);
    }
    __ fmind($dst$$FloatRegister, $dst$$FloatRegister, $dsrc$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// reduction min - predicated

instruct reduce_minI_masked(iRegINoSp dst, iRegIorL2I isrc, vReg vsrc, pRegGov pg,
                            vRegD tmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            (Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_BYTE ||
             Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_SHORT ||
             Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_INT));
  match(Set dst (MinReductionV (Binary isrc vsrc) pg));
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  format %{ "reduce_minI_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp, cr" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister,
                           $pg$$PRegister, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_minL_masked(iRegLNoSp dst, iRegL isrc, vReg vsrc, pRegGov pg,
                            vRegD tmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_LONG);
  match(Set dst (MinReductionV (Binary isrc vsrc) pg));
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  format %{ "reduce_minL_masked $dst, $isrc, $pg, $vsrc\t# KILL $tmp, cr" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $vsrc);
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $isrc$$Register, $vsrc$$FloatRegister,
                           $pg$$PRegister, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_minF_masked(vRegF dst, vRegF fsrc, vReg vsrc, pRegGov pg) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_FLOAT);
  match(Set dst (MinReductionV (Binary fsrc vsrc) pg));
  effect(TEMP_DEF dst);
  format %{ "reduce_minF_masked $dst, $fsrc, $pg, $vsrc" %}
  ins_encode %{
    __ sve_fminv($dst$$FloatRegister, __ S, $pg$$PRegister, $vsrc$$FloatRegister);
    __ fmins($dst$$FloatRegister, $dst$$FloatRegister, $fsrc$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_minD_masked(vRegD dst, vRegD dsrc, vReg vsrc, pRegGov pg) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n->in(1)->in(2)) == T_DOUBLE);
  match(Set dst (MinReductionV (Binary dsrc vsrc) pg));
  effect(TEMP_DEF dst);
  format %{ "reduce_minD_masked $dst, $dsrc, $pg, $vsrc" %}
  ins_encode %{
    __ sve_fminv($dst$$FloatRegister, __ D, $pg$$PRegister, $vsrc$$FloatRegister);
    __ fmind($dst$$FloatRegister, $dst$$FloatRegister, $dsrc$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector reinterpret ---------------------------

instruct reinterpret_same_size(vReg dst_src) %{
  predicate(Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)));
  match(Set dst_src (VectorReinterpret dst_src));
  ins_cost(0);
  format %{ "reinterpret_same_size $dst_src\t# do nothing" %}
  ins_encode(/* empty encoding */);
  ins_pipe(pipe_class_empty);
%}

instruct reinterpret_resize_le128b(vReg dst, vReg src) %{
  predicate(Matcher::vector_length_in_bytes(n) != Matcher::vector_length_in_bytes(n->in(1)) &&
            Matcher::vector_length_in_bytes(n) <= 16 &&
            Matcher::vector_length_in_bytes(n->in(1)) <= 16);
  match(Set dst (VectorReinterpret src));
  format %{ "reinterpret_resize_le128b $dst, $src\t# vector <= 128 bits." %}
  ins_encode %{
    uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
    uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
    // The higher bits in "dst" register must be cleared to zero.
    if ((length_in_bytes_src == 4 && length_in_bytes_dst == 8) ||
        (length_in_bytes_src == 8 && length_in_bytes_dst == 4)) {
      // Reinterpret between 32 bits and 64 bits
      __ dup($dst$$FloatRegister, __ S, $src$$FloatRegister);
    } else if ((length_in_bytes_src == 4 && length_in_bytes_dst == 16) ||
               (length_in_bytes_src == 16 && length_in_bytes_dst == 4)) {
      // Reinterpret between 32 bits and 128 bits
      __ dup($dst$$FloatRegister, __ S, $src$$FloatRegister);
    } else if ((length_in_bytes_src == 8 && length_in_bytes_dst == 16) ||
               (length_in_bytes_src == 16 && length_in_bytes_dst == 8)) {
      // Reinterpret between 64 bits and 128 bits
      __ orr($dst$$FloatRegister, __ T8B, $src$$FloatRegister, $src$$FloatRegister);
    } else {
      assert(false, "invalid vector length");
      ShouldNotReachHere();
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct reinterpret_resize_gt128b(vReg dst, vReg src, pReg ptmp, rFlagsReg cr) %{
  predicate(Matcher::vector_length_in_bytes(n) != Matcher::vector_length_in_bytes(n->in(1)) &&
            (Matcher::vector_length_in_bytes(n) > 16 ||
             Matcher::vector_length_in_bytes(n->in(1)) > 16));
  match(Set dst (VectorReinterpret src));
  effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
  format %{ "reinterpret_resize_gt128b $dst, $src\t# vector > 128 bits. KILL $ptmp, cr" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
    uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
    uint length_in_bytes_resize = length_in_bytes_src < length_in_bytes_dst ?
                                  length_in_bytes_src : length_in_bytes_dst;
    assert(length_in_bytes_src <= MaxVectorSize && length_in_bytes_dst <= MaxVectorSize,
           "invalid vector length");
    __ sve_gen_mask_imm($ptmp$$PRegister, T_BYTE, length_in_bytes_resize);
    __ sve_dup($dst$$FloatRegister, __ B, 0);
    __ sve_sel($dst$$FloatRegister, __ B, $ptmp$$PRegister,
               $src$$FloatRegister, $dst$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ---------------------------- Vector zero extend --------------------------------

instruct vzeroExtBtoX(vReg dst, vReg src) %{
  match(Set dst (VectorUCastB2X src));
  format %{ "vzeroExtBtoX $dst, $src" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    assert(bt == T_SHORT || bt == T_INT || bt == T_LONG, "must be");
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      // 4B to 4S/4I, 8B to 8S
      __ neon_vector_extend($dst$$FloatRegister, bt, length_in_bytes,
                            $src$$FloatRegister, T_BYTE, /* is_unsigned */ true);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_vector_extend($dst$$FloatRegister, __ elemType_to_regVariant(bt),
                           $src$$FloatRegister, __ B, /* is_unsigned */ true);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vzeroExtStoX(vReg dst, vReg src) %{
  match(Set dst (VectorUCastS2X src));
  format %{ "vzeroExtStoX $dst, $src" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    assert(bt == T_INT || bt == T_LONG, "must be");
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      // 4S to 4I
      __ neon_vector_extend($dst$$FloatRegister, T_INT, length_in_bytes,
                            $src$$FloatRegister, T_SHORT, /* is_unsigned */ true);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_vector_extend($dst$$FloatRegister, __ elemType_to_regVariant(bt),
                           $src$$FloatRegister, __ H, /* is_unsigned */ true);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vzeroExtItoX(vReg dst, vReg src) %{
  match(Set dst (VectorUCastI2X src));
  format %{ "vzeroExtItoX $dst, $src" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    assert(bt == T_LONG, "must be");
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      // 2I to 2L
      __ neon_vector_extend($dst$$FloatRegister, T_LONG, length_in_bytes,
                            $src$$FloatRegister, T_INT, /* is_unsigned */ true);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_vector_extend($dst$$FloatRegister, __ D,
                           $src$$FloatRegister, __ S, /* is_unsigned */ true);
    }
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector cast ----------------------------------

// VectorCastB2X

instruct vcvtBtoX(vReg dst, vReg src) %{
  match(Set dst (VectorCastB2X src));
  format %{ "vcvtBtoX $dst, $src" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      // 4B to 4S/4I/4F, 8B to 8S
      __ neon_vector_extend($dst$$FloatRegister, bt == T_FLOAT ? T_INT : bt,
                            length_in_bytes, $src$$FloatRegister, T_BYTE);
      if (bt == T_FLOAT) {
        __ scvtfv(__ T4S, $dst$$FloatRegister, $dst$$FloatRegister);
      }
    } else {
      assert(UseSVE > 0, "must be sve");
      Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
      __ sve_vector_extend($dst$$FloatRegister, size, $src$$FloatRegister, __ B);
      if (is_floating_point_type(bt)) {
        __ sve_scvtf($dst$$FloatRegister, size, ptrue, $dst$$FloatRegister, size);
      }
    }
  %}
  ins_pipe(pipe_slow);
%}

// VectorCastS2X

instruct vcvtStoB_neon(vReg dst, vReg src) %{
  predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
            VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))));
  match(Set dst (VectorCastS2X src));
  format %{ "vcvtStoB_neon $dst, $src" %}
  ins_encode %{
    // 4S/8S to 4B/8B
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
    __ neon_vector_narrow($dst$$FloatRegister, T_BYTE,
                          $src$$FloatRegister, T_SHORT, length_in_bytes);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtStoB_sve(vReg dst, vReg src, vReg tmp) %{
  predicate(Matcher::vector_element_basic_type(n) == T_BYTE &&
            !VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))));
  match(Set dst (VectorCastS2X src));
  effect(TEMP tmp);
  format %{ "vcvtStoB_sve $dst, $src\t# KILL $tmp" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    __ sve_vector_narrow($dst$$FloatRegister, __ B,
                         $src$$FloatRegister, __ H, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtStoX_extend(vReg dst, vReg src) %{
  predicate(type2aelembytes(Matcher::vector_element_basic_type(n)) >= 4);
  match(Set dst (VectorCastS2X src));
  format %{ "vcvtStoX_extend $dst, $src" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      // 4S to 4I/4F
      __ neon_vector_extend($dst$$FloatRegister, T_INT, length_in_bytes,
                            $src$$FloatRegister, T_SHORT);
      if (bt == T_FLOAT) {
        __ scvtfv(__ T4S, $dst$$FloatRegister, $dst$$FloatRegister);
      }
    } else {
      assert(UseSVE > 0, "must be sve");
      Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
      __ sve_vector_extend($dst$$FloatRegister, size, $src$$FloatRegister, __ H);
      if (is_floating_point_type(bt)) {
        __ sve_scvtf($dst$$FloatRegister, size, ptrue, $dst$$FloatRegister, size);
      }
    }
  %}
  ins_pipe(pipe_slow);
%}

// VectorCastI2X

instruct vcvtItoX_narrow_neon(vReg dst, vReg src) %{
  predicate((Matcher::vector_element_basic_type(n) == T_BYTE ||
             Matcher::vector_element_basic_type(n) == T_SHORT) &&
            VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))));
  match(Set dst (VectorCastI2X src));
  effect(TEMP_DEF dst);
  format %{ "vcvtItoX_narrow_neon $dst, $src" %}
  ins_encode %{
    // 4I to 4B/4S
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
    __ neon_vector_narrow($dst$$FloatRegister, bt,
                          $src$$FloatRegister, T_INT, length_in_bytes);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtItoX_narrow_sve(vReg dst, vReg src, vReg tmp) %{
  predicate((Matcher::vector_element_basic_type(n) == T_BYTE ||
             Matcher::vector_element_basic_type(n) == T_SHORT) &&
            !VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))));
  match(Set dst (VectorCastI2X src));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "vcvtItoX_narrow_sve $dst, $src\t# KILL $tmp" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_vector_narrow($dst$$FloatRegister, __ elemType_to_regVariant(bt),
                         $src$$FloatRegister, __ S, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtItoX(vReg dst, vReg src) %{
  predicate(type2aelembytes(Matcher::vector_element_basic_type(n)) >= 4);
  match(Set dst (VectorCastI2X src));
  format %{ "vcvtItoX $dst, $src" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes =  Matcher::vector_length_in_bytes(this);
    if (bt == T_FLOAT) {
      if (VM_Version::use_neon_for_vector(length_in_bytes)) {
        // 2I/4I to 2F/4F
        __ scvtfv(get_arrangement(this), $dst$$FloatRegister, $src$$FloatRegister);
      } else {
        assert(UseSVE > 0, "must be sve");
        __ sve_scvtf($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister, __ S);
      }
    } else {
      assert(type2aelembytes(bt) == 8, "unsupported type");
      if (VM_Version::use_neon_for_vector(length_in_bytes)) {
        // 2I to 2L/2D
        __ neon_vector_extend($dst$$FloatRegister, T_LONG, length_in_bytes,
                              $src$$FloatRegister, T_INT);
        if (bt == T_DOUBLE) {
          __ scvtfv(__ T2D, $dst$$FloatRegister, $dst$$FloatRegister);
        }
      } else {
        assert(UseSVE > 0, "must be sve");
        __ sve_vector_extend($dst$$FloatRegister, __ D, $src$$FloatRegister, __ S);
        if (bt == T_DOUBLE) {
          __ sve_scvtf($dst$$FloatRegister, __ D, ptrue, $dst$$FloatRegister, __ D);
        }
      }
    }
  %}
  ins_pipe(pipe_slow);
%}

// VectorCastL2X

instruct vcvtLtoI_neon(vReg dst, vReg src) %{
  predicate(Matcher::vector_element_basic_type(n) == T_INT &&
            VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))));
  match(Set dst (VectorCastL2X src));
  format %{ "vcvtLtoI_neon $dst, $src" %}
  ins_encode %{
    // 2L to 2I
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
    __ neon_vector_narrow($dst$$FloatRegister, T_INT,
                          $src$$FloatRegister, T_LONG, length_in_bytes);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtLtoI_sve(vReg dst, vReg src, vReg tmp) %{
  predicate((Matcher::vector_element_basic_type(n) == T_INT &&
             !VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1)))) ||
            Matcher::vector_element_basic_type(n) == T_BYTE ||
            Matcher::vector_element_basic_type(n) == T_SHORT);
  match(Set dst (VectorCastL2X src));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "vcvtLtoI_sve $dst, $src\t# KILL $tmp" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_vector_narrow($dst$$FloatRegister, __ elemType_to_regVariant(bt),
                         $src$$FloatRegister, __ D, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtLtoF_neon(vReg dst, vReg src, vRegF tmp) %{
  predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n) == T_FLOAT);
  match(Set dst (VectorCastL2X src));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "vcvtLtoF_neon $dst, $src\t# 2L to 2F. KILL $tmp" %}
  ins_encode %{
    // 2L to 2F
    __ umov(rscratch1, $src$$FloatRegister, __ D, 0);
    __ scvtfs($dst$$FloatRegister, rscratch1);
    __ umov(rscratch1, $src$$FloatRegister, __ D, 1);
    __ scvtfs($tmp$$FloatRegister, rscratch1);
    __ ins($dst$$FloatRegister, __ S, $tmp$$FloatRegister, 1, 0);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtLtoF_sve(vReg dst, vReg src, vReg tmp) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_FLOAT);
  match(Set dst (VectorCastL2X src));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "vcvtLtoF_sve $dst, $src\t# KILL $tmp" %}
  ins_encode %{
    __ sve_scvtf($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister, __ D);
    __ sve_vector_narrow($dst$$FloatRegister, __ S,
                         $dst$$FloatRegister, __ D, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtLtoD(vReg dst, vReg src) %{
  predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
  match(Set dst (VectorCastL2X src));
  format %{ "vcvtLtoD $dst, $src" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      // 2L to 2D
      __ scvtfv(__ T2D, $dst$$FloatRegister, $src$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_scvtf($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister, __ D);
    }
  %}
  ins_pipe(pipe_slow);
%}

// VectorCastF2X

instruct vcvtFtoX_narrow_neon(vReg dst, vReg src) %{
  predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))) &&
            (Matcher::vector_element_basic_type(n) == T_BYTE ||
             Matcher::vector_element_basic_type(n) == T_SHORT));
  match(Set dst (VectorCastF2X src));
  effect(TEMP_DEF dst);
  format %{ "vcvtFtoX_narrow_neon $dst, $src" %}
  ins_encode %{
    // 4F to 4B/4S
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
    __ fcvtzs($dst$$FloatRegister, __ T4S, $src$$FloatRegister);
    __ neon_vector_narrow($dst$$FloatRegister, bt,
                          $dst$$FloatRegister, T_INT, length_in_bytes);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtFtoX_narrow_sve(vReg dst, vReg src, vReg tmp) %{
  predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))) &&
            (Matcher::vector_element_basic_type(n) == T_BYTE ||
             Matcher::vector_element_basic_type(n) == T_SHORT));
  match(Set dst (VectorCastF2X src));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "vcvtFtoX_narrow_sve $dst, $src\t# KILL $tmp" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_fcvtzs($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister, __ S);
    __ sve_vector_narrow($dst$$FloatRegister, __ elemType_to_regVariant(bt),
                         $dst$$FloatRegister, __ S, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtFtoX(vReg dst, vReg src) %{
  predicate(type2aelembytes(Matcher::vector_element_basic_type(n)) >= 4);
  match(Set dst (VectorCastF2X src));
  format %{ "vcvtFtoX $dst, $src" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes =  Matcher::vector_length_in_bytes(this);
    if (bt == T_INT) {
      if (VM_Version::use_neon_for_vector(length_in_bytes)) {
        // 2F/4F to 2I/4I
        __ fcvtzs($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
      } else {
        assert(UseSVE > 0, "must be sve");
        __ sve_fcvtzs($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister, __ S);
      }
    } else if (bt == T_LONG) {
      if (UseSVE == 0) {
        // 2F to 2L
        __ fcvtl($dst$$FloatRegister, __ T2D, $src$$FloatRegister, __ T2S);
        __ fcvtzs($dst$$FloatRegister, __ T2D, $dst$$FloatRegister);
      } else {
        __ sve_vector_extend($dst$$FloatRegister, __ D, $src$$FloatRegister, __ S);
        __ sve_fcvtzs($dst$$FloatRegister, __ D, ptrue, $dst$$FloatRegister, __ S);
      }
    } else {
      assert(bt == T_DOUBLE, "unsupported type");
      if (length_in_bytes == 16) {
        // 2F to 2D
        __ fcvtl($dst$$FloatRegister, __ T2D, $src$$FloatRegister, __ T2S);
      } else {
        assert(UseSVE > 0 && length_in_bytes > 16, "must be");
        __ sve_vector_extend($dst$$FloatRegister, __ D, $src$$FloatRegister, __ S);
        __ sve_fcvt($dst$$FloatRegister, __ D, ptrue, $dst$$FloatRegister, __ S);
      }
    }
  %}
  ins_pipe(pipe_slow);
%}

// VectorCastD2X

instruct vcvtDtoI_neon(vReg dst, vReg src) %{
  predicate(UseSVE == 0 && Matcher::vector_element_basic_type(n) == T_INT);
  match(Set dst (VectorCastD2X src));
  effect(TEMP_DEF dst);
  format %{ "vcvtDtoI_neon $dst, $src\t# 2D to 2I" %}
  ins_encode %{
    // 2D to 2I
    __ ins($dst$$FloatRegister, __ D, $src$$FloatRegister, 0, 1);
    // We can't use fcvtzs(vector, integer) instruction here because we need
    // saturation arithmetic. See JDK-8276151.
    __ fcvtzdw(rscratch1, $src$$FloatRegister);
    __ fcvtzdw(rscratch2, $dst$$FloatRegister);
    __ fmovs($dst$$FloatRegister, rscratch1);
    __ mov($dst$$FloatRegister, __ S, 1, rscratch2);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtDtoI_sve(vReg dst, vReg src, vReg tmp) %{
  predicate(UseSVE > 0 &&
            (Matcher::vector_element_basic_type(n) == T_BYTE ||
             Matcher::vector_element_basic_type(n) == T_SHORT ||
             Matcher::vector_element_basic_type(n) == T_INT));
  match(Set dst (VectorCastD2X src));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "vcvtDtoI_sve $dst, $src\t# KILL $tmp" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_fcvtzs($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister, __ D);
    __ sve_vector_narrow($dst$$FloatRegister, __ elemType_to_regVariant(bt),
                         $dst$$FloatRegister, __ D, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtDtoL(vReg dst, vReg src) %{
  predicate(Matcher::vector_element_basic_type(n) == T_LONG);
  match(Set dst (VectorCastD2X src));
  format %{ "vcvtDtoL $dst, $src" %}
  ins_encode %{
    uint length_in_bytes =  Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      // 2D to 2L
      __ fcvtzs($dst$$FloatRegister, __ T2D, $src$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_fcvtzs($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister, __ D);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtDtoF_64b(vReg dst, vReg src) %{
  predicate(Matcher::vector_element_basic_type(n) == T_FLOAT &&
            Matcher::vector_length_in_bytes(n) == 8);
  match(Set dst (VectorCastD2X src));
  format %{ "vcvtDtoF_64b $dst, $src\t# 2D to 2F" %}
  ins_encode %{
    // 2D to 2F
    __ fcvtn($dst$$FloatRegister, __ T2S, $src$$FloatRegister, __ T2D);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtDtoF_gt64b(vReg dst, vReg src, vReg tmp) %{
  predicate(Matcher::vector_element_basic_type(n) == T_FLOAT &&
            Matcher::vector_length_in_bytes(n) > 8);
  match(Set dst (VectorCastD2X src));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "vcvtDtoF_gt64b $dst, $src\t# vector > 64 bits. KILL $tmp" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    __ sve_fcvt($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister, __ D);
    __ sve_vector_narrow($dst$$FloatRegister, __ S,
                         $dst$$FloatRegister, __ D, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// VectorCastHF2F

instruct vcvtHFtoF(vReg dst, vReg src) %{
  match(Set dst (VectorCastHF2F src));
  format %{ "vcvtHFtoF $dst, $src" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      // 4HF to 4F
      __ fcvtl($dst$$FloatRegister, __ T4S, $src$$FloatRegister, __ T4H);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_vector_extend($dst$$FloatRegister, __ S, $src$$FloatRegister, __ H);
      __ sve_fcvt($dst$$FloatRegister, __ S, ptrue, $dst$$FloatRegister, __ H);
    }
  %}
  ins_pipe(pipe_slow);
%}

// VectorCastF2HF

instruct vcvtFtoHF_neon(vReg dst, vReg src) %{
  predicate(VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))));
  match(Set dst (VectorCastF2HF src));
  format %{ "vcvtFtoHF_neon $dst, $src\t# 4F to 4HF" %}
  ins_encode %{
    // 4F to 4HF
    __ fcvtn($dst$$FloatRegister, __ T4H, $src$$FloatRegister, __ T4S);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtFtoHF_sve(vReg dst, vReg src, vReg tmp) %{
  predicate(!VM_Version::use_neon_for_vector(Matcher::vector_length_in_bytes(n->in(1))));
  match(Set dst (VectorCastF2HF src));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "vcvtFtoHF_sve $dst, $src\t# KILL $tmp" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    __ sve_fcvt($dst$$FloatRegister, __ H, ptrue, $src$$FloatRegister, __ S);
    __ sve_vector_narrow($dst$$FloatRegister, __ H,
                         $dst$$FloatRegister, __ S, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Replicate ------------------------------------

// replicate from reg

instruct replicateI(vReg dst, iRegIorL2I src) %{
  match(Set dst (Replicate src));
  format %{ "replicateI $dst, $src" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ dup($dst$$FloatRegister, get_arrangement(this), $src$$Register);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_dup($dst$$FloatRegister, get_reg_variant(this), $src$$Register);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateL(vReg dst, iRegL src) %{
  match(Set dst (Replicate src));
  format %{ "replicateL $dst, $src" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ dup($dst$$FloatRegister, get_arrangement(this), $src$$Register);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_dup($dst$$FloatRegister, get_reg_variant(this), $src$$Register);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateF(vReg dst, vRegF src) %{
  match(Set dst (Replicate src));
  format %{ "replicateF $dst, $src" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ dup($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_cpy($dst$$FloatRegister, __ S, ptrue, $src$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateD(vReg dst, vRegD src) %{
  match(Set dst (Replicate src));
  format %{ "replicateD $dst, $src" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ dup($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_cpy($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

// replicate from imm

instruct replicateI_imm_le128b(vReg dst, immI con) %{
  predicate(Matcher::vector_length_in_bytes(n) <= 16 &&
            Matcher::is_non_long_integral_vector(n));
  match(Set dst (Replicate con));
  format %{ "replicateI_imm_le128b $dst, $con\t# vector <= 128 bits" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    int imm = (int)$con$$constant;
    if (type2aelembytes(bt) == 1) {
      // Refine imm for B
      imm = imm & 0xff;
    } else if (type2aelembytes(bt) == 2) {
      // Refine imm for S
      imm = imm & 0xffff;
    }
    __ mov($dst$$FloatRegister, get_arrangement(this), imm);
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateB_imm8_gt128b(vReg dst, immI8 con) %{
  predicate(Matcher::vector_length_in_bytes(n) > 16 &&
            Matcher::vector_element_basic_type(n) == T_BYTE);
  match(Set dst (Replicate con));
  format %{ "replicateB_imm8_gt128b $dst, $con\t# vector > 128 bits" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    __ sve_dup($dst$$FloatRegister, __ B, (int)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateI_imm8_gt128b(vReg dst, immI8_shift8 con) %{
  predicate(Matcher::vector_length_in_bytes(n) > 16 &&
            (Matcher::vector_element_basic_type(n) == T_SHORT ||
             Matcher::vector_element_basic_type(n) == T_INT));
  match(Set dst (Replicate con));
  format %{ "replicateI_imm8_gt128b $dst, $con\t# vector > 128 bits" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    __ sve_dup($dst$$FloatRegister, get_reg_variant(this), (int)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateL_imm_128b(vReg dst, immL con) %{
  predicate(Matcher::vector_length_in_bytes(n) == 16);
  match(Set dst (Replicate con));
  format %{ "replicateL_imm_128b $dst, $con\t# vector > 128 bits" %}
  ins_encode %{
    __ mov($dst$$FloatRegister, __ T2D, (uint64_t)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateL_imm8_gt128b(vReg dst, immL8_shift8 con) %{
  predicate(Matcher::vector_length_in_bytes(n) > 16);
  match(Set dst (Replicate con));
  format %{ "replicateL_imm8_gt128b $dst, $con\t# vector > 128 bits" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    __ sve_dup($dst$$FloatRegister, __ D, (int)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector insert --------------------------------

// BYTE, SHORT, INT

instruct insertI_le128b(vReg dst, vReg src, iRegIorL2I val, immI idx) %{
  predicate(Matcher::vector_length_in_bytes(n) <= 16 &&
            (Matcher::vector_element_basic_type(n) == T_BYTE ||
             Matcher::vector_element_basic_type(n) == T_SHORT ||
             Matcher::vector_element_basic_type(n) == T_INT));
  match(Set dst (VectorInsert (Binary src val) idx));
  format %{ "insertI_le128b $dst, $src, $val, $idx\t# vector <= 128 bits" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if ($dst$$FloatRegister != $src$$FloatRegister) {
      __ orr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
             $src$$FloatRegister, $src$$FloatRegister);
    }
    __ mov($dst$$FloatRegister, __ elemType_to_regVariant(bt),
           (int)($idx$$constant), $val$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct insertI_index_lt32(vReg dst, vReg src, iRegIorL2I val, immI idx,
                            vReg tmp, pRegGov pgtmp, rFlagsReg cr) %{
  predicate(n->in(2)->get_int() < 32 &&
            Matcher::vector_length_in_bytes(n) > 16 &&
            (Matcher::vector_element_basic_type(n) == T_BYTE ||
             Matcher::vector_element_basic_type(n) == T_SHORT ||
             Matcher::vector_element_basic_type(n) == T_INT));
  match(Set dst (VectorInsert (Binary src val) idx));
  effect(TEMP tmp, TEMP pgtmp, KILL cr);
  format %{ "insertI_index_lt32 $dst, $src, $val, $idx\t# vector > 128 bits, index < 31. KILL $tmp, $pgtmp, cr" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    BasicType bt = Matcher::vector_element_basic_type(this, $src);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ sve_index($tmp$$FloatRegister, size, -16, 1);
    __ sve_cmp(Assembler::EQ, $pgtmp$$PRegister, size, ptrue,
               $tmp$$FloatRegister, (int)($idx$$constant) - 16);
    if ($dst$$FloatRegister != $src$$FloatRegister) {
      __ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
    }
    __ sve_cpy($dst$$FloatRegister, size, $pgtmp$$PRegister, $val$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct insertI_index_ge32(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg tmp1,
                            vReg tmp2, pRegGov pgtmp, rFlagsReg cr) %{
  predicate(n->in(2)->get_int() >= 32 &&
            (Matcher::vector_element_basic_type(n) == T_BYTE ||
             Matcher::vector_element_basic_type(n) == T_SHORT ||
             Matcher::vector_element_basic_type(n) == T_INT));
  match(Set dst (VectorInsert (Binary src val) idx));
  effect(TEMP tmp1, TEMP tmp2, TEMP pgtmp, KILL cr);
  format %{ "insertI_index_ge32 $dst, $src, $val, $idx\t# index >= 32. KILL $tmp1, $tmp2, $pgtmp, cr" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    BasicType bt = Matcher::vector_element_basic_type(this, $src);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ sve_index($tmp1$$FloatRegister, size, 0, 1);
    __ sve_dup($tmp2$$FloatRegister, size, (int)($idx$$constant));
    __ sve_cmp(Assembler::EQ, $pgtmp$$PRegister, size, ptrue,
               $tmp1$$FloatRegister, $tmp2$$FloatRegister);
    if ($dst$$FloatRegister != $src$$FloatRegister) {
      __ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
    }
    __ sve_cpy($dst$$FloatRegister, size, $pgtmp$$PRegister, $val$$Register);
  %}
  ins_pipe(pipe_slow);
%}

// LONG

instruct insertL_128b(vReg dst, vReg src, iRegL val, immI idx) %{
  predicate(Matcher::vector_length_in_bytes(n) == 16 &&
            Matcher::vector_element_basic_type(n) == T_LONG);
  match(Set dst (VectorInsert (Binary src val) idx));
  format %{ "insertL_128b $dst, $src, $val, $idx\t# 2L" %}
  ins_encode %{
    if ($dst$$FloatRegister != $src$$FloatRegister) {
      __ orr($dst$$FloatRegister, __ T16B, $src$$FloatRegister, $src$$FloatRegister);
    }
    __ mov($dst$$FloatRegister, __ D, (int)($idx$$constant), $val$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct insertL_gt128b(vReg dst, vReg src, iRegL val, immI idx,
                        vReg tmp, pRegGov pgtmp, rFlagsReg cr) %{
  predicate(Matcher::vector_length_in_bytes(n) > 16 &&
            Matcher::vector_element_basic_type(n) == T_LONG);
  match(Set dst (VectorInsert (Binary src val) idx));
  effect(TEMP tmp, TEMP pgtmp, KILL cr);
  format %{ "insertL_gt128b $dst, $src, $val, $idx\t# vector > 128 bits. KILL $tmp, $pgtmp, cr" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    __ sve_index($tmp$$FloatRegister, __ D, -16, 1);
    __ sve_cmp(Assembler::EQ, $pgtmp$$PRegister, __ D, ptrue,
               $tmp$$FloatRegister, (int)($idx$$constant) - 16);
    if ($dst$$FloatRegister != $src$$FloatRegister) {
      __ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
    }
    __ sve_cpy($dst$$FloatRegister, __ D, $pgtmp$$PRegister, $val$$Register);
  %}
  ins_pipe(pipe_slow);
%}

// FLOAT

instruct insertF_le128b(vReg dst, vReg src, vRegF val, immI idx) %{
  predicate(Matcher::vector_length_in_bytes(n) <= 16 &&
            Matcher::vector_element_basic_type(n) == T_FLOAT);
  match(Set dst (VectorInsert (Binary src val) idx));
  effect(TEMP_DEF dst);
  format %{ "insertF_le128b $dst, $src, $val, $idx\t# vector <= 128 bits" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if ($dst$$FloatRegister != $src$$FloatRegister) {
      __ orr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
             $src$$FloatRegister, $src$$FloatRegister);
    }
    __ ins($dst$$FloatRegister, __ S, $val$$FloatRegister, (int)($idx$$constant), 0);
  %}
  ins_pipe(pipe_slow);
%}

instruct insertF_index_lt32(vReg dst, vReg src, vRegF val, immI idx,
                            pRegGov pgtmp, rFlagsReg cr) %{
  predicate(n->in(2)->get_int() < 32 &&
            Matcher::vector_length_in_bytes(n) > 16 &&
            Matcher::vector_element_basic_type(n) == T_FLOAT);
  match(Set dst (VectorInsert (Binary src val) idx));
  effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
  format %{ "insertF_index_lt32 $dst, $src, $val, $idx\t# vector > 128 bits, index < 32. KILL $pgtmp, cr" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    __ sve_index($dst$$FloatRegister, __ S, -16, 1);
    __ sve_cmp(Assembler::EQ, $pgtmp$$PRegister, __ S, ptrue,
               $dst$$FloatRegister, (int)($idx$$constant) - 16);
    __ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
    __ sve_cpy($dst$$FloatRegister, __ S, $pgtmp$$PRegister, $val$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct insertF_index_ge32(vReg dst, vReg src, vRegF val, immI idx, vReg tmp,
                            pRegGov pgtmp, rFlagsReg cr) %{
  predicate(n->in(2)->get_int() >= 32 &&
            Matcher::vector_element_basic_type(n) == T_FLOAT);
  match(Set dst (VectorInsert (Binary src val) idx));
  effect(TEMP_DEF dst, TEMP tmp, TEMP pgtmp, KILL cr);
  format %{ "insertF_index_ge32 $dst, $src, $val, $idx\t# index >= 32. KILL $tmp, $pgtmp, cr" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    __ sve_index($tmp$$FloatRegister, __ S, 0, 1);
    __ sve_dup($dst$$FloatRegister, __ S, (int)($idx$$constant));
    __ sve_cmp(Assembler::EQ, $pgtmp$$PRegister, __ S, ptrue,
               $tmp$$FloatRegister, $dst$$FloatRegister);
    __ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
    __ sve_cpy($dst$$FloatRegister, __ S, $pgtmp$$PRegister, $val$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// DOUBLE

instruct insertD_128b(vReg dst, vReg src, vRegD val, immI idx) %{
  predicate(Matcher::vector_length_in_bytes(n) == 16 &&
            Matcher::vector_element_basic_type(n) == T_DOUBLE);
  match(Set dst (VectorInsert (Binary src val) idx));
  effect(TEMP_DEF dst);
  format %{ "insertD_128b $dst, $src, $val, $idx\t# 2D" %}
  ins_encode %{
    if ($dst$$FloatRegister != $src$$FloatRegister) {
      __ orr($dst$$FloatRegister, __ T16B, $src$$FloatRegister, $src$$FloatRegister);
    }
    __ ins($dst$$FloatRegister, __ D, $val$$FloatRegister, (int)($idx$$constant), 0);
  %}
  ins_pipe(pipe_slow);
%}

instruct insertD_gt128b(vReg dst, vReg src, vRegD val, immI idx,
                        pRegGov pgtmp, rFlagsReg cr) %{
  predicate(Matcher::vector_length_in_bytes(n) > 16 &&
            Matcher::vector_element_basic_type(n) == T_DOUBLE);
  match(Set dst (VectorInsert (Binary src val) idx));
  effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
  format %{ "insertD_gt128b $dst, $src, $val, $idx\t# vector > 128 bits. KILL $pgtmp, cr" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    __ sve_index($dst$$FloatRegister, __ D, -16, 1);
    __ sve_cmp(Assembler::EQ, $pgtmp$$PRegister, __ D, ptrue,
               $dst$$FloatRegister, (int)($idx$$constant) - 16);
    __ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
    __ sve_cpy($dst$$FloatRegister, __ D, $pgtmp$$PRegister, $val$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Extract --------------------------------------

// BOOLEAN

instruct extractUB_ireg(iRegINoSp dst, vReg src, iRegI idx, vReg tmp) %{
  match(Set dst (ExtractUB src idx));
  effect(TEMP tmp);
  format %{ "extractUB_ireg $dst, $src, $idx\t# variable index. KILL $tmp" %}
  ins_encode %{
    // Input "src" is a vector of boolean represented as
    // bytes with 0x00/0x01 as element values.
    // "idx" is expected to be in range.

    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
    __ mov($tmp$$FloatRegister, __ B, 0, $idx$$Register);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ tbl($tmp$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
             $src$$FloatRegister, 1, $tmp$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_tbl($tmp$$FloatRegister, __ B, $src$$FloatRegister, $tmp$$FloatRegister);
    }
    __ smov($dst$$Register, $tmp$$FloatRegister, __ B, 0);
  %}
  ins_pipe(pipe_slow);
%}

instruct extractUB_index_lt16(iRegINoSp dst, vReg src, immI idx) %{
  predicate(n->in(2)->get_int() < 16);
  match(Set dst (ExtractUB src idx));
  format %{ "extractUB_index_lt16 $dst, $src, $idx\t# index < 16" %}
  ins_encode %{
    __ smov($dst$$Register, $src$$FloatRegister, __ B, (int)($idx$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct extractUB_index_ge16(iRegINoSp dst, vReg src, immI idx, vReg tmp) %{
  predicate(n->in(2)->get_int() >= 16);
  match(Set dst (ExtractUB src idx));
  effect(TEMP tmp);
  format %{ "extractUB_index_ge16 $dst, $src, $idx\t# index >=16. KILL $tmp" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    __ sve_extract_integral($dst$$Register, T_BYTE, $src$$FloatRegister,
                            (int)($idx$$constant), $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// BYTE

instruct extractB_index_lt16(iRegINoSp dst, vReg src, immI idx) %{
  predicate(n->in(2)->get_int() < 16);
  match(Set dst (ExtractB src idx));
  format %{ "extractB_index_lt16 $dst, $src, $idx\t# index < 16" %}
  ins_encode %{
    __ smov($dst$$Register, $src$$FloatRegister, __ B, (int)($idx$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct extractB_index_ge16(iRegINoSp dst, vReg src, immI idx, vReg tmp) %{
  predicate(n->in(2)->get_int() >= 16);
  match(Set dst (ExtractB src idx));
  effect(TEMP tmp);
  format %{ "extractB_index_ge16 $dst, $src, $idx\t# index >=16. KILL $tmp" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    __ sve_extract_integral($dst$$Register, T_BYTE, $src$$FloatRegister,
                            (int)($idx$$constant), $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// SHORT

instruct extractS_index_lt8(iRegINoSp dst, vReg src, immI idx) %{
  predicate(n->in(2)->get_int() < 8);
  match(Set dst (ExtractS src idx));
  format %{ "extractS_index_lt8 $dst, $src, $idx\t# index < 8" %}
  ins_encode %{
    __ smov($dst$$Register, $src$$FloatRegister, __ H, (int)($idx$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct extractS_index_ge8(iRegINoSp dst, vReg src, immI idx, vReg tmp) %{
  predicate(n->in(2)->get_int() >= 8);
  match(Set dst (ExtractS src idx));
  effect(TEMP tmp);
  format %{ "extractS_index_ge8 $dst, $src, $idx\t# index >=8. KILL $tmp" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    __ sve_extract_integral($dst$$Register, T_SHORT, $src$$FloatRegister,
                            (int)($idx$$constant), $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// INT

instruct extractI_index_lt4(iRegINoSp dst, vReg src, immI idx) %{
  predicate(n->in(2)->get_int() < 4);
  match(Set dst (ExtractI src idx));
  format %{ "extractI_index_lt4 $dst, $src, $idx\t# index < 4" %}
  ins_encode %{
    __ umov($dst$$Register, $src$$FloatRegister, __ S, (int)($idx$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct extractI_index_ge4(iRegINoSp dst, vReg src, immI idx, vReg tmp) %{
  predicate(n->in(2)->get_int() >= 4);
  match(Set dst (ExtractI src idx));
  effect(TEMP tmp);
  format %{ "extractI_index_ge4 $dst, $src, $idx\t# index >=4. KILL $tmp" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    __ sve_extract_integral($dst$$Register, T_INT, $src$$FloatRegister,
                            (int)($idx$$constant), $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// LONG

instruct extractL_index_lt2(iRegLNoSp dst, vReg src, immI idx) %{
  predicate(n->in(2)->get_int() < 2);
  match(Set dst (ExtractL src idx));
  format %{ "extractL_index_lt2 $dst, $src, $idx\t# index < 2" %}
  ins_encode %{
    __ umov($dst$$Register, $src$$FloatRegister, __ D, (int)($idx$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct extractL_index_ge2(iRegLNoSp dst, vReg src, immI idx, vReg tmp) %{
  predicate(n->in(2)->get_int() >= 2);
  match(Set dst (ExtractL src idx));
  effect(TEMP tmp);
  format %{ "extractL_index_ge2 $dst, $src, $idx\t# index >=2. KILL $tmp" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    __ sve_extract_integral($dst$$Register, T_LONG, $src$$FloatRegister,
                            (int)($idx$$constant), $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// FLOAT

instruct extractF(vRegF dst, vReg src, immI idx) %{
  match(Set dst (ExtractF src idx));
  effect(TEMP_DEF dst);
  format %{ "extractF $dst, $src, $idx" %}
  ins_encode %{
    int index = (int)$idx$$constant;
    if (index == 0) {
      __ fmovs($dst$$FloatRegister, $src$$FloatRegister);
    } else if (index < 4) {
      __ ins($dst$$FloatRegister, __ S, $src$$FloatRegister, 0, index);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
      __ sve_ext($dst$$FloatRegister, $dst$$FloatRegister, index << 2);
    }
  %}
  ins_pipe(pipe_slow);
%}

// DOUBLE

instruct extractD(vRegD dst, vReg src, immI idx) %{
  match(Set dst (ExtractD src idx));
  effect(TEMP_DEF dst);
  format %{ "extractD $dst, $src, $idx" %}
  ins_encode %{
    int index = (int)$idx$$constant;
    if (index == 0) {
      __ fmovd($dst$$FloatRegister, $src$$FloatRegister);
    } else if (index < 2) {
      __ ins($dst$$FloatRegister, __ D, $src$$FloatRegister, 0, index);
    } else {
      assert(UseSVE > 0, "must be sve");
      __ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
      __ sve_ext($dst$$FloatRegister, $dst$$FloatRegister, index << 3);
    }
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector mask load/store -----------------------

// vector load mask

instruct vloadmask_neon(vReg dst, vReg src) %{
  predicate(UseSVE == 0 &&
            (Matcher::vector_length_in_bytes(n) == 8 ||
             Matcher::vector_length_in_bytes(n) == 16));
  match(Set dst (VectorLoadMask src ));
  format %{ "vloadmask_neon $dst, $src" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    if (bt == T_BYTE) {
      uint length_in_bytes = Matcher::vector_length_in_bytes(this);
      __ negr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
              $src$$FloatRegister);
    } else {
      __ uxtl($dst$$FloatRegister, __ T8H, $src$$FloatRegister, __ T8B);
      if (type2aelembytes(bt) >= 4) {
        __ uxtl($dst$$FloatRegister, __ T4S, $dst$$FloatRegister, __ T4H);
      }
      if (type2aelembytes(bt) == 8) {
        __ uxtl($dst$$FloatRegister, __ T2D, $dst$$FloatRegister, __ T2S);
      }
      __ negr($dst$$FloatRegister, get_arrangement(this), $dst$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vloadmaskB_sve(pReg dst, vReg src, rFlagsReg cr) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE);
  match(Set dst (VectorLoadMask src));
  effect(KILL cr);
  format %{ "vloadmaskB_sve $dst, $src\t# KILL cr" %}
  ins_encode %{
    __ sve_cmp(Assembler::NE, $dst$$PRegister, __ B,
               ptrue, $src$$FloatRegister, 0);
  %}
  ins_pipe(pipe_slow);
%}

instruct vloadmask_extend_sve(pReg dst, vReg src, vReg tmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) != T_BYTE);
  match(Set dst (VectorLoadMask src));
  effect(TEMP tmp, KILL cr);
  format %{ "vloadmask_extend_sve $dst, $src\t# KILL $tmp, cr" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ sve_vector_extend($tmp$$FloatRegister, size, $src$$FloatRegister, __ B);
    __ sve_cmp(Assembler::NE, $dst$$PRegister, size, ptrue, $tmp$$FloatRegister, 0);
  %}
  ins_pipe(pipe_slow);
%}

instruct vloadmaskB_masked(pReg dst, vReg src, pRegGov pg, rFlagsReg cr) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE);
  match(Set dst (VectorLoadMask src pg));
  effect(KILL cr);
  format %{ "vloadmaskB_masked $dst, $pg, $src\t# KILL cr" %}
  ins_encode %{
    __ sve_cmp(Assembler::NE, $dst$$PRegister, __ B,
               $pg$$PRegister, $src$$FloatRegister, 0);
  %}
  ins_pipe(pipe_slow);
%}

instruct vloadmask_extend_masked(pReg dst, vReg src, pRegGov pg, vReg tmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) != T_BYTE);
  match(Set dst (VectorLoadMask src pg));
  effect(TEMP tmp, KILL cr);
  format %{ "vloadmask_extend_masked $dst, $pg, $src\t# KILL $tmp, cr" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ sve_vector_extend($tmp$$FloatRegister, size, $src$$FloatRegister, __ B);
    __ sve_cmp(Assembler::NE, $dst$$PRegister, size,
               $pg$$PRegister, $tmp$$FloatRegister, 0);
  %}
  ins_pipe(pipe_slow);
%}

// vector store mask - neon

instruct vstoremaskB_neon(vReg dst, vReg src, immI_1 size) %{
  predicate(UseSVE == 0);
  match(Set dst (VectorStoreMask src size));
  format %{ "vstoremaskB_neon $dst, $src" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    assert(length_in_bytes == 8 || length_in_bytes == 16, "must be");
    __ negr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
            $src$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vstoremask_narrow_neon(vReg dst, vReg src, immI_gt_1 size) %{
  predicate(UseSVE == 0);
  match(Set dst (VectorStoreMask src size));
  format %{ "vstoremask_narrow_neon $dst, $src" %}
  ins_encode %{
    int esize = (int)$size$$constant;
    if (esize == 2) {
      __ xtn($dst$$FloatRegister, __ T8B, $src$$FloatRegister, __ T8H);
    } else if (esize == 4) {
      __ xtn($dst$$FloatRegister, __ T4H, $src$$FloatRegister, __ T4S);
      __ xtn($dst$$FloatRegister, __ T8B, $dst$$FloatRegister, __ T8H);
    } else {
      assert(esize == 8, "must be");
      __ xtn($dst$$FloatRegister, __ T2S, $src$$FloatRegister, __ T2D);
      __ xtn($dst$$FloatRegister, __ T4H, $dst$$FloatRegister, __ T4S);
      __ xtn($dst$$FloatRegister, __ T8B, $dst$$FloatRegister, __ T8H);
    }
    __ negr($dst$$FloatRegister, __ T8B, $dst$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector store mask - sve

instruct vstoremaskB_sve(vReg dst, pReg src, immI_1 size) %{
  predicate(UseSVE > 0);
  match(Set dst (VectorStoreMask src size));
  format %{ "vstoremaskB_sve $dst, $src" %}
  ins_encode %{
    __ sve_cpy($dst$$FloatRegister, __ B, $src$$PRegister, 1, false);
  %}
  ins_pipe(pipe_slow);
%}

instruct vstoremask_narrow_sve(vReg dst, pReg src, immI_gt_1 size, vReg tmp) %{
  predicate(UseSVE > 0);
  match(Set dst (VectorStoreMask src size));
  effect(TEMP_DEF dst, TEMP tmp);
  format %{ "vstoremask_narrow_sve $dst, $src\t# KILL $tmp" %}
  ins_encode %{
    Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant((int)$size$$constant);
    __ sve_cpy($dst$$FloatRegister, size, $src$$PRegister, 1, false);
    __ sve_vector_narrow($dst$$FloatRegister, __ B,
                         $dst$$FloatRegister, size, $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// Combined rules for vector mask load when the vector element type is not T_BYTE

// VectorLoadMask+LoadVector, and the VectorLoadMask is unpredicated.
instruct vloadmask_loadV(pReg dst, indirect mem, vReg tmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            type2aelembytes(Matcher::vector_element_basic_type(n)) > 1);
  match(Set dst (VectorLoadMask (LoadVector mem)));
  effect(TEMP tmp, KILL cr);
  format %{ "vloadmask_loadV $dst, $mem\t# KILL $tmp, cr" %}
  ins_encode %{
    // Load mask values which are boolean type, and extend them to the
    // defined vector element type. Convert the vector to predicate.
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    loadStoreA_predicated(masm, false, $tmp$$FloatRegister,
                          ptrue, T_BOOLEAN, bt, $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
    __ sve_cmp(Assembler::NE, $dst$$PRegister, __ elemType_to_regVariant(bt),
               ptrue, $tmp$$FloatRegister, 0);
  %}
  ins_pipe(pipe_slow);
%}

// VectorLoadMask+LoadVector, and the VectorLoadMask is predicated.
instruct vloadmask_loadV_masked(pReg dst, indirect mem, pRegGov pg,
                                vReg tmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            type2aelembytes(Matcher::vector_element_basic_type(n)) > 1);
  match(Set dst (VectorLoadMask (LoadVector mem) pg));
  effect(TEMP tmp, KILL cr);
  format %{ "vloadmask_loadV_masked $dst, $pg, $mem\t# KILL $tmp, cr" %}
  ins_encode %{
    // Load valid mask values which are boolean type, and extend them to the
    // defined vector element type. Convert the vector to predicate.
    BasicType bt = Matcher::vector_element_basic_type(this);
    loadStoreA_predicated(masm, false, $tmp$$FloatRegister,
                          $pg$$PRegister, T_BOOLEAN, bt, $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
    __ sve_cmp(Assembler::NE, $dst$$PRegister, __ elemType_to_regVariant(bt),
               $pg$$PRegister, $tmp$$FloatRegister, 0);
  %}
  ins_pipe(pipe_slow);
%}

// VectorLoadMask+LoadVectorMasked, and the VectorLoadMask is unpredicated.
instruct vloadmask_loadVMasked(pReg dst, vmemA mem, pRegGov pg, vReg tmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            type2aelembytes(Matcher::vector_element_basic_type(n)) > 1);
  match(Set dst (VectorLoadMask (LoadVectorMasked mem pg)));
  effect(TEMP tmp, KILL cr);
  format %{ "vloadmask_loadVMasked $dst, $mem\t# KILL $tmp, cr" %}
  ins_encode %{
    // Load mask values which are boolean type, and extend them to the
    // defined vector element type. Convert the vector to predicate.
    //
    // Note that we cannot use "pg" here, since it is the predicate used
    // for the vector load with boolean type. But the predicate used in
    // the extending "sve_ld1b" is based on the final extended vector type,
    // which is the full-sized predicate (ptrue) used in VectorLoadMask.
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    loadStoreA_predicated(masm, false, $tmp$$FloatRegister,
                          ptrue, T_BOOLEAN, bt, $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
    __ sve_cmp(Assembler::NE, $dst$$PRegister, __ elemType_to_regVariant(bt),
               ptrue, $tmp$$FloatRegister, 0);
  %}
  ins_pipe(pipe_slow);
%}

// VectorLoadMask+LoadVectorMasked, and the VectorLoadMask is predicated.
instruct vloadmask_loadVMasked_masked(pReg dst, vmemA mem, pRegGov pg1, pRegGov pg2,
                                      vReg tmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            type2aelembytes(Matcher::vector_element_basic_type(n)) > 1);
  match(Set dst (VectorLoadMask (LoadVectorMasked mem pg1) pg2));
  effect(TEMP tmp, KILL cr);
  format %{ "vloadmask_loadVMasked_masked $dst, $pg2, $mem\t# KILL $tmp, cr" %}
  ins_encode %{
    // Load valid mask values which are boolean type, and extend them to the
    // defined vector element type. Convert the vector to predicate.
    //
    // Note that we cannot use "pg1" here, since it is the predicate used
    // for the vector load with boolean type. But the predicate used in
    // the extending "sve_ld1b" is based on the final extended vector type,
    // which is the "pg2" used in VectorLoadMask.
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    loadStoreA_predicated(masm, false, $tmp$$FloatRegister,
                          $pg2$$PRegister, T_BOOLEAN, bt, $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
    __ sve_cmp(Assembler::NE, $dst$$PRegister, __ elemType_to_regVariant(bt),
               $pg2$$PRegister, $tmp$$FloatRegister, 0);
  %}
  ins_pipe(pipe_slow);
%}

// Combined rules for vector mask store when the vector element type is not T_BYTE

// StoreVector+VectorStoreMask, and the vector size of "src" is equal to the MaxVectorSize.
instruct storeV_vstoremask(indirect mem, pReg src, immI_gt_1 esize, vReg tmp) %{
  predicate(UseSVE > 0 &&
            Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize);
  match(Set mem (StoreVector mem (VectorStoreMask src esize)));
  effect(TEMP tmp);
  format %{ "storeV_vstoremask $mem, $src\t# KILL $tmp" %}
  ins_encode %{
    // Convert the valid src predicate to vector, and store the vector elements
    // as boolean values.
    BasicType bt = Matcher::vector_element_basic_type(this, $src);
    assert(type2aelembytes(bt) == (int)$esize$$constant, "unsupported type");
    Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant($esize$$constant);
    __ sve_cpy($tmp$$FloatRegister, size, $src$$PRegister, 1, false);
    loadStoreA_predicated(masm, true, $tmp$$FloatRegister,
                          ptrue, T_BOOLEAN, bt, $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
  %}
  ins_pipe(pipe_slow);
%}

// StoreVector+VectorStoreMask, and the vector size of "src" is less than the MaxVectorSize.
instruct storeV_vstoremask_masked(indirect mem, pReg src, immI_gt_1 esize,
                                  vReg tmp, pRegGov pgtmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize);
  match(Set mem (StoreVector mem (VectorStoreMask src esize)));
  effect(TEMP tmp, TEMP pgtmp, KILL cr);
  format %{ "storeV_vstoremask_masked $mem, $src\t# KILL $tmp, $pgtmp, cr" %}
  ins_encode %{
    // Convert the valid src predicate to vector, and store the vector elements
    // as boolean values.
    BasicType bt = Matcher::vector_element_basic_type(this, $src);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ sve_cpy($tmp$$FloatRegister, size, $src$$PRegister, 1, false);
    __ sve_gen_mask_imm($pgtmp$$PRegister, bt, Matcher::vector_length(this, $src));
    loadStoreA_predicated(masm, true, $tmp$$FloatRegister,
                          $pgtmp$$PRegister, T_BOOLEAN, bt, $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
  %}
  ins_pipe(pipe_slow);
%}

// StoreVectorMasked+VectorStoreMask, and the vector size of "src" is equal to the MaxVectorSize.
instruct storeVMasked_vstoremask(vmemA mem, pReg src, pRegGov pg, immI_gt_1 esize, vReg tmp) %{
  predicate(UseSVE > 0 &&
            Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize);
  match(Set mem (StoreVectorMasked mem (Binary (VectorStoreMask src esize) pg)));
  effect(TEMP tmp);
  format %{ "storeVMasked_vstoremask $mem, $src\t# KILL $tmp" %}
  ins_encode %{
    // Convert the valid src predicate to vector, and store the vector elements
    // as boolean values.
    //
    // Note that we cannot use "pg" here, since it is the predicate used
    // for the vector store with boolean type. But the predicate used in
    // the narrowing "sve_st1b" is based on the "src" vector type, which
    // is the full-sized predicate (ptrue) here.
    BasicType bt = Matcher::vector_element_basic_type(this, $src);
    assert(type2aelembytes(bt) == (int)$esize$$constant, "unsupported type.");
    Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant($esize$$constant);
    __ sve_cpy($tmp$$FloatRegister, size, $src$$PRegister, 1, false);
    loadStoreA_predicated(masm, true, $tmp$$FloatRegister,
                          ptrue, T_BOOLEAN, bt, $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
  %}
  ins_pipe(pipe_slow);
%}

// StoreVectorMasked+VectorStoreMask, and the vector size of "src" is less than the MaxVectorSize.
instruct storeVMasked_vstoremask_masked(vmemA mem, pReg src, pRegGov pg, immI_gt_1 esize,
                                        vReg tmp, pRegGov pgtmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize);
  match(Set mem (StoreVectorMasked mem (Binary (VectorStoreMask src esize) pg)));
  effect(TEMP tmp, TEMP pgtmp, KILL cr);
  format %{ "storeVMasked_vstoremask_masked $mem, $src\t# KILL $tmp, $pgtmp, cr" %}
  ins_encode %{
    // Convert the valid src predicate to vector, and store the vector elements
    // as boolean values.
    //
    // Note that we cannot use "pg" here, since it is the predicate used for the
    // vector store with boolean type. But the predicate used in the narrowing
    // "sve_st1b" is based on the "src" vector type, which needed to be generated
    // here.
    BasicType bt = Matcher::vector_element_basic_type(this, $src);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ sve_cpy($tmp$$FloatRegister, size, $src$$PRegister, 1, false);
    __ sve_gen_mask_imm($pgtmp$$PRegister, bt, Matcher::vector_length(this, $src));
    loadStoreA_predicated(masm, true, $tmp$$FloatRegister,
                          $pgtmp$$PRegister, T_BOOLEAN, bt, $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector mask basic OPs ------------------------

// vector mask logical ops: and/or/xor/and_not

instruct vmask_and(pReg pd, pReg pn, pReg pm) %{
  predicate(UseSVE > 0);
  match(Set pd (AndVMask pn pm));
  format %{ "vmask_and $pd, $pn, $pm" %}
  ins_encode %{
    __ sve_and($pd$$PRegister, ptrue, $pn$$PRegister, $pm$$PRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmask_or(pReg pd, pReg pn, pReg pm) %{
  predicate(UseSVE > 0);
  match(Set pd (OrVMask pn pm));
  format %{ "vmask_or $pd, $pn, $pm" %}
  ins_encode %{
    __ sve_orr($pd$$PRegister, ptrue, $pn$$PRegister, $pm$$PRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmask_xor(pReg pd, pReg pn, pReg pm) %{
  predicate(UseSVE > 0);
  match(Set pd (XorVMask pn pm));
  format %{ "vmask_xor $pd, $pn, $pm" %}
  ins_encode %{
    __ sve_eor($pd$$PRegister, ptrue, $pn$$PRegister, $pm$$PRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmask_and_notI(pReg pd, pReg pn, pReg pm, immI_M1 m1) %{
  predicate(UseSVE > 0);
  match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
  format %{ "vmask_and_notI $pd, $pn, $pm" %}
  ins_encode %{
    __ sve_bic($pd$$PRegister, ptrue, $pn$$PRegister, $pm$$PRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmask_and_notL(pReg pd, pReg pn, pReg pm, immL_M1 m1) %{
  predicate(UseSVE > 0);
  match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
  format %{ "vmask_and_notL $pd, $pn, $pm" %}
  ins_encode %{
    __ sve_bic($pd$$PRegister, ptrue, $pn$$PRegister, $pm$$PRegister);
  %}
  ins_pipe(pipe_slow);
%}

// vector mask compare

instruct vmaskcmp_neon(vReg dst, vReg src1, vReg src2, immI cond) %{
  predicate(UseSVE == 0 &&
            (Matcher::vector_length_in_bytes(n) == 8 ||
             Matcher::vector_length_in_bytes(n) == 16));
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  format %{ "vmaskcmp_neon $dst, $src1, $src2, $cond" %}
  ins_encode %{
    Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    __ neon_compare($dst$$FloatRegister, bt, $src1$$FloatRegister,
                    $src2$$FloatRegister, condition, /* isQ */ length_in_bytes == 16);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmaskcmp_zeroI_neon(vReg dst, vReg src, immI0 zero, immI_cmp_cond cond) %{
  predicate(UseSVE == 0);
  match(Set dst (VectorMaskCmp (Binary src (Replicate zero)) cond));
  format %{ "vmaskcmp_zeroI_neon $dst, $src, #0, $cond" %}
  ins_encode %{
    Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    __ neon_compare_zero($dst$$FloatRegister, Matcher::vector_element_basic_type(this),
                         $src$$FloatRegister,
                         condition, /* isQ */ length_in_bytes == 16);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmaskcmp_zeroL_neon(vReg dst, vReg src, immL0 zero, immI_cmp_cond cond) %{
  predicate(UseSVE == 0);
  match(Set dst (VectorMaskCmp (Binary src (Replicate zero)) cond));
  format %{ "vmaskcmp_zeroL_neon $dst, $src, #0, $cond" %}
  ins_encode %{
    Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    __ neon_compare_zero($dst$$FloatRegister, T_LONG,
                         $src$$FloatRegister,
                         condition, /* isQ */ length_in_bytes == 16);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmaskcmp_zeroF_neon(vReg dst, vReg src, immF0 zero, immI_cmp_cond cond) %{
  predicate(UseSVE == 0);
  match(Set dst (VectorMaskCmp (Binary src (Replicate zero)) cond));
  format %{ "vmaskcmp_zeroF_neon $dst, $src, #0, $cond" %}
  ins_encode %{
    Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    __ neon_compare_zero($dst$$FloatRegister, T_FLOAT,
                         $src$$FloatRegister,
                         condition, /* isQ */ length_in_bytes == 16);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmaskcmp_zeroD_neon(vReg dst, vReg src, immD0 zero, immI_cmp_cond cond) %{
  predicate(UseSVE == 0);
  match(Set dst (VectorMaskCmp (Binary src (Replicate zero)) cond));
  format %{ "vmaskcmp_zeroD_neon $dst, $src, #0, $cond" %}
  ins_encode %{
    Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    __ neon_compare_zero($dst$$FloatRegister, T_DOUBLE,
                         $src$$FloatRegister,
                         condition, /* isQ */ length_in_bytes == 16);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmaskcmp_sve(pReg dst, vReg src1, vReg src2, immI cond, rFlagsReg cr) %{
  predicate(UseSVE > 0);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  effect(KILL cr);
  format %{ "vmaskcmp_sve $dst, $src1, $src2, $cond\t# KILL cr" %}
  ins_encode %{
    Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    __ sve_compare($dst$$PRegister, bt, ptrue, $src1$$FloatRegister,
                   $src2$$FloatRegister, condition);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmaskcmp_immI_sve(pReg dst, vReg src, immI5 imm, immI_cmp_cond cond, rFlagsReg cr) %{
  predicate(UseSVE > 0);
  match(Set dst (VectorMaskCmp (Binary src (Replicate imm)) cond));
  effect(KILL cr);
  format %{ "vmaskcmp_immI_sve $dst, $src, $imm, $cond\t# KILL cr" %}
  ins_encode %{
    Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    __ sve_cmp(condition, $dst$$PRegister, get_reg_variant(in(operand_index($src))),
               ptrue, $src$$FloatRegister, (int)$imm$$constant);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmaskcmpU_immI_sve(pReg dst, vReg src, immIU7 imm, immI_cmpU_cond cond, rFlagsReg cr) %{
  predicate(UseSVE > 0);
  match(Set dst (VectorMaskCmp (Binary src (Replicate imm)) cond));
  effect(KILL cr);
  format %{ "vmaskcmpU_immI_sve $dst, $src, $imm, $cond\t# KILL cr" %}
  ins_encode %{
    Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    __ sve_cmp(condition, $dst$$PRegister, get_reg_variant(in(operand_index($src))),
               ptrue, $src$$FloatRegister, (int)$imm$$constant);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmaskcmp_immL_sve(pReg dst, vReg src, immL5 imm, immI_cmp_cond cond, rFlagsReg cr) %{
  predicate(UseSVE > 0);
  match(Set dst (VectorMaskCmp (Binary src (Replicate imm)) cond));
  effect(KILL cr);
  format %{ "vmaskcmp_immL_sve $dst, $src, $imm, $cond\t# KILL cr" %}
  ins_encode %{
    Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    __ sve_cmp(condition, $dst$$PRegister, get_reg_variant(in(operand_index($src))),
               ptrue, $src$$FloatRegister, (int)$imm$$constant);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmaskcmpU_immL_sve(pReg dst, vReg src, immLU7 imm, immI_cmpU_cond cond, rFlagsReg cr) %{
  predicate(UseSVE > 0);
  match(Set dst (VectorMaskCmp (Binary src (Replicate imm)) cond));
  effect(KILL cr);
  format %{ "vmaskcmpU_immL_sve $dst, $src, $imm, $cond\t# KILL cr" %}
  ins_encode %{
    Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    __ sve_cmp(condition, $dst$$PRegister, get_reg_variant(in(operand_index($src))),
               ptrue, $src$$FloatRegister, (int)$imm$$constant);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmaskcmp_masked(pReg dst, vReg src1, vReg src2, immI cond,
                         pRegGov pg, rFlagsReg cr) %{
  predicate(UseSVE > 0);
  match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond pg)));
  effect(KILL cr);
  format %{ "vmaskcmp_masked $dst, $pg, $src1, $src2, $cond\t# KILL cr" %}
  ins_encode %{
    Assembler::Condition condition = to_assembler_cond((BoolTest::mask)$cond$$constant);
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_compare($dst$$PRegister, bt, $pg$$PRegister, $src1$$FloatRegister,
                   $src2$$FloatRegister, condition);
  %}
  ins_pipe(pipe_slow);
%}

// vector mask cast

instruct vmaskcast_same_esize_neon(vReg dst_src) %{
  predicate(UseSVE == 0 &&
            Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)) &&
            (Matcher::vector_length_in_bytes(n) == 8 || Matcher::vector_length_in_bytes(n) == 16));
  match(Set dst_src (VectorMaskCast dst_src));
  ins_cost(0);
  format %{ "vmaskcast_same_esize_neon $dst_src\t# do nothing" %}
  ins_encode(/* empty encoding */);
  ins_pipe(pipe_class_empty);
%}

instruct vmaskcast_extend_neon(vReg dst, vReg src) %{
  predicate(UseSVE == 0 &&
            Matcher::vector_length_in_bytes(n) > Matcher::vector_length_in_bytes(n->in(1)));
  match(Set dst (VectorMaskCast src));
  format %{ "vmaskcast_extend_neon $dst, $src" %}
  ins_encode %{
    BasicType dst_bt = Matcher::vector_element_basic_type(this);
    if (is_floating_point_type(dst_bt)) {
      dst_bt = (dst_bt == T_FLOAT) ? T_INT : T_LONG;
    }
    uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
    BasicType src_bt = Matcher::vector_element_basic_type(this, $src);
    if (is_floating_point_type(src_bt)) {
      src_bt = (src_bt == T_FLOAT) ? T_INT : T_LONG;
    }
    __ neon_vector_extend($dst$$FloatRegister, dst_bt, length_in_bytes_dst,
                          $src$$FloatRegister, src_bt);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmaskcast_narrow_neon(vReg dst, vReg src) %{
  predicate(UseSVE == 0 &&
            Matcher::vector_length_in_bytes(n) < Matcher::vector_length_in_bytes(n->in(1)));
  match(Set dst (VectorMaskCast src));
  format %{ "vmaskcast_narrow_neon $dst, $src" %}
  ins_encode %{
    BasicType dst_bt = Matcher::vector_element_basic_type(this);
    if (is_floating_point_type(dst_bt)) {
      dst_bt = (dst_bt == T_FLOAT) ? T_INT : T_LONG;
    }
    BasicType src_bt = Matcher::vector_element_basic_type(this, $src);
    if (is_floating_point_type(src_bt)) {
      src_bt = (src_bt == T_FLOAT) ? T_INT : T_LONG;
    }
    uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
    __ neon_vector_narrow($dst$$FloatRegister, dst_bt,
                          $src$$FloatRegister, src_bt, length_in_bytes_src);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmaskcast_same_esize_sve(pReg dst_src) %{
  predicate(UseSVE > 0 &&
            Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)));
  match(Set dst_src (VectorMaskCast dst_src));
  ins_cost(0);
  format %{ "vmaskcast_same_esize_sve $dst_src\t# do nothing" %}
  ins_encode(/* empty encoding */);
  ins_pipe(pipe_class_empty);
%}

instruct vmaskcast_extend_sve(pReg dst, pReg src) %{
  predicate(UseSVE > 0 &&
            Matcher::vector_length_in_bytes(n) > Matcher::vector_length_in_bytes(n->in(1)));
  match(Set dst (VectorMaskCast src));
  format %{ "vmaskcast_extend_sve $dst, $src" %}
  ins_encode %{
    uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
    uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
    assert(length_in_bytes_dst == 2 * length_in_bytes_src ||
           length_in_bytes_dst == 4 * length_in_bytes_src ||
           length_in_bytes_dst == 8 * length_in_bytes_src, "invalid vector length");
    __ sve_vmaskcast_extend($dst$$PRegister, $src$$PRegister,
                            length_in_bytes_dst, length_in_bytes_src);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmaskcast_narrow_sve(pReg dst, pReg src, pReg ptmp) %{
  predicate(UseSVE > 0 &&
            Matcher::vector_length_in_bytes(n) < Matcher::vector_length_in_bytes(n->in(1)));
  match(Set dst (VectorMaskCast src));
  effect(TEMP_DEF dst, TEMP ptmp);
  format %{ "vmaskcast_narrow_sve $dst, $src\t# KILL $ptmp" %}
  ins_encode %{
    uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
    uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
    assert(length_in_bytes_dst * 2 == length_in_bytes_src ||
           length_in_bytes_dst * 4 == length_in_bytes_src ||
           length_in_bytes_dst * 8 == length_in_bytes_src, "invalid vector length");
    __ sve_vmaskcast_narrow($dst$$PRegister, $src$$PRegister, $ptmp$$PRegister,
                            length_in_bytes_dst, length_in_bytes_src);
  %}
  ins_pipe(pipe_slow);
%}

// vector mask reinterpret

instruct vmask_reinterpret_same_esize(pReg dst_src) %{
  predicate(UseSVE > 0 &&
            Matcher::vector_length(n) == Matcher::vector_length(n->in(1)) &&
            Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)));
  match(Set dst_src (VectorReinterpret dst_src));
  ins_cost(0);
  format %{ "vmask_reinterpret_same_esize $dst_src\t# do nothing" %}
  ins_encode(/* empty encoding */);
  ins_pipe(pipe_class_empty);
%}

instruct vmask_reinterpret_diff_esize(pReg dst, pReg src, vReg tmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            Matcher::vector_length(n) != Matcher::vector_length(n->in(1)) &&
            Matcher::vector_length_in_bytes(n) == Matcher::vector_length_in_bytes(n->in(1)));
  match(Set dst (VectorReinterpret src));
  effect(TEMP tmp, KILL cr);
  format %{ "vmask_reinterpret_diff_esize $dst, $src\t# KILL $tmp, cr" %}
  ins_encode %{
    BasicType from_bt = Matcher::vector_element_basic_type(this, $src);
    Assembler::SIMD_RegVariant from_size = __ elemType_to_regVariant(from_bt);
    BasicType to_bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
    __ sve_cpy($tmp$$FloatRegister, from_size, $src$$PRegister, -1, false);
    __ sve_cmp(Assembler::EQ, $dst$$PRegister, to_size, ptrue, $tmp$$FloatRegister, -1);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector mask reductions -----------------------

// true count

instruct vmask_truecount_neon(iRegINoSp dst, vReg src, vReg tmp) %{
  predicate(UseSVE == 0);
  match(Set dst (VectorMaskTrueCount src));
  effect(TEMP tmp);
  format %{ "vmask_truecount_neon $dst, $src\t# KILL $tmp" %}
  ins_encode %{
    // Input "src" is a vector of boolean represented as bytes with
    // 0x00/0x01 as element values.
    BasicType bt = Matcher::vector_element_basic_type(this, $src);
    assert(bt == T_BOOLEAN, "unsupported type");
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
    __ addv($tmp$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
            $src$$FloatRegister);
    __ umov($dst$$Register, $tmp$$FloatRegister, __ B, 0);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmask_truecount_sve(iRegINoSp dst, pReg src) %{
  predicate(UseSVE > 0);
  match(Set dst (VectorMaskTrueCount src));
  format %{ "vmask_truecount_sve $dst, $src" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src);
    __ sve_cntp($dst$$Register, __ elemType_to_regVariant(bt),
                ptrue, $src$$PRegister);
  %}
  ins_pipe(pipe_slow);
%}

// Combined rule for VectorMaskTrueCount (VectorStoreMask) when the vector element type is not T_BYTE.

instruct vstoremask_truecount_neon(iRegINoSp dst, vReg src, immI_gt_1 size, vReg vtmp) %{
  match(Set dst (VectorMaskTrueCount (VectorStoreMask src size)));
  effect(TEMP vtmp);
  format %{ "vstoremask_truecount_neon $dst, $src\t# KILL $vtmp" %}
  ins_encode %{
    // Input "src" is a vector mask represented as lanes with
    // 0/-1 as element values.
    uint esize = (uint)$size$$constant;
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
    Assembler::SIMD_Arrangement arrangement = Assembler::esize2arrangement(esize,
                                                                           /* isQ */ length_in_bytes == 16);
    if (arrangement == __ T2D || arrangement == __ T2S) {
      __ addpv($vtmp$$FloatRegister, arrangement, $src$$FloatRegister, $src$$FloatRegister);
    } else {
      __ addv($vtmp$$FloatRegister, arrangement, $src$$FloatRegister);
    }
    __ smov($dst$$Register, $vtmp$$FloatRegister, __ B, 0);
    __ neg($dst$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

// first true

instruct vmask_firsttrue_neon(iRegINoSp dst, vReg src) %{
  predicate(UseSVE == 0);
  match(Set dst (VectorMaskFirstTrue src));
  format %{ "vmask_firsttrue_neon $dst, $src" %}
  ins_encode %{
    // Returns the index of the first active lane of the
    // vector mask, or VLENGTH if no lane is active.
    //
    // Input "src" is a vector of boolean represented as
    // bytes with 0x00/0x01 as element values.
    //
    // Computed by reversing the bits and counting the leading
    // zero bytes.

    BasicType bt = Matcher::vector_element_basic_type(this, $src);
    assert(bt == T_BOOLEAN, "unsupported type");
    uint vlength = Matcher::vector_length(this, $src);
    if (vlength <= 8) {
      __ fmovd($dst$$Register, $src$$FloatRegister);
      if (vlength == 2 || vlength == 4) {
        // Special handling for 2B or 4B cases:
        // Vector mask is moved to a 64-bit general register, but only the low 16/32 bits are
        // significant for 2B/4B cases. We initialize the 16th/32nd bit as bit 1, so as to generate
        // the expected result (i.e. VLENGTH) for the case that all lanes are zero.
        __ orr($dst$$Register, $dst$$Register, vlength == 2 ? 0x10000 : 0x100000000);
      }
      __ rbit($dst$$Register, $dst$$Register);
      __ clz($dst$$Register, $dst$$Register);
      __ lsrw($dst$$Register, $dst$$Register, 3);
    } else {
      assert(vlength == 16, "must be");
      Label FIRST_TRUE_INDEX;

      // Try to compute the result from lower 64 bits.
      __ fmovd($dst$$Register, $src$$FloatRegister);
      __ movw(rscratch1, zr);
      __ cbnz($dst$$Register, FIRST_TRUE_INDEX);

      // Compute the result from the higher 64 bits.
      __ fmovhid($dst$$Register, $src$$FloatRegister);
      __ movw(rscratch1, 8);

      // Reverse the bits and count the leading zero bytes.
      __ bind(FIRST_TRUE_INDEX);
      __ rbit($dst$$Register, $dst$$Register);
      __ clz($dst$$Register, $dst$$Register);
      __ addw($dst$$Register, rscratch1, $dst$$Register, Assembler::LSR, 3);
    }
  %}
  ins_pipe(pipe_slow);
%}

// Return the index of the first mask lane that is set, or vector length if none of
// them are set.

instruct vmask_firsttrue_sve(iRegINoSp dst, pReg src, pReg ptmp) %{
  predicate(UseSVE > 0);
  match(Set dst (VectorMaskFirstTrue src));
  effect(TEMP ptmp);
  format %{ "vmask_firsttrue_sve $dst, $src\t# KILL $ptmp" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    __ sve_brkb($ptmp$$PRegister, ptrue, $src$$PRegister, false);
    __ sve_cntp($dst$$Register, __ elemType_to_regVariant(bt), ptrue, $ptmp$$PRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmask_firsttrue_masked(iRegINoSp dst, pReg src, pReg pg, pReg ptmp) %{
  predicate(UseSVE > 0);
  match(Set dst (VectorMaskFirstTrue src pg));
  effect(TEMP ptmp);
  format %{ "vmask_firsttrue_masked $dst, $pg, $src\t# KILL $ptmp" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src);
    __ sve_brkb($ptmp$$PRegister, $pg$$PRegister, $src$$PRegister, false);
    __ sve_cntp($dst$$Register, __ elemType_to_regVariant(bt), ptrue, $ptmp$$PRegister);
  %}
  ins_pipe(pipe_slow);
%}

// last true

instruct vmask_lasttrue_neon(iRegINoSp dst, vReg src) %{
  predicate(UseSVE == 0);
  match(Set dst (VectorMaskLastTrue src));
  format %{ "vmask_lasttrue_neon $dst, $src" %}
  ins_encode %{
    // Returns the index of the last active lane of the
    // vector mask, or -1 if no lane is active.
    //
    // Input "src" is a vector of boolean represented as
    // bytes with 0x00/0x01 as element values.

    BasicType bt = Matcher::vector_element_basic_type(this, $src);
    assert(bt == T_BOOLEAN, "unsupported type");
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
    if (length_in_bytes <= 8) {
      // Computed by counting the leading zero bytes and
      // subtracting it by 7 (VLENGTH - 1).
      __ fmovd($dst$$Register, $src$$FloatRegister);
      __ clz($dst$$Register, $dst$$Register);
      __ movw(rscratch1, 7);
      __ subw($dst$$Register, rscratch1, $dst$$Register, Assembler::LSR, 3);
    } else {
      assert(length_in_bytes == 16, "must be");
      Label LAST_TRUE_INDEX;

      // Try to compute the result from higher 64 bits.
      __ fmovhid($dst$$Register, $src$$FloatRegister);
      __ movw(rscratch1, 16 - 1);
      __ cbnz($dst$$Register, LAST_TRUE_INDEX);

      // Compute the result from the lower 64 bits.
      __ fmovd($dst$$Register, $src$$FloatRegister);
      __ movw(rscratch1, 8 - 1);

      // Count the leading zero bytes and subtract it by 15 (VLENGTH - 1).
      __ bind(LAST_TRUE_INDEX);
      __ clz($dst$$Register, $dst$$Register);
      __ subw($dst$$Register, rscratch1, $dst$$Register, Assembler::LSR, 3);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vmask_lasttrue_sve(iRegINoSp dst, pReg src, pReg ptmp) %{
  predicate(UseSVE > 0);
  match(Set dst (VectorMaskLastTrue src));
  effect(TEMP ptmp);
  format %{ "vmask_lasttrue_sve $dst, $src\t# KILL $ptmp" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src);
    __ sve_vmask_lasttrue($dst$$Register, bt, $src$$PRegister, $ptmp$$PRegister);
  %}
  ins_pipe(pipe_slow);
%}

// tolong

instruct vmask_tolong_neon(iRegLNoSp dst, vReg src) %{
  predicate(UseSVE == 0);
  match(Set dst (VectorMaskToLong src));
  format %{ "vmask_tolong_neon $dst, $src" %}
  ins_encode %{
    // Input "src" is a vector of boolean represented as
    // bytes with 0x00/0x01 as element values.

    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
    if (length_in_bytes <= 8) {
      __ fmovd($dst$$Register, $src$$FloatRegister);
      __ bytemask_compress($dst$$Register);
    } else {
      assert(length_in_bytes == 16, "must be");
      __ umov($dst$$Register, $src$$FloatRegister, __ D, 0);
      __ umov(rscratch1, $src$$FloatRegister, __ D, 1);
      __ bytemask_compress($dst$$Register);
      __ bytemask_compress(rscratch1);
      __ orr($dst$$Register, $dst$$Register, rscratch1, Assembler::LSL, 8);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vmask_tolong_sve(iRegLNoSp dst, pReg src, vReg tmp1, vReg tmp2) %{
  predicate(UseSVE > 0);
  match(Set dst (VectorMaskToLong src));
  effect(TEMP tmp1, TEMP tmp2);
  format %{ "vmask_tolong_sve $dst, $src\t# KILL $tmp1, $tmp2" %}
  ins_encode %{
    __ sve_vmask_tolong($dst$$Register, $src$$PRegister,
                        Matcher::vector_element_basic_type(this, $src),
                        Matcher::vector_length(this, $src),
                        $tmp1$$FloatRegister, $tmp2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// fromlong

instruct vmask_fromlong(pReg dst, iRegL src, vReg tmp1, vReg tmp2) %{
  match(Set dst (VectorLongToMask src));
  effect(TEMP tmp1, TEMP tmp2);
  format %{ "vmask_fromlong $dst, $src\t# vector (sve2). KILL $tmp1, $tmp2" %}
  ins_encode %{
    __ sve_vmask_fromlong($dst$$PRegister, $src$$Register,
                          Matcher::vector_element_basic_type(this),
                          Matcher::vector_length(this),
                          $tmp1$$FloatRegister, $tmp2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector mask generation -----------------------

// maskAll

instruct vmaskAll_immI(pReg dst, immI src, rFlagsReg cr) %{
  predicate(UseSVE > 0);
  match(Set dst (MaskAll src));
  effect(KILL cr);
  format %{ "vmaskAll_immI $dst, $src\t# KILL cr" %}
  ins_encode %{
    int con = (int)$src$$constant;
    if (con == 0) {
      __ sve_pfalse($dst$$PRegister);
    } else {
      assert(con == -1, "invalid constant value for mask");
      BasicType bt = Matcher::vector_element_basic_type(this);
      __ sve_gen_mask_imm($dst$$PRegister, bt, Matcher::vector_length(this));
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vmaskAllI(pReg dst, iRegIorL2I src, vReg tmp, rFlagsReg cr) %{
  predicate(UseSVE > 0);
  match(Set dst (MaskAll src));
  effect(TEMP tmp, KILL cr);
  format %{ "vmaskAllI $dst, $src\t# KILL $tmp, cr" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    __ sve_dup($tmp$$FloatRegister, size, $src$$Register);
    __ sve_cmp(Assembler::NE, $dst$$PRegister, size, ptrue, $tmp$$FloatRegister, 0);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmaskAllI_masked(pReg dst, iRegIorL2I src, pRegGov pg, vReg tmp, rFlagsReg cr) %{
  predicate(UseSVE > 0);
  match(Set dst (MaskAll src pg));
  effect(TEMP tmp, KILL cr);
  format %{ "vmaskAllI_masked $dst, $pg, $src\t# KILL $tmp, cr" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ sve_dup($tmp$$FloatRegister, size, $src$$Register);
    __ sve_cmp(Assembler::NE, $dst$$PRegister, size,
               $pg$$PRegister, $tmp$$FloatRegister, 0);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmaskAll_immL(pReg dst, immL src, rFlagsReg cr) %{
  predicate(UseSVE > 0);
  match(Set dst (MaskAll src));
  effect(KILL cr);
  format %{ "vmaskAll_immL $dst, $src\t# KILL cr" %}
  ins_encode %{
    long con = (long)$src$$constant;
    if (con == 0) {
      __ sve_pfalse($dst$$PRegister);
    } else {
      assert(con == -1, "invalid constant value for mask");
      BasicType bt = Matcher::vector_element_basic_type(this);
      __ sve_gen_mask_imm($dst$$PRegister, bt, Matcher::vector_length(this));
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vmaskAllL(pReg dst, iRegL src, vReg tmp, rFlagsReg cr) %{
  predicate(UseSVE > 0);
  match(Set dst (MaskAll src));
  effect(TEMP tmp, KILL cr);
  format %{ "vmaskAllL $dst, $src\t# KILL $tmp, cr" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    __ sve_dup($tmp$$FloatRegister, size, $src$$Register);
    __ sve_cmp(Assembler::NE, $dst$$PRegister, size, ptrue, $tmp$$FloatRegister, 0);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmaskAllL_masked(pReg dst, iRegL src, pRegGov pg, vReg tmp, rFlagsReg cr) %{
  predicate(UseSVE > 0);
  match(Set dst (MaskAll src pg));
  effect(TEMP tmp, KILL cr);
  format %{ "vmaskAllL_masked $dst, $pg, $src\t# KILL $tmp, cr" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ sve_dup($tmp$$FloatRegister, size, $src$$Register);
    __ sve_cmp(Assembler::NE, $dst$$PRegister, size,
               $pg$$PRegister, $tmp$$FloatRegister, 0);
  %}
  ins_pipe(pipe_slow);
%}

// vetcor mask generation

instruct vmask_gen_I(pReg pd, iRegIorL2I src, rFlagsReg cr) %{
  predicate(UseSVE > 0);
  match(Set pd (VectorMaskGen (ConvI2L src)));
  effect(KILL cr);
  format %{ "vmask_gen_I $pd, $src\t# KILL cr" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_whileltw($pd$$PRegister, __ elemType_to_regVariant(bt), zr, $src$$Register);
  %}
  ins_pipe(pipe_class_default);
%}

instruct vmask_gen_L(pReg pd, iRegL src, rFlagsReg cr) %{
  predicate(UseSVE > 0);
  match(Set pd (VectorMaskGen src));
  effect(KILL cr);
  format %{ "vmask_gen_L $pd, $src\t# KILL cr" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_whilelt($pd$$PRegister, __ elemType_to_regVariant(bt), zr, $src$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct vmask_gen_imm(pReg pd, immL con, rFlagsReg cr) %{
  predicate(UseSVE > 0);
  match(Set pd (VectorMaskGen con));
  effect(KILL cr);
  format %{ "vmask_gen_imm $pd, $con\t# KILL cr" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_gen_mask_imm($pd$$PRegister, bt, (uint)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmask_gen_sub(pReg pd, iRegL src1, iRegL src2, rFlagsReg cr) %{
  predicate(UseSVE > 0);
  match(Set pd (VectorMaskGen (SubL src1 src2)));
  effect(KILL cr);
  format %{ "vmask_gen_sub $pd, $src2, $src1\t# KILL cr" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_whilelt($pd$$PRegister, __ elemType_to_regVariant(bt), $src2$$Register, $src1$$Register);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Popcount vector ------------------------------

// vector popcount - INT

instruct vpopcountI(vReg dst, vReg src) %{
  match(Set dst (PopCountVI src));
  format %{ "vpopcountI $dst, $src" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (bt == T_BYTE) {
      if (VM_Version::use_neon_for_vector(length_in_bytes)) {
        __ cnt($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
               $src$$FloatRegister);
      } else {
        assert(UseSVE > 0, "must be sve");
        __ sve_cnt($dst$$FloatRegister, __ B, ptrue, $src$$FloatRegister);
      }
    } else {
      assert(bt == T_SHORT || bt == T_INT, "unsupported");
      if (UseSVE == 0) {
        assert(length_in_bytes == 8 || length_in_bytes == 16, "unsupported");
        __ cnt($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
               $src$$FloatRegister);
        __ uaddlp($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
                  $dst$$FloatRegister);
        if (bt == T_INT) {
          __ uaddlp($dst$$FloatRegister, length_in_bytes == 16 ? __ T8H : __ T4H,
                    $dst$$FloatRegister);
        }
      } else {
        __ sve_cnt($dst$$FloatRegister, __ elemType_to_regVariant(bt),
                   ptrue, $src$$FloatRegister);
      }
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector popcount - LONG

instruct vpopcountL(vReg dst, vReg src) %{
  match(Set dst (PopCountVL src));
  format %{ "vpopcountL $dst, $src" %}
  ins_encode %{
    if (UseSVE == 0) {
      __ cnt($dst$$FloatRegister, __ T16B, $src$$FloatRegister);
      __ uaddlp($dst$$FloatRegister, __ T16B, $dst$$FloatRegister);
      __ uaddlp($dst$$FloatRegister, __ T8H, $dst$$FloatRegister);
      __ uaddlp($dst$$FloatRegister, __ T4S, $dst$$FloatRegister);
    } else {
      __ sve_cnt($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector popcount - predicated

instruct vpopcountI_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (PopCountVI dst_src pg));
  format %{ "vpopcountI_masked $dst_src, $pg, $dst_src" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_cnt($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
               $pg$$PRegister, $dst_src$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vpopcountL_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (PopCountVL dst_src pg));
  format %{ "vpopcountL_masked $dst_src, $pg, $dst_src" %}
  ins_encode %{
    __ sve_cnt($dst_src$$FloatRegister, __ D,
               $pg$$PRegister, $dst_src$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector blend ---------------------------------

instruct vblend_neon(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE == 0);
  match(Set dst (VectorBlend (Binary src1 src2) dst));
  format %{ "vblend_neon $dst, $src1, $src2" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    assert(length_in_bytes == 8 || length_in_bytes == 16, "must be");
    __ bsl($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
           $src2$$FloatRegister, $src1$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vblend_sve(vReg dst, vReg src1, vReg src2, pReg pg) %{
  predicate(UseSVE > 0);
  match(Set dst (VectorBlend (Binary src1 src2) pg));
  format %{ "vblend_sve $dst, $pg, $src1, $src2" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_sel($dst$$FloatRegister, __ elemType_to_regVariant(bt),
               $pg$$PRegister, $src2$$FloatRegister, $src1$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector round ---------------------------------

// vector Math.round

instruct vround_le128b(vReg dst, vReg src, vReg tmp1, vReg tmp2,
                       vReg tmp3, rFlagsReg cr) %{
  predicate(Matcher::vector_length_in_bytes(n) <= 16);
  match(Set dst (RoundVF src));
  match(Set dst (RoundVD src));
  effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, KILL cr);
  format %{ "vround_le128b $dst, $src\t# vector <= 128 bits. KILL $tmp1, $tmp2, $tmp3, cr" %}
  ins_encode %{
    __ vector_round_neon($dst$$FloatRegister, $src$$FloatRegister,
                         $tmp1$$FloatRegister, $tmp2$$FloatRegister,
                         $tmp3$$FloatRegister, get_arrangement(this));
  %}
  ins_pipe(pipe_slow);
%}

instruct vround_gt128b(vReg dst, vReg src, vReg tmp1, vReg tmp2,
                       pRegGov pgtmp, rFlagsReg cr) %{
  predicate(Matcher::vector_length_in_bytes(n) > 16);
  match(Set dst (RoundVF src));
  match(Set dst (RoundVD src));
  effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP pgtmp, KILL cr);
  format %{ "vround_gt128b $dst, $src\t# vector > 128 bits. KILL $tmp1, $tmp2, $pgtmp, cr" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ vector_round_sve($dst$$FloatRegister, $src$$FloatRegister,
                        $tmp1$$FloatRegister, $tmp2$$FloatRegister,
                        $pgtmp$$PRegister, __ elemType_to_regVariant(bt));
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ RoundDouble ----------------------------------

// vector Math.rint, floor, ceil

instruct vroundD(vReg dst, vReg src, immI rmode) %{
  predicate(Matcher::vector_element_basic_type(n) == T_DOUBLE);
  match(Set dst (RoundDoubleModeV src rmode));
  format %{ "vroundD $dst, $src, $rmode" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      switch ($rmode$$constant) {
        case RoundDoubleModeNode::rmode_rint:
          __ frintn($dst$$FloatRegister, __ T2D, $src$$FloatRegister);
          break;
        case RoundDoubleModeNode::rmode_floor:
          __ frintm($dst$$FloatRegister, __ T2D, $src$$FloatRegister);
          break;
        case RoundDoubleModeNode::rmode_ceil:
          __ frintp($dst$$FloatRegister, __ T2D, $src$$FloatRegister);
          break;
        default:
          assert(false, "unsupported");
          ShouldNotReachHere();
      }
    } else {
      assert(UseSVE > 0, "must be sve");
      switch ($rmode$$constant) {
        case RoundDoubleModeNode::rmode_rint:
          __ sve_frintn($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
          break;
        case RoundDoubleModeNode::rmode_floor:
          __ sve_frintm($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
          break;
        case RoundDoubleModeNode::rmode_ceil:
          __ sve_frintp($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
          break;
        default:
          assert(false, "unsupported");
          ShouldNotReachHere();
      }
    }
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ VectorTest -----------------------------------

// anytrue

instruct vtest_anytrue_neon(rFlagsReg cr, vReg src1, vReg src2, vReg tmp) %{
  predicate(UseSVE == 0 &&
            static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
  match(Set cr (VectorTest src1 src2));
  effect(TEMP tmp);
  format %{ "vtest_anytrue_neon $src1\t# KILL $tmp" %}
  ins_encode %{
    // No need to use src2.
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src1);
    assert(length_in_bytes == 8 || length_in_bytes == 16, "must be");
    __ addv($tmp$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B, $src1$$FloatRegister);
    __ umov(rscratch1, $tmp$$FloatRegister, __ B, 0);
    __ cmpw(rscratch1, zr);
  %}
  ins_pipe(pipe_slow);
%}

instruct vtest_anytrue_sve(rFlagsReg cr, pReg src1, pReg src2) %{
  predicate(UseSVE > 0 &&
            static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
  match(Set cr (VectorTest src1 src2));
  format %{ "vtest_anytrue_sve $src1" %}
  ins_encode %{
    // "src2" is not used for sve.
    __ sve_ptest(ptrue, $src1$$PRegister);
  %}
  ins_pipe(pipe_slow);
%}

// alltrue

instruct vtest_alltrue_neon(rFlagsReg cr, vReg src1, vReg src2, vReg tmp) %{
  predicate(UseSVE == 0 &&
            static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
  match(Set cr (VectorTest src1 src2));
  effect(TEMP tmp);
  format %{ "vtest_alltrue_neon $src1\t# KILL $tmp" %}
  ins_encode %{
    // No need to use src2.
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src1);
    assert(length_in_bytes == 8 || length_in_bytes == 16, "must be");
    __ uminv($tmp$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B, $src1$$FloatRegister);
    __ umov(rscratch1, $tmp$$FloatRegister, __ B, 0);
    __ cmpw(rscratch1, 0xff);
  %}
  ins_pipe(pipe_slow);
%}

instruct vtest_alltrue_sve(rFlagsReg cr, pReg src1, pReg src2, pReg ptmp) %{
  predicate(UseSVE > 0 &&
            static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
  match(Set cr (VectorTest src1 src2));
  effect(TEMP ptmp);
  format %{ "vtest_alltrue_sve $src1, $src2\t# KILL $ptmp" %}
  ins_encode %{
    __ sve_eors($ptmp$$PRegister, ptrue, $src1$$PRegister, $src2$$PRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector rearrange -----------------------------

// Here is an example that rearranges a NEON vector with 4 ints:
// Rearrange V1 int[a0, a1, a2, a3] to V2 int[a2, a3, a0, a1]
//   1. Get the indices of V1 and store them as Vi byte[0, 1, 2, 3].
//   2. Convert Vi byte[0, 1, 2, 3] to the indices of V2 and also store them as Vi byte[2, 3, 0, 1].
//   3. Unsigned extend Long Vi from byte[2, 3, 0, 1] to int[2, 3, 0, 1].
//   4. Multiply Vi int[2, 3, 0, 1] with constant int[0x04040404, 0x04040404, 0x04040404, 0x04040404]
//      and get tbl base Vm int[0x08080808, 0x0c0c0c0c, 0x00000000, 0x04040404].
//   5. Add Vm with constant int[0x03020100, 0x03020100, 0x03020100, 0x03020100]
//      and get tbl index Vm int[0x0b0a0908, 0x0f0e0d0c, 0x03020100, 0x07060504]
//   6. Use Vm as index register, and use V1 as table register.
//      Then get V2 as the result by tbl NEON instructions.
// Notes:
//   Step 1 matches VectorLoadConst.
//   Step 3 matches VectorLoadShuffle.
//   Step 4, 5, 6 match VectorRearrange.
//   For VectorRearrange short/int, the reason why such complex calculation is
//   required is because NEON tbl supports bytes table only, so for short/int, we
//   need to lookup 2/4 bytes as a group. For VectorRearrange long, we use bsl
//   to implement rearrange.

// Maybe move the shuffle preparation to VectorLoadShuffle
instruct rearrange_HS_neon(vReg dst, vReg src, vReg shuffle, vReg tmp1, vReg tmp2) %{
  predicate(UseSVE == 0 &&
            (Matcher::vector_element_basic_type(n) == T_SHORT ||
             (type2aelembytes(Matcher::vector_element_basic_type(n)) == 4 &&
              Matcher::vector_length_in_bytes(n) == 16)));
  match(Set dst (VectorRearrange src shuffle));
  effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2);
  format %{ "rearrange_HS_neon $dst, $src, $shuffle\t# vector (4S/8S/4I/4F). KILL $tmp1, $tmp2" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    if (bt == T_SHORT) {
      uint length_in_bytes = Matcher::vector_length_in_bytes(this);
      assert(length_in_bytes == 8 || length_in_bytes == 16, "must be");
      Assembler::SIMD_Arrangement size1 = length_in_bytes == 16 ? __ T16B : __ T8B;
      Assembler::SIMD_Arrangement size2 = length_in_bytes == 16 ? __ T8H : __ T4H;
      __ mov($tmp1$$FloatRegister, size1, 0x02);
      __ mov($tmp2$$FloatRegister, size2, 0x0100);
      __ mulv($dst$$FloatRegister, size2, $shuffle$$FloatRegister, $tmp1$$FloatRegister);
      __ addv($dst$$FloatRegister, size1, $dst$$FloatRegister, $tmp2$$FloatRegister);
      __ tbl($dst$$FloatRegister, size1, $src$$FloatRegister, 1, $dst$$FloatRegister);
    } else {
      assert(bt == T_INT || bt == T_FLOAT, "unsupported type");
      __ mov($tmp1$$FloatRegister, __ T16B, 0x04);
      __ mov($tmp2$$FloatRegister, __ T4S, 0x03020100);
      __ mulv($dst$$FloatRegister, __ T4S, $shuffle$$FloatRegister, $tmp1$$FloatRegister);
      __ addv($dst$$FloatRegister, __ T16B, $dst$$FloatRegister, $tmp2$$FloatRegister);
      __ tbl($dst$$FloatRegister, __ T16B, $src$$FloatRegister, 1, $dst$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct rearrange(vReg dst, vReg src, vReg shuffle) %{
  predicate(Matcher::vector_element_basic_type(n) == T_BYTE || UseSVE > 0);
  match(Set dst (VectorRearrange src shuffle));
  format %{ "rearrange $dst, $src, $shuffle" %}
  ins_encode %{
    BasicType bt_dst = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (bt_dst == T_BYTE && VM_Version::use_neon_for_vector(length_in_bytes)) {
      __ tbl($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
             $src$$FloatRegister, 1, $shuffle$$FloatRegister);
    } else {
      assert(UseSVE > 0, "must be sve");
      BasicType bt_src = Matcher::vector_element_basic_type(this, $src);
      __ sve_tbl($dst$$FloatRegister, __ elemType_to_regVariant(bt_src),
                 $src$$FloatRegister, $shuffle$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector Load Gather ---------------------------

instruct gather_loadS(vReg dst, indirect mem, vReg idx) %{
  predicate(UseSVE > 0 &&
            type2aelembytes(Matcher::vector_element_basic_type(n)) == 4);
  match(Set dst (LoadVectorGather mem idx));
  format %{ "gather_loadS $dst, $mem, $idx\t# vector (sve)" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    __ sve_ld1w_gather($dst$$FloatRegister, ptrue,
                       as_Register($mem$$base), $idx$$FloatRegister);
 %}
  ins_pipe(pipe_slow);
%}

instruct gather_loadD(vReg dst, indirect mem, vReg idx, vReg tmp) %{
  predicate(UseSVE > 0 &&
            type2aelembytes(Matcher::vector_element_basic_type(n)) == 8);
  match(Set dst (LoadVectorGather mem idx));
  effect(TEMP tmp);
  format %{ "gather_loadD $dst, $mem, $idx\t# vector (sve). KILL $tmp" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    __ sve_uunpklo($tmp$$FloatRegister, __ D, $idx$$FloatRegister);
    __ sve_ld1d_gather($dst$$FloatRegister, ptrue, as_Register($mem$$base),
                       $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct gather_loadS_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
  predicate(UseSVE > 0 &&
            type2aelembytes(Matcher::vector_element_basic_type(n)) == 4);
  match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
  format %{ "gather_loadS_masked $dst, $pg, $mem, $idx" %}
  ins_encode %{
    __ sve_ld1w_gather($dst$$FloatRegister, $pg$$PRegister,
                       as_Register($mem$$base), $idx$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct gather_loadD_masked(vReg dst, indirect mem, vReg idx, pRegGov pg, vReg tmp) %{
  predicate(UseSVE > 0 &&
            type2aelembytes(Matcher::vector_element_basic_type(n)) == 8);
  match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
  effect(TEMP tmp);
  format %{ "gather_loadD_masked $dst, $pg, $mem, $idx\t# KILL $tmp" %}
  ins_encode %{
    __ sve_uunpklo($tmp$$FloatRegister, __ D, $idx$$FloatRegister);
    __ sve_ld1d_gather($dst$$FloatRegister, $pg$$PRegister,
                       as_Register($mem$$base), $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector Store Scatter -------------------------

instruct scatter_storeS(indirect mem, vReg src, vReg idx) %{
  predicate(UseSVE > 0 &&
            type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 4);
  match(Set mem (StoreVectorScatter mem (Binary src idx)));
  format %{ "scatter_storeS $mem, $idx, $src\t# vector (sve)" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    __ sve_st1w_scatter($src$$FloatRegister, ptrue,
                        as_Register($mem$$base), $idx$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct scatter_storeD(indirect mem, vReg src, vReg idx, vReg tmp) %{
  predicate(UseSVE > 0 &&
            type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 8);
  match(Set mem (StoreVectorScatter mem (Binary src idx)));
  effect(TEMP tmp);
  format %{ "scatter_storeD $mem, $idx, $src\t# vector (sve). KILL $tmp" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
    assert(length_in_bytes == MaxVectorSize, "invalid vector length");
    __ sve_uunpklo($tmp$$FloatRegister, __ D, $idx$$FloatRegister);
    __ sve_st1d_scatter($src$$FloatRegister, ptrue,
                        as_Register($mem$$base), $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct scatter_storeS_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
  predicate(UseSVE > 0 &&
            type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 4);
  match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
  format %{ "scatter_storeS_masked $mem, $pg, $idx, $src" %}
  ins_encode %{
    __ sve_st1w_scatter($src$$FloatRegister, $pg$$PRegister,
                        as_Register($mem$$base), $idx$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct scatter_storeD_masked(indirect mem, vReg src, vReg idx, pRegGov pg, vReg tmp) %{
  predicate(UseSVE > 0 &&
            type2aelembytes(Matcher::vector_element_basic_type(n->in(3)->in(1))) == 8);
  match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
  effect(TEMP tmp);
  format %{ "scatter_storeD_masked $mem, $pg, $idx, $src\t# KILL $tmp" %}
  ins_encode %{
    __ sve_uunpklo($tmp$$FloatRegister, __ D, $idx$$FloatRegister);
    __ sve_st1d_scatter($src$$FloatRegister, $pg$$PRegister,
                        as_Register($mem$$base), $tmp$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ CountLeadingZerosV ---------------------------

instruct vcountLeadingZeros(vReg dst, vReg src) %{
  match(Set dst (CountLeadingZerosV src));
  format %{ "vcountLeadingZeros $dst, $src" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (bt == T_LONG) {
      if (UseSVE == 0) {
        __ umov(rscratch1, $src$$FloatRegister, __ D, 0);
        __ clz(rscratch1, rscratch1);
        __ mov($dst$$FloatRegister, __ D, 0, rscratch1);
        __ umov(rscratch1, $src$$FloatRegister, __ D, 1);
        __ clz(rscratch1, rscratch1);
        __ mov($dst$$FloatRegister, __ D, 1, rscratch1);
      } else {
        __ sve_clz($dst$$FloatRegister, __ D, ptrue, $src$$FloatRegister);
      }
    } else {
      if (VM_Version::use_neon_for_vector(length_in_bytes)) {
        __ clz($dst$$FloatRegister, get_arrangement(this), $src$$FloatRegister);
      } else {
        assert(UseSVE > 0, "must be sve");
        __ sve_clz($dst$$FloatRegister, __ elemType_to_regVariant(bt),
                   ptrue, $src$$FloatRegister);
      }
    }
  %}
  ins_pipe(pipe_slow);
%}

// The dst and src should use the same register to make sure the
// inactive lanes in dst save the same elements as src.

instruct vcountLeadingZeros_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (CountLeadingZerosV dst_src pg));
  format %{ "vcountLeadingZeros_masked $dst_src, $pg, $dst_src" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_clz($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
               $pg$$PRegister, $dst_src$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ CountTrailingZerosV --------------------------

instruct vcountTrailingZeros(vReg dst, vReg src) %{
  match(Set dst (CountTrailingZerosV src));
  format %{ "vcountTrailingZeros $dst, $src" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (bt == T_BYTE) {
      if (VM_Version::use_neon_for_vector(length_in_bytes)) {
        __ rbit($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
                $src$$FloatRegister);
        __ clz($dst$$FloatRegister, get_arrangement(this), $dst$$FloatRegister);
      } else {
        assert(UseSVE > 0, "must be sve");
        __ sve_rbit($dst$$FloatRegister, __ B, ptrue, $src$$FloatRegister);
        __ sve_clz($dst$$FloatRegister, __ B, ptrue, $dst$$FloatRegister);
      }
    } else {
      assert(bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported type");
      if (UseSVE == 0) {
        assert(length_in_bytes == 8 || length_in_bytes == 16, "unsupported");
        __ neon_reverse_bits($dst$$FloatRegister, $src$$FloatRegister,
                             bt, /* isQ */ length_in_bytes == 16);
        if (bt != T_LONG) {
          __ clz($dst$$FloatRegister, get_arrangement(this), $dst$$FloatRegister);
        } else {
          __ umov(rscratch1, $dst$$FloatRegister, __ D, 0);
          __ clz(rscratch1, rscratch1);
          __ mov($dst$$FloatRegister, __ D, 0, rscratch1);
          __ umov(rscratch1, $dst$$FloatRegister, __ D, 1);
          __ clz(rscratch1, rscratch1);
          __ mov($dst$$FloatRegister, __ D, 1, rscratch1);
        }
      } else {
        Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
        __ sve_rbit($dst$$FloatRegister, size, ptrue, $src$$FloatRegister);
        __ sve_clz($dst$$FloatRegister, size, ptrue, $dst$$FloatRegister);
      }
    }
  %}
  ins_pipe(pipe_slow);
%}

// The dst and src should use the same register to make sure the
// inactive lanes in dst save the same elements as src.
instruct vcountTrailingZeros_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (CountTrailingZerosV dst_src pg));
  format %{ "vcountTrailingZeros_masked $dst_src, $pg, $dst_src" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ sve_rbit($dst_src$$FloatRegister, size,
                $pg$$PRegister, $dst_src$$FloatRegister);
    __ sve_clz($dst_src$$FloatRegister, size,
               $pg$$PRegister, $dst_src$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ ReverseV -------------------------------------

instruct vreverse(vReg dst, vReg src) %{
  match(Set dst (ReverseV src));
  format %{ "vreverse $dst, $src" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (bt == T_BYTE) {
      if (VM_Version::use_neon_for_vector(length_in_bytes)) {
        __ rbit($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
                $src$$FloatRegister);
      } else {
        assert(UseSVE > 0, "must be sve");
        __ sve_rbit($dst$$FloatRegister, __ B, ptrue, $src$$FloatRegister);
      }
    } else {
      assert(bt == T_SHORT || bt == T_INT || bt == T_LONG, "unsupported type");
      if (UseSVE == 0) {
        assert(length_in_bytes == 8 || length_in_bytes == 16, "unsupported");
        __ neon_reverse_bits($dst$$FloatRegister, $src$$FloatRegister,
                             bt, /* isQ */ length_in_bytes == 16);
      } else {
        __ sve_rbit($dst$$FloatRegister, __ elemType_to_regVariant(bt),
                    ptrue, $src$$FloatRegister);
      }
    }
  %}
  ins_pipe(pipe_slow);
%}

// The dst and src should use the same register to make sure the
// inactive lanes in dst save the same elements as src.

instruct vreverse_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (ReverseV dst_src pg));
  format %{ "vreverse_masked $dst_src, $pg, $dst_src" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_rbit($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
               $pg$$PRegister, $dst_src$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ ReverseBytesV --------------------------------

instruct vreverseBytes(vReg dst, vReg src) %{
  match(Set dst (ReverseBytesV src));
  format %{ "vreverseBytes $dst, $src" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    if (VM_Version::use_neon_for_vector(length_in_bytes)) {
      assert(length_in_bytes == 8 || length_in_bytes == 16, "unsupported");
      if (bt == T_BYTE) {
        if ($dst$$FloatRegister != $src$$FloatRegister) {
          __ orr($dst$$FloatRegister, length_in_bytes == 16 ? __ T16B : __ T8B,
                 $src$$FloatRegister, $src$$FloatRegister);
        }
      } else {
        __ neon_reverse_bytes($dst$$FloatRegister, $src$$FloatRegister,
                              bt, /* isQ */ length_in_bytes == 16);
      }
    } else {
      assert(UseSVE > 0, "must be sve");
      if (bt == T_BYTE) {
        if ($dst$$FloatRegister != $src$$FloatRegister) {
          __ sve_orr($dst$$FloatRegister, $src$$FloatRegister, $src$$FloatRegister);
        }
      } else {
        __ sve_revb($dst$$FloatRegister, __ elemType_to_regVariant(bt),
                    ptrue, $src$$FloatRegister);
      }
    }
  %}
  ins_pipe(pipe_slow);
%}

// The dst and src should use the same register to make sure the
// inactive lanes in dst save the same elements as src.
instruct vreverseBytes_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (ReverseBytesV dst_src pg));
  format %{ "vreverseBytes_masked $dst_src, $pg, $dst_src" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    if (bt == T_BYTE) {
      // do nothing
    } else {
      __ sve_revb($dst_src$$FloatRegister, __ elemType_to_regVariant(bt),
                  $pg$$PRegister, $dst_src$$FloatRegister);
    }
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Populate Index to a Vector -------------------

instruct populateindex(vReg dst, iRegIorL2I src1, immI src2) %{
  predicate(UseSVE > 0);
  match(Set dst (PopulateIndex src1 src2));
  format %{ "populateindex $dst, $src1, $src2\t # populate index (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_index($dst$$FloatRegister, __ elemType_to_regVariant(bt),
                 $src1$$Register, (int)($src2$$constant));
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Compress/Expand Operations -------------------

instruct mcompress(pReg dst, pReg pg, rFlagsReg cr) %{
  predicate(UseSVE > 0);
  match(Set dst (CompressM pg));
  effect(KILL cr);
  format %{ "mcompress $dst, $pg\t# KILL cr" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ sve_cntp(rscratch1, size, ptrue, $pg$$PRegister);
    __ sve_whilelt(as_PRegister($dst$$reg), size, zr, rscratch1);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcompress(vReg dst, vReg src, pRegGov pg) %{
  predicate(UseSVE > 0 &&
            !is_subword_type(Matcher::vector_element_basic_type(n)));
  match(Set dst (CompressV src pg));
  format %{ "vcompress $dst, $src, $pg" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_compact($dst$$FloatRegister, __ elemType_to_regVariant(bt),
                   $src$$FloatRegister, $pg$$PRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcompressB(vReg dst, vReg src, pReg pg, vReg tmp1, vReg tmp2,
                    vReg tmp3, vReg tmp4, pReg ptmp, pRegGov pgtmp) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_BYTE);
  effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP tmp4, TEMP ptmp, TEMP pgtmp);
  match(Set dst (CompressV src pg));
  format %{ "vcompressB $dst, $src, $pg\t# KILL $tmp1, $tmp2, $tmp3, tmp4, $ptmp, $pgtmp" %}
  ins_encode %{
    __ sve_compress_byte($dst$$FloatRegister, $src$$FloatRegister, $pg$$PRegister,
                         $tmp1$$FloatRegister,$tmp2$$FloatRegister,
                         $tmp3$$FloatRegister,$tmp4$$FloatRegister,
                         $ptmp$$PRegister, $pgtmp$$PRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcompressS(vReg dst, vReg src, pReg pg,
                    vReg tmp1, vReg tmp2, pRegGov pgtmp) %{
  predicate(UseSVE > 0 && Matcher::vector_element_basic_type(n) == T_SHORT);
  effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP pgtmp);
  match(Set dst (CompressV src pg));
  format %{ "vcompressS $dst, $src, $pg\t# KILL $tmp1, $tmp2, $pgtmp" %}
  ins_encode %{
    __ sve_compress_short($dst$$FloatRegister, $src$$FloatRegister, $pg$$PRegister,
                          $tmp1$$FloatRegister,$tmp2$$FloatRegister, $pgtmp$$PRegister);
  %}
  ins_pipe(pipe_slow);
%}

instruct vexpand(vReg dst, vReg src, pRegGov pg) %{
  match(Set dst (ExpandV src pg));
  effect(TEMP_DEF dst);
  format %{ "vexpand $dst, $pg, $src" %}
  ins_encode %{
    // Example input:   src   = 1 2 3 4 5 6 7 8
    //                  pg    = 1 0 0 1 1 0 1 1
    // Expected result: dst   = 4 0 0 5 6 0 7 8

    // The basic idea is to use TBL which can shuffle the elements in the given
    // vector flexibly. HISTCNT + SUB is used to generate the second source input
    // for TBL whose value is used to select the indexed element from src vector.

    BasicType bt = Matcher::vector_element_basic_type(this);
    assert(UseSVE == 2 && !is_subword_type(bt), "unsupported");
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    // dst = 0 0 0 0 0 0 0 0
    __ sve_dup($dst$$FloatRegister, size, 0);
    // dst = 5 0 0 4 3 0 2 1
    __ sve_histcnt($dst$$FloatRegister, size, $pg$$PRegister,
                   $dst$$FloatRegister, $dst$$FloatRegister);
    // dst = 4 -1 -1 3 2 -1 1 0
    __ sve_sub($dst$$FloatRegister, size, 1);
    // dst = 4 0 0 5 6 0 7 8
    __ sve_tbl($dst$$FloatRegister, size, $src$$FloatRegister, $dst$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector signum --------------------------------

// Vector Math.signum

instruct vsignum_le128b(vReg dst, vReg src, vReg zero, vReg one) %{
  predicate(Matcher::vector_length_in_bytes(n) <= 16);
  match(Set dst (SignumVF src (Binary zero one)));
  match(Set dst (SignumVD src (Binary zero one)));
  effect(TEMP_DEF dst);
  format %{ "vsignum_le128b $dst, $src\t# vector <= 128 bits" %}
  ins_encode %{
    __ vector_signum_neon($dst$$FloatRegister, $src$$FloatRegister, $zero$$FloatRegister,
                          $one$$FloatRegister, get_arrangement(this));
  %}
  ins_pipe(pipe_slow);
%}

instruct vsignum_gt128b(vReg dst, vReg src, vReg zero, vReg one, vReg tmp, pRegGov pgtmp) %{
  predicate(Matcher::vector_length_in_bytes(n) > 16);
  match(Set dst (SignumVF src (Binary zero one)));
  match(Set dst (SignumVD src (Binary zero one)));
  effect(TEMP_DEF dst, TEMP tmp, TEMP pgtmp);
  format %{ "vsignum_gt128b $dst, $src\t# vector > 128 bits. KILL $tmp, $pgtmp" %}
  ins_encode %{
    assert(UseSVE > 0, "must be sve");
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ vector_signum_sve($dst$$FloatRegister, $src$$FloatRegister, $zero$$FloatRegister,
                         $one$$FloatRegister, $tmp$$FloatRegister, $pgtmp$$PRegister,
                         __ elemType_to_regVariant(bt));
  %}
  ins_pipe(pipe_slow);
%}

// ---------------------------------- CompressBitsV --------------------------------

instruct vcompressBits(vReg dst, vReg src1, vReg src2) %{
  match(Set dst (CompressBitsV src1 src2));
  format %{ "vcompressBits $dst, $src1, $src2\t# vector (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ sve_bext($dst$$FloatRegister, size,
                $src1$$FloatRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}

// ----------------------------------- ExpandBitsV ---------------------------------

instruct vexpandBits(vReg dst, vReg src1, vReg src2) %{
  match(Set dst (ExpandBitsV src1 src2));
  format %{ "vexpandBits $dst, $src1, $src2\t# vector (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ sve_bdep($dst$$FloatRegister, size,
                $src1$$FloatRegister, $src2$$FloatRegister);
  %}
  ins_pipe(pipe_slow);
%}