diff --git a/0001-ELF-Simplify-AArch64-relocateAlloc.-NFC.patch b/0001-ELF-Simplify-AArch64-relocateAlloc.-NFC.patch new file mode 100644 index 0000000..662a9d2 --- /dev/null +++ b/0001-ELF-Simplify-AArch64-relocateAlloc.-NFC.patch @@ -0,0 +1,47 @@ +From e710ff28456c1accbeb3a7e503163233bc615b16 Mon Sep 17 00:00:00 2001 +From: Fangrui Song +Date: Wed, 11 Feb 2026 21:23:34 -0800 +Subject: [PATCH] [ELF] Simplify AArch64::relocateAlloc. NFC + +--- + lld/ELF/Arch/AArch64.cpp | 11 ++++++----- + 1 file changed, 6 insertions(+), 5 deletions(-) + +diff --git a/lld/ELF/Arch/AArch64.cpp b/lld/ELF/Arch/AArch64.cpp +index 55434ae2151c..3c0d4ca64555 100644 +--- a/lld/ELF/Arch/AArch64.cpp ++++ b/lld/ELF/Arch/AArch64.cpp +@@ -931,9 +931,10 @@ static bool needsGotForMemtag(const Relocation &rel) { + + void AArch64::relocateAlloc(InputSection &sec, uint8_t *buf) const { + uint64_t secAddr = sec.getOutputSection()->addr + sec.outSecOff; +- AArch64Relaxer relaxer(ctx, sec.relocs()); +- for (size_t i = 0, size = sec.relocs().size(); i != size; ++i) { +- const Relocation &rel = sec.relocs()[i]; ++ const ArrayRef relocs = sec.relocs(); ++ AArch64Relaxer relaxer(ctx, relocs); ++ for (size_t i = 0, size = relocs.size(); i != size; ++i) { ++ const Relocation &rel = relocs[i]; + if (rel.expr == R_NONE) // See finalizeAddressDependentContent() + continue; + uint8_t *loc = buf + rel.offset; +@@ -947,14 +948,14 @@ void AArch64::relocateAlloc(InputSection &sec, uint8_t *buf) const { + switch (rel.expr) { + case RE_AARCH64_GOT_PAGE_PC: + if (i + 1 < size && +- relaxer.tryRelaxAdrpLdr(rel, sec.relocs()[i + 1], secAddr, buf)) { ++ relaxer.tryRelaxAdrpLdr(rel, relocs[i + 1], secAddr, buf)) { + ++i; + continue; + } + break; + case RE_AARCH64_PAGE_PC: + if (i + 1 < size && +- relaxer.tryRelaxAdrpAdd(rel, sec.relocs()[i + 1], secAddr, buf)) { ++ relaxer.tryRelaxAdrpAdd(rel, relocs[i + 1], secAddr, buf)) { + ++i; + continue; + } +-- +2.50.1 + diff --git a/0001-LLD-AArch64-Make-adrp-ldr-relaxation-per-symbol-all-.patch b/0001-LLD-AArch64-Make-adrp-ldr-relaxation-per-symbol-all-.patch new file mode 100644 index 0000000..d46ab12 --- /dev/null +++ b/0001-LLD-AArch64-Make-adrp-ldr-relaxation-per-symbol-all-.patch @@ -0,0 +1,416 @@ +From 8d590969badd9f3fed18a99fffbd9afdf3a975c6 Mon Sep 17 00:00:00 2001 +From: Nikita Popov +Date: Fri, 10 Jul 2026 09:09:08 +0200 +Subject: [PATCH] [LLD][AArch64] Make adrp+ldr relaxation per-symbol + all-or-nothing (#208396) + +We can't relax only some adrp+ldr pairs for a symbol, because there may +be a branch target between the adrp and ldr of this form: + + adrp x1, :got:sym + .Lfoo: + ldr x1, [x1, :got_lo12:sym] + # ... + + adrp x1, :got:sym + ldr x2, [x1, :got_lo12:sym] + b .Lfoo + +Relaxing the first adrp+ldr here would be invalid. This was clarified in +the ARM ABI in: +https://github.com/ARM-software/abi-aa/commit/11fb4ef42898060189d6a34ee96966e696ecbd20. + +The implementation already performed a pre-scan to check that the +relevant relocations occur in pairs. Change this scan to a) check all +the preconditions for the relaxation and b) make the decision per +symbol. + +Fixes https://github.com/llvm/llvm-project/issues/138254. +--- + bolt/test/AArch64/got-load-symbolization.s | 11 ++- + bolt/test/AArch64/lite-mode.s | 17 +++-- + lld/ELF/Arch/AArch64.cpp | 83 +++++++++++++-------- + lld/test/ELF/aarch64-adrp-ldr-got.s | 84 +++++++++++++++++++--- + 4 files changed, 149 insertions(+), 46 deletions(-) + +diff --git a/bolt/test/AArch64/got-load-symbolization.s b/bolt/test/AArch64/got-load-symbolization.s +index b4d682688758..399859fd7bfd 100644 +--- a/bolt/test/AArch64/got-load-symbolization.s ++++ b/bolt/test/AArch64/got-load-symbolization.s +@@ -38,9 +38,9 @@ _start: + # CHECK-NEXT: adrp x2, __BOLT_got_zero + # CHECK-NEXT: nop + # CHECK-NEXT: ldr x2, [x2, :lo12:__BOLT_got_zero{{.*}}] +- adrp x2, :got:near ++ adrp x2, :got:near2 + nop +- ldr x2, [x2, :got_lo12:near] ++ ldr x2, [x2, :got_lo12:near2] + + ## Load data object with local visibility. Relaxable into adrp+add. + # CHECK-NEXT: adrp x3, "local_far_data/1" +@@ -58,6 +58,7 @@ _start: + .size _start, .-_start + + .weak near ++.weak near2 + .weak far + .weak far_data + +@@ -77,6 +78,12 @@ near: + ret + .size near, .-near + ++ .globl near2 ++ .type near2, @function ++near2: ++ ret ++.size near2, .-near2 ++ + #--- far.s + + .text +diff --git a/bolt/test/AArch64/lite-mode.s b/bolt/test/AArch64/lite-mode.s +index f2d06219f7a2..24c31772b847 100644 +--- a/bolt/test/AArch64/lite-mode.s ++++ b/bolt/test/AArch64/lite-mode.s +@@ -24,12 +24,12 @@ + # CHECK-COMPACT-NOT: <_start.org.0> + + ## Verify that the number of FDEs matches the number of functions in the output +-## binary. There are three original functions and two optimized. ++## binary. There are four original functions and three optimized. + ## NOTE: at the moment we are emitting extra FDEs for patched functions, thus + ## there is one more FDE for _start. + # RUN: llvm-readelf -u %t.bolt | grep -wc FDE \ + # RUN: | FileCheck --check-prefix=CHECK-FDE %s +-# CHECK-FDE: 6 ++# CHECK-FDE: 8 + + ## In lite mode, optimized code will be separated from the original .text by + ## over 128MB, making it impossible for call/bl instructions in cold functions +@@ -106,9 +106,9 @@ cold_function: + # CHECK-NEXT: add x4 + + ## Check that non-relaxable GOT load is left intact. +- adrp x5, :got:far_func ++ adrp x5, :got:far_func2 + nop +- ldr x5, [x5, #:got_lo12:far_func] ++ ldr x5, [x5, #:got_lo12:far_func2] + # CHECK-INPUT-NEXT: adrp x5 + # CHECK-INPUT-NEXT: nop + # CHECK-INPUT-NEXT: ldr x5 +@@ -155,3 +155,12 @@ far_func: + ret x30 + .cfi_endproc + .size far_func, .-far_func ++ ++ .globl far_func2 ++ .type far_func2, %function ++far_func2: ++# FDATA: 0 [unknown] 0 1 far_func2 0 0 100 ++ .cfi_startproc ++ ret x30 ++ .cfi_endproc ++ .size far_func2, .-far_func2 +diff --git a/lld/ELF/Arch/AArch64.cpp b/lld/ELF/Arch/AArch64.cpp +index ed6d42fd4c05..11c3dca1a358 100644 +--- a/lld/ELF/Arch/AArch64.cpp ++++ b/lld/ELF/Arch/AArch64.cpp +@@ -106,13 +106,17 @@ private: + + struct AArch64Relaxer { + Ctx &ctx; +- bool safeToRelaxAdrpLdr = false; ++ SmallPtrSet unsafeToRelaxAdrpLdr; + +- AArch64Relaxer(Ctx &ctx, ArrayRef relocs); ++ AArch64Relaxer(Ctx &ctx, ArrayRef relocs, uint64_t secAddr, ++ uint8_t *buf); + bool tryRelaxAdrpAdd(const Relocation &adrpRel, const Relocation &addRel, + uint64_t secAddr, uint8_t *buf) const; + bool tryRelaxAdrpLdr(const Relocation &adrpRel, const Relocation &ldrRel, + uint64_t secAddr, uint8_t *buf) const; ++ bool isLegalAdrpLdrRelaxationCandidate(const Relocation &adrpRel, ++ const Relocation &ldrRel, ++ uint64_t secAddr, uint8_t *buf) const; + }; + } // namespace + +@@ -896,26 +900,30 @@ void AArch64::relaxTlsIeToLe(uint8_t *loc, const Relocation &rel, + llvm_unreachable("invalid relocation for TLS IE to LE relaxation"); + } + +-AArch64Relaxer::AArch64Relaxer(Ctx &ctx, ArrayRef relocs) ++AArch64Relaxer::AArch64Relaxer(Ctx &ctx, ArrayRef relocs, ++ uint64_t secAddr, uint8_t *buf) + : ctx(ctx) { + if (!ctx.arg.relax) + return; +- // Check if R_AARCH64_ADR_GOT_PAGE and R_AARCH64_LD64_GOT_LO12_NC +- // always appear in pairs. ++ // For a given symbol R_AARCH64_ADR_GOT_PAGE and R_AARCH64_LD64_GOT_LO12_NC ++ // relaxation is all-or-nothing. We can't relax only some of them, as there ++ // may be a jump destination between the two relocations. + size_t i = 0; + const size_t size = relocs.size(); + for (; i != size; ++i) { + if (relocs[i].type == R_AARCH64_ADR_GOT_PAGE) { +- if (i + 1 < size && relocs[i + 1].type == R_AARCH64_LD64_GOT_LO12_NC) { ++ if (i + 1 < size && relocs[i + 1].type == R_AARCH64_LD64_GOT_LO12_NC && ++ !unsafeToRelaxAdrpLdr.contains(relocs[i].sym) && ++ isLegalAdrpLdrRelaxationCandidate(relocs[i], relocs[i + 1], secAddr, ++ buf)) { + ++i; + continue; + } +- break; ++ unsafeToRelaxAdrpLdr.insert(relocs[i].sym); + } else if (relocs[i].type == R_AARCH64_LD64_GOT_LO12_NC) { +- break; ++ unsafeToRelaxAdrpLdr.insert(relocs[i].sym); + } + } +- safeToRelaxAdrpLdr = i == size; + } + + bool AArch64Relaxer::tryRelaxAdrpAdd(const Relocation &adrpRel, +@@ -966,23 +974,9 @@ bool AArch64Relaxer::tryRelaxAdrpAdd(const Relocation &adrpRel, + return true; + } + +-bool AArch64Relaxer::tryRelaxAdrpLdr(const Relocation &adrpRel, +- const Relocation &ldrRel, uint64_t secAddr, +- uint8_t *buf) const { +- if (!safeToRelaxAdrpLdr) +- return false; +- +- // When the definition of sym is not preemptible then we may +- // be able to relax +- // ADRP xn, :got: sym +- // LDR xn, [ xn :got_lo12: sym] +- // to +- // ADRP xn, sym +- // ADD xn, xn, :lo_12: sym +- +- if (adrpRel.type != R_AARCH64_ADR_GOT_PAGE || +- ldrRel.type != R_AARCH64_LD64_GOT_LO12_NC) +- return false; ++bool AArch64Relaxer::isLegalAdrpLdrRelaxationCandidate( ++ const Relocation &adrpRel, const Relocation &ldrRel, uint64_t secAddr, ++ uint8_t *buf) const { + // Check if the relocations apply to consecutive instructions. + if (adrpRel.offset + 4 != ldrRel.offset) + return false; +@@ -1022,10 +1016,37 @@ bool AArch64Relaxer::tryRelaxAdrpLdr(const Relocation &adrpRel, + if (val != llvm::SignExtend64(val, 33)) + return false; + ++ return true; ++} ++ ++bool AArch64Relaxer::tryRelaxAdrpLdr(const Relocation &adrpRel, ++ const Relocation &ldrRel, uint64_t secAddr, ++ uint8_t *buf) const { ++ // When the definition of sym is not preemptible then we may ++ // be able to relax ++ // ADRP xn, :got: sym ++ // LDR xn, [ xn :got_lo12: sym] ++ // to ++ // ADRP xn, sym ++ // ADD xn, xn, :lo_12: sym ++ ++ if (!ctx.arg.relax || adrpRel.type != R_AARCH64_ADR_GOT_PAGE || ++ ldrRel.type != R_AARCH64_LD64_GOT_LO12_NC) ++ return false; ++ ++ Symbol *sym = adrpRel.sym; ++ if (unsafeToRelaxAdrpLdr.contains(sym)) ++ return false; ++ ++ assert(isLegalAdrpLdrRelaxationCandidate(adrpRel, ldrRel, secAddr, buf) && ++ "Should have been marked as unsafe"); ++ ++ uint32_t adrpInstr = read32le(buf + adrpRel.offset); ++ uint32_t adrpDestReg = adrpInstr & 0x1f; + Relocation adrpSymRel = {RE_AARCH64_PAGE_PC, R_AARCH64_ADR_PREL_PG_HI21, +- adrpRel.offset, /*addend=*/0, &sym}; ++ adrpRel.offset, /*addend=*/0, sym}; + Relocation addRel = {R_ABS, R_AARCH64_ADD_ABS_LO12_NC, ldrRel.offset, +- /*addend=*/0, &sym}; ++ /*addend=*/0, sym}; + + // adrp x_ + write32le(buf + adrpSymRel.offset, 0x90000000 | adrpDestReg); +@@ -1034,11 +1055,11 @@ bool AArch64Relaxer::tryRelaxAdrpLdr(const Relocation &adrpRel, + + ctx.target->relocate( + buf + adrpSymRel.offset, adrpSymRel, +- SignExtend64(getAArch64Page(sym.getVA(ctx)) - ++ SignExtend64(getAArch64Page(sym->getVA(ctx)) - + getAArch64Page(secAddr + adrpSymRel.offset), + 64)); + ctx.target->relocate(buf + addRel.offset, addRel, +- SignExtend64(sym.getVA(ctx), 64)); ++ SignExtend64(sym->getVA(ctx), 64)); + tryRelaxAdrpAdd(adrpSymRel, addRel, secAddr, buf); + return true; + } +@@ -1052,7 +1073,7 @@ static bool needsGotForMemtag(const Relocation &rel) { + void AArch64::relocateAlloc(InputSection &sec, uint8_t *buf) const { + uint64_t secAddr = sec.getOutputSection()->addr + sec.outSecOff; + const ArrayRef relocs = sec.relocs(); +- AArch64Relaxer relaxer(ctx, relocs); ++ AArch64Relaxer relaxer(ctx, relocs, secAddr, buf); + for (size_t i = 0, size = relocs.size(); i != size; ++i) { + const Relocation &rel = relocs[i]; + if (rel.expr == R_NONE) // See finalizeAddressDependentContent() +diff --git a/lld/test/ELF/aarch64-adrp-ldr-got.s b/lld/test/ELF/aarch64-adrp-ldr-got.s +index 56a90aac3876..a8216da9f20c 100644 +--- a/lld/test/ELF/aarch64-adrp-ldr-got.s ++++ b/lld/test/ELF/aarch64-adrp-ldr-got.s +@@ -4,6 +4,7 @@ + # RUN: llvm-mc -filetype=obj -triple=aarch64 %t/a.s -o %t/a.o + # RUN: llvm-mc -filetype=obj -triple=aarch64 %t/unpaired.s -o %t/unpaired.o + # RUN: llvm-mc -filetype=obj -triple=aarch64 %t/lone-ldr.s -o %t/lone-ldr.o ++# RUN: llvm-mc -filetype=obj -triple=aarch64 %t/all-or-nothing.s -o %t/all-or-nothing.o + + # RUN: ld.lld %t/a.o -T %t/out-of-adr-range.t -o %t/a + # RUN: llvm-objdump --no-show-raw-insn -d %t/a | FileCheck %s +@@ -49,7 +50,8 @@ + # RUN: llvm-objdump --no-show-raw-insn -d %t/out-of-range | \ + # RUN: FileCheck --check-prefix=X1-NO-RELAX %s + +-## Relocations do not appear in pairs, no relaxations should be applied. ++## Relocations do not appear in pairs, no relaxations should be applied for ++## that symbol. We can still relax other symbols. + # RUN: ld.lld %t/unpaired.o -o %t/unpaired + # RUN: llvm-objdump --no-show-raw-insn -d %t/unpaired | \ + # RUN: FileCheck --check-prefix=UNPAIRED %s +@@ -58,6 +60,9 @@ + # UNPAIRED-NEXT: b + # UNPAIRED-NEXT: adrp x0 + # UNPAIRED: ldr x0 ++## This is a different symbol. ++# UNPAIRED: nop ++# UNPAIRED: adr x1 + + ## Relocations do not appear in pairs, no relaxations should be applied. + # RUN: ld.lld %t/lone-ldr.o -o %t/lone-ldr +@@ -66,6 +71,26 @@ + + # LONE-LDR: ldr x0 + ++## Make sure that relaxation is not applied if not all adrp+ldr pairs for ++## a given symbol can be relaxed. This is not legal, because there may be ++## a branch destination between the adrp and ldr instructions. We can still ++## perform the relaxation for other symbols, or the same symbol in a different ++## section. ++# RUN: ld.lld %t/all-or-nothing.o -o %t/all-or-nothing ++# RUN: llvm-objdump --no-show-raw-insn -d %t/all-or-nothing | \ ++# RUN: FileCheck --check-prefix=ALL-OR-NOTHING %s ++ ++# ALL-OR-NOTHING-LABEL: <_start>: ++# ALL-OR-NOTHING: adrp x1 ++# ALL-OR-NOTHING: ldr x1 ++# ALL-OR-NOTHING: adrp x1 ++# ALL-OR-NOTHING: ldr x2 ++# ALL-OR-NOTHING: nop ++# ALL-OR-NOTHING: adr x1 ++# ALL-OR-NOTHING-LABEL: : ++# ALL-OR-NOTHING: nop ++# ALL-OR-NOTHING: adr x1 ++ + ## This linker script ensures that .rodata and .text are sufficiently (>1M) + ## far apart so that the adrp + ldr pair cannot be relaxed to adr + nop. + #--- out-of-adr-range.t +@@ -95,25 +120,40 @@ SECTIONS { + .hidden x + x: + .word 10 ++.hidden y ++y: ++.word 10 ++.hidden z ++z: ++.word 10 ++.hidden u ++u: ++.word 10 ++.hidden v ++v: ++.word 10 + .text + .global _start + _start: + adrp x1, :got:x + ldr x1, [x1, #:got_lo12:x] +- adrp x2, :got:x+1 +- ldr x2, [x2, #:got_lo12:x] +- adrp x3, :got:x +- ldr x3, [x3, #:got_lo12:x+8] +- adrp x4, :got:x +- ldr x5, [x4, #:got_lo12:x] +- adrp x6, :got:x +- ldr x6, [x0, #:got_lo12:x] ++ adrp x2, :got:y+1 ++ ldr x2, [x2, #:got_lo12:y] ++ adrp x3, :got:z ++ ldr x3, [x3, #:got_lo12:z+8] ++ adrp x4, :got:u ++ ldr x5, [x4, #:got_lo12:u] ++ adrp x6, :got:v ++ ldr x6, [x0, #:got_lo12:v] + + #--- unpaired.s + .text + .hidden x + x: + nop ++.hidden y ++y: ++ nop + .global _start + _start: + adrp x0, :got:x +@@ -121,6 +161,8 @@ _start: + adrp x0, :got:x + L: + ldr x0, [x0, #:got_lo12:x] ++ adrp x1, :got:y ++ ldr x1, [x1, #:got_lo12:y] + + #--- lone-ldr.s + .text +@@ -130,3 +172,27 @@ x: + .global _start + _start: + ldr x0, [x0, #:got_lo12:x] ++ ++#--- all-or-nothing.s ++.rodata ++.hidden x ++x: ++.word 10 ++.hidden y ++y: ++.word 10 ++.text ++.global _start ++_start: ++ adrp x1, :got:x ++ ldr x1, [x1, #:got_lo12:x] ++ adrp x1, :got:x ++ ldr x2, [x1, #:got_lo12:x] ++ adrp x1, :got:y ++ ldr x1, [x1, #:got_lo12:y] ++ ++.section .text.foo ++.global foo ++foo: ++ adrp x1, :got:x ++ ldr x1, [x1, #:got_lo12:x] +-- +2.50.1 + diff --git a/0001-LLVM-Verifier-Fix-buffer-overflow-when-verifying-gc..patch b/0001-LLVM-Verifier-Fix-buffer-overflow-when-verifying-gc..patch new file mode 100644 index 0000000..1f79e82 --- /dev/null +++ b/0001-LLVM-Verifier-Fix-buffer-overflow-when-verifying-gc..patch @@ -0,0 +1,88 @@ +From 7bb9626d5ab901e5c1a8e9acbbb1684c982401b4 Mon Sep 17 00:00:00 2001 +From: Tulio Magno Quites Machado Filho +Date: Fri, 10 Jul 2026 15:45:02 -0300 +Subject: [PATCH] [LLVM][Verifier] Fix buffer overflow when verifying + gc.statepoint (#208278) + +When a negative base index is passed, the verification detects the +out-of-bounds value and start printing information that helps to +identify what caused the error. In order to print all the information, +it tries to dereference the Base pointer at +GCRelocateInst::getBasePtr(), causing the buffer overflow. Add a bounds +check to getBasePTR() and getDerivedPtr() in order to avoid this. + +Improve the test in order to validate negative values passed as indexes. +They're based on the reproducer from issue #199191. + +Fixes #199191 +--- + llvm/lib/IR/AsmWriter.cpp | 10 ++++++++-- + llvm/lib/IR/IntrinsicInst.cpp | 19 ++++++++++++++++--- + 3 files changed, 40 insertions(+), 5 deletions(-) + +diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp +index e90630a8cae5..6752d6aa344d 100644 +--- a/llvm/lib/IR/AsmWriter.cpp ++++ b/llvm/lib/IR/AsmWriter.cpp +@@ -4372,9 +4372,15 @@ void AssemblyWriter::printInstructionLine(const Instruction &I) { + /// intrinsic indicating base and derived pointer names. + void AssemblyWriter::printGCRelocateComment(const GCRelocateInst &Relocate) { + Out << " ; ("; +- writeOperand(Relocate.getBasePtr(), false); ++ if (Value *BasePtr = Relocate.getBasePtr()) ++ writeOperand(BasePtr, false); ++ else ++ Out << "invalid"; + Out << ", "; +- writeOperand(Relocate.getDerivedPtr(), false); ++ if (Value *DerivedPtr = Relocate.getDerivedPtr()) ++ writeOperand(DerivedPtr, false); ++ else ++ Out << "invalid"; + Out << ")"; + } + +diff --git a/llvm/lib/IR/IntrinsicInst.cpp b/llvm/lib/IR/IntrinsicInst.cpp +index 3e9f3257956a..eb964d566f29 100644 +--- a/llvm/lib/IR/IntrinsicInst.cpp ++++ b/llvm/lib/IR/IntrinsicInst.cpp +@@ -867,10 +867,16 @@ Value *GCRelocateInst::getBasePtr() const { + auto Statepoint = getStatepoint(); + if (isa(Statepoint)) + return UndefValue::get(Statepoint->getType()); +- ++ // Handle too few (bundle) arguments to avoid crashes when printing invalid ++ // IR, e.g. in the verifier. + auto *GCInst = cast(Statepoint); +- if (auto Opt = GCInst->getOperandBundle(LLVMContext::OB_gc_live)) ++ if (auto Opt = GCInst->getOperandBundle(LLVMContext::OB_gc_live)) { ++ if (getBasePtrIndex() > Opt->Inputs.size()) ++ return nullptr; + return *(Opt->Inputs.begin() + getBasePtrIndex()); ++ } ++ if (getBasePtrIndex() > GCInst->arg_size()) ++ return nullptr; + return *(GCInst->arg_begin() + getBasePtrIndex()); + } + +@@ -879,9 +885,16 @@ Value *GCRelocateInst::getDerivedPtr() const { + if (isa(Statepoint)) + return UndefValue::get(Statepoint->getType()); + ++ // Handle too few (bundle) arguments to avoid crashes when printing invalid ++ // IR, e.g. in the verifier. + auto *GCInst = cast(Statepoint); +- if (auto Opt = GCInst->getOperandBundle(LLVMContext::OB_gc_live)) ++ if (auto Opt = GCInst->getOperandBundle(LLVMContext::OB_gc_live)) { ++ if (getDerivedPtrIndex() > Opt->Inputs.size()) ++ return nullptr; + return *(Opt->Inputs.begin() + getDerivedPtrIndex()); ++ } ++ if (getDerivedPtrIndex() > GCInst->arg_size()) ++ return nullptr; + return *(GCInst->arg_begin() + getDerivedPtrIndex()); + } + +-- +2.50.1 + diff --git a/0001-SDAG-Freeze-condition-in-select-of-load-fold-208683.patch b/0001-SDAG-Freeze-condition-in-select-of-load-fold-208683.patch new file mode 100644 index 0000000..d15a946 --- /dev/null +++ b/0001-SDAG-Freeze-condition-in-select-of-load-fold-208683.patch @@ -0,0 +1,3977 @@ +From abcef279cd33492fe8301c8873fc535fa4dbf0d5 Mon Sep 17 00:00:00 2001 +From: Nikita Popov +Date: Fri, 10 Jul 2026 16:08:02 +0200 +Subject: [PATCH] [SDAG] Freeze condition in select of load fold (#208683) + +When converting `select cond, (load p1), (load p2)` to `load (select +cond, p1, p2)`, if `cond` is poison, originally this would result in a +`poison` result, while after the transform it would result in a load of +poison, which is immediate UB. Fix this by freezing the condition. + +Fixes https://github.com/llvm/llvm-project/issues/208611. + +(cherry picked from commit f2dfbf06f7db1a3cace4beb9f65d5a7f6a8b6235) +--- + llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 16 +- + llvm/test/CodeGen/AArch64/icmp.ll | 26 +- + llvm/test/CodeGen/ARM/fp16-promote.ll | 2 +- + llvm/test/CodeGen/Mips/cconv/vector.ll | 10 +- + llvm/test/CodeGen/Mips/cmov.ll | 2 +- + llvm/test/CodeGen/NVPTX/i1-select.ll | 14 +- + .../CodeGen/X86/fp-strict-scalar-cmp-fp16.ll | 112 +- + llvm/test/CodeGen/X86/fp-strict-scalar-cmp.ll | 968 +++++++++++++----- + .../test/CodeGen/X86/fp128-libcalls-strict.ll | 2 + + .../CodeGen/X86/fp80-strict-scalar-cmp.ll | 148 ++- + llvm/test/CodeGen/X86/isel-select-cmov.ll | 16 +- + llvm/test/CodeGen/X86/select-constant-xor.ll | 12 +- + llvm/test/CodeGen/X86/select-mmx.ll | 4 +- + .../test/CodeGen/X86/select-of-load-poison.ll | 20 + + 14 files changed, 994 insertions(+), 358 deletions(-) + create mode 100644 llvm/test/CodeGen/X86/select-of-load-poison.ll + +diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +index ee9238753735a..dec92eff5bc19 100644 +--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp ++++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +@@ -29318,10 +29318,12 @@ bool DAGCombiner::SimplifySelectOps(SDNode *TheSelect, SDValue LHS, + SDNode::hasPredecessorHelper(RLD, Visited, Worklist))) + return false; + +- Addr = DAG.getSelect(SDLoc(TheSelect), +- LLD->getBasePtr().getValueType(), +- TheSelect->getOperand(0), LLD->getBasePtr(), +- RLD->getBasePtr()); ++ // If the condition is poison, originally this would result in a poison ++ // result. After the transform, this would result in a load of poison, ++ // which is UB. Freeze the condition to prevent this. ++ Addr = DAG.getSelect(SDLoc(TheSelect), LLD->getBasePtr().getValueType(), ++ DAG.getFreeze(TheSelect->getOperand(0)), ++ LLD->getBasePtr(), RLD->getBasePtr()); + } else { // Otherwise SELECT_CC + // We cannot do this optimization if any pair of {RLD, LLD} is a + // predecessor to {RLD, LLD, CondLHS, CondRHS}. As we've already compared +@@ -29340,10 +29342,10 @@ bool DAGCombiner::SimplifySelectOps(SDNode *TheSelect, SDValue LHS, + SDNode::hasPredecessorHelper(RLD, Visited, Worklist))) + return false; + ++ SDValue FrozenOp0 = DAG.getFreeze(TheSelect->getOperand(0)); ++ SDValue FrozenOp1 = DAG.getFreeze(TheSelect->getOperand(1)); + Addr = DAG.getNode(ISD::SELECT_CC, SDLoc(TheSelect), +- LLD->getBasePtr().getValueType(), +- TheSelect->getOperand(0), +- TheSelect->getOperand(1), ++ LLD->getBasePtr().getValueType(), FrozenOp0, FrozenOp1, + LLD->getBasePtr(), RLD->getBasePtr(), + TheSelect->getOperand(4)); + } +diff --git a/llvm/test/CodeGen/AArch64/icmp.ll b/llvm/test/CodeGen/AArch64/icmp.ll +index 7195e2b2f1255..920d5a090c303 100644 +--- a/llvm/test/CodeGen/AArch64/icmp.ll ++++ b/llvm/test/CodeGen/AArch64/icmp.ll +@@ -1379,26 +1379,22 @@ entry: + define <2 x i128> @v2i128_i128(<2 x i128> %a, <2 x i128> %b, <2 x i128> %d, <2 x i128> %e) { + ; CHECK-SD-LABEL: v2i128_i128: + ; CHECK-SD: // %bb.0: // %entry ++; CHECK-SD-NEXT: cmp x2, x6 + ; CHECK-SD-NEXT: add x10, sp, #32 + ; CHECK-SD-NEXT: mov x11, sp ++; CHECK-SD-NEXT: sbcs xzr, x3, x7 ++; CHECK-SD-NEXT: cset w8, lt + ; CHECK-SD-NEXT: cmp x0, x4 +-; CHECK-SD-NEXT: orr x12, x10, #0x8 +-; CHECK-SD-NEXT: orr x13, x11, #0x8 + ; CHECK-SD-NEXT: sbcs xzr, x1, x5 ++; CHECK-SD-NEXT: cset w9, lt ++; CHECK-SD-NEXT: cmp w9, #0 ++; CHECK-SD-NEXT: csel x9, x11, x10, ne ++; CHECK-SD-NEXT: cmp w8, #0 + ; CHECK-SD-NEXT: add x8, sp, #48 +-; CHECK-SD-NEXT: add x9, sp, #16 +-; CHECK-SD-NEXT: csel x12, x13, x12, lt +-; CHECK-SD-NEXT: csel x10, x11, x10, lt +-; CHECK-SD-NEXT: cmp x2, x6 +-; CHECK-SD-NEXT: orr x11, x8, #0x8 +-; CHECK-SD-NEXT: orr x13, x9, #0x8 +-; CHECK-SD-NEXT: sbcs xzr, x3, x7 +-; CHECK-SD-NEXT: ldr x0, [x10] +-; CHECK-SD-NEXT: csel x8, x9, x8, lt +-; CHECK-SD-NEXT: csel x9, x13, x11, lt +-; CHECK-SD-NEXT: ldr x1, [x12] +-; CHECK-SD-NEXT: ldr x2, [x8] +-; CHECK-SD-NEXT: ldr x3, [x9] ++; CHECK-SD-NEXT: add x10, sp, #16 ++; CHECK-SD-NEXT: ldp x0, x1, [x9] ++; CHECK-SD-NEXT: csel x8, x10, x8, ne ++; CHECK-SD-NEXT: ldp x2, x3, [x8] + ; CHECK-SD-NEXT: ret + ; + ; CHECK-GI-LABEL: v2i128_i128: +diff --git a/llvm/test/CodeGen/ARM/fp16-promote.ll b/llvm/test/CodeGen/ARM/fp16-promote.ll +index 27a0bf2eb9037..5b3539def98a0 100644 +--- a/llvm/test/CodeGen/ARM/fp16-promote.ll ++++ b/llvm/test/CodeGen/ARM/fp16-promote.ll +@@ -348,7 +348,7 @@ define half @test_tailcall_flipped(half %a, half %b) #0 { + ; No conversion is needed + define void @test_select(ptr %p, ptr %q, i1 zeroext %c) #0 { + ; CHECK-ALL-LABEL: test_select: +-; CHECK-ALL: cmp r2, #0 ++; CHECK-ALL: tst r2, #1 + ; CHECK-ALL-NEXT: movne r1, r0 + ; CHECK-ALL-NEXT: ldrh r1, [r1] + ; CHECK-ALL-NEXT: strh r1, [r0] +diff --git a/llvm/test/CodeGen/Mips/cconv/vector.ll b/llvm/test/CodeGen/Mips/cconv/vector.ll +index c64c60ddef73d..5cd2b77615fcd 100644 +--- a/llvm/test/CodeGen/Mips/cconv/vector.ll ++++ b/llvm/test/CodeGen/Mips/cconv/vector.ll +@@ -6038,15 +6038,15 @@ entry: + define <4 x float> @select(<4 x i32> %cond, <4 x float> %arg1, <4 x float> %arg2) { + ; MIPS32-LABEL: select: + ; MIPS32: # %bb.0: # %entry +-; MIPS32-NEXT: andi $1, $7, 1 +-; MIPS32-NEXT: lw $2, 16($sp) +-; MIPS32-NEXT: andi $2, $2, 1 ++; MIPS32-NEXT: lw $1, 16($sp) ++; MIPS32-NEXT: andi $2, $7, 1 ++; MIPS32-NEXT: andi $1, $1, 1 + ; MIPS32-NEXT: addiu $3, $sp, 44 + ; MIPS32-NEXT: addiu $5, $sp, 28 + ; MIPS32-NEXT: addiu $7, $sp, 48 + ; MIPS32-NEXT: addiu $8, $sp, 32 +-; MIPS32-NEXT: movn $7, $8, $2 +-; MIPS32-NEXT: movn $3, $5, $1 ++; MIPS32-NEXT: movn $7, $8, $1 ++; MIPS32-NEXT: movn $3, $5, $2 + ; MIPS32-NEXT: andi $1, $6, 1 + ; MIPS32-NEXT: addiu $2, $sp, 40 + ; MIPS32-NEXT: addiu $5, $sp, 24 +diff --git a/llvm/test/CodeGen/Mips/cmov.ll b/llvm/test/CodeGen/Mips/cmov.ll +index bb3c7c27a122d..ee60b353b86b6 100644 +--- a/llvm/test/CodeGen/Mips/cmov.ll ++++ b/llvm/test/CodeGen/Mips/cmov.ll +@@ -65,7 +65,7 @@ entry: + + ; 64-CMOV: daddiu $[[R1:[0-9]+]], ${{[0-9]+}}, %got_disp(d) + ; 64-CMOV: daddiu $[[R0:[0-9]+]], ${{[0-9]+}}, %got_disp(c) +-; 64-CMOV: movn $[[R1]], $[[R0]], $4 ++; 64-CMOV: movn $[[R1]], $[[R0]], $2 + + ; 64-CMP-DAG: daddiu $[[R1:[0-9]+]], ${{[0-9]+}}, %got_disp(d) + ; 64-CMP-DAG: daddiu $[[R0:[0-9]+]], ${{[0-9]+}}, %got_disp(c) +diff --git a/llvm/test/CodeGen/NVPTX/i1-select.ll b/llvm/test/CodeGen/NVPTX/i1-select.ll +index c91a3df204d80..c13e494c3077a 100644 +--- a/llvm/test/CodeGen/NVPTX/i1-select.ll ++++ b/llvm/test/CodeGen/NVPTX/i1-select.ll +@@ -70,9 +70,9 @@ define i32 @test_select_i1_trunc_2(i64 %a, i16 %b, i32 %c, i32 %true, i32 %false + define i32 @test_select_i1_basic(i32 %v1, i32 %v2, i32 %v3, i32 %true, i32 %false) { + ; CHECK-LABEL: test_select_i1_basic( + ; CHECK: { +-; CHECK-NEXT: .reg .pred %p<4>; ++; CHECK-NEXT: .reg .pred %p<6>; + ; CHECK-NEXT: .reg .b32 %r<6>; +-; CHECK-NEXT: .reg .b64 %rd<6>; ++; CHECK-NEXT: .reg .b64 %rd<4>; + ; CHECK-EMPTY: + ; CHECK-NEXT: // %bb.0: + ; CHECK-NEXT: ld.param.b32 %r1, [test_select_i1_basic_param_0]; +@@ -81,13 +81,13 @@ define i32 @test_select_i1_basic(i32 %v1, i32 %v2, i32 %v3, i32 %true, i32 %fals + ; CHECK-NEXT: setp.ne.b32 %p1, %r1, 0; + ; CHECK-NEXT: ld.param.b32 %r4, [test_select_i1_basic_param_2]; + ; CHECK-NEXT: setp.eq.b32 %p2, %r4, 0; +-; CHECK-NEXT: setp.eq.b32 %p3, %r3, 0; ++; CHECK-NEXT: and.pred %p3, %p1, %p2; ++; CHECK-NEXT: setp.eq.b32 %p4, %r3, 0; ++; CHECK-NEXT: or.pred %p5, %p4, %p3; + ; CHECK-NEXT: mov.b64 %rd1, test_select_i1_basic_param_4; + ; CHECK-NEXT: mov.b64 %rd2, test_select_i1_basic_param_3; +-; CHECK-NEXT: selp.b64 %rd3, %rd2, %rd1, %p2; +-; CHECK-NEXT: selp.b64 %rd4, %rd3, %rd1, %p1; +-; CHECK-NEXT: selp.b64 %rd5, %rd2, %rd4, %p3; +-; CHECK-NEXT: ld.param.b32 %r5, [%rd5]; ++; CHECK-NEXT: selp.b64 %rd3, %rd2, %rd1, %p5; ++; CHECK-NEXT: ld.param.b32 %r5, [%rd3]; + ; CHECK-NEXT: st.param.b32 [func_retval0], %r5; + ; CHECK-NEXT: ret; + %b1 = icmp eq i32 %v1, 0 +diff --git a/llvm/test/CodeGen/X86/fp-strict-scalar-cmp-fp16.ll b/llvm/test/CodeGen/X86/fp-strict-scalar-cmp-fp16.ll +index 6a6b86e8efa7c..1c06160584f92 100644 +--- a/llvm/test/CodeGen/X86/fp-strict-scalar-cmp-fp16.ll ++++ b/llvm/test/CodeGen/X86/fp-strict-scalar-cmp-fp16.ll +@@ -49,10 +49,13 @@ define i32 @test_f16_oeq_q(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vucomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: setnp %al ++; X86-FP16-NEXT: sete %cl ++; X86-FP16-NEXT: andb %al, %cl ++; X86-FP16-NEXT: testb $1, %cl + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; X86-FP16-NEXT: cmovnel %eax, %ecx +-; X86-FP16-NEXT: cmovpl %eax, %ecx + ; X86-FP16-NEXT: movl (%ecx), %eax + ; X86-FP16-NEXT: retl + ; +@@ -112,9 +115,11 @@ define i32 @test_f16_ogt_q(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vucomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: seta %al ++; X86-FP16-NEXT: testb $1, %al + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X86-FP16-NEXT: cmoval %eax, %ecx ++; X86-FP16-NEXT: cmovnel %eax, %ecx + ; X86-FP16-NEXT: movl (%ecx), %eax + ; X86-FP16-NEXT: retl + ; +@@ -173,9 +178,11 @@ define i32 @test_f16_oge_q(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vucomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: setae %al ++; X86-FP16-NEXT: testb $1, %al + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X86-FP16-NEXT: cmovael %eax, %ecx ++; X86-FP16-NEXT: cmovnel %eax, %ecx + ; X86-FP16-NEXT: movl (%ecx), %eax + ; X86-FP16-NEXT: retl + ; +@@ -236,9 +243,11 @@ define i32 @test_f16_olt_q(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vucomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: seta %al ++; X86-FP16-NEXT: testb $1, %al + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X86-FP16-NEXT: cmoval %eax, %ecx ++; X86-FP16-NEXT: cmovnel %eax, %ecx + ; X86-FP16-NEXT: movl (%ecx), %eax + ; X86-FP16-NEXT: retl + ; +@@ -299,9 +308,11 @@ define i32 @test_f16_ole_q(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vucomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: setae %al ++; X86-FP16-NEXT: testb $1, %al + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X86-FP16-NEXT: cmovael %eax, %ecx ++; X86-FP16-NEXT: cmovnel %eax, %ecx + ; X86-FP16-NEXT: movl (%ecx), %eax + ; X86-FP16-NEXT: retl + ; +@@ -360,6 +371,8 @@ define i32 @test_f16_one_q(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vucomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: setne %al ++; X86-FP16-NEXT: testb $1, %al + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; X86-FP16-NEXT: cmovnel %eax, %ecx +@@ -421,9 +434,11 @@ define i32 @test_f16_ord_q(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vucomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: setnp %al ++; X86-FP16-NEXT: testb $1, %al + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X86-FP16-NEXT: cmovnpl %eax, %ecx ++; X86-FP16-NEXT: cmovnel %eax, %ecx + ; X86-FP16-NEXT: movl (%ecx), %eax + ; X86-FP16-NEXT: retl + ; +@@ -482,9 +497,11 @@ define i32 @test_f16_ueq_q(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vucomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: sete %al ++; X86-FP16-NEXT: testb $1, %al + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X86-FP16-NEXT: cmovel %eax, %ecx ++; X86-FP16-NEXT: cmovnel %eax, %ecx + ; X86-FP16-NEXT: movl (%ecx), %eax + ; X86-FP16-NEXT: retl + ; +@@ -545,9 +562,11 @@ define i32 @test_f16_ugt_q(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vucomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: setb %al ++; X86-FP16-NEXT: testb $1, %al + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X86-FP16-NEXT: cmovbl %eax, %ecx ++; X86-FP16-NEXT: cmovnel %eax, %ecx + ; X86-FP16-NEXT: movl (%ecx), %eax + ; X86-FP16-NEXT: retl + ; +@@ -608,9 +627,11 @@ define i32 @test_f16_uge_q(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vucomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: setbe %al ++; X86-FP16-NEXT: testb $1, %al + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X86-FP16-NEXT: cmovbel %eax, %ecx ++; X86-FP16-NEXT: cmovnel %eax, %ecx + ; X86-FP16-NEXT: movl (%ecx), %eax + ; X86-FP16-NEXT: retl + ; +@@ -669,9 +690,11 @@ define i32 @test_f16_ult_q(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vucomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: setb %al ++; X86-FP16-NEXT: testb $1, %al + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X86-FP16-NEXT: cmovbl %eax, %ecx ++; X86-FP16-NEXT: cmovnel %eax, %ecx + ; X86-FP16-NEXT: movl (%ecx), %eax + ; X86-FP16-NEXT: retl + ; +@@ -730,9 +753,11 @@ define i32 @test_f16_ule_q(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vucomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: setbe %al ++; X86-FP16-NEXT: testb $1, %al + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X86-FP16-NEXT: cmovbel %eax, %ecx ++; X86-FP16-NEXT: cmovnel %eax, %ecx + ; X86-FP16-NEXT: movl (%ecx), %eax + ; X86-FP16-NEXT: retl + ; +@@ -793,10 +818,13 @@ define i32 @test_f16_une_q(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vucomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: setp %al ++; X86-FP16-NEXT: setne %cl ++; X86-FP16-NEXT: orb %al, %cl ++; X86-FP16-NEXT: testb $1, %cl + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; X86-FP16-NEXT: cmovnel %eax, %ecx +-; X86-FP16-NEXT: cmovpl %eax, %ecx + ; X86-FP16-NEXT: movl (%ecx), %eax + ; X86-FP16-NEXT: retl + ; +@@ -856,9 +884,11 @@ define i32 @test_f16_uno_q(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vucomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: setp %al ++; X86-FP16-NEXT: testb $1, %al + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X86-FP16-NEXT: cmovpl %eax, %ecx ++; X86-FP16-NEXT: cmovnel %eax, %ecx + ; X86-FP16-NEXT: movl (%ecx), %eax + ; X86-FP16-NEXT: retl + ; +@@ -919,10 +949,13 @@ define i32 @test_f16_oeq_s(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vcomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: setnp %al ++; X86-FP16-NEXT: sete %cl ++; X86-FP16-NEXT: andb %al, %cl ++; X86-FP16-NEXT: testb $1, %cl + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; X86-FP16-NEXT: cmovnel %eax, %ecx +-; X86-FP16-NEXT: cmovpl %eax, %ecx + ; X86-FP16-NEXT: movl (%ecx), %eax + ; X86-FP16-NEXT: retl + ; +@@ -982,9 +1015,11 @@ define i32 @test_f16_ogt_s(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vcomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: seta %al ++; X86-FP16-NEXT: testb $1, %al + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X86-FP16-NEXT: cmoval %eax, %ecx ++; X86-FP16-NEXT: cmovnel %eax, %ecx + ; X86-FP16-NEXT: movl (%ecx), %eax + ; X86-FP16-NEXT: retl + ; +@@ -1043,9 +1078,11 @@ define i32 @test_f16_oge_s(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vcomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: setae %al ++; X86-FP16-NEXT: testb $1, %al + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X86-FP16-NEXT: cmovael %eax, %ecx ++; X86-FP16-NEXT: cmovnel %eax, %ecx + ; X86-FP16-NEXT: movl (%ecx), %eax + ; X86-FP16-NEXT: retl + ; +@@ -1106,9 +1143,11 @@ define i32 @test_f16_olt_s(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vcomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: seta %al ++; X86-FP16-NEXT: testb $1, %al + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X86-FP16-NEXT: cmoval %eax, %ecx ++; X86-FP16-NEXT: cmovnel %eax, %ecx + ; X86-FP16-NEXT: movl (%ecx), %eax + ; X86-FP16-NEXT: retl + ; +@@ -1169,9 +1208,11 @@ define i32 @test_f16_ole_s(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vcomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: setae %al ++; X86-FP16-NEXT: testb $1, %al + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X86-FP16-NEXT: cmovael %eax, %ecx ++; X86-FP16-NEXT: cmovnel %eax, %ecx + ; X86-FP16-NEXT: movl (%ecx), %eax + ; X86-FP16-NEXT: retl + ; +@@ -1230,6 +1271,8 @@ define i32 @test_f16_one_s(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vcomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: setne %al ++; X86-FP16-NEXT: testb $1, %al + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; X86-FP16-NEXT: cmovnel %eax, %ecx +@@ -1291,9 +1334,11 @@ define i32 @test_f16_ord_s(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vcomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: setnp %al ++; X86-FP16-NEXT: testb $1, %al + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X86-FP16-NEXT: cmovnpl %eax, %ecx ++; X86-FP16-NEXT: cmovnel %eax, %ecx + ; X86-FP16-NEXT: movl (%ecx), %eax + ; X86-FP16-NEXT: retl + ; +@@ -1352,9 +1397,11 @@ define i32 @test_f16_ueq_s(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vcomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: sete %al ++; X86-FP16-NEXT: testb $1, %al + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X86-FP16-NEXT: cmovel %eax, %ecx ++; X86-FP16-NEXT: cmovnel %eax, %ecx + ; X86-FP16-NEXT: movl (%ecx), %eax + ; X86-FP16-NEXT: retl + ; +@@ -1415,9 +1462,11 @@ define i32 @test_f16_ugt_s(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vcomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: setb %al ++; X86-FP16-NEXT: testb $1, %al + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X86-FP16-NEXT: cmovbl %eax, %ecx ++; X86-FP16-NEXT: cmovnel %eax, %ecx + ; X86-FP16-NEXT: movl (%ecx), %eax + ; X86-FP16-NEXT: retl + ; +@@ -1478,9 +1527,11 @@ define i32 @test_f16_uge_s(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vcomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: setbe %al ++; X86-FP16-NEXT: testb $1, %al + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X86-FP16-NEXT: cmovbel %eax, %ecx ++; X86-FP16-NEXT: cmovnel %eax, %ecx + ; X86-FP16-NEXT: movl (%ecx), %eax + ; X86-FP16-NEXT: retl + ; +@@ -1539,9 +1590,11 @@ define i32 @test_f16_ult_s(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vcomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: setb %al ++; X86-FP16-NEXT: testb $1, %al + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X86-FP16-NEXT: cmovbl %eax, %ecx ++; X86-FP16-NEXT: cmovnel %eax, %ecx + ; X86-FP16-NEXT: movl (%ecx), %eax + ; X86-FP16-NEXT: retl + ; +@@ -1600,9 +1653,11 @@ define i32 @test_f16_ule_s(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vcomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: setbe %al ++; X86-FP16-NEXT: testb $1, %al + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X86-FP16-NEXT: cmovbel %eax, %ecx ++; X86-FP16-NEXT: cmovnel %eax, %ecx + ; X86-FP16-NEXT: movl (%ecx), %eax + ; X86-FP16-NEXT: retl + ; +@@ -1663,10 +1718,13 @@ define i32 @test_f16_une_s(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vcomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: setp %al ++; X86-FP16-NEXT: setne %cl ++; X86-FP16-NEXT: orb %al, %cl ++; X86-FP16-NEXT: testb $1, %cl + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; X86-FP16-NEXT: cmovnel %eax, %ecx +-; X86-FP16-NEXT: cmovpl %eax, %ecx + ; X86-FP16-NEXT: movl (%ecx), %eax + ; X86-FP16-NEXT: retl + ; +@@ -1726,9 +1784,11 @@ define i32 @test_f16_uno_s(i32 %a, i32 %b, half %f1, half %f2) #0 { + ; X86-FP16: # %bb.0: + ; X86-FP16-NEXT: vmovsh {{.*#+}} xmm0 = mem[0],zero,zero,zero,zero,zero,zero,zero + ; X86-FP16-NEXT: vcomish {{[0-9]+}}(%esp), %xmm0 ++; X86-FP16-NEXT: setp %al ++; X86-FP16-NEXT: testb $1, %al + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-FP16-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X86-FP16-NEXT: cmovpl %eax, %ecx ++; X86-FP16-NEXT: cmovnel %eax, %ecx + ; X86-FP16-NEXT: movl (%ecx), %eax + ; X86-FP16-NEXT: retl + ; +diff --git a/llvm/test/CodeGen/X86/fp-strict-scalar-cmp.ll b/llvm/test/CodeGen/X86/fp-strict-scalar-cmp.ll +index e3e2b6225a7ba..b9823fd5646cc 100644 +--- a/llvm/test/CodeGen/X86/fp-strict-scalar-cmp.ll ++++ b/llvm/test/CodeGen/X86/fp-strict-scalar-cmp.ll +@@ -13,10 +13,13 @@ define i32 @test_f32_oeq_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: ucomiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setnp %al ++; SSE-32-NEXT: sete %cl ++; SSE-32-NEXT: andb %al, %cl ++; SSE-32-NEXT: testb $1, %cl + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; SSE-32-NEXT: cmovnel %eax, %ecx +-; SSE-32-NEXT: cmovpl %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -32,10 +35,13 @@ define i32 @test_f32_oeq_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vucomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setnp %al ++; AVX-32-NEXT: sete %cl ++; AVX-32-NEXT: andb %al, %cl ++; AVX-32-NEXT: testb $1, %cl + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; AVX-32-NEXT: cmovnel %eax, %ecx +-; AVX-32-NEXT: cmovpl %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -56,13 +62,17 @@ define i32 @test_f32_oeq_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: leal {{[0-9]+}}(%esp), %eax +-; X87-NEXT: jne .LBB0_3 +-; X87-NEXT: # %bb.1: +-; X87-NEXT: jp .LBB0_3 ++; X87-NEXT: setnp %al ++; X87-NEXT: sete %cl ++; X87-NEXT: andb %al, %cl ++; X87-NEXT: testb $1, %cl ++; X87-NEXT: jne .LBB0_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax +-; X87-NEXT: .LBB0_3: ++; X87-NEXT: movl (%eax), %eax ++; X87-NEXT: retl ++; X87-NEXT: .LBB0_1: ++; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax + ; X87-NEXT: retl + ; +@@ -73,10 +83,13 @@ define i32 @test_f32_oeq_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setnp %al ++; X87-CMOV-NEXT: sete %cl ++; X87-CMOV-NEXT: andb %al, %cl ++; X87-CMOV-NEXT: testb $1, %cl + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; X87-CMOV-NEXT: cmovnel %eax, %ecx +-; X87-CMOV-NEXT: cmovpl %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmp.f32( +@@ -91,9 +104,11 @@ define i32 @test_f32_ogt_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: ucomiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: seta %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmoval %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -108,9 +123,11 @@ define i32 @test_f32_ogt_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vucomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: seta %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmoval %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -130,7 +147,9 @@ define i32 @test_f32_ogt_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: ja .LBB1_1 ++; X87-NEXT: seta %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB1_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -147,9 +166,11 @@ define i32 @test_f32_ogt_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: seta %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmoval %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmp.f32( +@@ -164,9 +185,11 @@ define i32 @test_f32_oge_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: ucomiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setae %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovael %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -181,9 +204,11 @@ define i32 @test_f32_oge_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vucomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setae %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovael %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -203,7 +228,9 @@ define i32 @test_f32_oge_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jae .LBB2_1 ++; X87-NEXT: setae %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB2_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -220,9 +247,11 @@ define i32 @test_f32_oge_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setae %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovael %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmp.f32( +@@ -237,9 +266,11 @@ define i32 @test_f32_olt_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: ucomiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: seta %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmoval %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -254,9 +285,11 @@ define i32 @test_f32_olt_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vucomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: seta %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmoval %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -276,7 +309,9 @@ define i32 @test_f32_olt_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: ja .LBB3_1 ++; X87-NEXT: seta %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB3_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -293,9 +328,11 @@ define i32 @test_f32_olt_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: seta %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmoval %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmp.f32( +@@ -310,9 +347,11 @@ define i32 @test_f32_ole_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: ucomiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setae %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovael %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -327,9 +366,11 @@ define i32 @test_f32_ole_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vucomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setae %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovael %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -349,7 +390,9 @@ define i32 @test_f32_ole_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jae .LBB4_1 ++; X87-NEXT: setae %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB4_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -366,9 +409,11 @@ define i32 @test_f32_ole_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setae %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovael %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmp.f32( +@@ -383,6 +428,8 @@ define i32 @test_f32_one_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: ucomiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setne %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; SSE-32-NEXT: cmovnel %eax, %ecx +@@ -400,6 +447,8 @@ define i32 @test_f32_one_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vucomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setne %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; AVX-32-NEXT: cmovnel %eax, %ecx +@@ -422,6 +471,8 @@ define i32 @test_f32_one_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf ++; X87-NEXT: setne %al ++; X87-NEXT: testb $1, %al + ; X87-NEXT: jne .LBB5_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax +@@ -439,6 +490,8 @@ define i32 @test_f32_one_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setne %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; X87-CMOV-NEXT: cmovnel %eax, %ecx +@@ -456,9 +509,11 @@ define i32 @test_f32_ord_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: ucomiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setnp %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovnpl %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -473,9 +528,11 @@ define i32 @test_f32_ord_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vucomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setnp %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovnpl %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -495,7 +552,9 @@ define i32 @test_f32_ord_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jnp .LBB6_1 ++; X87-NEXT: setnp %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB6_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -512,9 +571,11 @@ define i32 @test_f32_ord_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setnp %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovnpl %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmp.f32( +@@ -529,9 +590,11 @@ define i32 @test_f32_ueq_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: ucomiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: sete %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovel %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -546,9 +609,11 @@ define i32 @test_f32_ueq_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vucomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: sete %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovel %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -568,7 +633,9 @@ define i32 @test_f32_ueq_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: je .LBB7_1 ++; X87-NEXT: sete %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB7_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -585,9 +652,11 @@ define i32 @test_f32_ueq_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: sete %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovel %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmp.f32( +@@ -602,9 +671,11 @@ define i32 @test_f32_ugt_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: ucomiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setb %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovbl %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -619,9 +690,11 @@ define i32 @test_f32_ugt_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vucomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setb %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovbl %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -641,7 +714,9 @@ define i32 @test_f32_ugt_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jb .LBB8_1 ++; X87-NEXT: setb %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB8_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -658,9 +733,11 @@ define i32 @test_f32_ugt_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setb %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovbl %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmp.f32( +@@ -675,9 +752,11 @@ define i32 @test_f32_uge_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: ucomiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setbe %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovbel %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -692,9 +771,11 @@ define i32 @test_f32_uge_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vucomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setbe %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovbel %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -714,7 +795,9 @@ define i32 @test_f32_uge_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jbe .LBB9_1 ++; X87-NEXT: setbe %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB9_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -731,9 +814,11 @@ define i32 @test_f32_uge_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setbe %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovbel %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmp.f32( +@@ -748,9 +833,11 @@ define i32 @test_f32_ult_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: ucomiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setb %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovbl %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -765,9 +852,11 @@ define i32 @test_f32_ult_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vucomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setb %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovbl %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -787,7 +876,9 @@ define i32 @test_f32_ult_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jb .LBB10_1 ++; X87-NEXT: setb %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB10_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -804,9 +895,11 @@ define i32 @test_f32_ult_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setb %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovbl %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmp.f32( +@@ -821,9 +914,11 @@ define i32 @test_f32_ule_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: ucomiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setbe %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovbel %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -838,9 +933,11 @@ define i32 @test_f32_ule_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vucomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setbe %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovbel %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -860,7 +957,9 @@ define i32 @test_f32_ule_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jbe .LBB11_1 ++; X87-NEXT: setbe %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB11_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -877,9 +976,11 @@ define i32 @test_f32_ule_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setbe %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovbel %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmp.f32( +@@ -894,10 +995,13 @@ define i32 @test_f32_une_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: ucomiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setp %al ++; SSE-32-NEXT: setne %cl ++; SSE-32-NEXT: orb %al, %cl ++; SSE-32-NEXT: testb $1, %cl + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; SSE-32-NEXT: cmovnel %eax, %ecx +-; SSE-32-NEXT: cmovpl %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -913,10 +1017,13 @@ define i32 @test_f32_une_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vucomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setp %al ++; AVX-32-NEXT: setne %cl ++; AVX-32-NEXT: orb %al, %cl ++; AVX-32-NEXT: testb $1, %cl + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; AVX-32-NEXT: cmovnel %eax, %ecx +-; AVX-32-NEXT: cmovpl %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -937,13 +1044,17 @@ define i32 @test_f32_une_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: leal {{[0-9]+}}(%esp), %eax +-; X87-NEXT: jne .LBB12_3 +-; X87-NEXT: # %bb.1: +-; X87-NEXT: jp .LBB12_3 ++; X87-NEXT: setp %al ++; X87-NEXT: setne %cl ++; X87-NEXT: orb %al, %cl ++; X87-NEXT: testb $1, %cl ++; X87-NEXT: jne .LBB12_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax +-; X87-NEXT: .LBB12_3: ++; X87-NEXT: movl (%eax), %eax ++; X87-NEXT: retl ++; X87-NEXT: .LBB12_1: ++; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax + ; X87-NEXT: retl + ; +@@ -954,10 +1065,13 @@ define i32 @test_f32_une_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setp %al ++; X87-CMOV-NEXT: setne %cl ++; X87-CMOV-NEXT: orb %al, %cl ++; X87-CMOV-NEXT: testb $1, %cl + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; X87-CMOV-NEXT: cmovnel %eax, %ecx +-; X87-CMOV-NEXT: cmovpl %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmp.f32( +@@ -972,9 +1086,11 @@ define i32 @test_f32_uno_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: ucomiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setp %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovpl %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -989,9 +1105,11 @@ define i32 @test_f32_uno_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vucomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setp %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovpl %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -1011,7 +1129,9 @@ define i32 @test_f32_uno_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jp .LBB13_1 ++; X87-NEXT: setp %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB13_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -1028,9 +1148,11 @@ define i32 @test_f32_uno_q(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setp %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovpl %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmp.f32( +@@ -1045,10 +1167,13 @@ define i32 @test_f64_oeq_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: ucomisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setnp %al ++; SSE-32-NEXT: sete %cl ++; SSE-32-NEXT: andb %al, %cl ++; SSE-32-NEXT: testb $1, %cl + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; SSE-32-NEXT: cmovnel %eax, %ecx +-; SSE-32-NEXT: cmovpl %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -1064,10 +1189,13 @@ define i32 @test_f64_oeq_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vucomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setnp %al ++; AVX-32-NEXT: sete %cl ++; AVX-32-NEXT: andb %al, %cl ++; AVX-32-NEXT: testb $1, %cl + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; AVX-32-NEXT: cmovnel %eax, %ecx +-; AVX-32-NEXT: cmovpl %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -1088,13 +1216,17 @@ define i32 @test_f64_oeq_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: leal {{[0-9]+}}(%esp), %eax +-; X87-NEXT: jne .LBB14_3 +-; X87-NEXT: # %bb.1: +-; X87-NEXT: jp .LBB14_3 ++; X87-NEXT: setnp %al ++; X87-NEXT: sete %cl ++; X87-NEXT: andb %al, %cl ++; X87-NEXT: testb $1, %cl ++; X87-NEXT: jne .LBB14_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax +-; X87-NEXT: .LBB14_3: ++; X87-NEXT: movl (%eax), %eax ++; X87-NEXT: retl ++; X87-NEXT: .LBB14_1: ++; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax + ; X87-NEXT: retl + ; +@@ -1105,10 +1237,13 @@ define i32 @test_f64_oeq_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setnp %al ++; X87-CMOV-NEXT: sete %cl ++; X87-CMOV-NEXT: andb %al, %cl ++; X87-CMOV-NEXT: testb $1, %cl + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; X87-CMOV-NEXT: cmovnel %eax, %ecx +-; X87-CMOV-NEXT: cmovpl %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmp.f64( +@@ -1123,9 +1258,11 @@ define i32 @test_f64_ogt_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: ucomisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: seta %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmoval %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -1140,9 +1277,11 @@ define i32 @test_f64_ogt_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vucomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: seta %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmoval %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -1162,7 +1301,9 @@ define i32 @test_f64_ogt_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: ja .LBB15_1 ++; X87-NEXT: seta %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB15_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -1179,9 +1320,11 @@ define i32 @test_f64_ogt_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: seta %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmoval %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmp.f64( +@@ -1196,9 +1339,11 @@ define i32 @test_f64_oge_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: ucomisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setae %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovael %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -1213,9 +1358,11 @@ define i32 @test_f64_oge_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vucomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setae %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovael %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -1235,7 +1382,9 @@ define i32 @test_f64_oge_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jae .LBB16_1 ++; X87-NEXT: setae %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB16_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -1252,9 +1401,11 @@ define i32 @test_f64_oge_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setae %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovael %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmp.f64( +@@ -1269,9 +1420,11 @@ define i32 @test_f64_olt_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: ucomisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: seta %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmoval %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -1286,9 +1439,11 @@ define i32 @test_f64_olt_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vucomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: seta %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmoval %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -1308,7 +1463,9 @@ define i32 @test_f64_olt_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: ja .LBB17_1 ++; X87-NEXT: seta %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB17_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -1325,9 +1482,11 @@ define i32 @test_f64_olt_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: seta %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmoval %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmp.f64( +@@ -1342,9 +1501,11 @@ define i32 @test_f64_ole_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: ucomisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setae %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovael %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -1359,9 +1520,11 @@ define i32 @test_f64_ole_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vucomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setae %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovael %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -1381,7 +1544,9 @@ define i32 @test_f64_ole_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jae .LBB18_1 ++; X87-NEXT: setae %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB18_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -1398,9 +1563,11 @@ define i32 @test_f64_ole_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setae %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovael %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmp.f64( +@@ -1415,6 +1582,8 @@ define i32 @test_f64_one_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: ucomisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setne %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; SSE-32-NEXT: cmovnel %eax, %ecx +@@ -1432,6 +1601,8 @@ define i32 @test_f64_one_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vucomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setne %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; AVX-32-NEXT: cmovnel %eax, %ecx +@@ -1454,6 +1625,8 @@ define i32 @test_f64_one_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf ++; X87-NEXT: setne %al ++; X87-NEXT: testb $1, %al + ; X87-NEXT: jne .LBB19_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax +@@ -1471,6 +1644,8 @@ define i32 @test_f64_one_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setne %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; X87-CMOV-NEXT: cmovnel %eax, %ecx +@@ -1488,9 +1663,11 @@ define i32 @test_f64_ord_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: ucomisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setnp %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovnpl %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -1505,9 +1682,11 @@ define i32 @test_f64_ord_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vucomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setnp %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovnpl %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -1527,7 +1706,9 @@ define i32 @test_f64_ord_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jnp .LBB20_1 ++; X87-NEXT: setnp %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB20_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -1544,9 +1725,11 @@ define i32 @test_f64_ord_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setnp %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovnpl %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmp.f64( +@@ -1561,9 +1744,11 @@ define i32 @test_f64_ueq_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: ucomisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: sete %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovel %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -1578,9 +1763,11 @@ define i32 @test_f64_ueq_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vucomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: sete %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovel %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -1600,7 +1787,9 @@ define i32 @test_f64_ueq_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: je .LBB21_1 ++; X87-NEXT: sete %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB21_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -1617,9 +1806,11 @@ define i32 @test_f64_ueq_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: sete %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovel %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmp.f64( +@@ -1634,9 +1825,11 @@ define i32 @test_f64_ugt_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: ucomisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setb %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovbl %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -1651,9 +1844,11 @@ define i32 @test_f64_ugt_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vucomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setb %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovbl %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -1673,7 +1868,9 @@ define i32 @test_f64_ugt_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jb .LBB22_1 ++; X87-NEXT: setb %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB22_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -1690,9 +1887,11 @@ define i32 @test_f64_ugt_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setb %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovbl %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmp.f64( +@@ -1707,9 +1906,11 @@ define i32 @test_f64_uge_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: ucomisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setbe %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovbel %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -1724,9 +1925,11 @@ define i32 @test_f64_uge_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vucomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setbe %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovbel %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -1746,7 +1949,9 @@ define i32 @test_f64_uge_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jbe .LBB23_1 ++; X87-NEXT: setbe %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB23_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -1763,9 +1968,11 @@ define i32 @test_f64_uge_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setbe %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovbel %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmp.f64( +@@ -1780,9 +1987,11 @@ define i32 @test_f64_ult_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: ucomisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setb %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovbl %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -1797,9 +2006,11 @@ define i32 @test_f64_ult_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vucomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setb %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovbl %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -1819,7 +2030,9 @@ define i32 @test_f64_ult_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jb .LBB24_1 ++; X87-NEXT: setb %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB24_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -1836,9 +2049,11 @@ define i32 @test_f64_ult_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setb %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovbl %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmp.f64( +@@ -1853,9 +2068,11 @@ define i32 @test_f64_ule_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: ucomisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setbe %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovbel %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -1870,9 +2087,11 @@ define i32 @test_f64_ule_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vucomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setbe %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovbel %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -1892,7 +2111,9 @@ define i32 @test_f64_ule_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jbe .LBB25_1 ++; X87-NEXT: setbe %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB25_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -1909,9 +2130,11 @@ define i32 @test_f64_ule_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setbe %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovbel %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmp.f64( +@@ -1926,10 +2149,13 @@ define i32 @test_f64_une_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: ucomisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setp %al ++; SSE-32-NEXT: setne %cl ++; SSE-32-NEXT: orb %al, %cl ++; SSE-32-NEXT: testb $1, %cl + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; SSE-32-NEXT: cmovnel %eax, %ecx +-; SSE-32-NEXT: cmovpl %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -1945,10 +2171,13 @@ define i32 @test_f64_une_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vucomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setp %al ++; AVX-32-NEXT: setne %cl ++; AVX-32-NEXT: orb %al, %cl ++; AVX-32-NEXT: testb $1, %cl + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; AVX-32-NEXT: cmovnel %eax, %ecx +-; AVX-32-NEXT: cmovpl %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -1969,13 +2198,17 @@ define i32 @test_f64_une_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: leal {{[0-9]+}}(%esp), %eax +-; X87-NEXT: jne .LBB26_3 +-; X87-NEXT: # %bb.1: +-; X87-NEXT: jp .LBB26_3 ++; X87-NEXT: setp %al ++; X87-NEXT: setne %cl ++; X87-NEXT: orb %al, %cl ++; X87-NEXT: testb $1, %cl ++; X87-NEXT: jne .LBB26_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax +-; X87-NEXT: .LBB26_3: ++; X87-NEXT: movl (%eax), %eax ++; X87-NEXT: retl ++; X87-NEXT: .LBB26_1: ++; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax + ; X87-NEXT: retl + ; +@@ -1986,10 +2219,13 @@ define i32 @test_f64_une_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setp %al ++; X87-CMOV-NEXT: setne %cl ++; X87-CMOV-NEXT: orb %al, %cl ++; X87-CMOV-NEXT: testb $1, %cl + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; X87-CMOV-NEXT: cmovnel %eax, %ecx +-; X87-CMOV-NEXT: cmovpl %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmp.f64( +@@ -2004,9 +2240,11 @@ define i32 @test_f64_uno_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: ucomisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setp %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovpl %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -2021,9 +2259,11 @@ define i32 @test_f64_uno_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vucomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setp %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovpl %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -2043,7 +2283,9 @@ define i32 @test_f64_uno_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jp .LBB27_1 ++; X87-NEXT: setp %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB27_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -2060,9 +2302,11 @@ define i32 @test_f64_uno_q(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fucompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setp %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovpl %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmp.f64( +@@ -2077,10 +2321,13 @@ define i32 @test_f32_oeq_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: comiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setnp %al ++; SSE-32-NEXT: sete %cl ++; SSE-32-NEXT: andb %al, %cl ++; SSE-32-NEXT: testb $1, %cl + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; SSE-32-NEXT: cmovnel %eax, %ecx +-; SSE-32-NEXT: cmovpl %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -2096,10 +2343,13 @@ define i32 @test_f32_oeq_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vcomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setnp %al ++; AVX-32-NEXT: sete %cl ++; AVX-32-NEXT: andb %al, %cl ++; AVX-32-NEXT: testb $1, %cl + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; AVX-32-NEXT: cmovnel %eax, %ecx +-; AVX-32-NEXT: cmovpl %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -2120,13 +2370,17 @@ define i32 @test_f32_oeq_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: leal {{[0-9]+}}(%esp), %eax +-; X87-NEXT: jne .LBB28_3 +-; X87-NEXT: # %bb.1: +-; X87-NEXT: jp .LBB28_3 ++; X87-NEXT: setnp %al ++; X87-NEXT: sete %cl ++; X87-NEXT: andb %al, %cl ++; X87-NEXT: testb $1, %cl ++; X87-NEXT: jne .LBB28_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax +-; X87-NEXT: .LBB28_3: ++; X87-NEXT: movl (%eax), %eax ++; X87-NEXT: retl ++; X87-NEXT: .LBB28_1: ++; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax + ; X87-NEXT: retl + ; +@@ -2137,10 +2391,13 @@ define i32 @test_f32_oeq_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setnp %al ++; X87-CMOV-NEXT: sete %cl ++; X87-CMOV-NEXT: andb %al, %cl ++; X87-CMOV-NEXT: testb $1, %cl + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; X87-CMOV-NEXT: cmovnel %eax, %ecx +-; X87-CMOV-NEXT: cmovpl %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmps.f32( +@@ -2155,9 +2412,11 @@ define i32 @test_f32_ogt_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: comiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: seta %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmoval %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -2172,9 +2431,11 @@ define i32 @test_f32_ogt_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vcomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: seta %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmoval %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -2194,7 +2455,9 @@ define i32 @test_f32_ogt_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: ja .LBB29_1 ++; X87-NEXT: seta %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB29_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -2211,9 +2474,11 @@ define i32 @test_f32_ogt_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: seta %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmoval %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmps.f32( +@@ -2228,9 +2493,11 @@ define i32 @test_f32_oge_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: comiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setae %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovael %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -2245,9 +2512,11 @@ define i32 @test_f32_oge_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vcomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setae %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovael %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -2267,7 +2536,9 @@ define i32 @test_f32_oge_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jae .LBB30_1 ++; X87-NEXT: setae %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB30_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -2284,9 +2555,11 @@ define i32 @test_f32_oge_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setae %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovael %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmps.f32( +@@ -2301,9 +2574,11 @@ define i32 @test_f32_olt_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: comiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: seta %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmoval %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -2318,9 +2593,11 @@ define i32 @test_f32_olt_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vcomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: seta %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmoval %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -2340,7 +2617,9 @@ define i32 @test_f32_olt_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: ja .LBB31_1 ++; X87-NEXT: seta %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB31_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -2357,9 +2636,11 @@ define i32 @test_f32_olt_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: seta %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmoval %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmps.f32( +@@ -2374,9 +2655,11 @@ define i32 @test_f32_ole_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: comiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setae %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovael %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -2391,9 +2674,11 @@ define i32 @test_f32_ole_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vcomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setae %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovael %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -2413,7 +2698,9 @@ define i32 @test_f32_ole_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jae .LBB32_1 ++; X87-NEXT: setae %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB32_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -2430,9 +2717,11 @@ define i32 @test_f32_ole_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setae %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovael %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmps.f32( +@@ -2447,6 +2736,8 @@ define i32 @test_f32_one_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: comiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setne %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; SSE-32-NEXT: cmovnel %eax, %ecx +@@ -2464,6 +2755,8 @@ define i32 @test_f32_one_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vcomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setne %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; AVX-32-NEXT: cmovnel %eax, %ecx +@@ -2486,6 +2779,8 @@ define i32 @test_f32_one_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf ++; X87-NEXT: setne %al ++; X87-NEXT: testb $1, %al + ; X87-NEXT: jne .LBB33_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax +@@ -2503,6 +2798,8 @@ define i32 @test_f32_one_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setne %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; X87-CMOV-NEXT: cmovnel %eax, %ecx +@@ -2520,9 +2817,11 @@ define i32 @test_f32_ord_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: comiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setnp %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovnpl %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -2537,9 +2836,11 @@ define i32 @test_f32_ord_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vcomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setnp %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovnpl %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -2559,7 +2860,9 @@ define i32 @test_f32_ord_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jnp .LBB34_1 ++; X87-NEXT: setnp %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB34_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -2576,9 +2879,11 @@ define i32 @test_f32_ord_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setnp %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovnpl %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmps.f32( +@@ -2593,9 +2898,11 @@ define i32 @test_f32_ueq_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: comiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: sete %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovel %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -2610,9 +2917,11 @@ define i32 @test_f32_ueq_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vcomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: sete %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovel %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -2632,7 +2941,9 @@ define i32 @test_f32_ueq_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: je .LBB35_1 ++; X87-NEXT: sete %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB35_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -2649,9 +2960,11 @@ define i32 @test_f32_ueq_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: sete %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovel %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmps.f32( +@@ -2666,9 +2979,11 @@ define i32 @test_f32_ugt_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: comiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setb %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovbl %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -2683,9 +2998,11 @@ define i32 @test_f32_ugt_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vcomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setb %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovbl %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -2705,7 +3022,9 @@ define i32 @test_f32_ugt_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jb .LBB36_1 ++; X87-NEXT: setb %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB36_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -2722,9 +3041,11 @@ define i32 @test_f32_ugt_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setb %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovbl %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmps.f32( +@@ -2739,9 +3060,11 @@ define i32 @test_f32_uge_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: comiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setbe %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovbel %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -2756,9 +3079,11 @@ define i32 @test_f32_uge_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vcomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setbe %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovbel %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -2778,7 +3103,9 @@ define i32 @test_f32_uge_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jbe .LBB37_1 ++; X87-NEXT: setbe %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB37_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -2795,9 +3122,11 @@ define i32 @test_f32_uge_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setbe %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovbel %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmps.f32( +@@ -2812,9 +3141,11 @@ define i32 @test_f32_ult_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: comiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setb %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovbl %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -2829,9 +3160,11 @@ define i32 @test_f32_ult_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vcomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setb %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovbl %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -2851,7 +3184,9 @@ define i32 @test_f32_ult_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jb .LBB38_1 ++; X87-NEXT: setb %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB38_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -2868,9 +3203,11 @@ define i32 @test_f32_ult_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setb %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovbl %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmps.f32( +@@ -2885,9 +3222,11 @@ define i32 @test_f32_ule_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: comiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setbe %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovbel %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -2902,9 +3241,11 @@ define i32 @test_f32_ule_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vcomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setbe %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovbel %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -2924,7 +3265,9 @@ define i32 @test_f32_ule_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jbe .LBB39_1 ++; X87-NEXT: setbe %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB39_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -2941,9 +3284,11 @@ define i32 @test_f32_ule_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setbe %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovbel %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmps.f32( +@@ -2958,10 +3303,13 @@ define i32 @test_f32_une_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: comiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setp %al ++; SSE-32-NEXT: setne %cl ++; SSE-32-NEXT: orb %al, %cl ++; SSE-32-NEXT: testb $1, %cl + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; SSE-32-NEXT: cmovnel %eax, %ecx +-; SSE-32-NEXT: cmovpl %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -2977,10 +3325,13 @@ define i32 @test_f32_une_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vcomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setp %al ++; AVX-32-NEXT: setne %cl ++; AVX-32-NEXT: orb %al, %cl ++; AVX-32-NEXT: testb $1, %cl + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; AVX-32-NEXT: cmovnel %eax, %ecx +-; AVX-32-NEXT: cmovpl %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -3001,13 +3352,17 @@ define i32 @test_f32_une_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: leal {{[0-9]+}}(%esp), %eax +-; X87-NEXT: jne .LBB40_3 +-; X87-NEXT: # %bb.1: +-; X87-NEXT: jp .LBB40_3 ++; X87-NEXT: setp %al ++; X87-NEXT: setne %cl ++; X87-NEXT: orb %al, %cl ++; X87-NEXT: testb $1, %cl ++; X87-NEXT: jne .LBB40_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax +-; X87-NEXT: .LBB40_3: ++; X87-NEXT: movl (%eax), %eax ++; X87-NEXT: retl ++; X87-NEXT: .LBB40_1: ++; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax + ; X87-NEXT: retl + ; +@@ -3018,10 +3373,13 @@ define i32 @test_f32_une_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setp %al ++; X87-CMOV-NEXT: setne %cl ++; X87-CMOV-NEXT: orb %al, %cl ++; X87-CMOV-NEXT: testb $1, %cl + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; X87-CMOV-NEXT: cmovnel %eax, %ecx +-; X87-CMOV-NEXT: cmovpl %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmps.f32( +@@ -3036,9 +3394,11 @@ define i32 @test_f32_uno_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; SSE-32-NEXT: comiss {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setp %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovpl %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -3053,9 +3413,11 @@ define i32 @test_f32_uno_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero + ; AVX-32-NEXT: vcomiss {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setp %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovpl %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -3075,7 +3437,9 @@ define i32 @test_f32_uno_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jp .LBB41_1 ++; X87-NEXT: setp %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB41_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -3092,9 +3456,11 @@ define i32 @test_f32_uno_s(i32 %a, i32 %b, float %f1, float %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setp %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovpl %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmps.f32( +@@ -3109,10 +3475,13 @@ define i32 @test_f64_oeq_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: comisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setnp %al ++; SSE-32-NEXT: sete %cl ++; SSE-32-NEXT: andb %al, %cl ++; SSE-32-NEXT: testb $1, %cl + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; SSE-32-NEXT: cmovnel %eax, %ecx +-; SSE-32-NEXT: cmovpl %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -3128,10 +3497,13 @@ define i32 @test_f64_oeq_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vcomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setnp %al ++; AVX-32-NEXT: sete %cl ++; AVX-32-NEXT: andb %al, %cl ++; AVX-32-NEXT: testb $1, %cl + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; AVX-32-NEXT: cmovnel %eax, %ecx +-; AVX-32-NEXT: cmovpl %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -3152,13 +3524,17 @@ define i32 @test_f64_oeq_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: leal {{[0-9]+}}(%esp), %eax +-; X87-NEXT: jne .LBB42_3 +-; X87-NEXT: # %bb.1: +-; X87-NEXT: jp .LBB42_3 ++; X87-NEXT: setnp %al ++; X87-NEXT: sete %cl ++; X87-NEXT: andb %al, %cl ++; X87-NEXT: testb $1, %cl ++; X87-NEXT: jne .LBB42_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax +-; X87-NEXT: .LBB42_3: ++; X87-NEXT: movl (%eax), %eax ++; X87-NEXT: retl ++; X87-NEXT: .LBB42_1: ++; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax + ; X87-NEXT: retl + ; +@@ -3169,10 +3545,13 @@ define i32 @test_f64_oeq_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setnp %al ++; X87-CMOV-NEXT: sete %cl ++; X87-CMOV-NEXT: andb %al, %cl ++; X87-CMOV-NEXT: testb $1, %cl + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; X87-CMOV-NEXT: cmovnel %eax, %ecx +-; X87-CMOV-NEXT: cmovpl %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmps.f64( +@@ -3187,9 +3566,11 @@ define i32 @test_f64_ogt_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: comisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: seta %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmoval %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -3204,9 +3585,11 @@ define i32 @test_f64_ogt_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vcomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: seta %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmoval %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -3226,7 +3609,9 @@ define i32 @test_f64_ogt_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: ja .LBB43_1 ++; X87-NEXT: seta %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB43_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -3243,9 +3628,11 @@ define i32 @test_f64_ogt_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: seta %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmoval %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmps.f64( +@@ -3260,9 +3647,11 @@ define i32 @test_f64_oge_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: comisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setae %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovael %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -3277,9 +3666,11 @@ define i32 @test_f64_oge_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vcomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setae %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovael %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -3299,7 +3690,9 @@ define i32 @test_f64_oge_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jae .LBB44_1 ++; X87-NEXT: setae %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB44_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -3316,9 +3709,11 @@ define i32 @test_f64_oge_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setae %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovael %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmps.f64( +@@ -3333,9 +3728,11 @@ define i32 @test_f64_olt_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: comisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: seta %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmoval %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -3350,9 +3747,11 @@ define i32 @test_f64_olt_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vcomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: seta %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmoval %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -3372,7 +3771,9 @@ define i32 @test_f64_olt_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: ja .LBB45_1 ++; X87-NEXT: seta %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB45_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -3389,9 +3790,11 @@ define i32 @test_f64_olt_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: seta %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmoval %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmps.f64( +@@ -3406,9 +3809,11 @@ define i32 @test_f64_ole_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: comisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setae %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovael %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -3423,9 +3828,11 @@ define i32 @test_f64_ole_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vcomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setae %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovael %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -3445,7 +3852,9 @@ define i32 @test_f64_ole_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jae .LBB46_1 ++; X87-NEXT: setae %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB46_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -3462,9 +3871,11 @@ define i32 @test_f64_ole_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setae %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovael %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmps.f64( +@@ -3479,6 +3890,8 @@ define i32 @test_f64_one_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: comisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setne %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; SSE-32-NEXT: cmovnel %eax, %ecx +@@ -3496,6 +3909,8 @@ define i32 @test_f64_one_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vcomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setne %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; AVX-32-NEXT: cmovnel %eax, %ecx +@@ -3518,6 +3933,8 @@ define i32 @test_f64_one_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf ++; X87-NEXT: setne %al ++; X87-NEXT: testb $1, %al + ; X87-NEXT: jne .LBB47_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax +@@ -3535,6 +3952,8 @@ define i32 @test_f64_one_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setne %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; X87-CMOV-NEXT: cmovnel %eax, %ecx +@@ -3552,9 +3971,11 @@ define i32 @test_f64_ord_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: comisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setnp %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovnpl %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -3569,9 +3990,11 @@ define i32 @test_f64_ord_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vcomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setnp %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovnpl %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -3591,7 +4014,9 @@ define i32 @test_f64_ord_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jnp .LBB48_1 ++; X87-NEXT: setnp %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB48_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -3608,9 +4033,11 @@ define i32 @test_f64_ord_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setnp %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovnpl %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmps.f64( +@@ -3625,9 +4052,11 @@ define i32 @test_f64_ueq_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: comisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: sete %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovel %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -3642,9 +4071,11 @@ define i32 @test_f64_ueq_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vcomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: sete %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovel %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -3664,7 +4095,9 @@ define i32 @test_f64_ueq_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: je .LBB49_1 ++; X87-NEXT: sete %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB49_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -3681,9 +4114,11 @@ define i32 @test_f64_ueq_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: sete %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovel %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmps.f64( +@@ -3698,9 +4133,11 @@ define i32 @test_f64_ugt_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: comisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setb %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovbl %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -3715,9 +4152,11 @@ define i32 @test_f64_ugt_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vcomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setb %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovbl %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -3737,7 +4176,9 @@ define i32 @test_f64_ugt_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jb .LBB50_1 ++; X87-NEXT: setb %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB50_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -3754,9 +4195,11 @@ define i32 @test_f64_ugt_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setb %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovbl %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmps.f64( +@@ -3771,9 +4214,11 @@ define i32 @test_f64_uge_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: comisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setbe %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovbel %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -3788,9 +4233,11 @@ define i32 @test_f64_uge_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vcomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setbe %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovbel %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -3810,7 +4257,9 @@ define i32 @test_f64_uge_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jbe .LBB51_1 ++; X87-NEXT: setbe %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB51_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -3827,9 +4276,11 @@ define i32 @test_f64_uge_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setbe %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovbel %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmps.f64( +@@ -3844,9 +4295,11 @@ define i32 @test_f64_ult_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: comisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setb %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovbl %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -3861,9 +4314,11 @@ define i32 @test_f64_ult_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vcomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setb %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovbl %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -3883,7 +4338,9 @@ define i32 @test_f64_ult_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jb .LBB52_1 ++; X87-NEXT: setb %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB52_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -3900,9 +4357,11 @@ define i32 @test_f64_ult_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setb %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovbl %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmps.f64( +@@ -3917,9 +4376,11 @@ define i32 @test_f64_ule_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: comisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setbe %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovbel %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -3934,9 +4395,11 @@ define i32 @test_f64_ule_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vcomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setbe %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovbel %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -3956,7 +4419,9 @@ define i32 @test_f64_ule_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jbe .LBB53_1 ++; X87-NEXT: setbe %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB53_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -3973,9 +4438,11 @@ define i32 @test_f64_ule_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setbe %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovbel %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmps.f64( +@@ -3990,10 +4457,13 @@ define i32 @test_f64_une_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: comisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setp %al ++; SSE-32-NEXT: setne %cl ++; SSE-32-NEXT: orb %al, %cl ++; SSE-32-NEXT: testb $1, %cl + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; SSE-32-NEXT: cmovnel %eax, %ecx +-; SSE-32-NEXT: cmovpl %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -4009,10 +4479,13 @@ define i32 @test_f64_une_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vcomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setp %al ++; AVX-32-NEXT: setne %cl ++; AVX-32-NEXT: orb %al, %cl ++; AVX-32-NEXT: testb $1, %cl + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; AVX-32-NEXT: cmovnel %eax, %ecx +-; AVX-32-NEXT: cmovpl %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -4033,13 +4506,17 @@ define i32 @test_f64_une_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: leal {{[0-9]+}}(%esp), %eax +-; X87-NEXT: jne .LBB54_3 +-; X87-NEXT: # %bb.1: +-; X87-NEXT: jp .LBB54_3 ++; X87-NEXT: setp %al ++; X87-NEXT: setne %cl ++; X87-NEXT: orb %al, %cl ++; X87-NEXT: testb $1, %cl ++; X87-NEXT: jne .LBB54_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax +-; X87-NEXT: .LBB54_3: ++; X87-NEXT: movl (%eax), %eax ++; X87-NEXT: retl ++; X87-NEXT: .LBB54_1: ++; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax + ; X87-NEXT: retl + ; +@@ -4050,10 +4527,13 @@ define i32 @test_f64_une_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setp %al ++; X87-CMOV-NEXT: setne %cl ++; X87-CMOV-NEXT: orb %al, %cl ++; X87-CMOV-NEXT: testb $1, %cl + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; X87-CMOV-NEXT: cmovnel %eax, %ecx +-; X87-CMOV-NEXT: cmovpl %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmps.f64( +@@ -4068,9 +4548,11 @@ define i32 @test_f64_uno_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; SSE-32: # %bb.0: + ; SSE-32-NEXT: movsd {{.*#+}} xmm0 = mem[0],zero + ; SSE-32-NEXT: comisd {{[0-9]+}}(%esp), %xmm0 ++; SSE-32-NEXT: setp %al ++; SSE-32-NEXT: testb $1, %al + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SSE-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; SSE-32-NEXT: cmovpl %eax, %ecx ++; SSE-32-NEXT: cmovnel %eax, %ecx + ; SSE-32-NEXT: movl (%ecx), %eax + ; SSE-32-NEXT: retl + ; +@@ -4085,9 +4567,11 @@ define i32 @test_f64_uno_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; AVX-32: # %bb.0: + ; AVX-32-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero + ; AVX-32-NEXT: vcomisd {{[0-9]+}}(%esp), %xmm0 ++; AVX-32-NEXT: setp %al ++; AVX-32-NEXT: testb $1, %al + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %eax + ; AVX-32-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; AVX-32-NEXT: cmovpl %eax, %ecx ++; AVX-32-NEXT: cmovnel %eax, %ecx + ; AVX-32-NEXT: movl (%ecx), %eax + ; AVX-32-NEXT: retl + ; +@@ -4107,7 +4591,9 @@ define i32 @test_f64_uno_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-NEXT: fnstsw %ax + ; X87-NEXT: # kill: def $ah killed $ah killed $ax + ; X87-NEXT: sahf +-; X87-NEXT: jp .LBB55_1 ++; X87-NEXT: setp %al ++; X87-NEXT: testb $1, %al ++; X87-NEXT: jne .LBB55_1 + ; X87-NEXT: # %bb.2: + ; X87-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-NEXT: movl (%eax), %eax +@@ -4124,9 +4610,11 @@ define i32 @test_f64_uno_s(i32 %a, i32 %b, double %f1, double %f2) #0 { + ; X87-CMOV-NEXT: fcompi %st(1), %st + ; X87-CMOV-NEXT: fstp %st(0) + ; X87-CMOV-NEXT: wait ++; X87-CMOV-NEXT: setp %al ++; X87-CMOV-NEXT: testb $1, %al + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X87-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx +-; X87-CMOV-NEXT: cmovpl %eax, %ecx ++; X87-CMOV-NEXT: cmovnel %eax, %ecx + ; X87-CMOV-NEXT: movl (%ecx), %eax + ; X87-CMOV-NEXT: retl + %cond = call i1 @llvm.experimental.constrained.fcmps.f64( +diff --git a/llvm/test/CodeGen/X86/fp128-libcalls-strict.ll b/llvm/test/CodeGen/X86/fp128-libcalls-strict.ll +index ad2d690fd7ed0..5dbc28230d5af 100644 +--- a/llvm/test/CodeGen/X86/fp128-libcalls-strict.ll ++++ b/llvm/test/CodeGen/X86/fp128-libcalls-strict.ll +@@ -3898,6 +3898,7 @@ define i64 @cmp_ueq_q(i64 %a, i64 %b, fp128 %x, fp128 %y) #0 { + ; X86-NEXT: calll __unordtf2 + ; X86-NEXT: addl $32, %esp + ; X86-NEXT: orb %bl, %al ++; X86-NEXT: testb $1, %al + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; X86-NEXT: cmovnel %eax, %ecx +@@ -3981,6 +3982,7 @@ define i64 @cmp_ueq_q(i64 %a, i64 %b, fp128 %x, fp128 %y) #0 { + ; WIN-X86-NEXT: calll ___unordtf2 + ; WIN-X86-NEXT: addl $32, %esp + ; WIN-X86-NEXT: orb %bl, %al ++; WIN-X86-NEXT: testb $1, %al + ; WIN-X86-NEXT: jne LBB39_1 + ; WIN-X86-NEXT: # %bb.2: + ; WIN-X86-NEXT: leal 16(%ebp), %ecx +diff --git a/llvm/test/CodeGen/X86/fp80-strict-scalar-cmp.ll b/llvm/test/CodeGen/X86/fp80-strict-scalar-cmp.ll +index cb2361fbb8d37..f95738d9e546e 100644 +--- a/llvm/test/CodeGen/X86/fp80-strict-scalar-cmp.ll ++++ b/llvm/test/CodeGen/X86/fp80-strict-scalar-cmp.ll +@@ -12,13 +12,17 @@ define i32 @test_oeq_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf +-; X86-NEXT: leal {{[0-9]+}}(%esp), %eax +-; X86-NEXT: jne .LBB0_3 +-; X86-NEXT: # %bb.1: +-; X86-NEXT: jp .LBB0_3 ++; X86-NEXT: setnp %al ++; X86-NEXT: sete %cl ++; X86-NEXT: andb %al, %cl ++; X86-NEXT: testb $1, %cl ++; X86-NEXT: jne .LBB0_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax +-; X86-NEXT: .LBB0_3: ++; X86-NEXT: movl (%eax), %eax ++; X86-NEXT: retl ++; X86-NEXT: .LBB0_1: ++; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax + ; X86-NEXT: retl + ; +@@ -50,7 +54,9 @@ define i32 @test_ogt_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf +-; X86-NEXT: ja .LBB1_1 ++; X86-NEXT: seta %al ++; X86-NEXT: testb $1, %al ++; X86-NEXT: jne .LBB1_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax +@@ -87,7 +93,9 @@ define i32 @test_oge_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf +-; X86-NEXT: jae .LBB2_1 ++; X86-NEXT: setae %al ++; X86-NEXT: testb $1, %al ++; X86-NEXT: jne .LBB2_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax +@@ -124,7 +132,9 @@ define i32 @test_olt_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf +-; X86-NEXT: ja .LBB3_1 ++; X86-NEXT: seta %al ++; X86-NEXT: testb $1, %al ++; X86-NEXT: jne .LBB3_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax +@@ -161,7 +171,9 @@ define i32 @test_ole_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf +-; X86-NEXT: jae .LBB4_1 ++; X86-NEXT: setae %al ++; X86-NEXT: testb $1, %al ++; X86-NEXT: jne .LBB4_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax +@@ -198,6 +210,8 @@ define i32 @test_one_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf ++; X86-NEXT: setne %al ++; X86-NEXT: testb $1, %al + ; X86-NEXT: jne .LBB5_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax +@@ -235,7 +249,9 @@ define i32 @test_ord_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf +-; X86-NEXT: jnp .LBB6_1 ++; X86-NEXT: setnp %al ++; X86-NEXT: testb $1, %al ++; X86-NEXT: jne .LBB6_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax +@@ -272,7 +288,9 @@ define i32 @test_ueq_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf +-; X86-NEXT: je .LBB7_1 ++; X86-NEXT: sete %al ++; X86-NEXT: testb $1, %al ++; X86-NEXT: jne .LBB7_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax +@@ -309,7 +327,9 @@ define i32 @test_ugt_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf +-; X86-NEXT: jb .LBB8_1 ++; X86-NEXT: setb %al ++; X86-NEXT: testb $1, %al ++; X86-NEXT: jne .LBB8_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax +@@ -346,7 +366,9 @@ define i32 @test_uge_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf +-; X86-NEXT: jbe .LBB9_1 ++; X86-NEXT: setbe %al ++; X86-NEXT: testb $1, %al ++; X86-NEXT: jne .LBB9_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax +@@ -383,7 +405,9 @@ define i32 @test_ult_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf +-; X86-NEXT: jb .LBB10_1 ++; X86-NEXT: setb %al ++; X86-NEXT: testb $1, %al ++; X86-NEXT: jne .LBB10_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax +@@ -420,7 +444,9 @@ define i32 @test_ule_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf +-; X86-NEXT: jbe .LBB11_1 ++; X86-NEXT: setbe %al ++; X86-NEXT: testb $1, %al ++; X86-NEXT: jne .LBB11_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax +@@ -457,13 +483,17 @@ define i32 @test_une_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf +-; X86-NEXT: leal {{[0-9]+}}(%esp), %eax +-; X86-NEXT: jne .LBB12_3 +-; X86-NEXT: # %bb.1: +-; X86-NEXT: jp .LBB12_3 ++; X86-NEXT: setp %al ++; X86-NEXT: setne %cl ++; X86-NEXT: orb %al, %cl ++; X86-NEXT: testb $1, %cl ++; X86-NEXT: jne .LBB12_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax +-; X86-NEXT: .LBB12_3: ++; X86-NEXT: movl (%eax), %eax ++; X86-NEXT: retl ++; X86-NEXT: .LBB12_1: ++; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax + ; X86-NEXT: retl + ; +@@ -495,7 +525,9 @@ define i32 @test_uno_q(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf +-; X86-NEXT: jp .LBB13_1 ++; X86-NEXT: setp %al ++; X86-NEXT: testb $1, %al ++; X86-NEXT: jne .LBB13_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax +@@ -532,13 +564,17 @@ define i32 @test_oeq_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf +-; X86-NEXT: leal {{[0-9]+}}(%esp), %eax +-; X86-NEXT: jne .LBB14_3 +-; X86-NEXT: # %bb.1: +-; X86-NEXT: jp .LBB14_3 ++; X86-NEXT: setnp %al ++; X86-NEXT: sete %cl ++; X86-NEXT: andb %al, %cl ++; X86-NEXT: testb $1, %cl ++; X86-NEXT: jne .LBB14_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax +-; X86-NEXT: .LBB14_3: ++; X86-NEXT: movl (%eax), %eax ++; X86-NEXT: retl ++; X86-NEXT: .LBB14_1: ++; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax + ; X86-NEXT: retl + ; +@@ -570,7 +606,9 @@ define i32 @test_ogt_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf +-; X86-NEXT: ja .LBB15_1 ++; X86-NEXT: seta %al ++; X86-NEXT: testb $1, %al ++; X86-NEXT: jne .LBB15_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax +@@ -607,7 +645,9 @@ define i32 @test_oge_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf +-; X86-NEXT: jae .LBB16_1 ++; X86-NEXT: setae %al ++; X86-NEXT: testb $1, %al ++; X86-NEXT: jne .LBB16_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax +@@ -644,7 +684,9 @@ define i32 @test_olt_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf +-; X86-NEXT: ja .LBB17_1 ++; X86-NEXT: seta %al ++; X86-NEXT: testb $1, %al ++; X86-NEXT: jne .LBB17_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax +@@ -681,7 +723,9 @@ define i32 @test_ole_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf +-; X86-NEXT: jae .LBB18_1 ++; X86-NEXT: setae %al ++; X86-NEXT: testb $1, %al ++; X86-NEXT: jne .LBB18_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax +@@ -718,6 +762,8 @@ define i32 @test_one_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf ++; X86-NEXT: setne %al ++; X86-NEXT: testb $1, %al + ; X86-NEXT: jne .LBB19_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax +@@ -755,7 +801,9 @@ define i32 @test_ord_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf +-; X86-NEXT: jnp .LBB20_1 ++; X86-NEXT: setnp %al ++; X86-NEXT: testb $1, %al ++; X86-NEXT: jne .LBB20_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax +@@ -792,7 +840,9 @@ define i32 @test_ueq_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf +-; X86-NEXT: je .LBB21_1 ++; X86-NEXT: sete %al ++; X86-NEXT: testb $1, %al ++; X86-NEXT: jne .LBB21_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax +@@ -829,7 +879,9 @@ define i32 @test_ugt_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf +-; X86-NEXT: jb .LBB22_1 ++; X86-NEXT: setb %al ++; X86-NEXT: testb $1, %al ++; X86-NEXT: jne .LBB22_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax +@@ -866,7 +918,9 @@ define i32 @test_uge_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf +-; X86-NEXT: jbe .LBB23_1 ++; X86-NEXT: setbe %al ++; X86-NEXT: testb $1, %al ++; X86-NEXT: jne .LBB23_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax +@@ -903,7 +957,9 @@ define i32 @test_ult_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf +-; X86-NEXT: jb .LBB24_1 ++; X86-NEXT: setb %al ++; X86-NEXT: testb $1, %al ++; X86-NEXT: jne .LBB24_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax +@@ -940,7 +996,9 @@ define i32 @test_ule_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf +-; X86-NEXT: jbe .LBB25_1 ++; X86-NEXT: setbe %al ++; X86-NEXT: testb $1, %al ++; X86-NEXT: jne .LBB25_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax +@@ -977,13 +1035,17 @@ define i32 @test_une_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf +-; X86-NEXT: leal {{[0-9]+}}(%esp), %eax +-; X86-NEXT: jne .LBB26_3 +-; X86-NEXT: # %bb.1: +-; X86-NEXT: jp .LBB26_3 ++; X86-NEXT: setp %al ++; X86-NEXT: setne %cl ++; X86-NEXT: orb %al, %cl ++; X86-NEXT: testb $1, %cl ++; X86-NEXT: jne .LBB26_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax +-; X86-NEXT: .LBB26_3: ++; X86-NEXT: movl (%eax), %eax ++; X86-NEXT: retl ++; X86-NEXT: .LBB26_1: ++; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax + ; X86-NEXT: retl + ; +@@ -1015,7 +1077,9 @@ define i32 @test_uno_s(i32 %a, i32 %b, x86_fp80 %f1, x86_fp80 %f2) #0 { + ; X86-NEXT: fnstsw %ax + ; X86-NEXT: # kill: def $ah killed $ah killed $ax + ; X86-NEXT: sahf +-; X86-NEXT: jp .LBB27_1 ++; X86-NEXT: setp %al ++; X86-NEXT: testb $1, %al ++; X86-NEXT: jne .LBB27_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax +diff --git a/llvm/test/CodeGen/X86/isel-select-cmov.ll b/llvm/test/CodeGen/X86/isel-select-cmov.ll +index d013ad2c7fbff..6ced7f45b6233 100644 +--- a/llvm/test/CodeGen/X86/isel-select-cmov.ll ++++ b/llvm/test/CodeGen/X86/isel-select-cmov.ll +@@ -47,7 +47,7 @@ define zeroext i8 @select_cmov_i8(i1 zeroext %cond, i8 zeroext %a, i8 zeroext %b + ; + ; SDAG-X86-LABEL: select_cmov_i8: + ; SDAG-X86: ## %bb.0: +-; SDAG-X86-NEXT: cmpb $0, {{[0-9]+}}(%esp) ++; SDAG-X86-NEXT: testb $1, {{[0-9]+}}(%esp) + ; SDAG-X86-NEXT: jne LBB0_1 + ; SDAG-X86-NEXT: ## %bb.2: + ; SDAG-X86-NEXT: leal {{[0-9]+}}(%esp), %eax +@@ -60,7 +60,7 @@ define zeroext i8 @select_cmov_i8(i1 zeroext %cond, i8 zeroext %a, i8 zeroext %b + ; + ; SDAG-X86-CMOV-LABEL: select_cmov_i8: + ; SDAG-X86-CMOV: ## %bb.0: +-; SDAG-X86-CMOV-NEXT: cmpb $0, {{[0-9]+}}(%esp) ++; SDAG-X86-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp) + ; SDAG-X86-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SDAG-X86-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; SDAG-X86-CMOV-NEXT: cmovnel %eax, %ecx +@@ -155,7 +155,7 @@ define zeroext i16 @select_cmov_i16(i1 zeroext %cond, i16 zeroext %a, i16 zeroex + ; + ; SDAG-X86-LABEL: select_cmov_i16: + ; SDAG-X86: ## %bb.0: +-; SDAG-X86-NEXT: cmpb $0, {{[0-9]+}}(%esp) ++; SDAG-X86-NEXT: testb $1, {{[0-9]+}}(%esp) + ; SDAG-X86-NEXT: jne LBB1_1 + ; SDAG-X86-NEXT: ## %bb.2: + ; SDAG-X86-NEXT: leal {{[0-9]+}}(%esp), %eax +@@ -168,7 +168,7 @@ define zeroext i16 @select_cmov_i16(i1 zeroext %cond, i16 zeroext %a, i16 zeroex + ; + ; SDAG-X86-CMOV-LABEL: select_cmov_i16: + ; SDAG-X86-CMOV: ## %bb.0: +-; SDAG-X86-CMOV-NEXT: cmpb $0, {{[0-9]+}}(%esp) ++; SDAG-X86-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp) + ; SDAG-X86-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SDAG-X86-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; SDAG-X86-CMOV-NEXT: cmovnel %eax, %ecx +@@ -360,7 +360,7 @@ define i32 @select_cmov_i32(i1 zeroext %cond, i32 %a, i32 %b) { + ; + ; SDAG-X86-LABEL: select_cmov_i32: + ; SDAG-X86: ## %bb.0: +-; SDAG-X86-NEXT: cmpb $0, {{[0-9]+}}(%esp) ++; SDAG-X86-NEXT: testb $1, {{[0-9]+}}(%esp) + ; SDAG-X86-NEXT: jne LBB3_1 + ; SDAG-X86-NEXT: ## %bb.2: + ; SDAG-X86-NEXT: leal {{[0-9]+}}(%esp), %eax +@@ -373,7 +373,7 @@ define i32 @select_cmov_i32(i1 zeroext %cond, i32 %a, i32 %b) { + ; + ; SDAG-X86-CMOV-LABEL: select_cmov_i32: + ; SDAG-X86-CMOV: ## %bb.0: +-; SDAG-X86-CMOV-NEXT: cmpb $0, {{[0-9]+}}(%esp) ++; SDAG-X86-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp) + ; SDAG-X86-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SDAG-X86-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; SDAG-X86-CMOV-NEXT: cmovnel %eax, %ecx +@@ -549,7 +549,7 @@ define i64 @select_cmov_i64(i1 zeroext %cond, i64 %a, i64 %b) { + ; + ; SDAG-X86-LABEL: select_cmov_i64: + ; SDAG-X86: ## %bb.0: +-; SDAG-X86-NEXT: cmpb $0, {{[0-9]+}}(%esp) ++; SDAG-X86-NEXT: testb $1, {{[0-9]+}}(%esp) + ; SDAG-X86-NEXT: jne LBB5_1 + ; SDAG-X86-NEXT: ## %bb.2: + ; SDAG-X86-NEXT: leal {{[0-9]+}}(%esp), %ecx +@@ -563,7 +563,7 @@ define i64 @select_cmov_i64(i1 zeroext %cond, i64 %a, i64 %b) { + ; + ; SDAG-X86-CMOV-LABEL: select_cmov_i64: + ; SDAG-X86-CMOV: ## %bb.0: +-; SDAG-X86-CMOV-NEXT: cmpb $0, {{[0-9]+}}(%esp) ++; SDAG-X86-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp) + ; SDAG-X86-CMOV-NEXT: leal {{[0-9]+}}(%esp), %eax + ; SDAG-X86-CMOV-NEXT: leal {{[0-9]+}}(%esp), %ecx + ; SDAG-X86-CMOV-NEXT: cmovnel %eax, %ecx +diff --git a/llvm/test/CodeGen/X86/select-constant-xor.ll b/llvm/test/CodeGen/X86/select-constant-xor.ll +index efc367d204cf1..5d80e04fef38c 100644 +--- a/llvm/test/CodeGen/X86/select-constant-xor.ll ++++ b/llvm/test/CodeGen/X86/select-constant-xor.ll +@@ -172,8 +172,10 @@ define i32 @selecti8i32(i8 %a) { + define i32 @icmpasreq(i32 %input, i32 %a, i32 %b) { + ; X86-LABEL: icmpasreq: + ; X86: # %bb.0: +-; X86-NEXT: cmpl $0, {{[0-9]+}}(%esp) +-; X86-NEXT: js .LBB8_1 ++; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ++; X86-NEXT: sarl $31, %eax ++; X86-NEXT: cmpl $-1, %eax ++; X86-NEXT: je .LBB8_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax +@@ -198,8 +200,10 @@ define i32 @icmpasreq(i32 %input, i32 %a, i32 %b) { + define i32 @icmpasrne(i32 %input, i32 %a, i32 %b) { + ; X86-LABEL: icmpasrne: + ; X86: # %bb.0: +-; X86-NEXT: cmpl $0, {{[0-9]+}}(%esp) +-; X86-NEXT: jns .LBB9_1 ++; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ++; X86-NEXT: sarl $31, %eax ++; X86-NEXT: cmpl $-1, %eax ++; X86-NEXT: jne .LBB9_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal {{[0-9]+}}(%esp), %eax + ; X86-NEXT: movl (%eax), %eax +diff --git a/llvm/test/CodeGen/X86/select-mmx.ll b/llvm/test/CodeGen/X86/select-mmx.ll +index 8a4308a5af64b..cc6603382082c 100644 +--- a/llvm/test/CodeGen/X86/select-mmx.ll ++++ b/llvm/test/CodeGen/X86/select-mmx.ll +@@ -85,8 +85,8 @@ define i64 @test49(i64 %arg, i64 %x, i64 %y) { + ; X86-NEXT: .cfi_def_cfa_register %ebp + ; X86-NEXT: andl $-8, %esp + ; X86-NEXT: subl $8, %esp +-; X86-NEXT: movl 8(%ebp), %eax +-; X86-NEXT: orl 12(%ebp), %eax ++; X86-NEXT: movl 12(%ebp), %eax ++; X86-NEXT: orl 8(%ebp), %eax + ; X86-NEXT: je .LBB1_1 + ; X86-NEXT: # %bb.2: + ; X86-NEXT: leal 24(%ebp), %eax +diff --git a/llvm/test/CodeGen/X86/select-of-load-poison.ll b/llvm/test/CodeGen/X86/select-of-load-poison.ll +new file mode 100644 +index 0000000000000..1f71f812d2327 +--- /dev/null ++++ b/llvm/test/CodeGen/X86/select-of-load-poison.ll +@@ -0,0 +1,20 @@ ++; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 ++; RUN: llc -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s ++ ++; Make sure there is an and 1, otherwise a case where the select condition ++; was poison could result in an out of bounds read instead. ++define i32 @test(i32 %x, ptr %p) { ++; CHECK-LABEL: test: ++; CHECK: # %bb.0: ++; CHECK-NEXT: # kill: def $edi killed $edi def $rdi ++; CHECK-NEXT: andl $1, %edi ++; CHECK-NEXT: movl 4(%rsi,%rdi,4), %eax ++; CHECK-NEXT: retq ++ %p1 = getelementptr i8, ptr %p, i64 4 ++ %v1 = load i32, ptr %p1 ++ %p2 = getelementptr i8, ptr %p, i64 8 ++ %v2 = load i32, ptr %p2 ++ %c = trunc nuw i32 %x to i1 ++ %v = select i1 %c, i32 %v2, i32 %v1 ++ ret i32 %v ++} diff --git a/0001-SystemZ-Avoid-unaligned-VL-VST-s-with-memcpy-memmove.patch b/0001-SystemZ-Avoid-unaligned-VL-VST-s-with-memcpy-memmove.patch new file mode 100644 index 0000000..95a7dfb --- /dev/null +++ b/0001-SystemZ-Avoid-unaligned-VL-VST-s-with-memcpy-memmove.patch @@ -0,0 +1,4156 @@ +From deb79107cdc640de8ccf4512a01512e7a8dcb72e Mon Sep 17 00:00:00 2001 +From: Jonas Paulsson +Date: Tue, 28 Apr 2026 19:09:09 +0200 +Subject: [PATCH] [SystemZ] Avoid unaligned VL/VST:s with + memcpy/memmove/memset. (#187100) + +This is a squash of the proposed backport PR #196359 and its dependencies +to llvm 22: + +[SystemZ] Improved testing for memcpy/memmove/memset. (#194682) +[SystemZ] Avoid unaligned VL/VST:s with memcpy/memmove/memset. (#187100) +[SystemZ] Remove superfluous args in tests. (#196022) +Fix memmove-01.ll test (to use libcalls in some cases). +--- + .../Target/SystemZ/SystemZISelLowering.cpp | 26 +- + llvm/test/CodeGen/SystemZ/memcpy-03.ll | 648 +++++- + llvm/test/CodeGen/SystemZ/memmove-01.ll | 1090 +++++++++ + llvm/test/CodeGen/SystemZ/memset-08.ll | 2018 +++++++++++++++-- + 4 files changed, 3453 insertions(+), 329 deletions(-) + create mode 100644 llvm/test/CodeGen/SystemZ/memmove-01.ll + +diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp +index c3ded986fd68..78862759c880 100644 +--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp ++++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp +@@ -830,6 +830,10 @@ SystemZTargetLowering::SystemZTargetLowering(const TargetMachine &TM, + MaxStoresPerMemcpy = Subtarget.hasVector() ? 2 : 0; + MaxStoresPerMemcpyOptSize = 0; + ++ // Same with memmove. ++ MaxStoresPerMemmove = Subtarget.hasVector() ? 2 : 0; ++ MaxStoresPerMemmoveOptSize = 0; ++ + // The main memset sequence is a byte store followed by an MVC. + // Two STC or MV..I stores win over that, but the kind of fused stores + // generated by target-independent code don't when the byte value is +@@ -1473,17 +1477,21 @@ bool SystemZTargetLowering::findOptimalMemOpLowering( + LLVMContext &Context, std::vector &MemOps, unsigned Limit, + const MemOp &Op, unsigned DstAS, unsigned SrcAS, + const AttributeList &FuncAttributes) const { ++ ++ assert(Limit != ~0U && ++ "Expected EmitTargetCodeForMemXXX() to handle AlwaysInline cases."); ++ ++ if (Op.isZeroMemset()) ++ return false; // Memset zero: Use XC. ++ + const int MVCFastLen = 16; ++ // Use MVC up to 16 bytes. Small memset uses STC/MVI for first byte. ++ if ((Op.isMemset() ? Op.size() - 1 : Op.size()) <= MVCFastLen) ++ return false; + +- if (Limit != ~unsigned(0)) { +- // Don't expand Op into scalar loads/stores in these cases: +- if (Op.isMemcpy() && Op.allowOverlap() && Op.size() <= MVCFastLen) +- return false; // Small memcpy: Use MVC +- if (Op.isMemset() && Op.size() - 1 <= MVCFastLen) +- return false; // Small memset (first byte with STC/MVI): Use MVC +- if (Op.isZeroMemset()) +- return false; // Memset zero: Use XC +- } ++ // Avoid unaligned VL/VST:s. ++ if (!Op.isAligned(Align(8)) || (Op.size() >= 25 && Op.size() <= 31)) ++ return false; + + return TargetLowering::findOptimalMemOpLowering(Context, MemOps, Limit, Op, + DstAS, SrcAS, FuncAttributes); +diff --git a/llvm/test/CodeGen/SystemZ/memcpy-03.ll b/llvm/test/CodeGen/SystemZ/memcpy-03.ll +index c703aef27532..213764e79ffb 100644 +--- a/llvm/test/CodeGen/SystemZ/memcpy-03.ll ++++ b/llvm/test/CodeGen/SystemZ/memcpy-03.ll +@@ -1,217 +1,661 @@ + ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +-; RUN: llc -mcpu=z15 < %s -mtriple=s390x-linux-gnu | FileCheck %s ++; RUN: llc -mcpu=z17 < %s -mtriple=s390x-linux-gnu | FileCheck %s + ; +-; Test memcpys of small constant lengths that should not be done with MVC. ++; Test non-volatile memcpys of small constant lengths in both aligned and ++; unaligned cases. + + declare void @llvm.memcpy.p0.p0.i64(ptr nocapture, ptr nocapture, i64, i1) nounwind + +-define void @fun16(ptr %Src, ptr %Dst, i8 %val) { ++define void @fun1(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun1: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(1,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 1, i1 false) ++ ret void ++} ++ ++define void @fun1_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun1_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(1,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 1, i1 false) ++ ret void ++} ++ ++define void @fun2(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun2: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(2,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 2, i1 false) ++ ret void ++} ++ ++define void @fun2_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun2_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(2,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 2, i1 false) ++ ret void ++} ++ ++define void @fun3(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun3: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(3,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 3, i1 false) ++ ret void ++} ++ ++define void @fun3_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun3_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(3,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 3, i1 false) ++ ret void ++} ++ ++define void @fun4(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun4: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(4,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 4, i1 false) ++ ret void ++} ++ ++define void @fun4_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun4_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(4,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 4, i1 false) ++ ret void ++} ++ ++define void @fun5(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun5: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(5,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 5, i1 false) ++ ret void ++} ++ ++define void @fun5_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun5_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(5,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 5, i1 false) ++ ret void ++} ++ ++define void @fun6(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun6: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(6,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 6, i1 false) ++ ret void ++} ++ ++define void @fun6_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun6_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(6,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 6, i1 false) ++ ret void ++} ++ ++define void @fun7(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun7: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(7,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 7, i1 false) ++ ret void ++} ++ ++define void @fun7_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun7_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(7,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 7, i1 false) ++ ret void ++} ++ ++define void @fun8(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun8: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(8,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 8, i1 false) ++ ret void ++} ++ ++define void @fun8_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun8_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(8,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 8, i1 false) ++ ret void ++} ++ ++define void @fun9(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun9: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(9,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 9, i1 false) ++ ret void ++} ++ ++define void @fun9_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun9_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(9,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 9, i1 false) ++ ret void ++} ++ ++define void @fun10(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun10: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(10,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 10, i1 false) ++ ret void ++} ++ ++define void @fun10_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun10_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(10,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 10, i1 false) ++ ret void ++} ++ ++define void @fun11(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun11: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(11,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 11, i1 false) ++ ret void ++} ++ ++define void @fun11_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun11_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(11,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 11, i1 false) ++ ret void ++} ++ ++define void @fun12(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun12: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(12,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 12, i1 false) ++ ret void ++} ++ ++define void @fun12_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun12_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(12,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 12, i1 false) ++ ret void ++} ++ ++define void @fun13(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun13: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(13,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 13, i1 false) ++ ret void ++} ++ ++define void @fun13_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun13_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(13,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 13, i1 false) ++ ret void ++} ++ ++define void @fun14(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun14: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(14,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 14, i1 false) ++ ret void ++} ++ ++define void @fun14_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun14_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(14,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 14, i1 false) ++ ret void ++} ++ ++define void @fun15(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun15: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(15,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 15, i1 false) ++ ret void ++} ++ ++define void @fun15_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun15_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(15,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 15, i1 false) ++ ret void ++} ++ ++define void @fun16(ptr %Dst, ptr %Src) { + ; CHECK-LABEL: fun16: + ; CHECK: # %bb.0: +-; CHECK-NEXT: mvc 0(16,%r3), 0(%r2) ++; CHECK-NEXT: mvc 0(16,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 16, i1 false) ++ ret void ++} ++ ++define void @fun16_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun16_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(16,%r2), 0(%r3) + ; CHECK-NEXT: br %r14 +- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 16, i1 false) ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 16, i1 false) + ret void + } + +-define void @fun17(ptr %Src, ptr %Dst, i8 %val) { ++define void @fun17(ptr %Dst, ptr %Src) { + ; CHECK-LABEL: fun17: + ; CHECK: # %bb.0: +-; CHECK-NEXT: lb %r0, 16(%r2) +-; CHECK-NEXT: stc %r0, 16(%r3) +-; CHECK-NEXT: vl %v0, 0(%r2), 4 +-; CHECK-NEXT: vst %v0, 0(%r3), 4 ++; CHECK-NEXT: lb %r0, 16(%r3) ++; CHECK-NEXT: stc %r0, 16(%r2) ++; CHECK-NEXT: vl %v0, 0(%r3), 3 ++; CHECK-NEXT: vst %v0, 0(%r2), 3 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 17, i1 false) ++ ret void ++} ++ ++define void @fun17_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun17_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(17,%r2), 0(%r3) + ; CHECK-NEXT: br %r14 +- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 17, i1 false) ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 17, i1 false) + ret void + } + +-define void @fun18(ptr %Src, ptr %Dst, i8 %val) { ++define void @fun17_unalignedSrc(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun17_unalignedSrc: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(17,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 4 %Src, i64 17, i1 false) ++ ret void ++} ++ ++define void @fun18(ptr %Dst, ptr %Src) { + ; CHECK-LABEL: fun18: + ; CHECK: # %bb.0: +-; CHECK-NEXT: lh %r0, 16(%r2) +-; CHECK-NEXT: sth %r0, 16(%r3) +-; CHECK-NEXT: vl %v0, 0(%r2), 4 +-; CHECK-NEXT: vst %v0, 0(%r3), 4 ++; CHECK-NEXT: lh %r0, 16(%r3) ++; CHECK-NEXT: sth %r0, 16(%r2) ++; CHECK-NEXT: vl %v0, 0(%r3), 3 ++; CHECK-NEXT: vst %v0, 0(%r2), 3 + ; CHECK-NEXT: br %r14 +- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 18, i1 false) ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 18, i1 false) + ret void + } + +-define void @fun19(ptr %Src, ptr %Dst, i8 %val) { ++define void @fun18_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun18_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(18,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 18, i1 false) ++ ret void ++} ++ ++define void @fun18_unalignedDst(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun18_unalignedDst: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(18,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 2 %Dst, ptr align 16 %Src, i64 18, i1 false) ++ ret void ++} ++ ++define void @fun19(ptr %Dst, ptr %Src) { + ; CHECK-LABEL: fun19: + ; CHECK: # %bb.0: +-; CHECK-NEXT: l %r0, 15(%r2) +-; CHECK-NEXT: st %r0, 15(%r3) +-; CHECK-NEXT: vl %v0, 0(%r2), 4 +-; CHECK-NEXT: vst %v0, 0(%r3), 4 ++; CHECK-NEXT: l %r0, 15(%r3) ++; CHECK-NEXT: st %r0, 15(%r2) ++; CHECK-NEXT: vl %v0, 0(%r3), 3 ++; CHECK-NEXT: vst %v0, 0(%r2), 3 + ; CHECK-NEXT: br %r14 +- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 19, i1 false) ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 19, i1 false) + ret void + } + +-define void @fun20(ptr %Src, ptr %Dst, i8 %val) { ++define void @fun19_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun19_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(19,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 19, i1 false) ++ ret void ++} ++ ++define void @fun20(ptr %Dst, ptr %Src) { + ; CHECK-LABEL: fun20: + ; CHECK: # %bb.0: +-; CHECK-NEXT: l %r0, 16(%r2) +-; CHECK-NEXT: st %r0, 16(%r3) ++; CHECK-NEXT: l %r0, 16(%r3) ++; CHECK-NEXT: st %r0, 16(%r2) ++; CHECK-NEXT: vl %v0, 0(%r3), 3 ++; CHECK-NEXT: vst %v0, 0(%r2), 3 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 20, i1 false) ++ ret void ++} ++ ++define void @fun20_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun20_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(20,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 20, i1 false) ++ ret void ++} ++ ++define void @fun20_localDst(ptr %Src) { ++; CHECK-LABEL: fun20_localDst: ++; CHECK: # %bb.0: ++; CHECK-NEXT: aghi %r15, -184 ++; CHECK-NEXT: .cfi_def_cfa_offset 344 + ; CHECK-NEXT: vl %v0, 0(%r2), 4 +-; CHECK-NEXT: vst %v0, 0(%r3), 4 ++; CHECK-NEXT: vst %v0, 164(%r15), 4 ++; CHECK-NEXT: mvc 180(4,%r15), 16(%r2) ++; CHECK-NEXT: aghi %r15, 184 + ; CHECK-NEXT: br %r14 ++ %Dst = alloca [20 x i8] + call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 20, i1 false) + ret void + } + +-define void @fun21(ptr %Src, ptr %Dst, i8 %val) { ++define void @fun21(ptr %Dst, ptr %Src) { + ; CHECK-LABEL: fun21: + ; CHECK: # %bb.0: +-; CHECK-NEXT: lg %r0, 13(%r2) +-; CHECK-NEXT: stg %r0, 13(%r3) +-; CHECK-NEXT: vl %v0, 0(%r2), 4 +-; CHECK-NEXT: vst %v0, 0(%r3), 4 ++; CHECK-NEXT: lg %r0, 13(%r3) ++; CHECK-NEXT: stg %r0, 13(%r2) ++; CHECK-NEXT: vl %v0, 0(%r3), 3 ++; CHECK-NEXT: vst %v0, 0(%r2), 3 + ; CHECK-NEXT: br %r14 +- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 21, i1 false) ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 21, i1 false) + ret void + } + +-define void @fun22(ptr %Src, ptr %Dst, i8 %val) { ++define void @fun21_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun21_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(21,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 21, i1 false) ++ ret void ++} ++ ++define void @fun22(ptr %Dst, ptr %Src) { + ; CHECK-LABEL: fun22: + ; CHECK: # %bb.0: +-; CHECK-NEXT: lg %r0, 14(%r2) +-; CHECK-NEXT: stg %r0, 14(%r3) +-; CHECK-NEXT: vl %v0, 0(%r2), 4 +-; CHECK-NEXT: vst %v0, 0(%r3), 4 ++; CHECK-NEXT: lg %r0, 14(%r3) ++; CHECK-NEXT: stg %r0, 14(%r2) ++; CHECK-NEXT: vl %v0, 0(%r3), 3 ++; CHECK-NEXT: vst %v0, 0(%r2), 3 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 22, i1 false) ++ ret void ++} ++ ++define void @fun22_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun22_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(22,%r2), 0(%r3) + ; CHECK-NEXT: br %r14 +- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 22, i1 false) ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 22, i1 false) + ret void + } + +-define void @fun23(ptr %Src, ptr %Dst, i8 %val) { ++define void @fun23(ptr %Dst, ptr %Src) { + ; CHECK-LABEL: fun23: + ; CHECK: # %bb.0: +-; CHECK-NEXT: lg %r0, 15(%r2) +-; CHECK-NEXT: stg %r0, 15(%r3) +-; CHECK-NEXT: vl %v0, 0(%r2), 4 +-; CHECK-NEXT: vst %v0, 0(%r3), 4 ++; CHECK-NEXT: lg %r0, 15(%r3) ++; CHECK-NEXT: stg %r0, 15(%r2) ++; CHECK-NEXT: vl %v0, 0(%r3), 3 ++; CHECK-NEXT: vst %v0, 0(%r2), 3 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 23, i1 false) ++ ret void ++} ++ ++define void @fun23_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun23_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(23,%r2), 0(%r3) + ; CHECK-NEXT: br %r14 +- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 23, i1 false) ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 23, i1 false) + ret void + } + +-define void @fun24(ptr %Src, ptr %Dst, i8 %val) { ++define void @fun24(ptr %Dst, ptr %Src) { + ; CHECK-LABEL: fun24: + ; CHECK: # %bb.0: +-; CHECK-NEXT: lg %r0, 16(%r2) +-; CHECK-NEXT: stg %r0, 16(%r3) +-; CHECK-NEXT: vl %v0, 0(%r2), 4 +-; CHECK-NEXT: vst %v0, 0(%r3), 4 ++; CHECK-NEXT: lg %r0, 16(%r3) ++; CHECK-NEXT: stg %r0, 16(%r2) ++; CHECK-NEXT: vl %v0, 0(%r3), 3 ++; CHECK-NEXT: vst %v0, 0(%r2), 3 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 24, i1 false) ++ ret void ++} ++ ++define void @fun24_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun24_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(24,%r2), 0(%r3) + ; CHECK-NEXT: br %r14 +- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 24, i1 false) ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 24, i1 false) + ret void + } + +-define void @fun25(ptr %Src, ptr %Dst, i8 %val) { ++define void @fun25(ptr %Dst, ptr %Src) { + ; CHECK-LABEL: fun25: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vl %v0, 9(%r2) +-; CHECK-NEXT: vst %v0, 9(%r3) +-; CHECK-NEXT: vl %v0, 0(%r2), 4 +-; CHECK-NEXT: vst %v0, 0(%r3), 4 ++; CHECK-NEXT: mvc 0(25,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 25, i1 false) ++ ret void ++} ++ ++define void @fun25_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun25_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(25,%r2), 0(%r3) + ; CHECK-NEXT: br %r14 +- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 25, i1 false) ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 25, i1 false) + ret void + } + +-define void @fun26(ptr %Src, ptr %Dst, i8 %val) { ++define void @fun26(ptr %Dst, ptr %Src) { + ; CHECK-LABEL: fun26: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vl %v0, 10(%r2) +-; CHECK-NEXT: vst %v0, 10(%r3) +-; CHECK-NEXT: vl %v0, 0(%r2), 4 +-; CHECK-NEXT: vst %v0, 0(%r3), 4 ++; CHECK-NEXT: mvc 0(26,%r2), 0(%r3) + ; CHECK-NEXT: br %r14 +- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 26, i1 false) ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 26, i1 false) + ret void + } + +-define void @fun27(ptr %Src, ptr %Dst, i8 %val) { ++define void @fun26_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun26_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(26,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 26, i1 false) ++ ret void ++} ++ ++define void @fun27(ptr %Dst, ptr %Src) { + ; CHECK-LABEL: fun27: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vl %v0, 11(%r2) +-; CHECK-NEXT: vst %v0, 11(%r3) +-; CHECK-NEXT: vl %v0, 0(%r2), 4 +-; CHECK-NEXT: vst %v0, 0(%r3), 4 ++; CHECK-NEXT: mvc 0(27,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 27, i1 false) ++ ret void ++} ++ ++define void @fun27_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun27_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(27,%r2), 0(%r3) + ; CHECK-NEXT: br %r14 +- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 27, i1 false) ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 27, i1 false) + ret void + } + +-define void @fun28(ptr %Src, ptr %Dst, i8 %val) { ++define void @fun28(ptr %Dst, ptr %Src) { + ; CHECK-LABEL: fun28: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vl %v0, 12(%r2) +-; CHECK-NEXT: vst %v0, 12(%r3) +-; CHECK-NEXT: vl %v0, 0(%r2), 4 +-; CHECK-NEXT: vst %v0, 0(%r3), 4 ++; CHECK-NEXT: mvc 0(28,%r2), 0(%r3) + ; CHECK-NEXT: br %r14 +- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 28, i1 false) ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 28, i1 false) + ret void + } + +-define void @fun29(ptr %Src, ptr %Dst, i8 %val) { ++define void @fun28_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun28_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(28,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 28, i1 false) ++ ret void ++} ++ ++define void @fun29(ptr %Dst, ptr %Src) { + ; CHECK-LABEL: fun29: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vl %v0, 13(%r2) +-; CHECK-NEXT: vst %v0, 13(%r3) +-; CHECK-NEXT: vl %v0, 0(%r2), 4 +-; CHECK-NEXT: vst %v0, 0(%r3), 4 ++; CHECK-NEXT: mvc 0(29,%r2), 0(%r3) + ; CHECK-NEXT: br %r14 +- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 29, i1 false) ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 29, i1 false) + ret void + } + +-define void @fun30(ptr %Src, ptr %Dst, i8 %val) { ++define void @fun29_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun29_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(29,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 29, i1 false) ++ ret void ++} ++ ++define void @fun30(ptr %Dst, ptr %Src) { + ; CHECK-LABEL: fun30: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vl %v0, 14(%r2) +-; CHECK-NEXT: vst %v0, 14(%r3) +-; CHECK-NEXT: vl %v0, 0(%r2), 4 +-; CHECK-NEXT: vst %v0, 0(%r3), 4 ++; CHECK-NEXT: mvc 0(30,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 30, i1 false) ++ ret void ++} ++ ++define void @fun30_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun30_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(30,%r2), 0(%r3) + ; CHECK-NEXT: br %r14 +- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 30, i1 false) ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 30, i1 false) + ret void + } + +-define void @fun31(ptr %Src, ptr %Dst, i8 %val) { ++define void @fun31(ptr %Dst, ptr %Src) { + ; CHECK-LABEL: fun31: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vl %v0, 15(%r2) +-; CHECK-NEXT: vst %v0, 15(%r3) +-; CHECK-NEXT: vl %v0, 0(%r2), 4 +-; CHECK-NEXT: vst %v0, 0(%r3), 4 ++; CHECK-NEXT: mvc 0(31,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 31, i1 false) ++ ret void ++} ++ ++define void @fun31_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun31_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(31,%r2), 0(%r3) + ; CHECK-NEXT: br %r14 +- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 31, i1 false) ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 31, i1 false) + ret void + } + +-define void @fun32(ptr %Src, ptr %Dst, i8 %val) { ++define void @fun32(ptr %Dst, ptr %Src) { + ; CHECK-LABEL: fun32: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vl %v0, 16(%r2), 4 +-; CHECK-NEXT: vst %v0, 16(%r3), 4 +-; CHECK-NEXT: vl %v0, 0(%r2), 4 +-; CHECK-NEXT: vst %v0, 0(%r3), 4 ++; CHECK-NEXT: vl %v0, 16(%r3), 3 ++; CHECK-NEXT: vst %v0, 16(%r2), 3 ++; CHECK-NEXT: vl %v0, 0(%r3), 3 ++; CHECK-NEXT: vst %v0, 0(%r2), 3 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 32, i1 false) ++ ret void ++} ++ ++define void @fun32_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun32_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(32,%r2), 0(%r3) + ; CHECK-NEXT: br %r14 +- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 32, i1 false) ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 32, i1 false) + ret void + } + +-define void @fun33(ptr %Src, ptr %Dst, i8 %val) { ++define void @fun33(ptr %Dst, ptr %Src) { + ; CHECK-LABEL: fun33: + ; CHECK: # %bb.0: +-; CHECK-NEXT: mvc 0(33,%r3), 0(%r2) ++; CHECK-NEXT: mvc 0(33,%r2), 0(%r3) + ; CHECK-NEXT: br %r14 +- call void @llvm.memcpy.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 33, i1 false) ++ call void @llvm.memcpy.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 33, i1 false) + ret void + } + ++define void @fun33_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun33_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvc 0(33,%r2), 0(%r3) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memcpy.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 33, i1 false) ++ ret void ++} +diff --git a/llvm/test/CodeGen/SystemZ/memmove-01.ll b/llvm/test/CodeGen/SystemZ/memmove-01.ll +new file mode 100644 +index 000000000000..b9c60fbe59a7 +--- /dev/null ++++ b/llvm/test/CodeGen/SystemZ/memmove-01.ll +@@ -0,0 +1,1090 @@ ++; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ++; RUN: llc -mcpu=z17 < %s -mtriple=s390x-linux-gnu | FileCheck %s ++; ++; Test non-volatile memmoves of small constant lengths in both aligned and ++; unaligned cases. ++ ++declare void @llvm.memmove.p0.p0.i64(ptr nocapture, ptr nocapture, i64, i1) nounwind ++ ++define void @fun1(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun1: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 1 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 1, i1 false) ++ ret void ++} ++ ++define void @fun1_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun1_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 1 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 1, i1 false) ++ ret void ++} ++ ++define void @fun2(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun2: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 2 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 2, i1 false) ++ ret void ++} ++ ++define void @fun2_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun2_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 2 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 2, i1 false) ++ ret void ++} ++ ++define void @fun3(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun3: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 3 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 3, i1 false) ++ ret void ++} ++ ++define void @fun3_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun3_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 3 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 3, i1 false) ++ ret void ++} ++ ++define void @fun4(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun4: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 4 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 4, i1 false) ++ ret void ++} ++ ++define void @fun4_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun4_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 4 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 4, i1 false) ++ ret void ++} ++ ++define void @fun5(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun5: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 5 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 5, i1 false) ++ ret void ++} ++ ++define void @fun5_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun5_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 5 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 5, i1 false) ++ ret void ++} ++ ++define void @fun6(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun6: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 6 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 6, i1 false) ++ ret void ++} ++ ++define void @fun6_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun6_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 6 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 6, i1 false) ++ ret void ++} ++ ++define void @fun7(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun7: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 7 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 7, i1 false) ++ ret void ++} ++ ++define void @fun7_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun7_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 7 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 7, i1 false) ++ ret void ++} ++ ++define void @fun8(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun8: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 8 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 8, i1 false) ++ ret void ++} ++ ++define void @fun8_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun8_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 8 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 8, i1 false) ++ ret void ++} ++ ++define void @fun9(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun9: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 9 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 9, i1 false) ++ ret void ++} ++ ++define void @fun9_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun9_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 9 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 9, i1 false) ++ ret void ++} ++ ++define void @fun10(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun10: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 10 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 10, i1 false) ++ ret void ++} ++ ++define void @fun10_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun10_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 10 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 10, i1 false) ++ ret void ++} ++ ++define void @fun11(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun11: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 11 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 11, i1 false) ++ ret void ++} ++ ++define void @fun11_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun11_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 11 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 11, i1 false) ++ ret void ++} ++ ++define void @fun12(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun12: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 12 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 12, i1 false) ++ ret void ++} ++ ++define void @fun12_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun12_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 12 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 12, i1 false) ++ ret void ++} ++ ++define void @fun13(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun13: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 13 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 13, i1 false) ++ ret void ++} ++ ++define void @fun13_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun13_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 13 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 13, i1 false) ++ ret void ++} ++ ++define void @fun14(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun14: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 14 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 14, i1 false) ++ ret void ++} ++ ++define void @fun14_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun14_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 14 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 14, i1 false) ++ ret void ++} ++ ++define void @fun15(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun15: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 15 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 15, i1 false) ++ ret void ++} ++ ++define void @fun15_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun15_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 15 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 15, i1 false) ++ ret void ++} ++ ++define void @fun16(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun16: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 16 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 16, i1 false) ++ ret void ++} ++ ++define void @fun16_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun16_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 16 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 16, i1 false) ++ ret void ++} ++ ++define void @fun17(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun17: ++; CHECK: # %bb.0: ++; CHECK-NEXT: lb %r0, 16(%r3) ++; CHECK-NEXT: vl %v0, 0(%r3), 3 ++; CHECK-NEXT: stc %r0, 16(%r2) ++; CHECK-NEXT: vst %v0, 0(%r2), 3 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 17, i1 false) ++ ret void ++} ++ ++define void @fun17_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun17_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 17 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 17, i1 false) ++ ret void ++} ++ ++define void @fun17_unalignedSrc(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun17_unalignedSrc: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 17 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 16 %Dst, ptr align 4 %Src, i64 17, i1 false) ++ ret void ++} ++ ++define void @fun18(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun18: ++; CHECK: # %bb.0: ++; CHECK-NEXT: lh %r0, 16(%r3) ++; CHECK-NEXT: vl %v0, 0(%r3), 3 ++; CHECK-NEXT: sth %r0, 16(%r2) ++; CHECK-NEXT: vst %v0, 0(%r2), 3 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 18, i1 false) ++ ret void ++} ++ ++define void @fun18_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun18_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 18 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 18, i1 false) ++ ret void ++} ++ ++define void @fun18_unalignedDst(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun18_unalignedDst: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 18 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 2 %Dst, ptr align 16 %Src, i64 18, i1 false) ++ ret void ++} ++ ++define void @fun19(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun19: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 19 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 19, i1 false) ++ ret void ++} ++ ++define void @fun19_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun19_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 19 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 19, i1 false) ++ ret void ++} ++ ++define void @fun20(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun20: ++; CHECK: # %bb.0: ++; CHECK-NEXT: vl %v0, 0(%r3), 3 ++; CHECK-NEXT: l %r0, 16(%r3) ++; CHECK-NEXT: st %r0, 16(%r2) ++; CHECK-NEXT: vst %v0, 0(%r2), 3 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 20, i1 false) ++ ret void ++} ++ ++define void @fun20_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun20_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 20 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 20, i1 false) ++ ret void ++} ++ ++define void @fun20_localDst(ptr %Src) { ++; CHECK-LABEL: fun20_localDst: ++; CHECK: # %bb.0: ++; CHECK-NEXT: aghi %r15, -184 ++; CHECK-NEXT: .cfi_def_cfa_offset 344 ++; CHECK-NEXT: vl %v0, 0(%r2), 4 ++; CHECK-NEXT: vst %v0, 164(%r15), 4 ++; CHECK-NEXT: mvc 180(4,%r15), 16(%r2) ++; CHECK-NEXT: aghi %r15, 184 ++; CHECK-NEXT: br %r14 ++ %Dst = alloca [20 x i8] ++ call void @llvm.memmove.p0.p0.i64(ptr align 16 %Dst, ptr align 16 %Src, i64 20, i1 false) ++ ret void ++} ++ ++define void @fun21(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun21: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 21 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 21, i1 false) ++ ret void ++} ++ ++define void @fun21_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun21_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 21 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 21, i1 false) ++ ret void ++} ++ ++define void @fun22(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun22: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 22 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 22, i1 false) ++ ret void ++} ++ ++define void @fun22_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun22_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 22 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 22, i1 false) ++ ret void ++} ++ ++define void @fun23(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun23: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 23 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 23, i1 false) ++ ret void ++} ++ ++define void @fun23_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun23_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 23 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 23, i1 false) ++ ret void ++} ++ ++define void @fun24(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun24: ++; CHECK: # %bb.0: ++; CHECK-NEXT: vl %v0, 0(%r3), 3 ++; CHECK-NEXT: lg %r0, 16(%r3) ++; CHECK-NEXT: stg %r0, 16(%r2) ++; CHECK-NEXT: vst %v0, 0(%r2), 3 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 24, i1 false) ++ ret void ++} ++ ++define void @fun24_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun24_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 24 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 24, i1 false) ++ ret void ++} ++ ++define void @fun25(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun25: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 25 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 25, i1 false) ++ ret void ++} ++ ++define void @fun25_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun25_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 25 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 25, i1 false) ++ ret void ++} ++ ++define void @fun26(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun26: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 26 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 26, i1 false) ++ ret void ++} ++ ++define void @fun26_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun26_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 26 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 26, i1 false) ++ ret void ++} ++ ++define void @fun27(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun27: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 27 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 27, i1 false) ++ ret void ++} ++ ++define void @fun27_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun27_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 27 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 27, i1 false) ++ ret void ++} ++ ++define void @fun28(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun28: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 28 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 28, i1 false) ++ ret void ++} ++ ++define void @fun28_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun28_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 28 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 28, i1 false) ++ ret void ++} ++ ++define void @fun29(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun29: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 29 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 29, i1 false) ++ ret void ++} ++ ++define void @fun29_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun29_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 29 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 29, i1 false) ++ ret void ++} ++ ++define void @fun30(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun30: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 30 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 30, i1 false) ++ ret void ++} ++ ++define void @fun30_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun30_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 30 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 30, i1 false) ++ ret void ++} ++ ++define void @fun31(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun31: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 31 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 31, i1 false) ++ ret void ++} ++ ++define void @fun31_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun31_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 31 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 31, i1 false) ++ ret void ++} ++ ++define void @fun32(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun32: ++; CHECK: # %bb.0: ++; CHECK-NEXT: vl %v0, 0(%r3), 3 ++; CHECK-NEXT: vl %v1, 16(%r3), 3 ++; CHECK-NEXT: vst %v1, 16(%r2), 3 ++; CHECK-NEXT: vst %v0, 0(%r2), 3 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 32, i1 false) ++ ret void ++} ++ ++define void @fun32_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun32_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 32 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 32, i1 false) ++ ret void ++} ++ ++define void @fun33(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun33: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 33 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 8 %Dst, ptr align 8 %Src, i64 33, i1 false) ++ ret void ++} ++ ++define void @fun33_unaligned(ptr %Dst, ptr %Src) { ++; CHECK-LABEL: fun33_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stmg %r14, %r15, 112(%r15) ++; CHECK-NEXT: .cfi_offset %r14, -48 ++; CHECK-NEXT: .cfi_offset %r15, -40 ++; CHECK-NEXT: aghi %r15, -160 ++; CHECK-NEXT: .cfi_def_cfa_offset 320 ++; CHECK-NEXT: lghi %r4, 33 ++; CHECK-NEXT: brasl %r14, memmove@PLT ++; CHECK-NEXT: lmg %r14, %r15, 272(%r15) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memmove.p0.p0.i64(ptr align 1 %Dst, ptr align 1 %Src, i64 33, i1 false) ++ ret void ++} +diff --git a/llvm/test/CodeGen/SystemZ/memset-08.ll b/llvm/test/CodeGen/SystemZ/memset-08.ll +index 931230983368..43c50320e011 100644 +--- a/llvm/test/CodeGen/SystemZ/memset-08.ll ++++ b/llvm/test/CodeGen/SystemZ/memset-08.ll +@@ -1,230 +1,1702 @@ + ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py + ; RUN: llc -mcpu=z15 %s -mtriple=s390x-linux-gnu -o - | FileCheck %s + ; +-; Test memsets of small constant lengths, that should not be done with MVC. ++; Test non-volatile memsets of small constant lengths in both aligned and ++; unaligned cases. + + declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg) + ++define void @reg1(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg1: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 1, i1 false) ++ ret void ++} ++ ++define void @reg1_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg1_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 1, i1 false) ++ ret void ++} ++ ++define void @reg2(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg2: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 1(%r2) ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 2, i1 false) ++ ret void ++} ++ ++define void @reg2_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg2_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 1(%r2) ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 2, i1 false) ++ ret void ++} ++ ++define void @reg3(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg3: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(2,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 3, i1 false) ++ ret void ++} ++ ++define void @reg3_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg3_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(2,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 3, i1 false) ++ ret void ++} ++ ++define void @reg4(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg4: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(3,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 4, i1 false) ++ ret void ++} ++ ++define void @reg4_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg4_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(3,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 4, i1 false) ++ ret void ++} ++ ++define void @reg5(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg5: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(4,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 5, i1 false) ++ ret void ++} ++ ++define void @reg5_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg5_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(4,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 5, i1 false) ++ ret void ++} ++ ++define void @reg6(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg6: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(5,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 6, i1 false) ++ ret void ++} ++ ++define void @reg6_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg6_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(5,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 6, i1 false) ++ ret void ++} ++ ++define void @reg7(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg7: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(6,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 7, i1 false) ++ ret void ++} ++ ++define void @reg7_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg7_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(6,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 7, i1 false) ++ ret void ++} ++ ++define void @reg8(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg8: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(7,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 8, i1 false) ++ ret void ++} ++ ++define void @reg8_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg8_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(7,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 8, i1 false) ++ ret void ++} ++ ++define void @reg9(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg9: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(8,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 9, i1 false) ++ ret void ++} ++ ++define void @reg9_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg9_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(8,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 9, i1 false) ++ ret void ++} ++ ++define void @reg10(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg10: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(9,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 10, i1 false) ++ ret void ++} ++ ++define void @reg10_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg10_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(9,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 10, i1 false) ++ ret void ++} ++ ++define void @reg11(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg11: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(10,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 11, i1 false) ++ ret void ++} ++ ++define void @reg11_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg11_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(10,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 11, i1 false) ++ ret void ++} ++ ++define void @reg12(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg12: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(11,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 12, i1 false) ++ ret void ++} ++ ++define void @reg12_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg12_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(11,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 12, i1 false) ++ ret void ++} ++ ++define void @reg13(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg13: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(12,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 13, i1 false) ++ ret void ++} ++ ++define void @reg13_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg13_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(12,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 13, i1 false) ++ ret void ++} ++ ++define void @reg14(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg14: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(13,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 14, i1 false) ++ ret void ++} ++ ++define void @reg14_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg14_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(13,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 14, i1 false) ++ ret void ++} ++ ++define void @reg15(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg15: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(14,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 15, i1 false) ++ ret void ++} ++ ++define void @reg15_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg15_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(14,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 15, i1 false) ++ ret void ++} ++ ++define void @reg16(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg16: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(15,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 16, i1 false) ++ ret void ++} ++ ++define void @reg16_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg16_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(15,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 16, i1 false) ++ ret void ++} ++ + define void @reg17(ptr %Dst, i8 %val) { + ; CHECK-LABEL: reg17: + ; CHECK: # %bb.0: +-; CHECK-NEXT: stc %r3, 0(%r2) +-; CHECK-NEXT: mvc 1(16,%r2), 0(%r2) ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(16,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 17, i1 false) ++ ret void ++} ++ ++define void @reg17_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg17_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(16,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 17, i1 false) ++ ret void ++} ++ ++define void @reg18(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg18: ++; CHECK: # %bb.0: ++; CHECK-NEXT: vlvgp %v0, %r3, %r3 ++; CHECK-NEXT: vrepb %v0, %v0, 7 ++; CHECK-NEXT: vst %v0, 0(%r2), 3 ++; CHECK-NEXT: vsteh %v0, 16(%r2), 0 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 18, i1 false) ++ ret void ++} ++ ++define void @reg18_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg18_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(17,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 18, i1 false) ++ ret void ++} ++ ++define void @reg19(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg19: ++; CHECK: # %bb.0: ++; CHECK-NEXT: vlvgp %v0, %r3, %r3 ++; CHECK-NEXT: vrepb %v0, %v0, 7 ++; CHECK-NEXT: vstef %v0, 15(%r2), 0 ++; CHECK-NEXT: vst %v0, 0(%r2), 3 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 19, i1 false) ++ ret void ++} ++ ++define void @reg19_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg19_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(18,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 19, i1 false) ++ ret void ++} ++ ++define void @reg20(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg20: ++; CHECK: # %bb.0: ++; CHECK-NEXT: vlvgp %v0, %r3, %r3 ++; CHECK-NEXT: vrepb %v0, %v0, 7 ++; CHECK-NEXT: vstef %v0, 16(%r2), 0 ++; CHECK-NEXT: vst %v0, 0(%r2), 3 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 20, i1 false) ++ ret void ++} ++ ++define void @reg20_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg20_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(19,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 20, i1 false) ++ ret void ++} ++ ++define void @reg20_localDst(i8 %val) { ++; CHECK-LABEL: reg20_localDst: ++; CHECK: # %bb.0: ++; CHECK-NEXT: aghi %r15, -184 ++; CHECK-NEXT: .cfi_def_cfa_offset 344 ++; CHECK-NEXT: vlvgp %v0, %r2, %r2 ++; CHECK-NEXT: vrepb %v0, %v0, 7 ++; CHECK-NEXT: vstef %v0, 180(%r15), 0 ++; CHECK-NEXT: vst %v0, 164(%r15), 4 ++; CHECK-NEXT: aghi %r15, 184 ++; CHECK-NEXT: br %r14 ++ %Dst = alloca [20 x i8] ++ call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 20, i1 false) ++ ret void ++} ++ ++define void @reg21(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg21: ++; CHECK: # %bb.0: ++; CHECK-NEXT: vlvgp %v0, %r3, %r3 ++; CHECK-NEXT: vrepb %v0, %v0, 7 ++; CHECK-NEXT: vsteg %v0, 13(%r2), 0 ++; CHECK-NEXT: vst %v0, 0(%r2), 3 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 21, i1 false) ++ ret void ++} ++ ++define void @reg21_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg21_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(20,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 21, i1 false) ++ ret void ++} ++ ++define void @reg22(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg22: ++; CHECK: # %bb.0: ++; CHECK-NEXT: vlvgp %v0, %r3, %r3 ++; CHECK-NEXT: vrepb %v0, %v0, 7 ++; CHECK-NEXT: vsteg %v0, 14(%r2), 0 ++; CHECK-NEXT: vst %v0, 0(%r2), 3 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 22, i1 false) ++ ret void ++} ++ ++define void @reg22_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg22_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(21,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 22, i1 false) ++ ret void ++} ++ ++define void @reg23(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg23: ++; CHECK: # %bb.0: ++; CHECK-NEXT: vlvgp %v0, %r3, %r3 ++; CHECK-NEXT: vrepb %v0, %v0, 7 ++; CHECK-NEXT: vsteg %v0, 15(%r2), 0 ++; CHECK-NEXT: vst %v0, 0(%r2), 3 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 23, i1 false) ++ ret void ++} ++ ++define void @reg23_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg23_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(22,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 23, i1 false) ++ ret void ++} ++ ++define void @reg24(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg24: ++; CHECK: # %bb.0: ++; CHECK-NEXT: vlvgp %v0, %r3, %r3 ++; CHECK-NEXT: vrepb %v0, %v0, 7 ++; CHECK-NEXT: vsteg %v0, 16(%r2), 0 ++; CHECK-NEXT: vst %v0, 0(%r2), 3 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 24, i1 false) ++ ret void ++} ++ ++define void @reg24_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg24_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(23,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 24, i1 false) ++ ret void ++} ++ ++define void @reg25(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg25: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(24,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 25, i1 false) ++ ret void ++} ++ ++define void @reg25_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg25_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(24,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 25, i1 false) ++ ret void ++} ++ ++define void @reg26(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg26: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(25,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 26, i1 false) ++ ret void ++} ++ ++define void @reg26_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg26_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(25,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 26, i1 false) ++ ret void ++} ++ ++define void @reg27(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg27: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(26,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 27, i1 false) ++ ret void ++} ++ ++define void @reg27_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg27_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(26,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 27, i1 false) ++ ret void ++} ++ ++define void @reg28(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg28: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(27,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 28, i1 false) ++ ret void ++} ++ ++define void @reg28_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg28_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(27,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 28, i1 false) ++ ret void ++} ++ ++define void @reg29(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg29: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(28,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 29, i1 false) ++ ret void ++} ++ ++define void @reg29_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg29_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(28,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 29, i1 false) ++ ret void ++} ++ ++define void @reg30(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg30: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(29,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 30, i1 false) ++ ret void ++} ++ ++define void @reg30_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg30_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(29,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 30, i1 false) ++ ret void ++} ++ ++define void @reg31(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg31: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(30,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 31, i1 false) ++ ret void ++} ++ ++define void @reg31_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg31_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(30,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 31, i1 false) ++ ret void ++} ++ ++define void @reg32(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg32: ++; CHECK: # %bb.0: ++; CHECK-NEXT: vlvgp %v0, %r3, %r3 ++; CHECK-NEXT: vrepb %v0, %v0, 7 ++; CHECK-NEXT: vst %v0, 16(%r2), 3 ++; CHECK-NEXT: vst %v0, 0(%r2), 3 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 32, i1 false) ++ ret void ++} ++ ++define void @reg32_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg32_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(31,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 32, i1 false) ++ ret void ++} ++ ++define void @reg33(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg33: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(32,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 %val, i64 33, i1 false) ++ ret void ++} ++ ++define void @reg33_unaligned(ptr %Dst, i8 %val) { ++; CHECK-LABEL: reg33_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: stc %r3, 0(%r2) ++; CHECK-NEXT: mvc 1(32,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 %val, i64 33, i1 false) ++ ret void ++} ++ ++;; Immediate value ++ ++define void @imm1(ptr %Dst) { ++; CHECK-LABEL: imm1: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 1 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 1, i64 1, i1 false) ++ ret void ++} ++ ++define void @imm1_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm1_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 255 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 -1, i64 1, i1 false) ++ ret void ++} ++ ++define void @imm2(ptr %Dst) { ++; CHECK-LABEL: imm2: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvhhi 0(%r2), 514 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 2, i64 2, i1 false) ++ ret void ++} ++ ++define void @imm2_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm2_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvhhi 0(%r2), 514 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 2, i64 2, i1 false) ++ ret void ++} ++ ++define void @imm3(ptr %Dst) { ++; CHECK-LABEL: imm3: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 2(%r2), 3 ++; CHECK-NEXT: mvhhi 0(%r2), 771 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 3, i64 3, i1 false) ++ ret void ++} ++ ++define void @imm3_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm3_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 2(%r2), 3 ++; CHECK-NEXT: mvhhi 0(%r2), 771 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 3, i64 3, i1 false) ++ ret void ++} ++ ++define void @imm4(ptr %Dst) { ++; CHECK-LABEL: imm4: ++; CHECK: # %bb.0: ++; CHECK-NEXT: vrepib %v0, 4 ++; CHECK-NEXT: vstef %v0, 0(%r2), 0 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 4, i64 4, i1 false) ++ ret void ++} ++ ++define void @imm4_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm4_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: vrepib %v0, 4 ++; CHECK-NEXT: vstef %v0, 0(%r2), 0 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 4, i64 4, i1 false) ++ ret void ++} ++ ++define void @imm5(ptr %Dst) { ++; CHECK-LABEL: imm5: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 5 ++; CHECK-NEXT: mvc 1(4,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 5, i64 5, i1 false) ++ ret void ++} ++ ++define void @imm5_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm5_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 5 ++; CHECK-NEXT: mvc 1(4,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 5, i64 5, i1 false) ++ ret void ++} ++ ++define void @imm6(ptr %Dst) { ++; CHECK-LABEL: imm6: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 6 ++; CHECK-NEXT: mvc 1(5,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 6, i64 6, i1 false) ++ ret void ++} ++ ++define void @imm6_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm6_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 6 ++; CHECK-NEXT: mvc 1(5,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 6, i64 6, i1 false) ++ ret void ++} ++ ++define void @imm7(ptr %Dst) { ++; CHECK-LABEL: imm7: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 7 ++; CHECK-NEXT: mvc 1(6,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 7, i64 7, i1 false) ++ ret void ++} ++ ++define void @imm7_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm7_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 7 ++; CHECK-NEXT: mvc 1(6,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 7, i64 7, i1 false) ++ ret void ++} ++ ++define void @imm8(ptr %Dst) { ++; CHECK-LABEL: imm8: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 8 ++; CHECK-NEXT: mvc 1(7,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 8, i64 8, i1 false) ++ ret void ++} ++ ++define void @imm8_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm8_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 8 ++; CHECK-NEXT: mvc 1(7,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 8, i64 8, i1 false) ++ ret void ++} ++ ++define void @imm9(ptr %Dst) { ++; CHECK-LABEL: imm9: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 9 ++; CHECK-NEXT: mvc 1(8,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 9, i64 9, i1 false) ++ ret void ++} ++ ++define void @imm9_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm9_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 9 ++; CHECK-NEXT: mvc 1(8,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 9, i64 9, i1 false) ++ ret void ++} ++ ++define void @imm10(ptr %Dst) { ++; CHECK-LABEL: imm10: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 10 ++; CHECK-NEXT: mvc 1(9,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 10, i64 10, i1 false) ++ ret void ++} ++ ++define void @imm10_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm10_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 10 ++; CHECK-NEXT: mvc 1(9,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 10, i64 10, i1 false) ++ ret void ++} ++ ++define void @imm11(ptr %Dst) { ++; CHECK-LABEL: imm11: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 11 ++; CHECK-NEXT: mvc 1(10,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 11, i64 11, i1 false) ++ ret void ++} ++ ++define void @imm11_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm11_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 11 ++; CHECK-NEXT: mvc 1(10,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 11, i64 11, i1 false) ++ ret void ++} ++ ++define void @imm12(ptr %Dst) { ++; CHECK-LABEL: imm12: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 12 ++; CHECK-NEXT: mvc 1(11,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 12, i64 12, i1 false) ++ ret void ++} ++ ++define void @imm12_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm12_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 12 ++; CHECK-NEXT: mvc 1(11,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 12, i64 12, i1 false) ++ ret void ++} ++ ++define void @imm13(ptr %Dst) { ++; CHECK-LABEL: imm13: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 13 ++; CHECK-NEXT: mvc 1(12,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 13, i64 13, i1 false) ++ ret void ++} ++ ++define void @imm13_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm13_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 13 ++; CHECK-NEXT: mvc 1(12,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 13, i64 13, i1 false) ++ ret void ++} ++ ++define void @imm14(ptr %Dst) { ++; CHECK-LABEL: imm14: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 14 ++; CHECK-NEXT: mvc 1(13,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 14, i64 14, i1 false) ++ ret void ++} ++ ++define void @imm14_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm14_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 14 ++; CHECK-NEXT: mvc 1(13,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 14, i64 14, i1 false) ++ ret void ++} ++ ++define void @imm15(ptr %Dst) { ++; CHECK-LABEL: imm15: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 15 ++; CHECK-NEXT: mvc 1(14,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 15, i64 15, i1 false) ++ ret void ++} ++ ++define void @imm15_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm15_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 15 ++; CHECK-NEXT: mvc 1(14,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 15, i64 15, i1 false) ++ ret void ++} ++ ++define void @imm16(ptr %Dst) { ++; CHECK-LABEL: imm16: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 16 ++; CHECK-NEXT: mvc 1(15,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 16, i64 16, i1 false) ++ ret void ++} ++ ++define void @imm16_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm16_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 16 ++; CHECK-NEXT: mvc 1(15,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 16, i64 16, i1 false) ++ ret void ++} ++ ++define void @imm17(ptr %Dst) { ++; CHECK-LABEL: imm17: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 17 ++; CHECK-NEXT: mvc 1(16,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 17, i64 17, i1 false) ++ ret void ++} ++ ++define void @imm17_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm17_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 17 ++; CHECK-NEXT: mvc 1(16,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 17, i64 17, i1 false) ++ ret void ++} ++ ++define void @imm18(ptr %Dst) { ++; CHECK-LABEL: imm18: ++; CHECK: # %bb.0: ++; CHECK-NEXT: vrepib %v0, 18 ++; CHECK-NEXT: vst %v0, 0(%r2), 3 ++; CHECK-NEXT: mvhhi 16(%r2), 4626 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 18, i64 18, i1 false) ++ ret void ++} ++ ++define void @imm18_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm18_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 18 ++; CHECK-NEXT: mvc 1(17,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 18, i64 18, i1 false) ++ ret void ++} ++ ++define void @imm19(ptr %Dst) { ++; CHECK-LABEL: imm19: ++; CHECK: # %bb.0: ++; CHECK-NEXT: vrepib %v0, 19 ++; CHECK-NEXT: vstef %v0, 15(%r2), 0 ++; CHECK-NEXT: vst %v0, 0(%r2), 3 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 19, i64 19, i1 false) ++ ret void ++} ++ ++define void @imm19_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm19_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 19 ++; CHECK-NEXT: mvc 1(18,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 19, i64 19, i1 false) ++ ret void ++} ++ ++define void @imm20(ptr %Dst) { ++; CHECK-LABEL: imm20: ++; CHECK: # %bb.0: ++; CHECK-NEXT: vrepib %v0, 20 ++; CHECK-NEXT: vstef %v0, 16(%r2), 0 ++; CHECK-NEXT: vst %v0, 0(%r2), 3 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 20, i64 20, i1 false) ++ ret void ++} ++ ++define void @imm20_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm20_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 20 ++; CHECK-NEXT: mvc 1(19,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 20, i64 20, i1 false) ++ ret void ++} ++ ++define void @imm20_localDst() { ++; CHECK-LABEL: imm20_localDst: ++; CHECK: # %bb.0: ++; CHECK-NEXT: aghi %r15, -184 ++; CHECK-NEXT: .cfi_def_cfa_offset 344 ++; CHECK-NEXT: vrepib %v0, 20 ++; CHECK-NEXT: vstef %v0, 180(%r15), 0 ++; CHECK-NEXT: vst %v0, 164(%r15), 4 ++; CHECK-NEXT: aghi %r15, 184 ++; CHECK-NEXT: br %r14 ++ %Dst = alloca [20 x i8] ++ call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 20, i64 20, i1 false) ++ ret void ++} ++ ++define void @imm21(ptr %Dst) { ++; CHECK-LABEL: imm21: ++; CHECK: # %bb.0: ++; CHECK-NEXT: vrepib %v0, 21 ++; CHECK-NEXT: vsteg %v0, 13(%r2), 0 ++; CHECK-NEXT: vst %v0, 0(%r2), 3 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 21, i64 21, i1 false) ++ ret void ++} ++ ++define void @imm21_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm21_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 21 ++; CHECK-NEXT: mvc 1(20,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 21, i64 21, i1 false) ++ ret void ++} ++ ++define void @imm22(ptr %Dst) { ++; CHECK-LABEL: imm22: ++; CHECK: # %bb.0: ++; CHECK-NEXT: vrepib %v0, 22 ++; CHECK-NEXT: vsteg %v0, 14(%r2), 0 ++; CHECK-NEXT: vst %v0, 0(%r2), 3 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 22, i64 22, i1 false) ++ ret void ++} ++ ++define void @imm22_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm22_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 22 ++; CHECK-NEXT: mvc 1(21,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 22, i64 22, i1 false) ++ ret void ++} ++ ++define void @imm23(ptr %Dst) { ++; CHECK-LABEL: imm23: ++; CHECK: # %bb.0: ++; CHECK-NEXT: vrepib %v0, 23 ++; CHECK-NEXT: vsteg %v0, 15(%r2), 0 ++; CHECK-NEXT: vst %v0, 0(%r2), 3 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 23, i64 23, i1 false) ++ ret void ++} ++ ++define void @imm23_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm23_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 23 ++; CHECK-NEXT: mvc 1(22,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 23, i64 23, i1 false) ++ ret void ++} ++ ++define void @imm24(ptr %Dst) { ++; CHECK-LABEL: imm24: ++; CHECK: # %bb.0: ++; CHECK-NEXT: vrepib %v0, 24 ++; CHECK-NEXT: vsteg %v0, 16(%r2), 0 ++; CHECK-NEXT: vst %v0, 0(%r2), 3 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 24, i64 24, i1 false) ++ ret void ++} ++ ++define void @imm24_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm24_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 24 ++; CHECK-NEXT: mvc 1(23,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 24, i64 24, i1 false) ++ ret void ++} ++ ++define void @imm25(ptr %Dst) { ++; CHECK-LABEL: imm25: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 25 ++; CHECK-NEXT: mvc 1(24,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 25, i64 25, i1 false) ++ ret void ++} ++ ++define void @imm25_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm25_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 25 ++; CHECK-NEXT: mvc 1(24,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 25, i64 25, i1 false) ++ ret void ++} ++ ++define void @imm26(ptr %Dst) { ++; CHECK-LABEL: imm26: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 26 ++; CHECK-NEXT: mvc 1(25,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 26, i64 26, i1 false) ++ ret void ++} ++ ++define void @imm26_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm26_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 26 ++; CHECK-NEXT: mvc 1(25,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 26, i64 26, i1 false) ++ ret void ++} ++ ++define void @imm27(ptr %Dst) { ++; CHECK-LABEL: imm27: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 27 ++; CHECK-NEXT: mvc 1(26,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 27, i64 27, i1 false) ++ ret void ++} ++ ++define void @imm27_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm27_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 27 ++; CHECK-NEXT: mvc 1(26,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 27, i64 27, i1 false) ++ ret void ++} ++ ++define void @imm28(ptr %Dst) { ++; CHECK-LABEL: imm28: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 28 ++; CHECK-NEXT: mvc 1(27,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 28, i64 28, i1 false) ++ ret void ++} ++ ++define void @imm28_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm28_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 28 ++; CHECK-NEXT: mvc 1(27,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 28, i64 28, i1 false) ++ ret void ++} ++ ++define void @imm29(ptr %Dst) { ++; CHECK-LABEL: imm29: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 29 ++; CHECK-NEXT: mvc 1(28,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 29, i64 29, i1 false) ++ ret void ++} ++ ++define void @imm29_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm29_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 29 ++; CHECK-NEXT: mvc 1(28,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 29, i64 29, i1 false) ++ ret void ++} ++ ++define void @imm30(ptr %Dst) { ++; CHECK-LABEL: imm30: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 30 ++; CHECK-NEXT: mvc 1(29,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 30, i64 30, i1 false) ++ ret void ++} ++ ++define void @imm30_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm30_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 30 ++; CHECK-NEXT: mvc 1(29,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 30, i64 30, i1 false) ++ ret void ++} ++ ++define void @imm31(ptr %Dst) { ++; CHECK-LABEL: imm31: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 31 ++; CHECK-NEXT: mvc 1(30,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 31, i64 31, i1 false) ++ ret void ++} ++ ++define void @imm31_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm31_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 31 ++; CHECK-NEXT: mvc 1(30,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 31, i64 31, i1 false) ++ ret void ++} ++ ++define void @imm32(ptr %Dst) { ++; CHECK-LABEL: imm32: ++; CHECK: # %bb.0: ++; CHECK-NEXT: vrepib %v0, 32 ++; CHECK-NEXT: vst %v0, 16(%r2), 3 ++; CHECK-NEXT: vst %v0, 0(%r2), 3 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 32, i64 32, i1 false) ++ ret void ++} ++ ++define void @imm32_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm32_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 32 ++; CHECK-NEXT: mvc 1(31,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 32, i64 32, i1 false) ++ ret void ++} ++ ++define void @imm33(ptr %Dst) { ++; CHECK-LABEL: imm33: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 33 ++; CHECK-NEXT: mvc 1(32,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 33, i64 33, i1 false) ++ ret void ++} ++ ++define void @imm33_unaligned(ptr %Dst) { ++; CHECK-LABEL: imm33_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 33 ++; CHECK-NEXT: mvc 1(32,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 33, i64 33, i1 false) ++ ret void ++} ++ ++;; zero ++ ++define void @zero1(ptr %Dst) { ++; CHECK-LABEL: zero1: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 0 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 1, i1 false) ++ ret void ++} ++ ++define void @zero1_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero1_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvi 0(%r2), 0 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 1, i1 false) ++ ret void ++} ++ ++define void @zero2(ptr %Dst) { ++; CHECK-LABEL: zero2: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvhhi 0(%r2), 0 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 2, i1 false) ++ ret void ++} ++ ++define void @zero2_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero2_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvhhi 0(%r2), 0 + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 17, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 2, i1 false) + ret void + } + +-define void @reg18(ptr %Dst, i8 %val) { +-; CHECK-LABEL: reg18: ++define void @zero3(ptr %Dst) { ++; CHECK-LABEL: zero3: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vlvgp %v0, %r3, %r3 +-; CHECK-NEXT: vrepb %v0, %v0, 7 +-; CHECK-NEXT: vst %v0, 0(%r2), 4 +-; CHECK-NEXT: vsteh %v0, 16(%r2), 0 ++; CHECK-NEXT: mvi 2(%r2), 0 ++; CHECK-NEXT: mvhhi 0(%r2), 0 + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 18, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 3, i1 false) + ret void + } + +-define void @reg19(ptr %Dst, i8 %val) { +-; CHECK-LABEL: reg19: ++define void @zero3_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero3_unaligned: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vlvgp %v0, %r3, %r3 +-; CHECK-NEXT: vrepb %v0, %v0, 7 +-; CHECK-NEXT: vstef %v0, 15(%r2), 0 +-; CHECK-NEXT: vst %v0, 0(%r2), 4 ++; CHECK-NEXT: mvi 2(%r2), 0 ++; CHECK-NEXT: mvhhi 0(%r2), 0 + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 19, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 3, i1 false) + ret void + } + +-define void @reg20(ptr %Dst, i8 %val) { +-; CHECK-LABEL: reg20: ++define void @zero4(ptr %Dst) { ++; CHECK-LABEL: zero4: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vlvgp %v0, %r3, %r3 +-; CHECK-NEXT: vrepb %v0, %v0, 7 +-; CHECK-NEXT: vstef %v0, 16(%r2), 0 +-; CHECK-NEXT: vst %v0, 0(%r2), 4 ++; CHECK-NEXT: mvhi 0(%r2), 0 + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 20, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 4, i1 false) + ret void + } + +-define void @reg21(ptr %Dst, i8 %val) { +-; CHECK-LABEL: reg21: ++define void @zero4_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero4_unaligned: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vlvgp %v0, %r3, %r3 +-; CHECK-NEXT: vrepb %v0, %v0, 7 +-; CHECK-NEXT: vsteg %v0, 13(%r2), 0 +-; CHECK-NEXT: vst %v0, 0(%r2), 4 ++; CHECK-NEXT: mvhi 0(%r2), 0 + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 21, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 4, i1 false) + ret void + } + +-define void @reg22(ptr %Dst, i8 %val) { +-; CHECK-LABEL: reg22: ++define void @zero5(ptr %Dst) { ++; CHECK-LABEL: zero5: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vlvgp %v0, %r3, %r3 +-; CHECK-NEXT: vrepb %v0, %v0, 7 +-; CHECK-NEXT: vsteg %v0, 14(%r2), 0 +-; CHECK-NEXT: vst %v0, 0(%r2), 4 ++; CHECK-NEXT: mvi 4(%r2), 0 ++; CHECK-NEXT: mvhi 0(%r2), 0 + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 22, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 5, i1 false) + ret void + } + +-define void @reg23(ptr %Dst, i8 %val) { +-; CHECK-LABEL: reg23: ++define void @zero5_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero5_unaligned: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vlvgp %v0, %r3, %r3 +-; CHECK-NEXT: vrepb %v0, %v0, 7 +-; CHECK-NEXT: vsteg %v0, 15(%r2), 0 +-; CHECK-NEXT: vst %v0, 0(%r2), 4 ++; CHECK-NEXT: mvi 4(%r2), 0 ++; CHECK-NEXT: mvhi 0(%r2), 0 + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 23, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 5, i1 false) + ret void + } + +-define void @reg24(ptr %Dst, i8 %val) { +-; CHECK-LABEL: reg24: ++define void @zero6(ptr %Dst) { ++; CHECK-LABEL: zero6: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vlvgp %v0, %r3, %r3 +-; CHECK-NEXT: vrepb %v0, %v0, 7 +-; CHECK-NEXT: vsteg %v0, 16(%r2), 0 +-; CHECK-NEXT: vst %v0, 0(%r2), 4 ++; CHECK-NEXT: mvhhi 4(%r2), 0 ++; CHECK-NEXT: mvhi 0(%r2), 0 + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 24, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 6, i1 false) + ret void + } + +-define void @reg25(ptr %Dst, i8 %val) { +-; CHECK-LABEL: reg25: ++define void @zero6_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero6_unaligned: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vlvgp %v0, %r3, %r3 +-; CHECK-NEXT: vrepb %v0, %v0, 7 +-; CHECK-NEXT: vst %v0, 9(%r2) +-; CHECK-NEXT: vst %v0, 0(%r2), 4 ++; CHECK-NEXT: mvhhi 4(%r2), 0 ++; CHECK-NEXT: mvhi 0(%r2), 0 + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 25, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 6, i1 false) + ret void + } + +-define void @reg26(ptr %Dst, i8 %val) { +-; CHECK-LABEL: reg26: ++define void @zero7(ptr %Dst) { ++; CHECK-LABEL: zero7: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vlvgp %v0, %r3, %r3 +-; CHECK-NEXT: vrepb %v0, %v0, 7 +-; CHECK-NEXT: vst %v0, 10(%r2) +-; CHECK-NEXT: vst %v0, 0(%r2), 4 ++; CHECK-NEXT: xc 0(7,%r2), 0(%r2) + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 26, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 7, i1 false) + ret void + } + +-define void @reg27(ptr %Dst, i8 %val) { +-; CHECK-LABEL: reg27: ++define void @zero7_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero7_unaligned: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vlvgp %v0, %r3, %r3 +-; CHECK-NEXT: vrepb %v0, %v0, 7 +-; CHECK-NEXT: vst %v0, 11(%r2) +-; CHECK-NEXT: vst %v0, 0(%r2), 4 ++; CHECK-NEXT: xc 0(7,%r2), 0(%r2) + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 27, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 7, i1 false) + ret void + } + +-define void @reg28(ptr %Dst, i8 %val) { +-; CHECK-LABEL: reg28: ++define void @zero8(ptr %Dst) { ++; CHECK-LABEL: zero8: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vlvgp %v0, %r3, %r3 +-; CHECK-NEXT: vrepb %v0, %v0, 7 +-; CHECK-NEXT: vst %v0, 12(%r2) +-; CHECK-NEXT: vst %v0, 0(%r2), 4 ++; CHECK-NEXT: mvghi 0(%r2), 0 + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 28, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 8, i1 false) + ret void + } + +-define void @reg29(ptr %Dst, i8 %val) { +-; CHECK-LABEL: reg29: ++define void @zero8_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero8_unaligned: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vlvgp %v0, %r3, %r3 +-; CHECK-NEXT: vrepb %v0, %v0, 7 +-; CHECK-NEXT: vst %v0, 13(%r2) +-; CHECK-NEXT: vst %v0, 0(%r2), 4 ++; CHECK-NEXT: mvi 0(%r2), 8 ++; CHECK-NEXT: mvc 1(7,%r2), 0(%r2) + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 29, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 8, i64 8, i1 false) + ret void + } + +-define void @reg30(ptr %Dst, i8 %val) { +-; CHECK-LABEL: reg30: ++define void @zero9(ptr %Dst) { ++; CHECK-LABEL: zero9: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vlvgp %v0, %r3, %r3 +-; CHECK-NEXT: vrepb %v0, %v0, 7 +-; CHECK-NEXT: vst %v0, 14(%r2) +-; CHECK-NEXT: vst %v0, 0(%r2), 4 ++; CHECK-NEXT: mvi 8(%r2), 0 ++; CHECK-NEXT: mvghi 0(%r2), 0 + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 30, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 9, i1 false) + ret void + } + +-define void @reg31(ptr %Dst, i8 %val) { +-; CHECK-LABEL: reg31: ++define void @zero9_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero9_unaligned: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vlvgp %v0, %r3, %r3 +-; CHECK-NEXT: vrepb %v0, %v0, 7 +-; CHECK-NEXT: vst %v0, 15(%r2) +-; CHECK-NEXT: vst %v0, 0(%r2), 4 ++; CHECK-NEXT: mvi 8(%r2), 0 ++; CHECK-NEXT: mvghi 0(%r2), 0 + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 31, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 9, i1 false) + ret void + } + +-define void @reg32(ptr %Dst, i8 %val) { +-; CHECK-LABEL: reg32: ++define void @zero10(ptr %Dst) { ++; CHECK-LABEL: zero10: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vlvgp %v0, %r3, %r3 +-; CHECK-NEXT: vrepb %v0, %v0, 7 +-; CHECK-NEXT: vst %v0, 16(%r2), 4 +-; CHECK-NEXT: vst %v0, 0(%r2), 4 ++; CHECK-NEXT: mvhhi 8(%r2), 0 ++; CHECK-NEXT: mvghi 0(%r2), 0 + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 32, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 10, i1 false) + ret void + } + +-define void @reg33(ptr %Dst, i8 %val) { +-; CHECK-LABEL: reg33: ++define void @zero10_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero10_unaligned: + ; CHECK: # %bb.0: +-; CHECK-NEXT: stc %r3, 0(%r2) +-; CHECK-NEXT: mvc 1(32,%r2), 0(%r2) ++; CHECK-NEXT: mvhhi 8(%r2), 0 ++; CHECK-NEXT: mvghi 0(%r2), 0 + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 %val, i64 33, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 10, i1 false) + ret void + } + +-;; Immediate value ++define void @zero11(ptr %Dst) { ++; CHECK-LABEL: zero11: ++; CHECK: # %bb.0: ++; CHECK-NEXT: xc 0(11,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 11, i1 false) ++ ret void ++} + +-define void @imm17(ptr %Dst) { +-; CHECK-LABEL: imm17: ++define void @zero11_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero11_unaligned: + ; CHECK: # %bb.0: +-; CHECK-NEXT: mvi 0(%r2), 1 +-; CHECK-NEXT: mvc 1(16,%r2), 0(%r2) ++; CHECK-NEXT: xc 0(11,%r2), 0(%r2) + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 1, i64 17, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 11, i1 false) + ret void + } + +-define void @imm18(ptr %Dst) { +-; CHECK-LABEL: imm18: ++define void @zero12(ptr %Dst) { ++; CHECK-LABEL: zero12: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvhi 8(%r2), 0 ++; CHECK-NEXT: mvghi 0(%r2), 0 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 12, i1 false) ++ ret void ++} ++ ++define void @zero12_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero12_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: mvhi 8(%r2), 0 ++; CHECK-NEXT: mvghi 0(%r2), 0 ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 12, i1 false) ++ ret void ++} ++ ++define void @zero13(ptr %Dst) { ++; CHECK-LABEL: zero13: ++; CHECK: # %bb.0: ++; CHECK-NEXT: xc 0(13,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 13, i1 false) ++ ret void ++} ++ ++define void @zero13_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero13_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: xc 0(13,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 13, i1 false) ++ ret void ++} ++ ++define void @zero14(ptr %Dst) { ++; CHECK-LABEL: zero14: ++; CHECK: # %bb.0: ++; CHECK-NEXT: xc 0(14,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 14, i1 false) ++ ret void ++} ++ ++define void @zero14_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero14_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: xc 0(14,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 14, i1 false) ++ ret void ++} ++ ++define void @zero15(ptr %Dst) { ++; CHECK-LABEL: zero15: ++; CHECK: # %bb.0: ++; CHECK-NEXT: xc 0(15,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 15, i1 false) ++ ret void ++} ++ ++define void @zero15_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero15_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: xc 0(15,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 15, i1 false) ++ ret void ++} ++ ++define void @zero16(ptr %Dst) { ++; CHECK-LABEL: zero16: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vgbm %v0, 65535 +-; CHECK-NEXT: vst %v0, 0(%r2), 4 +-; CHECK-NEXT: mvhhi 16(%r2), -1 ++; CHECK-NEXT: vgbm %v0, 0 ++; CHECK-NEXT: vst %v0, 0(%r2), 3 + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 -1, i64 18, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 16, i1 false) ++ ret void ++} ++ ++define void @zero16_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero16_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: vgbm %v0, 0 ++; CHECK-NEXT: vst %v0, 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 16, i1 false) ++ ret void ++} ++ ++define void @zero17(ptr %Dst) { ++; CHECK-LABEL: zero17: ++; CHECK: # %bb.0: ++; CHECK-NEXT: xc 0(17,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 17, i1 false) ++ ret void ++} ++ ++define void @zero17_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero17_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: xc 0(17,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 17, i1 false) + ret void + } + +@@ -233,95 +1705,155 @@ define void @zero18(ptr %Dst) { + ; CHECK: # %bb.0: + ; CHECK-NEXT: xc 0(18,%r2), 0(%r2) + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 0, i64 18, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 18, i1 false) + ret void + } + +-define void @imm19(ptr %Dst) { +-; CHECK-LABEL: imm19: ++define void @zero18_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero18_unaligned: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vrepib %v0, 1 +-; CHECK-NEXT: vstef %v0, 15(%r2), 0 +-; CHECK-NEXT: vst %v0, 0(%r2), 4 ++; CHECK-NEXT: xc 0(18,%r2), 0(%r2) + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 1, i64 19, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 18, i1 false) + ret void + } + +-define void @imm20(ptr %Dst) { +-; CHECK-LABEL: imm20: ++define void @zero19(ptr %Dst) { ++; CHECK-LABEL: zero19: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vgbm %v0, 65535 +-; CHECK-NEXT: vst %v0, 0(%r2), 4 +-; CHECK-NEXT: mvhi 16(%r2), -1 ++; CHECK-NEXT: xc 0(19,%r2), 0(%r2) + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 -1, i64 20, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 19, i1 false) + ret void + } + +-define void @imm21(ptr %Dst) { +-; CHECK-LABEL: imm21: ++define void @zero19_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero19_unaligned: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vrepib %v0, 1 +-; CHECK-NEXT: vsteg %v0, 13(%r2), 0 +-; CHECK-NEXT: vst %v0, 0(%r2), 4 ++; CHECK-NEXT: xc 0(19,%r2), 0(%r2) + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 1, i64 21, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 19, i1 false) + ret void + } + +-define void @imm22(ptr %Dst) { +-; CHECK-LABEL: imm22: ++define void @zero20(ptr %Dst) { ++; CHECK-LABEL: zero20: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vgbm %v0, 65535 +-; CHECK-NEXT: vst %v0, 0(%r2), 4 +-; CHECK-NEXT: mvghi 14(%r2), -1 ++; CHECK-NEXT: xc 0(20,%r2), 0(%r2) + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 -1, i64 22, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 20, i1 false) + ret void + } + +-define void @imm23(ptr %Dst) { +-; CHECK-LABEL: imm23: ++define void @zero20_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero20_unaligned: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vrepib %v0, 1 +-; CHECK-NEXT: vsteg %v0, 15(%r2), 0 +-; CHECK-NEXT: vst %v0, 0(%r2), 4 ++; CHECK-NEXT: xc 0(20,%r2), 0(%r2) + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 1, i64 23, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 20, i1 false) + ret void + } + +-define void @imm24(ptr %Dst) { +-; CHECK-LABEL: imm24: ++define void @zero20_localDst() { ++; CHECK-LABEL: zero20_localDst: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vgbm %v0, 65535 +-; CHECK-NEXT: vst %v0, 0(%r2), 4 +-; CHECK-NEXT: mvghi 16(%r2), -1 ++; CHECK-NEXT: aghi %r15, -184 ++; CHECK-NEXT: .cfi_def_cfa_offset 344 ++; CHECK-NEXT: xc 164(20,%r15), 164(%r15) ++; CHECK-NEXT: aghi %r15, 184 + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 -1, i64 24, i1 false) ++ %Dst = alloca [20 x i8] ++ call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 0, i64 20, i1 false) + ret void + } + +-define void @imm25(ptr %Dst) { +-; CHECK-LABEL: imm25: ++define void @zero21(ptr %Dst) { ++; CHECK-LABEL: zero21: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vrepib %v0, 1 +-; CHECK-NEXT: vst %v0, 9(%r2) +-; CHECK-NEXT: vst %v0, 0(%r2), 4 ++; CHECK-NEXT: xc 0(21,%r2), 0(%r2) + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 1, i64 25, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 21, i1 false) + ret void + } + +-define void @imm26(ptr %Dst) { +-; CHECK-LABEL: imm26: ++define void @zero21_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero21_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: xc 0(21,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 21, i1 false) ++ ret void ++} ++ ++define void @zero22(ptr %Dst) { ++; CHECK-LABEL: zero22: ++; CHECK: # %bb.0: ++; CHECK-NEXT: xc 0(22,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 22, i1 false) ++ ret void ++} ++ ++define void @zero22_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero22_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: xc 0(22,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 22, i1 false) ++ ret void ++} ++ ++define void @zero23(ptr %Dst) { ++; CHECK-LABEL: zero23: ++; CHECK: # %bb.0: ++; CHECK-NEXT: xc 0(23,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 23, i1 false) ++ ret void ++} ++ ++define void @zero23_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero23_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: xc 0(23,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 23, i1 false) ++ ret void ++} ++ ++define void @zero24(ptr %Dst) { ++; CHECK-LABEL: zero24: ++; CHECK: # %bb.0: ++; CHECK-NEXT: xc 0(24,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 24, i1 false) ++ ret void ++} ++ ++define void @zero24_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero24_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: xc 0(24,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 24, i1 false) ++ ret void ++} ++ ++define void @zero25(ptr %Dst) { ++; CHECK-LABEL: zero25: ++; CHECK: # %bb.0: ++; CHECK-NEXT: xc 0(25,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 25, i1 false) ++ ret void ++} ++ ++define void @zero25_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero25_unaligned: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vgbm %v0, 65535 +-; CHECK-NEXT: vst %v0, 10(%r2) +-; CHECK-NEXT: vst %v0, 0(%r2), 4 ++; CHECK-NEXT: xc 0(25,%r2), 0(%r2) + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 -1, i64 26, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 25, i1 false) + ret void + } + +@@ -330,73 +1862,106 @@ define void @zero26(ptr %Dst) { + ; CHECK: # %bb.0: + ; CHECK-NEXT: xc 0(26,%r2), 0(%r2) + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 0, i64 26, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 26, i1 false) + ret void + } + +-define void @imm27(ptr %Dst) { +-; CHECK-LABEL: imm27: ++define void @zero26_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero26_unaligned: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vrepib %v0, 1 +-; CHECK-NEXT: vst %v0, 11(%r2) +-; CHECK-NEXT: vst %v0, 0(%r2), 4 ++; CHECK-NEXT: xc 0(26,%r2), 0(%r2) + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 1, i64 27, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 26, i1 false) + ret void + } + +-define void @imm28(ptr %Dst) { +-; CHECK-LABEL: imm28: ++define void @zero27(ptr %Dst) { ++; CHECK-LABEL: zero27: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vgbm %v0, 65535 +-; CHECK-NEXT: vst %v0, 12(%r2) +-; CHECK-NEXT: vst %v0, 0(%r2), 4 ++; CHECK-NEXT: xc 0(27,%r2), 0(%r2) + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 -1, i64 28, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 27, i1 false) + ret void + } + +-define void @imm29(ptr %Dst) { +-; CHECK-LABEL: imm29: ++define void @zero27_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero27_unaligned: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vrepib %v0, 1 +-; CHECK-NEXT: vst %v0, 13(%r2) +-; CHECK-NEXT: vst %v0, 0(%r2), 4 ++; CHECK-NEXT: xc 0(27,%r2), 0(%r2) + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 1, i64 29, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 27, i1 false) + ret void + } + +-define void @imm30(ptr %Dst) { +-; CHECK-LABEL: imm30: ++define void @zero28(ptr %Dst) { ++; CHECK-LABEL: zero28: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vgbm %v0, 65535 +-; CHECK-NEXT: vst %v0, 14(%r2) +-; CHECK-NEXT: vst %v0, 0(%r2), 4 ++; CHECK-NEXT: xc 0(28,%r2), 0(%r2) + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 -1, i64 30, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 28, i1 false) + ret void + } + +-define void @imm31(ptr %Dst) { +-; CHECK-LABEL: imm31: ++define void @zero28_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero28_unaligned: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vrepib %v0, 1 +-; CHECK-NEXT: vst %v0, 15(%r2) +-; CHECK-NEXT: vst %v0, 0(%r2), 4 ++; CHECK-NEXT: xc 0(28,%r2), 0(%r2) + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 1, i64 31, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 28, i1 false) + ret void + } + +-define void @imm32(ptr %Dst) { +-; CHECK-LABEL: imm32: ++define void @zero29(ptr %Dst) { ++; CHECK-LABEL: zero29: ++; CHECK: # %bb.0: ++; CHECK-NEXT: xc 0(29,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 29, i1 false) ++ ret void ++} ++ ++define void @zero29_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero29_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: xc 0(29,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 29, i1 false) ++ ret void ++} ++ ++define void @zero30(ptr %Dst) { ++; CHECK-LABEL: zero30: ++; CHECK: # %bb.0: ++; CHECK-NEXT: xc 0(30,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 30, i1 false) ++ ret void ++} ++ ++define void @zero30_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero30_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: xc 0(30,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 30, i1 false) ++ ret void ++} ++ ++define void @zero31(ptr %Dst) { ++; CHECK-LABEL: zero31: ++; CHECK: # %bb.0: ++; CHECK-NEXT: xc 0(31,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 31, i1 false) ++ ret void ++} ++ ++define void @zero31_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero31_unaligned: + ; CHECK: # %bb.0: +-; CHECK-NEXT: vgbm %v0, 65535 +-; CHECK-NEXT: vst %v0, 16(%r2), 4 +-; CHECK-NEXT: vst %v0, 0(%r2), 4 ++; CHECK-NEXT: xc 0(31,%r2), 0(%r2) + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 -1, i64 32, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 31, i1 false) + ret void + } + +@@ -405,16 +1970,33 @@ define void @zero32(ptr %Dst) { + ; CHECK: # %bb.0: + ; CHECK-NEXT: xc 0(32,%r2), 0(%r2) + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 0, i64 32, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 32, i1 false) + ret void + } + +-define void @imm33(ptr %Dst) { +-; CHECK-LABEL: imm33: ++define void @zero32_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero32_unaligned: + ; CHECK: # %bb.0: +-; CHECK-NEXT: mvi 0(%r2), 1 +-; CHECK-NEXT: mvc 1(32,%r2), 0(%r2) ++; CHECK-NEXT: xc 0(32,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 32, i1 false) ++ ret void ++} ++ ++define void @zero33(ptr %Dst) { ++; CHECK-LABEL: zero33: ++; CHECK: # %bb.0: ++; CHECK-NEXT: xc 0(33,%r2), 0(%r2) ++; CHECK-NEXT: br %r14 ++ call void @llvm.memset.p0.i64(ptr align 8 %Dst, i8 0, i64 33, i1 false) ++ ret void ++} ++ ++define void @zero33_unaligned(ptr %Dst) { ++; CHECK-LABEL: zero33_unaligned: ++; CHECK: # %bb.0: ++; CHECK-NEXT: xc 0(33,%r2), 0(%r2) + ; CHECK-NEXT: br %r14 +- call void @llvm.memset.p0.i64(ptr align 16 %Dst, i8 1, i64 33, i1 false) ++ call void @llvm.memset.p0.i64(ptr align 1 %Dst, i8 0, i64 33, i1 false) + ret void + } +-- +2.54.0 + diff --git a/0001-X86-Fix-EVEX-compression-for-VPMOV-2M-KMOV-with-tied.patch b/0001-X86-Fix-EVEX-compression-for-VPMOV-2M-KMOV-with-tied.patch new file mode 100644 index 0000000..ede6fb3 --- /dev/null +++ b/0001-X86-Fix-EVEX-compression-for-VPMOV-2M-KMOV-with-tied.patch @@ -0,0 +1,81 @@ +From a04c1eced55f2f3ea8dbd3d17db0b6df271c0809 Mon Sep 17 00:00:00 2001 +From: Stefan Weigl-Bosker +Date: Mon, 18 May 2026 10:47:14 -0400 +Subject: [PATCH] [X86] Fix EVEX compression for VPMOV*2M + KMOV with tied mask + use (#198220) + +When scanning uses of the mask produced by `VPMOV*2M`, we previously bailed out as soon as we encountered a write. For tied read/write mask instructions such as `KSHIFTR*`, which both read and write the same mask register, the pass could miss the use, fold the earlier `KMOV`, and erase the `VPMOV*2M` def even though the mask was still live. + +Disclaimer: LLM came up with the MIR tests and explained this pass to me. + +Fixes #198197 + +(cherry picked from commit f0fc9d0abf7024ceb5cb827b16f02c10e54fe0fd) +--- + llvm/lib/Target/X86/X86CompressEVEX.cpp | 12 ++++++------ + llvm/test/CodeGen/X86/evex-to-vex-compress.mir | 12 ++++++++++++ + 2 files changed, 18 insertions(+), 6 deletions(-) + +diff --git a/llvm/lib/Target/X86/X86CompressEVEX.cpp b/llvm/lib/Target/X86/X86CompressEVEX.cpp +index c1faf7d1aa1e..b8fbcd2582de 100644 +--- a/llvm/lib/Target/X86/X86CompressEVEX.cpp ++++ b/llvm/lib/Target/X86/X86CompressEVEX.cpp +@@ -271,12 +271,6 @@ static bool tryCompressVPMOVPattern(MachineInstr &MI, MachineBasicBlock &MBB, + + for (MachineInstr &CurMI : llvm::make_range( + std::next(MachineBasicBlock::iterator(MI)), MBB.end())) { +- if (CurMI.modifiesRegister(MaskReg, TRI)) { +- if (!KMovMI) +- return false; // Mask clobbered before use +- break; +- } +- + if (CurMI.readsRegister(MaskReg, TRI)) { + if (KMovMI) + return false; // Fail: Mask has MULTIPLE uses +@@ -295,6 +289,12 @@ static bool tryCompressVPMOVPattern(MachineInstr &MI, MachineBasicBlock &MBB, + } + } + ++ if (CurMI.modifiesRegister(MaskReg, TRI)) { ++ if (!KMovMI) ++ return false; // Mask clobbered before use ++ break; ++ } ++ + if (!KMovMI && CurMI.modifiesRegister(SrcVecReg, TRI)) { + return false; // SrcVecReg modified before it could be used by MOVMSK + } +diff --git a/llvm/test/CodeGen/X86/evex-to-vex-compress.mir b/llvm/test/CodeGen/X86/evex-to-vex-compress.mir +index b33a1d571c81..575f0c7d6a4f 100644 +--- a/llvm/test/CodeGen/X86/evex-to-vex-compress.mir ++++ b/llvm/test/CodeGen/X86/evex-to-vex-compress.mir +@@ -914,6 +914,12 @@ body: | + $k0 = VPMOVD2MZ256kr $ymm0 + $eax = KMOVBrk $k0 + $ebx = KMOVBrk $k0 ++ ; CHECK: $k0 = VPMOVD2MZ256kr $ymm0 ++ ; CHECK: $eax = KMOVBrk $k0 ++ ; CHECK: $k0 = KSHIFTRBki $k0, 2 ++ $k0 = VPMOVD2MZ256kr $ymm0 ++ $eax = KMOVBrk $k0 ++ $k0 = KSHIFTRBki $k0, 2 + ; CHECK: $k0 = VPMOVB2MZ256kr $ymm0 + ; CHECK: $eax = KMOVWrk $k0 + $k0 = VPMOVB2MZ256kr $ymm0 +@@ -1803,6 +1809,12 @@ body: | + $k0 = VPMOVD2MZ128kr $xmm0 + $eax = KMOVBrk $k0 + $ebx = KMOVBrk $k0 ++ ; CHECK: $k0 = VPMOVD2MZ128kr $xmm0 ++ ; CHECK: $eax = KMOVBrk $k0 ++ ; CHECK: $k0 = KSHIFTRBki $k0, 2 ++ $k0 = VPMOVD2MZ128kr $xmm0 ++ $eax = KMOVBrk $k0 ++ $k0 = KSHIFTRBki $k0, 2 + ; CHECK: $k0 = VPMOVB2MZ128kr $xmm0 + ; CHECK: $eax = KMOVBrk $k0 + $k0 = VPMOVB2MZ128kr $xmm0 +-- +2.50.1 + diff --git a/0001-lld-ELF-Concatenate-.gnu.build.attributes.-sections-.patch b/0001-lld-ELF-Concatenate-.gnu.build.attributes.-sections-.patch new file mode 100644 index 0000000..e4d530a --- /dev/null +++ b/0001-lld-ELF-Concatenate-.gnu.build.attributes.-sections-.patch @@ -0,0 +1,87 @@ +From 2119e359b1f968da94ff27a4078d569e18903aef Mon Sep 17 00:00:00 2001 +From: Nikita Popov +Date: Mon, 13 Jul 2026 09:08:54 +0200 +Subject: [PATCH] [lld][ELF] Concatenate .gnu.build.attributes.* sections + (#208737) + +ld.bfd/ld.gold have been concatenating the GNU build attribute sections +since 2018: + +https://gitlab.com/gnutools/binutils-gdb/-/commit/7d8a31665739412395f6dd370d2279acd322e78e + +Do the same in LLD. These do not have a dedicated section type or flags, +so this is handled by the name-based logic. (Peculiarly, there used to +be SHF_GNU_BUILD_NOTE, but it was removed again.) + +Not concatenating these results in a huge number of sections, which +breaks tools like `file`. +--- + lld/ELF/LinkerScript.cpp | 2 +- + lld/test/ELF/gnu-build-attributes.s | 42 +++++++++++++++++++++++++++++ + 2 files changed, 43 insertions(+), 1 deletion(-) + create mode 100644 lld/test/ELF/gnu-build-attributes.s + +diff --git a/lld/ELF/LinkerScript.cpp b/lld/ELF/LinkerScript.cpp +index 1c0a49a73962..64fb717e17d9 100644 +--- a/lld/ELF/LinkerScript.cpp ++++ b/lld/ELF/LinkerScript.cpp +@@ -119,7 +119,7 @@ StringRef LinkerScript::getOutputSectionName(const InputSectionBase *s) const { + ".init_array", ".fini_array", ".tbss", + ".tdata", ".ARM.exidx", ".ARM.extab", + ".ctors", ".dtors", ".sbss", +- ".sdata", ".srodata"}) ++ ".sdata", ".srodata", ".gnu.build.attributes"}) + if (isSectionPrefix(v, s->name)) + return v; + +diff --git a/lld/test/ELF/gnu-build-attributes.s b/lld/test/ELF/gnu-build-attributes.s +new file mode 100644 +index 000000000000..79ad6c40ae50 +--- /dev/null ++++ b/lld/test/ELF/gnu-build-attributes.s +@@ -0,0 +1,42 @@ ++# REQUIRES: x86 ++ ++## Check that .gnu.build.attributes.* sections are concatenated into a single ++## .gnu.build.attributes section. ++ ++# RUN: llvm-mc -filetype=obj -triple=x86_64-linux-gnu %s -o %t.o ++# RUN: ld.lld %t.o -o %t ++# RUN: llvm-readobj -n %t | FileCheck %s ++ ++# CHECK: NoteSections [ ++# CHECK-NEXT: NoteSection { ++# CHECK-NEXT: Name: .gnu.build.attributes ++# CHECK-NEXT: Offset: 0x120 ++# CHECK-NEXT: Size: 0x28 ++# CHECK-NEXT: Notes [ ++# CHECK-NEXT: { ++# CHECK-NEXT: Owner: GA${{.*}}:a1 ++# CHECK-NEXT: Data size: 0x0 ++# CHECK-NEXT: Type: OPEN ++# CHECK-NEXT: } ++# CHECK-NEXT: { ++# CHECK-NEXT: Owner: GA${{.*}}:b1 ++# CHECK-NEXT: Data size: 0x0 ++# CHECK-NEXT: Type: OPEN ++# CHECK-NEXT: } ++# CHECK-NEXT: ] ++# CHECK-NEXT: } ++# CHECK-NEXT: ] ++ ++.section ".gnu.build.attributes.text.foo", "", @note ++.balign 4 ++.long 8 ++.long 0 ++.long 0x100 ++.asciz "GA$\x03:a1" ++ ++.section ".gnu.build.attributes.text.bar", "", @note ++.balign 4 ++.long 8 ++.long 0 ++.long 0x100 ++.asciz "GA$\x03:b1" +-- +2.50.1 + diff --git a/0b6a1ef429.patch b/0b6a1ef429.patch new file mode 100644 index 0000000..8f6e2db --- /dev/null +++ b/0b6a1ef429.patch @@ -0,0 +1,70 @@ +From 0b6a1ef4297bb839fe26041602d32411358e0032 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= +Date: Tue, 26 May 2026 01:41:01 +0200 +Subject: [PATCH] [lit] Normalize RLIM_INFINITY to "infinity" in + print_limits.py for Python 3.15+ (#190953) + +Python 3.15 changed resource.getrlimit() to return the platform's +maximum value (e.g., 18446744073709551615 on 64-bit systems) instead of +-1 for RLIM_INFINITY. This breaks lit tests that expect -1 for unlimited +resource limits. + +This patch normalizes the return value to "infinity" when it equals +RLIM_INFINITY to maintain compatibility with existing tests across all +Python versions. + +Fixes test failure: shtest-ulimit-nondarwin.py +Expected: RLIMIT_FSIZE=-1 +Got: RLIMIT_FSIZE=18446744073709551615 + +Reference: +https://github.com/python/cpython/commit/0324c726dea702282a0300225e989b19ae23b759 +Reference: https://bugzilla.redhat.com/show_bug.cgi?id=2448969 + +Analysis and testing assisted by AI. + +Assisted-by: Claude Sonnet 4.5 + +--------- + +Co-authored-by: Alexander Richardson +Co-authored-by: Tulio Magno Quites Machado Filho +--- + .../tests/Inputs/shtest-ulimit/print_limits.py | 17 +++++++++++++---- + llvm/utils/lit/tests/shtest-ulimit-nondarwin.py | 2 +- + 2 files changed, 14 insertions(+), 5 deletions(-) + +diff --git a/llvm/utils/lit/tests/Inputs/shtest-ulimit/print_limits.py b/llvm/utils/lit/tests/Inputs/shtest-ulimit/print_limits.py +index c732c0429e661..6c03721baf36d 100644 +--- a/llvm/utils/lit/tests/Inputs/shtest-ulimit/print_limits.py ++++ b/llvm/utils/lit/tests/Inputs/shtest-ulimit/print_limits.py +@@ -1,6 +1,15 @@ + import resource + +-print("RLIMIT_AS=" + str(resource.getrlimit(resource.RLIMIT_AS)[0])) +-print("RLIMIT_NOFILE=" + str(resource.getrlimit(resource.RLIMIT_NOFILE)[0])) +-print("RLIMIT_STACK=" + str(resource.getrlimit(resource.RLIMIT_STACK)[0])) +-print("RLIMIT_FSIZE=" + str(resource.getrlimit(resource.RLIMIT_FSIZE)[0])) ++ ++def normalize_limit(limit_value): ++ """Normalize RLIM_INFINITY to "infinity" for consistency across Python versions. ++ ++ Python 3.15+ returns the platform's max value (e.g., 2^64-1) instead of -1. ++ """ ++ return "infinity" if limit_value == resource.RLIM_INFINITY else str(limit_value) ++ ++ ++print("RLIMIT_AS=" + normalize_limit(resource.getrlimit(resource.RLIMIT_AS)[0])) ++print("RLIMIT_NOFILE=" + normalize_limit(resource.getrlimit(resource.RLIMIT_NOFILE)[0])) ++print("RLIMIT_STACK=" + normalize_limit(resource.getrlimit(resource.RLIMIT_STACK)[0])) ++print("RLIMIT_FSIZE=" + normalize_limit(resource.getrlimit(resource.RLIMIT_FSIZE)[0])) +diff --git a/llvm/utils/lit/tests/shtest-ulimit-nondarwin.py b/llvm/utils/lit/tests/shtest-ulimit-nondarwin.py +index 43811db750f80..80844d1d79460 100644 +--- a/llvm/utils/lit/tests/shtest-ulimit-nondarwin.py ++++ b/llvm/utils/lit/tests/shtest-ulimit-nondarwin.py +@@ -18,4 +18,4 @@ + # CHECK: ulimit -f 5 + # CHECK: RLIMIT_FSIZE=5 + # CHECK: ulimit -f unlimited +-# CHECK: RLIMIT_FSIZE=-1 ++# CHECK: RLIMIT_FSIZE=infinity diff --git a/22-185375.patch b/22-185375.patch deleted file mode 100644 index 82c3af9..0000000 --- a/22-185375.patch +++ /dev/null @@ -1,86 +0,0 @@ -From f463bef09be73ae9a415fcd3fd49689bd95b0f0a Mon Sep 17 00:00:00 2001 -From: Congcong Cai -Date: Fri, 20 Feb 2026 07:03:27 +0800 -Subject: [PATCH] [SimplifyCFG] process prof data when remove case in umin - (#182261) - -In #164097, we introduce a optimization for umin. But it does not handle -profile data correctly. -This PR remove profile data when remove cases. -Fixed: #181837 - -(cherry picked from commit 31e5f86a3cdc960ef7b2f0a533c4a37cf526cacd) ---- - llvm/lib/Transforms/Utils/SimplifyCFG.cpp | 2 +- - .../Transforms/SimplifyCFG/switch-umin.ll | 43 +++++++++++++++++++ - 2 files changed, 44 insertions(+), 1 deletion(-) - -diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp -index 5f4807242581d..a16f274a4ed5a 100644 ---- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp -+++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp -@@ -7724,7 +7724,7 @@ static bool simplifySwitchWhenUMin(SwitchInst *SI, DomTreeUpdater *DTU) { - BasicBlock *DeadCaseBB = I->getCaseSuccessor(); - DeadCaseBB->removePredecessor(BB); - Updates.push_back({DominatorTree::Delete, BB, DeadCaseBB}); -- I = SIW->removeCase(I); -+ I = SIW.removeCase(I); - E = SIW->case_end(); - } - -diff --git a/llvm/test/Transforms/SimplifyCFG/switch-umin.ll b/llvm/test/Transforms/SimplifyCFG/switch-umin.ll -index 44665365dc222..ff958e4d04147 100644 ---- a/llvm/test/Transforms/SimplifyCFG/switch-umin.ll -+++ b/llvm/test/Transforms/SimplifyCFG/switch-umin.ll -@@ -239,8 +239,51 @@ case4: - - } - -+define void @switch_remove_dead_cases(i32 %x) { -+; CHECK-LABEL: define void @switch_remove_dead_cases( -+; CHECK-SAME: i32 [[X:%.*]]) { -+; CHECK-NEXT: [[MIN:%.*]] = call i32 @llvm.umin.i32(i32 [[X]], i32 4) -+; CHECK-NEXT: switch i32 [[X]], label %[[COMMON_RET:.*]] [ -+; CHECK-NEXT: i32 2, label %[[CASE_A:.*]] -+; CHECK-NEXT: i32 3, label %[[CASE_B:.*]] -+; CHECK-NEXT: ], !prof [[PROF1:![0-9]+]] -+; CHECK: [[COMMON_RET]]: -+; CHECK-NEXT: ret void -+; CHECK: [[CASE_A]]: -+; CHECK-NEXT: call void @a() -+; CHECK-NEXT: br label %[[COMMON_RET]] -+; CHECK: [[CASE_B]]: -+; CHECK-NEXT: call void @b() -+; CHECK-NEXT: br label %[[COMMON_RET]] -+; -+ %min = call i32 @llvm.umin.i32(i32 %x, i32 4) -+ switch i32 %min, label %unreachable [ -+ i32 2, label %case_a -+ i32 3, label %case_b -+ i32 4, label %case_ret -+ i32 5, label %case_ret -+ ], !prof !1 -+ -+case_a: -+ call void @a() -+ ret void -+ -+case_b: -+ call void @b() -+ ret void -+ -+case_ret: -+ ret void -+ -+unreachable: -+ unreachable -+} - - !0 = !{!"branch_weights", i32 1, i32 2, i32 3, i32 99, i32 5} - ;. - ; CHECK: [[PROF0]] = !{!"branch_weights", i32 5, i32 2, i32 3, i32 99} - ;. -+!1 = !{!"branch_weights", i32 11, i32 12, i32 13, i32 14, i32 15} -+;. -+; CHECK: [[PROF1]] = !{!"branch_weights", i32 14, i32 12, i32 13} -+;. diff --git a/22-185922.patch b/22-185922.patch deleted file mode 100644 index 4513df1..0000000 --- a/22-185922.patch +++ /dev/null @@ -1,55 +0,0 @@ -From ccf0ee68b86f65a6a4e83756f717faad7c779cb1 Mon Sep 17 00:00:00 2001 -From: Nikita Popov -Date: Wed, 11 Mar 2026 18:03:05 +0100 -Subject: [PATCH] [SystemZ] Limit depth of findCCUse() - -The recursion here has potentially exponential complexity. Avoid -this by limiting the depth of recursion. - -An alternative would be to memoize the results. I went with the -simpler depth limit on the assumption that we don't particularly -care about very deep value chains here. ---- - llvm/lib/Target/SystemZ/SystemZISelLowering.cpp | 13 +++++++++---- - 1 file changed, 9 insertions(+), 4 deletions(-) - -diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp -index 2a9cb903f3921..84d66f88a812d 100644 ---- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp -+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp -@@ -8692,7 +8692,12 @@ SDValue SystemZTargetLowering::combineSETCC( - return SDValue(); - } - --static std::pair findCCUse(const SDValue &Val) { -+static std::pair findCCUse(const SDValue &Val, -+ unsigned Depth = 0) { -+ // Limit depth of potentially exponential walk. -+ if (Depth > 5) -+ return std::make_pair(SDValue(), SystemZ::CCMASK_NONE); -+ - switch (Val.getOpcode()) { - default: - return std::make_pair(SDValue(), SystemZ::CCMASK_NONE); -@@ -8705,7 +8710,7 @@ static std::pair findCCUse(const SDValue &Val) { - SDValue Op4CCReg = Val.getOperand(4); - if (Op4CCReg.getOpcode() == SystemZISD::ICMP || - Op4CCReg.getOpcode() == SystemZISD::TM) { -- auto [OpCC, OpCCValid] = findCCUse(Op4CCReg.getOperand(0)); -+ auto [OpCC, OpCCValid] = findCCUse(Op4CCReg.getOperand(0), Depth + 1); - if (OpCC != SDValue()) - return std::make_pair(OpCC, OpCCValid); - } -@@ -8722,10 +8727,10 @@ static std::pair findCCUse(const SDValue &Val) { - case ISD::SHL: - case ISD::SRA: - case ISD::SRL: -- auto [Op0CC, Op0CCValid] = findCCUse(Val.getOperand(0)); -+ auto [Op0CC, Op0CCValid] = findCCUse(Val.getOperand(0), Depth + 1); - if (Op0CC != SDValue()) - return std::make_pair(Op0CC, Op0CCValid); -- return findCCUse(Val.getOperand(1)); -+ return findCCUse(Val.getOperand(1), Depth + 1); - } - } - diff --git a/22-190701.patch b/22-190701.patch deleted file mode 100644 index 51c3246..0000000 --- a/22-190701.patch +++ /dev/null @@ -1,87 +0,0 @@ -From 3915d1efcdb1e9d10c8f6966acbe5c359d824ba1 Mon Sep 17 00:00:00 2001 -From: Josh Stone -Date: Mon, 6 Apr 2026 14:08:10 -0700 -Subject: [PATCH] [CodeGen] Preserve big-endian trunc in concat_vectors - -A transform from `concat_vectors(trunc(scalar), undef)` to -`scalar_to_vector(scalar)` is only equivalent for little-endian targets. -On big-endian, that would put the extra upper bytes ahead of the desired -truncated bytes. This problem was seen on Rust s390x in [RHEL-147748]. - -[RHEL-147748]: https://redhat.atlassian.net/browse/RHEL-147748 - -Assisted-by: Claude Code ---- - llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 4 +- - llvm/test/CodeGen/SystemZ/vec-trunc-to-i16.ll | 45 +++++++++++++++++++ - 2 files changed, 48 insertions(+), 1 deletion(-) - create mode 100644 llvm/test/CodeGen/SystemZ/vec-trunc-to-i16.ll - -diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp -index 383e45c5ea3a8..5485ee86251a5 100644 ---- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp -+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp -@@ -26513,9 +26513,11 @@ SDValue DAGCombiner::visitCONCAT_VECTORS(SDNode *N) { - // If the bitcast type isn't legal, it might be a trunc of a legal type; - // look through the trunc so we can still do the transform: - // concat_vectors(trunc(scalar), undef) -> scalar_to_vector(scalar) -+ // However, this is only equivalent on little-endian targets. - if (Scalar->getOpcode() == ISD::TRUNCATE && - !TLI.isTypeLegal(Scalar.getValueType()) && -- TLI.isTypeLegal(Scalar->getOperand(0).getValueType())) -+ TLI.isTypeLegal(Scalar->getOperand(0).getValueType()) && -+ DAG.getDataLayout().isLittleEndian()) - Scalar = Scalar->getOperand(0); - - EVT SclTy = Scalar.getValueType(); -diff --git a/llvm/test/CodeGen/SystemZ/vec-trunc-to-i16.ll b/llvm/test/CodeGen/SystemZ/vec-trunc-to-i16.ll -new file mode 100644 -index 0000000000000..42d787d945145 ---- /dev/null -+++ b/llvm/test/CodeGen/SystemZ/vec-trunc-to-i16.ll -@@ -0,0 +1,45 @@ -+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s -+ -+; Test that truncated scalars use the correct vector insert instruction. -+; On big-endian targets, concat_vectors should not skip truncates when -+; creating scalar_to_vector, as the bytes would be in the wrong position. -+ -+; This truncated i16 should use vlvgh (insert halfword), not vlvgf (insert fullword). -+define <16 x i8> @test_concat_trunc_i16(i32 %x) { -+; CHECK-LABEL: test_concat_trunc_i16: -+; CHECK: # %bb.0: -+; CHECK-NEXT: vlvgh %v24, %r2, 0 -+; CHECK-NEXT: br %r14 -+ %t = trunc i32 %x to i16 -+ %vec = bitcast i16 %t to <2 x i8> -+ %result = shufflevector <2 x i8> %vec, <2 x i8> poison, <16 x i32> -+ ret <16 x i8> %result -+} -+ -+; Test with a more complex shuffle pattern, reduced from a Rust bug report. -+define fastcc void @test_shuffle_with_trunc() { -+; CHECK-LABEL: test_shuffle_with_trunc: -+; CHECK: # %bb.0: -+; CHECK-NEXT: lh %r1, 0 -+; CHECK-NEXT: l %r0, 0 -+; CHECK-NEXT: vlvgh %v1, %r1, 0 -+; CHECK-NEXT: larl %r1, .LCPI1_0 -+; CHECK-NEXT: vl %v2, 0(%r1), 3 -+; CHECK-NEXT: vlvgf %v0, %r0, 0 -+; CHECK-NEXT: vperm %v0, %v0, %v1, %v2 -+; CHECK-NEXT: vst %v0, 0, 3 -+; CHECK-NEXT: br %r14 -+ %1 = load i32, ptr null, align 8 -+ %2 = load i16, ptr null, align 1 -+ br label %3 -+ -+3: -+ %4 = bitcast i32 %1 to <4 x i8> -+ %5 = shufflevector <4 x i8> %4, <4 x i8> zeroinitializer, <16 x i32> -+ %6 = bitcast i16 %2 to <2 x i8> -+ %7 = shufflevector <2 x i8> %6, <2 x i8> zeroinitializer, <16 x i32> -+ %8 = shufflevector <16 x i8> %5, <16 x i8> %7, <16 x i32> -+ store <16 x i8> %8, ptr null, align 8 -+ ret void -+} diff --git a/llvm.spec b/llvm.spec index af0ce53..088e99e 100644 --- a/llvm.spec +++ b/llvm.spec @@ -2,7 +2,7 @@ #region version %global maj_ver 22 %global min_ver 1 -%global patch_ver 3 +%global patch_ver 8 #global rc_ver rc3 %bcond_with snapshot_build @@ -28,6 +28,7 @@ %define bcond_override_default_offload 0 %define bcond_override_default_mlir 0 %define bcond_override_default_flang 0 +%define bcond_override_default_libclc 0 %define bcond_override_default_build_bolt 0 %define bcond_override_default_polly 0 %define bcond_override_default_pgo 0 @@ -41,7 +42,7 @@ %bcond_with compat_build # Bundle compat libraries for a previous LLVM version, as part of llvm-libs and # clang-libs. Used on RHEL. -%bcond_without bundle_compat_lib +%bcond_with bundle_compat_lib %bcond_without check %if %{with bundle_compat_lib} @@ -56,6 +57,14 @@ %bcond_without python_lit %endif +%if %{maj_ver} >= 23 +%global build_docs 0 +%global build_docs_toggle OFF +%else +%global build_docs 1 +%global build_docs_toggle ON +%endif + %bcond_without lldb %ifarch ppc64le @@ -109,6 +118,8 @@ # Set Fortran build flags to nil because they contain flags that don't apply to flang. %global build_fflags %{nil} +%endif +#endregion flang %{lua: @@ -145,8 +156,7 @@ function print_max_procs(per_proc_mem) print(cpu) end } -%endif -#endregion flang + # The libcxx build condition also enables libcxxabi and libunwind. %if %{without compat_build} && %{defined fedora} @@ -207,15 +217,16 @@ end %endif %endif -# Historically, LLD was used used at the same combinations that enabled PGO. -# If this changes, we need to update the following lines. -# However, we should be able to link using LLD even if PGO is disabled. -# Reminder: RHEL8 still builds with gcc + ld.bfd. -%if %{with pgo} -%bcond_without use_lld -%else + +%if 0%{?rhel} == 8 # RHEL8 still builds with gcc + ld.bfd. %bcond_with use_lld +%else +%bcond_without use_lld +%endif + +%if %{with pgo} && %{without use_lld} +%{error:PGO requires --with=lld} %endif # For PGO Disable LTO for now because of LLVMgold.so not found error @@ -224,6 +235,12 @@ end %global _lto_cflags %nil %endif +%if %{maj_ver} >= 23 && 0%{undefined rhel} && %{without compat_build} && %{with snapshot_build} +%bcond_without libclc +%else +%bcond_with libclc +%endif + # We are building with clang for faster/lower memory LTO builds. # See https://docs.fedoraproject.org/en-US/packaging-guidelines/#_compiler_macros # Reminder: This only works on Fedora and RHEL >= 9. @@ -255,6 +272,12 @@ end %bcond_without libedit %endif +%if %{defined rhel} && 0%{?rhel} >= 10 +%bcond_with multilib +%else +%bcond_without multilib +%endif + # Opt out of https://fedoraproject.org/wiki/Changes/fno-omit-frame-pointer # https://bugzilla.redhat.com/show_bug.cgi?id=2158587 %undefine _include_frame_pointers @@ -271,8 +294,10 @@ end # Suffixless tarball name (essentially: basename -s .tar.xz llvm-project-17.0.6.src.tar.xz) %if %{with snapshot_build} %global src_tarball_dir llvm-project-%{llvm_snapshot_git_revision} +%global src_manpage_tarball_dir llvm_man_pages-%{llvm_snapshot_yyyymmdd} %else %global src_tarball_dir llvm-project-%{maj_ver}.%{min_ver}.%{patch_ver}%{?rc_ver:-%{rc_ver}}.src +%global src_manpage_tarball_dir llvm_man_pages-%{maj_ver}.%{min_ver}.%{patch_ver} %endif # LLD uses "fast" as the algortithm for generating build-id @@ -405,6 +430,12 @@ end %global pkg_name_flang flang%{pkg_suffix} #endregion flang globals +#region libclc globals +%if %{with libclc} +%global pkg_name_libclc libclc%{pkg_suffix} +%endif +#endregion libclc globals + #endregion globals #region packages @@ -423,9 +454,15 @@ URL: http://llvm.org %if %{with snapshot_build} Source0: https://github.com/llvm/llvm-project/archive/%{llvm_snapshot_git_revision}.tar.gz +%if %{build_docs} == 0 +Source42: https://github.com/fedora-llvm-team/llvm-snapshots/releases/download/snapshot-version-sync/%{src_manpage_tarball_dir}.tar.xz +%endif %else Source0: https://github.com/llvm/llvm-project/releases/download/llvmorg-%{maj_ver}.%{min_ver}.%{patch_ver}%{?rc_ver:-%{rc_ver}}/%{src_tarball_dir}.tar.xz Source1: https://github.com/llvm/llvm-project/releases/download/llvmorg-%{maj_ver}.%{min_ver}.%{patch_ver}%{?rc_ver:-%{rc_ver}}/%{src_tarball_dir}.tar.xz.sig +%if %{build_docs} == 0 +Source42: https://github.com/llvm/llvm-project/releases/download/llvmorg-%{maj_ver}.%{min_ver}.%{patch_ver}%{?rc_ver:-%{rc_ver}}/%{src_manpage_tarball_dir}.tar.xz +%endif %endif Source6: release-keys.asc @@ -487,9 +524,28 @@ Patch103: 0001-Workaround-a-bug-in-ORC-on-ppc64le.patch Patch104: 0001-Driver-Give-devtoolset-path-precedence-over-Installe.patch #endregion CLANG patches +# Fix lit tests for Python 3.15+ https://bugzilla.redhat.com/show_bug.cgi?id=2448969 +Patch2209: https://github.com/llvm/llvm-project/commit/0b6a1ef429.patch + +# s390x fix for unaligned memory access performance regressions. +Patch2210: 0001-SystemZ-Avoid-unaligned-VL-VST-s-with-memcpy-memmove.patch + +# Fix a heap buffer overflow. +# https://bugzilla.redhat.com/show_bug.cgi?id=2496390 +Patch2211: 0001-LLVM-Verifier-Fix-buffer-overflow-when-verifying-gc..patch + +# Fix miscompilation. +# https://bugzilla.redhat.com/show_bug.cgi?id=2499684 +Patch2212: 0001-SDAG-Freeze-condition-in-select-of-load-fold-208683.patch +# Fix a vector miscompilation +Patch2213: 0001-X86-Fix-EVEX-compression-for-VPMOV-2M-KMOV-with-tied.patch + #region LLD patches Patch106: 0001-19-Always-build-shared-libs-for-LLD.patch Patch2103: 0001-lld-Adjust-compressed-debug-level-test-for-s390x-wit.patch +Patch2214: 0001-ELF-Simplify-AArch64-relocateAlloc.-NFC.patch +Patch2215: 0001-LLD-AArch64-Make-adrp-ldr-relaxation-per-symbol-all-.patch +Patch2216: 0001-lld-ELF-Concatenate-.gnu.build.attributes.-sections-.patch #endregion LLD patches #region polly patches @@ -525,12 +581,8 @@ Patch2105: 43cb4631c1f42dbfce78288b8ae30b5840ed59b3.patch # Fix for s390x vector miscompilation (rhbz#2430017) Patch2106: 0001-SystemZ-Fix-code-in-widening-vector-multiplication-1.patch -# Fix for s390x vector miscompilation (RHEL-147748) -Patch2203: 22-190701.patch - %if 0%{?rhel} == 8 %global python3_pkgversion 3.12 -%global python3_version 3.12 %global __python3 /usr/bin/python3.12 %endif @@ -578,20 +630,22 @@ BuildRequires: lld %endif %endif +%if %{build_docs} # This intentionally does not use python3_pkgversion. RHEL 8 does not have # python3.12-sphinx, and we are only using it as a binary anyway. BuildRequires: python3-sphinx -%if 0%{?rhel} != 8 -# RHEL 8 does not have these packages for python3.12. However, they are only -# needed for LLDB tests. -BuildRequires: python%{python3_pkgversion}-psutil -BuildRequires: python%{python3_pkgversion}-pexpect %endif -%if %{undefined rhel} +%if 0%{?rhel} != 8 +# RHEL 8 does not have these packages for python3.12. +BuildRequires: python%{python3_pkgversion}-psutil +%endif +%if %{undefined rhel} && %{build_docs} BuildRequires: python%{python3_pkgversion}-myst-parser %endif # Needed for %%multilib_fix_c_header +%if %{with multilib} BuildRequires: multilib-rpm-config +%endif %if %{with gold} BuildRequires: binutils-devel %if %{undefined rhel} || 0%{?rhel} > 8 @@ -912,7 +966,7 @@ Development header files for clang tools. %package -n git-clang-format%{pkg_suffix} Summary: Integration of clang-format for git Requires: %{pkg_name_clang}-tools-extra = %{version}-%{release} -Requires: git +Requires: git-core Requires: python%{python3_pkgversion} %description -n git-clang-format%{pkg_suffix} @@ -921,7 +975,9 @@ clang-format integration for git. %package -n python%{python3_pkgversion}-%{pkg_name_clang} Summary: Python3 bindings for clang Requires: %{pkg_name_clang}-devel%{?_isa} = %{version}-%{release} +%if "%{?python3_version}" != "" Requires: python(abi) = %{python3_version} +%endif Provides: python%{python3_pkgversion}-clang(major) = %{maj_ver} %if 0%{?rhel} == 8 # Became python3.12-clang in LLVM 19 @@ -1040,6 +1096,9 @@ License: Apache-2.0 WITH LLVM-exception OR NCSA URL: http://lldb.llvm.org/ Requires: %{pkg_name_clang}-libs%{?_isa} = %{version}-%{release} +%if 0%{?fedora} >= 45 || 0%{?rhel} >= 11 +Recommends: yama-ptrace-enable +%endif %if %{without compat_build} Requires: python%{python3_pkgversion}-lldb %endif @@ -1271,6 +1330,51 @@ Flang runtime libraries. %endif #endregion flang packages +#region libclc packages +%if %{with libclc} +%package -n %{pkg_name_libclc} +Summary: An open source implementation of the OpenCL 1.1 library requirements + +License: Apache-2.0 WITH LLVM-exception OR NCSA OR MIT +URL: https://libclc.llvm.org +Obsoletes: %{pkg_name_libclc}-devel < 23 + +%description -n %{pkg_name_libclc} +libclc is an open source, BSD licensed implementation of the library +requirements of the OpenCL C programming language, as specified by the +OpenCL 1.1 Specification. The following sections of the specification +impose library requirements: + + * 6.1: Supported Data Types + * 6.2.3: Explicit Conversions + * 6.2.4.2: Reinterpreting Types Using as_type() and as_typen() + * 6.9: Preprocessor Directives and Macros + * 6.11: Built-in Functionsj + * 9.3: Double Precision Floating-Point + * 9.4: 64-bit Atomics + * 9.5: Writing to 3D image memory objects + * 9.6: Half Precision Floating-Point + +libclc is intended to be used with the Clang compiler's OpenCL frontend. + +libclc is designed to be portable and extensible. To this end, it provides +generic implementations of most library requirements, allowing the target +to override the generic implementation at the granularity of individual +functions. + +libclc currently only supports the PTX target, but support for more +targets is welcome. + +%package -n %{pkg_name_libclc}-spirv +Summary: Spirv subset of %{name} + +%description -n %{pkg_name_libclc}-spirv +The %{pkg_name_libclc}-spirv package contains the spirv32-unknown-unknown/libclc.spv and +spirv64-unknown-unknown/libclc.spv files only, which are the subset required for upstream +Mesa OpenCL support with RustiCL. + +%endif +#endregion libclc packages #endregion packages #region prep @@ -1300,6 +1404,11 @@ Flang runtime libraries. %endif +# Unpack the man pages first +%if %{build_docs} == 0 +%autosetup -N -T -b 42 -n %{src_manpage_tarball_dir} +%endif + # -T : Do Not Perform Default Archive Unpacking (without this, the th source would be unpacked twice) # -b : Unpack The nth Sources Before Changing Directory # -n : Set Name of Build Directory @@ -1350,10 +1459,12 @@ Flang runtime libraries. #endregion COMPILER-RT preparation #region lldb preparation +%if %{build_docs} # Compat builds don't build python bindings, but should still build man pages. %if %{with compat_build} sed -i 's/LLDB_ENABLE_PYTHON/TRUE/' lldb/docs/CMakeLists.txt %endif +%endif #endregion #region libcxx preparation @@ -1395,6 +1506,8 @@ cd llvm/utils/lit %global projects clang;clang-tools-extra;lld %global runtimes compiler-rt;openmp +%global runtime_targets default + %if %{with lldb} %global projects %{projects};lldb %endif @@ -1422,6 +1535,15 @@ cd llvm/utils/lit %if %{with offload} %global runtimes %{runtimes};offload +%global runtime_targets %{runtime_targets};amdgcn-amd-amdhsa +%endif + +%if %{with libclc} || %{with offload} +%global runtime_targets %{runtime_targets};nvptx64-nvidia-cuda +%endif + +%if %{with libclc} +%global runtime_targets %{runtime_targets};spirv32-unknown-unknown;spirv64-unknown-unknown;amdgcn-amd-amdhsa-llvm %endif %global gcc_triple --gcc-triple=%{_target_cpu}-redhat-linux @@ -1539,7 +1661,7 @@ CLANG_LDFLAGS=$(strip_specs "$LDFLAGS $CLANG_LDFLAGS_EXTRA") -DCLANG_DEFAULT_UNWINDLIB=libgcc \\\ -DCLANG_ENABLE_ARCMT:BOOL=ON \\\ -DCLANG_ENABLE_STATIC_ANALYZER:BOOL=ON \\\ - -DCLANG_INCLUDE_DOCS:BOOL=ON \\\ + -DCLANG_INCLUDE_DOCS:BOOL=%{build_docs_toggle} \\\ -DCLANG_INCLUDE_TESTS:BOOL=ON \\\ -DCLANG_PLUGIN_SUPPORT:BOOL=ON \\\ -DCLANG_REPOSITORY_STRING="%{?dist_vendor} %{version}-%{release}" \\\ @@ -1566,15 +1688,15 @@ CLANG_LDFLAGS=$(strip_specs "$LDFLAGS $CLANG_LDFLAGS_EXTRA") # Add all *enabled* documentation targets (no doxygen but sphinx) %global cmake_config_args %{cmake_config_args} \\\ -DLLVM_ENABLE_DOXYGEN:BOOL=OFF \\\ - -DLLVM_ENABLE_SPHINX:BOOL=ON \\\ - -DLLVM_BUILD_DOCS:BOOL=ON + -DLLVM_ENABLE_SPHINX:BOOL=%{build_docs_toggle} \\\ + -DLLVM_BUILD_DOCS:BOOL=%{build_docs_toggle} # Configure sphinx: # Build man-pages but no HTML docs using sphinx %global cmake_config_args %{cmake_config_args} \\\ -DSPHINX_EXECUTABLE=/usr/bin/sphinx-build-3 \\\ -DSPHINX_OUTPUT_HTML:BOOL=OFF \\\ - -DSPHINX_OUTPUT_MAN:BOOL=ON \\\ + -DSPHINX_OUTPUT_MAN:BOOL=%{build_docs_toggle} \\\ -DSPHINX_WARNINGS_AS_ERRORS=OFF #endregion docs options @@ -1586,11 +1708,7 @@ CLANG_LDFLAGS=$(strip_specs "$LDFLAGS $CLANG_LDFLAGS_EXTRA") %ifarch ppc64le %global cmake_config_args %{cmake_config_args} -DLLDB_TEST_USER_ARGS=--skip-category=watchpoint %endif -%if 0%{?rhel} == 8 - %global cmake_config_args %{cmake_config_args} -DLLDB_INCLUDE_TESTS:BOOL=OFF -%else - %global cmake_config_args %{cmake_config_args} -DLLDB_ENFORCE_STRICT_TEST_REQUIREMENTS:BOOL=ON -%endif +%global cmake_config_args %{cmake_config_args} -DLLDB_INCLUDE_TESTS:BOOL=OFF %endif #endregion lldb options @@ -1652,7 +1770,7 @@ CLANG_LDFLAGS=$(strip_specs "$LDFLAGS $CLANG_LDFLAGS_EXTRA") #region mlir options %if %{with mlir} %global cmake_config_args %{cmake_config_args} \\\ - -DMLIR_INCLUDE_DOCS:BOOL=ON \\\ + -DMLIR_INCLUDE_DOCS:BOOL=%{build_docs_toggle} \\\ -DMLIR_INCLUDE_TESTS:BOOL=ON \\\ -DMLIR_INCLUDE_INTEGRATION_TESTS:BOOL=OFF \\\ -DMLIR_INSTALL_AGGREGATE_OBJECTS=OFF \\\ @@ -1671,16 +1789,22 @@ CLANG_LDFLAGS=$(strip_specs "$LDFLAGS $CLANG_LDFLAGS_EXTRA") # We reset the cxxflags to "" here because this is compiling for a GPU # target, where our cflags are either questionable or actively wrong. %global cmake_config_args %{cmake_config_args} \\\ - -DLLVM_RUNTIME_TARGETS='default;amdgcn-amd-amdhsa;nvptx64-nvidia-cuda' \\\ - -DRUNTIMES_nvptx64-nvidia-cuda_LLVM_ENABLE_RUNTIMES=openmp \\\ - -DRUNTIMES_amdgcn-amd-amdhsa_LLVM_ENABLE_RUNTIMES=openmp \\\ - -DRUNTIMES_amdgcn-amd-amdhsa_CMAKE_CXX_FLAGS="" \\\ - -DRUNTIMES_nvptx64-nvidia-cuda_CMAKE_CXX_FLAGS="" + -DRUNTIMES_amdgcn-amd-amdhsa_CMAKE_CXX_FLAGS="" \\\ + -DRUNTIMES_nvptx64-nvidia-cuda_CMAKE_CXX_FLAGS="" \\\ + -DRUNTIMES_amdgcn-amd-amdhsa_CMAKE_EXE_LINKER_FLAGS="" \\\ + -DRUNTIMES_nvptx64-nvidia-cuda_CMAKE_EXE_LINKER_FLAGS="" \\\ + -DRUNTIMES_amdgcn-amd-amdhsa_CMAKE_SHARED_LINKER_FLAGS="" \\\ + -DRUNTIMES_nvptx64-nvidia-cuda_CMAKE_SHARED_LINKER_FLAGS="" \\\ + -DRUNTIMES_amdgcn-amd-amdhsa_CMAKE_MODULE_LINKER_FLAGS="" \\\ + -DRUNTIMES_nvptx64-nvidia-cuda_CMAKE_MODULE_LINKER_FLAGS="" \\\ + -DRUNTIMES_amdgcn-amd-amdhsa_CMAKE_STATIC_LINKER_FLAGS="" \\\ + -DRUNTIMES_nvptx64-nvidia-cuda_CMAKE_STATIC_LINKER_FLAGS="" \\\ + -DRUNTIMES_amdgcn-amd-amdhsa_LLVM_ENABLE_RUNTIMES="openmp" -%if 0%{?__isa_bits} == 64 +%if 0%{?__isa_bits} == 64 && %{maj_ver} <= 22 # The following shouldn't be required, but due to a bug, we have to be # explicit about LLVM_LIBDIR_SUFFIX for nvptx64-nvidia-cuda. -# TODO: Remove this after fixing +# This got fixed in LLVM 23. # https://github.com/llvm/llvm-project/issues/159762 %global cmake_config_args %{cmake_config_args} \\\ -DRUNTIMES_nvptx64-nvidia-cuda_LLVM_LIBDIR_SUFFIX=64 @@ -1698,7 +1822,7 @@ CLANG_LDFLAGS=$(strip_specs "$LDFLAGS $CLANG_LDFLAGS_EXTRA") #region flang options %if %{with flang} %global cmake_config_args %{cmake_config_args} \\\ - -DFLANG_INCLUDE_DOCS:BOOL=ON + -DFLANG_INCLUDE_DOCS:BOOL=%{build_docs_toggle} # Build both, shared and static flang runtime objects. # See also https://llvm.org/devmtg/2025-04/slides/quick_talk/kruse_flang-rt.pdf %global cmake_config_args %{cmake_config_args} \\\ @@ -1712,6 +1836,32 @@ CLANG_LDFLAGS=$(strip_specs "$LDFLAGS $CLANG_LDFLAGS_EXTRA") %endif #endregion flang options +#region libclc options +%if %{with libclc} + +# Build SPIR-V targets with the SPIR-V backend, reset CXX flags and build libclc +# runtime +%global cmake_config_args %{cmake_config_args} \\\ + -DRUNTIMES_spirv32-unknown-unknown_LIBCLC_USE_SPIRV_BACKEND:BOOL=ON \\\ + -DRUNTIMES_spirv64-unknown-unknown_LIBCLC_USE_SPIRV_BACKEND:BOOL=ON \\\ + -DRUNTIMES_spirv32-unknown-unknown_CMAKE_CXX_FLAGS="" \\\ + -DRUNTIMES_spirv64-unknown-unknown_CMAKE_CXX_FLAGS="" \\\ + -DRUNTIMES_amdgcn-amd-amdhsa-llvm_CMAKE_CXX_FLAGS="" \\\ + -DRUNTIMES_spirv32-unknown-unknown_LLVM_ENABLE_RUNTIMES="libclc" \\\ + -DRUNTIMES_spirv64-unknown-unknown_LLVM_ENABLE_RUNTIMES="libclc" \\\ + -DRUNTIMES_amdgcn-amd-amdhsa-llvm_LLVM_ENABLE_RUNTIMES="libclc" +%endif +#endregion libclc options + +%if %{with offload} || %{with libclc} +# For the NVIDIA triple we potentially build both, openmp and libclc +%global combined_runtimes %{?with_offload:openmp%{?with_libclc:;}}%{?with_libclc:libclc} +%global cmake_config_args %{cmake_config_args} \\\ + -DRUNTIMES_nvptx64-nvidia-cuda_LLVM_ENABLE_RUNTIMES="%{combined_runtimes}" + +%global cmake_config_args %{cmake_config_args} \\\ + -DLLVM_RUNTIME_TARGETS="%{runtime_targets}" +%endif #region test options %global cmake_config_args %{cmake_config_args} \\\ @@ -2038,7 +2188,9 @@ install %{build_libdir}/libLLVMTestingSupport.a %{buildroot}%{install_libdir} install %{build_libdir}/libLLVMTestingAnnotations.a %{buildroot}%{install_libdir} # Fix multi-lib +%if %{with multilib} %multilib_fix_c_header --file %{install_includedir}/llvm/Config/llvm-config.h +%endif %if %{without compat_build} @@ -2151,7 +2303,9 @@ ln -s clang-%{maj_ver}.1 %{buildroot}%{install_mandir}/man1/clang++.1 chmod a+x %{buildroot}%{install_datadir}/scan-view/{Reporter.py,startfile.py} # multilib fix +%if %{with multilib} %multilib_fix_c_header --file %{install_includedir}/clang/Config/config.h +%endif # remove editor integrations (bbedit, sublime, emacs, vim) rm -vf %{buildroot}%{install_datadir}/clang/clang-format-bbedit.applescript @@ -2235,13 +2389,17 @@ rm %{buildroot}%{install_bindir}/llvm-omp-kernel-replay touch %{buildroot}%{_bindir}/ld %endif +%if %{build_docs} install -D -m 644 -t %{buildroot}%{install_mandir}/man1/ lld/docs/ld.lld.1 +%endif #endregion LLD installation #region LLDB installation %if %{with lldb} +%if %{with multilib} %multilib_fix_c_header --file %{install_includedir}/lldb/Host/Config.h +%endif %if %{without compat_build} # Move python package out of llvm prefix. @@ -2308,7 +2466,14 @@ rm -v %{buildroot}%{install_libdir}/libFIRAnalysis.a \ %{buildroot}%{install_libdir}/libFIROpenACCTransforms.a \ %{buildroot}%{install_libdir}/libMIFDialect.a + +%if %{maj_ver} < 23 find %{buildroot}%{install_includedir}/flang -type f -a ! -iname '*.mod' -delete +%else +# Remove header files that are only needed for writing plugins. +# TODO: Maybe we should package these in the future. +rm -Rf %{buildroot}%{install_includedir}/flang +%endif # this is a test binary rm -v %{buildroot}%{install_bindir}/f18-parse-demo @@ -2361,6 +2526,21 @@ move_and_replace_with_symlinks() { -exec ln -s --relative "$dest/{}" "$src/{}" \;) } +%if %{build_docs} == 0 +# Install the man pages before the symlinks below are created +cp -v ../%{src_manpage_tarball_dir}/* %{buildroot}%{install_mandir}/man1/ +%if %{without flang} +rm %{buildroot}%{install_mandir}/man1/flang.1 +%endif +%if %{without polly} +rm %{buildroot}%{install_mandir}/man1/polly.1 +%endif +%if %{without lldb} +rm %{buildroot}%{install_mandir}/man1/lldb.1 +rm %{buildroot}%{install_mandir}/man1/lldb-server.1 +%endif +%endif + %if %{without compat_build} # Move files from the llvm prefix to the system prefix and replace them with # symlinks. We do it this way around because symlinks between multilib packages @@ -2474,6 +2654,7 @@ function reset_test_opts() # Set to mark tests as expected to fail. # See https://llvm.org/docs/CommandGuide/lit.html#cmdoption-lit-xfail unset LIT_XFAIL + unset LIT_XFAIL_NOT # Set to mark tests to not even run. # See https://llvm.org/docs/CommandGuide/lit.html#cmdoption-lit-filter-out @@ -2543,10 +2724,18 @@ reset_test_opts %cmake_build --target check-lit #endregion Test LLVM lit +#region Test libclc +%if %{with libclc} +reset_test_opts +%cmake_build --target check-libclc +%endif +#endregion Test libclc + #region Test LLVM reset_test_opts # Xfail testing of update utility tools export LIT_XFAIL="tools/UpdateTestChecks" + %cmake_build --target check-llvm #endregion Test LLVM @@ -3117,6 +3306,7 @@ fi %if %{maj_ver} >= 23 %{expand_bins %{expand: llubi + llvm-extract-bundle-entry llvm-gpu-loader }} %else @@ -3187,6 +3377,7 @@ fi %if %{maj_ver} >= 23 %{expand_mans %{expand: llubi + llvm-extract-bundle-entry }} %else %{expand_mans %{expand: @@ -3297,12 +3488,6 @@ fi }} %{install_bindir}/clang-%{maj_ver} -%{_sysconfdir}/%{pkg_name_clang}/%{_target_platform}-clang.cfg -%{_sysconfdir}/%{pkg_name_clang}/%{_target_platform}-clang++.cfg -%ifarch x86_64 -%{_sysconfdir}/%{pkg_name_clang}/i386-redhat-linux-gnu-clang.cfg -%{_sysconfdir}/%{pkg_name_clang}/i386-redhat-linux-gnu-clang++.cfg -%endif %{expand_mans clang clang++} %if 0%{with pgo} @@ -3329,6 +3514,13 @@ fi %{_libdir}/libclang-cpp.so.%{compat_maj_ver}* %endif +%{_sysconfdir}/%{pkg_name_clang}/%{_target_platform}-clang.cfg +%{_sysconfdir}/%{pkg_name_clang}/%{_target_platform}-clang++.cfg +%ifarch x86_64 +%{_sysconfdir}/%{pkg_name_clang}/i386-redhat-linux-gnu-clang.cfg +%{_sysconfdir}/%{pkg_name_clang}/i386-redhat-linux-gnu-clang++.cfg +%endif + %files -n %{pkg_name_clang}-devel %license clang/LICENSE.TXT %{expand_libs %{expand: @@ -3416,6 +3608,7 @@ fi %if %{maj_ver} >= 23 %{expand_bins %{expand: + clang-ssaf-analyzer clang-ssaf-format clang-ssaf-linker }} @@ -3656,8 +3849,6 @@ fi %endif #endregion MLIR files -#region libcxx files - #region flang files %if %{with flang} %files -n %{pkg_name_flang} @@ -3672,9 +3863,15 @@ fi flang-new }} %{install_bindir}/flang-%{maj_ver} + +%if %{maj_ver} < 23 %{expand_includes %{expand: flang/*.mod }} +%else +%{_prefix}/lib/clang/%{maj_ver}/finclude/flang/%{llvm_triple}/*.mod +%{_prefix}/lib/clang/%{maj_ver}/finclude/flang/%{llvm_triple}/omp_lib.h +%endif %{_sysconfdir}/%{pkg_name_clang}/%{_target_platform}-flang.cfg %ifarch x86_64 @@ -3691,6 +3888,27 @@ fi %endif #region flang files +#region libclc files +%if %{with libclc} +%files -n %{pkg_name_libclc} +%license libclc/LICENSE.TXT +%doc libclc/README.md libclc/CREDITS.TXT +%{_prefix}/lib/clang/%{maj_ver}/lib/amdgcn-amd-amdhsa-llvm/libclc.bc +%{_prefix}/lib/clang/%{maj_ver}/lib/amdgcn-amd-amdhsa-llvm/libclc.a +%{_prefix}/lib/clang/%{maj_ver}/lib/nvptx64-nvidia-cuda/libclc.bc +%{_prefix}/lib/clang/%{maj_ver}/lib/nvptx64-nvidia-cuda/libclc.a + +%files -n %{pkg_name_libclc}-spirv +%license libclc/LICENSE.TXT +%doc libclc/README.md libclc/CREDITS.TXT +%{_prefix}/lib/clang/%{maj_ver}/lib/spirv32-unknown-unknown/libclc.spv +%{_prefix}/lib/clang/%{maj_ver}/lib/spirv32-unknown-unknown/libclc.a +%{_prefix}/lib/clang/%{maj_ver}/lib/spirv64-unknown-unknown/libclc.spv +%{_prefix}/lib/clang/%{maj_ver}/lib/spirv64-unknown-unknown/libclc.a +%endif +#endregion libclc files + +#region libcxx files %if %{with libcxx} %files -n %{pkg_name_libcxx} diff --git a/sources b/sources index 191f498..1b09eae 100644 --- a/sources +++ b/sources @@ -1,4 +1,2 @@ -SHA512 (llvm-project-22.1.3.src.tar.xz) = 3557a955d55471671ae2f7b9c809affd59a29a6fb1e70a2a5d040dc1c6376246deb0635be8ca36cae09112981760e9afb128c822e5554bd722589fb8dee3f0df -SHA512 (llvm-project-22.1.3.src.tar.xz.sig) = 153a0d174492a0facd061b5cfa3e18dbf946cc0c7d1fb50f4d961410d41cea1f355515fd3e892be676b8b34d61a21962c48acb90aa5d310d05cf6452053e52ad -SHA512 (llvm-project-21.1.8.src.tar.xz) = cae4c44e7bf678071723da63ad5839491d717a7233e7f4791aa408207f3ea42f52de939ad15189b112c02a0770f1bb8d59bae6ad31ef53417a6eea7770fe52ab -SHA512 (llvm-project-21.1.8.src.tar.xz.sig) = 10f58eff58ed6e701d0f123b15e68c82ab8cbdf99b1c86c0d83e3b8553e90ea51055e30327e8e442ded57c8f503e2a2de9ee075e9c28b5ba815a0f8922f8671c +SHA512 (llvm-project-22.1.8.src.tar.xz) = 2615b20ba08534f83ab8ecc7b5ba43b5f1dfcf9cdb2534a32fcdbf0ccdd9a008b46276e45ef26ed9377f65b5e4ae89ea798f3863fd034484b5715140f3a7b35c +SHA512 (llvm-project-22.1.8.src.tar.xz.sig) = 99a457b5b1fb409a5fe72b59ebd4ddae5cade3e5f2493e33b44d4f4b4625f7a1743f80106efb1134668842b15ea3400ce2c29263bec8ff986e05040910125e15