From 6d5697f7cb4e933d2f176c46b7ac05a9cbaeb8b6 Mon Sep 17 00:00:00 2001
From: Ulrich Weigand <ulrich.weigand@de.ibm.com>
Date: Thu, 23 Jan 2025 19:11:18 +0100
Subject: [PATCH] [SystemZ] Fix ICE with i128->i64 uaddo carry chain

We can only optimize a uaddo_carry via specialized instruction
if the carry was produced by another uaddo(_carry) instruction;
there is already a check for that.

However, i128 uaddo(_carry) use a completely different mechanism;
they indicate carry in a vector register instead of the CC flag.
Thus, we must also check that we don't mix those two - that check
has been missing.

Fixes: https://github.com/llvm/llvm-project/issues/124001
---
 .../Target/SystemZ/SystemZISelLowering.cpp    | 12 ++++++----
 llvm/test/CodeGen/SystemZ/pr124001.ll         | 23 +++++++++++++++++++
 2 files changed, 31 insertions(+), 4 deletions(-)
 create mode 100644 llvm/test/CodeGen/SystemZ/pr124001.ll

diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index 4040ab6d4510..1fb31c26e20d 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -4708,15 +4708,19 @@ SDValue SystemZTargetLowering::lowerXALUO(SDValue Op,
 }
 
 static bool isAddCarryChain(SDValue Carry) {
-  while (Carry.getOpcode() == ISD::UADDO_CARRY)
+  while (Carry.getOpcode() == ISD::UADDO_CARRY &&
+         Carry->getValueType(0) != MVT::i128)
     Carry = Carry.getOperand(2);
-  return Carry.getOpcode() == ISD::UADDO;
+  return Carry.getOpcode() == ISD::UADDO &&
+         Carry->getValueType(0) != MVT::i128;
 }
 
 static bool isSubBorrowChain(SDValue Carry) {
-  while (Carry.getOpcode() == ISD::USUBO_CARRY)
+  while (Carry.getOpcode() == ISD::USUBO_CARRY &&
+         Carry->getValueType(0) != MVT::i128)
     Carry = Carry.getOperand(2);
-  return Carry.getOpcode() == ISD::USUBO;
+  return Carry.getOpcode() == ISD::USUBO &&
+         Carry->getValueType(0) != MVT::i128;
 }
 
 // Lower UADDO_CARRY/USUBO_CARRY nodes.
diff --git a/llvm/test/CodeGen/SystemZ/pr124001.ll b/llvm/test/CodeGen/SystemZ/pr124001.ll
new file mode 100644
index 000000000000..9cf630a55dd6
--- /dev/null
+++ b/llvm/test/CodeGen/SystemZ/pr124001.ll
@@ -0,0 +1,23 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z13 | FileCheck %s
+
+define i64 @test(i128 %in) {
+; CHECK-LABEL: test:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    larl %r1, .LCPI0_0
+; CHECK-NEXT:    vl %v0, 0(%r2), 3
+; CHECK-NEXT:    vl %v1, 0(%r1), 3
+; CHECK-NEXT:    vaccq %v0, %v0, %v1
+; CHECK-NEXT:    vlgvg %r1, %v0, 1
+; CHECK-NEXT:    la %r2, 1(%r1)
+; CHECK-NEXT:    br %r14
+  %1 = tail call { i128, i1 } @llvm.uadd.with.overflow.i128(i128 %in, i128 1)
+  %2 = extractvalue { i128, i1 } %1, 1
+  %3 = zext i1 %2 to i64
+  %4 = add i64 %3, 1
+  ret i64 %4
+}
+
+declare { i128, i1 } @llvm.uadd.with.overflow.i128(i128, i128) #0
+
+attributes #0 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }
-- 
2.48.1