From 383b0114d3cf3162e02d01f52713b4cc52734f50 Mon Sep 17 00:00:00 2001 From: Sjoerd Meijer Date: Thu, 16 Jan 2025 08:32:03 -0800 Subject: [PATCH] [LoopInterchange] Remove "S" Scalar Dependencies We are not handling 'S' scalar dependencies correctly and have at least the following miscompiles related to that: [LoopInterchange] incorrect handling of scalar dependencies and dependence vectors starting with ">" #54176 [LoopInterchange] Interchange breaks program correctness #46867 [LoopInterchange] Loops should not interchanged due to dependencies #47259 [LoopInterchange] Loops should not interchanged due to control flow #47401 This patch does no longer insert the "S" dependency/direction into the dependency matrix, so a dependency is never "S". We seem to have forgotten what the exact meaning is of this dependency type, and don't see why it should be treated differently. We prefer correctness over incorrect and more aggressive results. I.e., this prevents the miscompiles at the expense of handling less cases, i.e. making interchange more pessimistic. However, some of the cases that are now rejected for dependence analysis reasons, were rejected before too but for other reasons (e.g. profitability). So at least for the llvm regression tests, the number of regression are very reasonable. This should be a stopgap. We would like to get interchange enabled by default and thus prefer correctness over unsafe transforms, and later see if we can get solve the regressions. --- .../lib/Transforms/Scalar/LoopInterchange.cpp | 28 ++++++++----------- .../LoopInterchange/gh54176-scalar-deps.ll | 8 ++++++ .../LoopInterchange/inner-only-reductions.ll | 11 +++++--- ...most-latch-uses-values-in-middle-header.ll | 8 ++++++ .../interchange-flow-dep-outer.ll | 8 ++++++ llvm/test/Transforms/LoopInterchange/lcssa.ll | 12 ++++++-- .../outer-header-jump-to-inner-latch.ll | 8 ++++++ .../pr43176-move-to-new-latch.ll | 8 +++--- ...97-lcssa-for-multiple-outer-loop-blocks.ll | 8 ++++++ .../LoopInterchange/profitability.ll | 8 ++++++ .../reductions-across-inner-and-outer-loop.ll | 2 +- .../LoopInterchange/unique-dep-matrix.ll | 4 +-- .../LoopInterchange/vector-gep-operand.ll | 8 ++++++ 13 files changed, 91 insertions(+), 30 deletions(-) diff --git a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp index a0c0080c0bda1..0d8f52e12d5e4 100644 --- a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp +++ b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp @@ -136,23 +136,17 @@ static bool populateDependencyMatrix(CharMatrix &DepMatrix, unsigned Level, unsigned Levels = D->getLevels(); char Direction; for (unsigned II = 1; II <= Levels; ++II) { - if (D->isScalar(II)) { - Direction = 'S'; - Dep.push_back(Direction); - } else { - unsigned Dir = D->getDirection(II); - if (Dir == Dependence::DVEntry::LT || - Dir == Dependence::DVEntry::LE) - Direction = '<'; - else if (Dir == Dependence::DVEntry::GT || - Dir == Dependence::DVEntry::GE) - Direction = '>'; - else if (Dir == Dependence::DVEntry::EQ) - Direction = '='; - else - Direction = '*'; - Dep.push_back(Direction); - } + unsigned Dir = D->getDirection(II); + if (Dir == Dependence::DVEntry::LT || Dir == Dependence::DVEntry::LE) + Direction = '<'; + else if (Dir == Dependence::DVEntry::GT || + Dir == Dependence::DVEntry::GE) + Direction = '>'; + else if (Dir == Dependence::DVEntry::EQ) + Direction = '='; + else + Direction = '*'; + Dep.push_back(Direction); } while (Dep.size() != Level) { Dep.push_back('I'); diff --git a/llvm/test/Transforms/LoopInterchange/gh54176-scalar-deps.ll b/llvm/test/Transforms/LoopInterchange/gh54176-scalar-deps.ll index bc9f16fbe58d6..6c2f367c2fd2b 100644 --- a/llvm/test/Transforms/LoopInterchange/gh54176-scalar-deps.ll +++ b/llvm/test/Transforms/LoopInterchange/gh54176-scalar-deps.ll @@ -1,3 +1,11 @@ +; Remove 'S' Scalar Dependencies #119345 +; Scalar dependencies are not handled correctly, so they were removed to avoid +; miscompiles. The loop nest in this test case used to be interchanged, but it's +; no longer triggering. XFAIL'ing this test to indicate that this test should +; interchanged if scalar deps are handled correctly. +; +; XFAIL: * + ; RUN: opt < %s -passes=loop-interchange -pass-remarks-output=%t -disable-output ; RUN: FileCheck -input-file %t %s diff --git a/llvm/test/Transforms/LoopInterchange/inner-only-reductions.ll b/llvm/test/Transforms/LoopInterchange/inner-only-reductions.ll index 79a1cb7116966..ee1a7f1619928 100644 --- a/llvm/test/Transforms/LoopInterchange/inner-only-reductions.ll +++ b/llvm/test/Transforms/LoopInterchange/inner-only-reductions.ll @@ -2,8 +2,11 @@ ; RUN: -verify-dom-info -verify-loop-info -verify-loop-lcssa 2>&1 | FileCheck -check-prefix=IR %s ; RUN: FileCheck --input-file=%t %s -; Inner loop only reductions are not supported currently. See discussion at -; D53027 for more information on the required checks. +; Both tests should be rejected as interchange candidates. For now, they are +; rejected for dependence analysis reasons, but that's because support for 'S' +; scalar dependencies was removed. When that is properly, the inner loop only +; reductions should still not be supported currently, see discussion at D53027 +; for more information on the required checks. @A = common global [500 x [500 x i32]] zeroinitializer @X = common global i32 0 @@ -18,7 +21,7 @@ ; CHECK: --- !Missed ; CHECK-NEXT: Pass: loop-interchange -; CHECK-NEXT: Name: UnsupportedPHI +; CHECK-NEXT: Name: Dependence ; CHECK-NEXT: Function: reduction_01 ; IR-LABEL: @reduction_01( @@ -71,7 +74,7 @@ for.end8: ; preds = %for.cond1.for.inc6_ ; CHECK: --- !Missed ; CHECK-NEXT: Pass: loop-interchange -; CHECK-NEXT: Name: UnsupportedPHIOuter +; CHECK-NEXT: Name: Dependence ; CHECK-NEXT: Function: reduction_03 ; IR-LABEL: @reduction_03( diff --git a/llvm/test/Transforms/LoopInterchange/innermost-latch-uses-values-in-middle-header.ll b/llvm/test/Transforms/LoopInterchange/innermost-latch-uses-values-in-middle-header.ll index 230f7dc2bcfad..da37395372e5f 100644 --- a/llvm/test/Transforms/LoopInterchange/innermost-latch-uses-values-in-middle-header.ll +++ b/llvm/test/Transforms/LoopInterchange/innermost-latch-uses-values-in-middle-header.ll @@ -1,3 +1,11 @@ +; Remove 'S' Scalar Dependencies #119345 +; Scalar dependencies are not handled correctly, so they were removed to avoid +; miscompiles. The loop nest in this test case used to be interchanged, but it's +; no longer triggering. XFAIL'ing this test to indicate that this test should +; interchanged if scalar deps are handled correctly. +; +; XFAIL: * + ; RUN: opt < %s -passes=loop-interchange -verify-dom-info -verify-loop-info -pass-remarks-output=%t -disable-output ; RUN: FileCheck -input-file %t %s diff --git a/llvm/test/Transforms/LoopInterchange/interchange-flow-dep-outer.ll b/llvm/test/Transforms/LoopInterchange/interchange-flow-dep-outer.ll index a208c1f46a705..77ab845846bd6 100644 --- a/llvm/test/Transforms/LoopInterchange/interchange-flow-dep-outer.ll +++ b/llvm/test/Transforms/LoopInterchange/interchange-flow-dep-outer.ll @@ -1,3 +1,11 @@ +; Remove 'S' Scalar Dependencies #119345 +; Scalar dependencies are not handled correctly, so they were removed to avoid +; miscompiles. The loop nest in this test case used to be interchanged, but it's +; no longer triggering. XFAIL'ing this test to indicate that this test should +; interchanged if scalar deps are handled correctly. +; +; XFAIL: * + ; RUN: opt < %s -passes=loop-interchange -cache-line-size=64 -verify-dom-info -verify-loop-info -pass-remarks-output=%t -disable-output ; RUN: FileCheck -input-file %t %s diff --git a/llvm/test/Transforms/LoopInterchange/lcssa.ll b/llvm/test/Transforms/LoopInterchange/lcssa.ll index 0a5aefd9e4911..e77efe3052b40 100644 --- a/llvm/test/Transforms/LoopInterchange/lcssa.ll +++ b/llvm/test/Transforms/LoopInterchange/lcssa.ll @@ -1,3 +1,11 @@ +; Remove 'S' Scalar Dependencies #119345 +; Scalar dependencies are not handled correctly, so they were removed to avoid +; miscompiles. The loop nest in this test case used to be interchanged, but it's +; no longer triggering. XFAIL'ing this test to indicate that this test should +; interchanged if scalar deps are handled correctly. +; +; XFAIL: * + ; RUN: opt < %s -passes=loop-interchange -cache-line-size=64 -pass-remarks-missed='loop-interchange' -verify-loop-lcssa -pass-remarks-output=%t -S ; RUN: FileCheck --input-file %t --check-prefix REMARK %s @@ -177,7 +185,7 @@ for.end16: ; preds = %for.exit } ; PHI node in inner latch with multiple predecessors. -; REMARK: Interchanged +; REMARK: Interchanged ; REMARK-NEXT: lcssa_05 define void @lcssa_05(ptr %ptr, i1 %arg) { @@ -222,7 +230,7 @@ for.end16: ; preds = %for.exit ret void } -; REMARK: UnsupportedExitPHI +; REMARK: UnsupportedExitPHI ; REMARK-NEXT: lcssa_06 define void @lcssa_06(ptr %ptr, ptr %ptr1, i1 %arg) { diff --git a/llvm/test/Transforms/LoopInterchange/outer-header-jump-to-inner-latch.ll b/llvm/test/Transforms/LoopInterchange/outer-header-jump-to-inner-latch.ll index aaf8b1daf0414..5ee1bddbe23fc 100644 --- a/llvm/test/Transforms/LoopInterchange/outer-header-jump-to-inner-latch.ll +++ b/llvm/test/Transforms/LoopInterchange/outer-header-jump-to-inner-latch.ll @@ -1,3 +1,11 @@ +; Remove 'S' Scalar Dependencies #119345 +; Scalar dependencies are not handled correctly, so they were removed to avoid +; miscompiles. The loop nest in this test case used to be interchanged, but it's +; no longer triggering. XFAIL'ing this test to indicate that this test should +; interchanged if scalar deps are handled correctly. +; +; XFAIL: * + ; RUN: opt -passes=loop-interchange -cache-line-size=64 -verify-dom-info -verify-loop-info -verify-loop-lcssa %s -pass-remarks-output=%t -disable-output ; RUN: FileCheck -input-file %t %s diff --git a/llvm/test/Transforms/LoopInterchange/pr43176-move-to-new-latch.ll b/llvm/test/Transforms/LoopInterchange/pr43176-move-to-new-latch.ll index 9d2e393937bd5..6b25c3bc9a4ba 100644 --- a/llvm/test/Transforms/LoopInterchange/pr43176-move-to-new-latch.ll +++ b/llvm/test/Transforms/LoopInterchange/pr43176-move-to-new-latch.ll @@ -14,10 +14,10 @@ ; CHECK: --- !Missed ; CHECK-NEXT: Pass: loop-interchange -; CHECK-NEXT: Name: InterchangeNotProfitable +; CHECK-NEXT: Name: Dependence ; CHECK-NEXT: Function: test1 ; CHECK-NEXT: Args: -; CHECK-NEXT: - String: Interchanging loops is not considered to improve cache locality nor vectorization. +; CHECK-NEXT: - String: Cannot interchange loops due to dependences. define void @test1() { entry: @@ -54,10 +54,10 @@ for.cond.for.end5_crit_edge: ; preds = %for.inc3 ; CHECK: --- !Missed ; CHECK-NEXT: Pass: loop-interchange -; CHECK-NEXT: Name: InterchangeNotProfitable +; CHECK-NEXT: Name: Dependence ; CHECK-NEXT: Function: test2 ; CHECK-NEXT: Args: -; CHECK-NEXT: - String: Interchanging loops is not considered to improve cache locality nor vectorization. +; CHECK-NEXT: - String: Cannot interchange loops due to dependences. define void @test2() { entry: diff --git a/llvm/test/Transforms/LoopInterchange/pr43797-lcssa-for-multiple-outer-loop-blocks.ll b/llvm/test/Transforms/LoopInterchange/pr43797-lcssa-for-multiple-outer-loop-blocks.ll index 38970354c3d1c..51493d72a827d 100644 --- a/llvm/test/Transforms/LoopInterchange/pr43797-lcssa-for-multiple-outer-loop-blocks.ll +++ b/llvm/test/Transforms/LoopInterchange/pr43797-lcssa-for-multiple-outer-loop-blocks.ll @@ -1,3 +1,11 @@ +; Remove 'S' Scalar Dependencies #119345 +; Scalar dependencies are not handled correctly, so they were removed to avoid +; miscompiles. The loop nest in this test case used to be interchanged, but it's +; no longer triggering. XFAIL'ing this test to indicate that this test should +; interchanged if scalar deps are handled correctly. +; +; XFAIL: * + ; RUN: opt -passes=loop-interchange -cache-line-size=64 -verify-loop-lcssa %s -pass-remarks-output=%t -disable-output ; RUN: FileCheck -input-file %t %s diff --git a/llvm/test/Transforms/LoopInterchange/profitability.ll b/llvm/test/Transforms/LoopInterchange/profitability.ll index 505c6c422beb6..4804ecfc8ed6f 100644 --- a/llvm/test/Transforms/LoopInterchange/profitability.ll +++ b/llvm/test/Transforms/LoopInterchange/profitability.ll @@ -1,3 +1,11 @@ +; Remove 'S' Scalar Dependencies #119345 +; Scalar dependencies are not handled correctly, so they were removed to avoid +; miscompiles. The loop nest in this test case used to be interchanged, but it's +; no longer triggering. XFAIL'ing this test to indicate that this test should +; interchanged if scalar deps are handled correctly. +; +; XFAIL: * + ; RUN: opt < %s -passes=loop-interchange -cache-line-size=64 -pass-remarks-output=%t -verify-dom-info -verify-loop-info \ ; RUN: -pass-remarks=loop-interchange -pass-remarks-missed=loop-interchange ; RUN: FileCheck -input-file %t %s diff --git a/llvm/test/Transforms/LoopInterchange/reductions-across-inner-and-outer-loop.ll b/llvm/test/Transforms/LoopInterchange/reductions-across-inner-and-outer-loop.ll index fa2021a15a080..eea0c2635d595 100644 --- a/llvm/test/Transforms/LoopInterchange/reductions-across-inner-and-outer-loop.ll +++ b/llvm/test/Transforms/LoopInterchange/reductions-across-inner-and-outer-loop.ll @@ -153,7 +153,7 @@ for1.loopexit: ; preds = %for1.inc ; Check that we do not interchange if reduction is stored in an invariant address inside inner loop ; REMARKS: --- !Missed ; REMARKS-NEXT: Pass: loop-interchange -; REMARKS-NEXT: Name: UnsupportedPHIOuter +; REMARKS-NEXT: Name: Dependence ; REMARKS-NEXT: Function: test4 define i64 @test4(ptr %Arr, ptr %dst) { diff --git a/llvm/test/Transforms/LoopInterchange/unique-dep-matrix.ll b/llvm/test/Transforms/LoopInterchange/unique-dep-matrix.ll index 6943e39cf163e..e5c18830784ac 100644 --- a/llvm/test/Transforms/LoopInterchange/unique-dep-matrix.ll +++ b/llvm/test/Transforms/LoopInterchange/unique-dep-matrix.ll @@ -3,8 +3,8 @@ ; CHECK: Dependency matrix before interchange: ; CHECK-NEXT: I I -; CHECK-NEXT: = S -; CHECK-NEXT: < S +; CHECK-NEXT: = * +; CHECK-NEXT: < * ; CHECK-NEXT: Processing InnerLoopId ; This example is taken from github issue #54176 diff --git a/llvm/test/Transforms/LoopInterchange/vector-gep-operand.ll b/llvm/test/Transforms/LoopInterchange/vector-gep-operand.ll index 022cdd44b7f50..478042f6f3c4d 100644 --- a/llvm/test/Transforms/LoopInterchange/vector-gep-operand.ll +++ b/llvm/test/Transforms/LoopInterchange/vector-gep-operand.ll @@ -1,3 +1,11 @@ +; Remove 'S' Scalar Dependencies #119345 +; Scalar dependencies are not handled correctly, so they were removed to avoid +; miscompiles. The loop nest in this test case used to be interchanged, but it's +; no longer triggering. XFAIL'ing this test to indicate that this test should +; interchanged if scalar deps are handled correctly. +; +; XFAIL: * + ; RUN: opt -passes=loop-interchange -cache-line-size=64 -loop-interchange-threshold=-10 %s -pass-remarks-output=%t -disable-output ; RUN: FileCheck -input-file %t %s