[LV][LAA] Vectorize loop invariant values stored into loop invariant address
Summary:
We are overly conservative in loop vectorizer with respect to stores to loop
invariant addresses.
More details in https://bugs.llvm.org/show_bug.cgi?id=38546
This is the first part of the fix where we start with vectorizing loop invariant
values to loop invariant addresses.
This also includes changes to ORE for stores to invariant address.
Reviewers: anemet, Ayal, mkuper, mssimpso
Subscribers: llvm-commits
Differential Revision: https://reviews.llvm.org/D50665
llvm-svn: 343028
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index a24d660..8312a0d 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -1862,10 +1862,21 @@
// writes and between reads and writes, but not between reads and reads.
ValueSet Seen;
+ // Record uniform store addresses to identify if we have multiple stores
+ // to the same address.
+ ValueSet UniformStores;
+
for (StoreInst *ST : Stores) {
Value *Ptr = ST->getPointerOperand();
- // Check for store to loop invariant address.
- StoreToLoopInvariantAddress |= isUniform(Ptr);
+
+ if (isUniform(Ptr)) {
+ // Consider multiple stores to the same uniform address as a store of a
+ // variant value.
+ bool MultipleStoresToUniformPtr = !UniformStores.insert(Ptr).second;
+ HasVariantStoreToLoopInvariantAddress |=
+ (!isUniform(ST->getValueOperand()) || MultipleStoresToUniformPtr);
+ }
+
// If we did *not* see this pointer before, insert it to the read-write
// list. At this phase it is only a 'write' list.
if (Seen.insert(Ptr).second) {
@@ -2265,7 +2276,7 @@
PtrRtChecking(llvm::make_unique<RuntimePointerChecking>(SE)),
DepChecker(llvm::make_unique<MemoryDepChecker>(*PSE, L)), TheLoop(L),
NumLoads(0), NumStores(0), MaxSafeDepDistBytes(-1), CanVecMem(false),
- StoreToLoopInvariantAddress(false) {
+ HasVariantStoreToLoopInvariantAddress(false) {
if (canAnalyzeLoop())
analyzeLoop(AA, LI, TLI, DT);
}
@@ -2297,8 +2308,8 @@
PtrRtChecking->print(OS, Depth);
OS << "\n";
- OS.indent(Depth) << "Store to invariant address was "
- << (StoreToLoopInvariantAddress ? "" : "not ")
+ OS.indent(Depth) << "Variant Store to invariant address was "
+ << (HasVariantStoreToLoopInvariantAddress ? "" : "not ")
<< "found in loop.\n";
OS.indent(Depth) << "SCEV assumptions:\n";