[MLIR] Add support for permutation_map
This CL hooks up and uses permutation_map in vector_transfer ops.
In particular, when going into the nuts and bolts of the implementation, it
became clear that cases arose that required supporting broadcast semantics.
Broadcast semantics are thus added to the general permutation_map.
The verify methods and tests are updated accordingly.
Examples of interest include.
Example 1:
The following MLIR snippet:
```mlir
for %i3 = 0 to %M {
for %i4 = 0 to %N {
for %i5 = 0 to %P {
%a5 = load %A[%i4, %i5, %i3] : memref<?x?x?xf32>
}}}
```
may vectorize with {permutation_map: (d0, d1, d2) -> (d2, d1)} into:
```mlir
for %i3 = 0 to %0 step 32 {
for %i4 = 0 to %1 {
for %i5 = 0 to %2 step 256 {
%4 = vector_transfer_read %arg0, %i4, %i5, %i3
{permutation_map: (d0, d1, d2) -> (d2, d1)} :
(memref<?x?x?xf32>, index, index) -> vector<32x256xf32>
}}}
````
Meaning that vector_transfer_read will be responsible for reading the 2-D slice:
`%arg0[%i4, %i5:%15+256, %i3:%i3+32]` into vector<32x256xf32>. This will
require a transposition when vector_transfer_read is further lowered.
Example 2:
The following MLIR snippet:
```mlir
%cst0 = constant 0 : index
for %i0 = 0 to %M {
%a0 = load %A[%cst0, %cst0] : memref<?x?xf32>
}
```
may vectorize with {permutation_map: (d0) -> (0)} into:
```mlir
for %i0 = 0 to %0 step 128 {
%3 = vector_transfer_read %arg0, %c0_0, %c0_0
{permutation_map: (d0, d1) -> (0)} :
(memref<?x?xf32>, index, index) -> vector<128xf32>
}
````
Meaning that vector_transfer_read will be responsible of reading the 0-D slice
`%arg0[%c0, %c0]` into vector<128xf32>. This will require a 1-D vector
broadcast when vector_transfer_read is further lowered.
Additionally, some minor cleanups and refactorings are performed.
One notable thing missing here is the composition with a projection map during
materialization. This is because I could not find an AffineMap composition
that operates on AffineMap directly: everything related to composition seems
to require going through SSAValue and only operates on AffinMap at a distance
via AffineValueMap. I have raised this concern a bunch of times already, the
followup CL will actually do something about it.
In the meantime, the projection is hacked at a minimum to pass verification
and materialiation tests are temporarily incorrect.
PiperOrigin-RevId: 224376828
diff --git a/lib/Analysis/VectorAnalysis.cpp b/lib/Analysis/VectorAnalysis.cpp
index 9c2160c..ebddaff 100644
--- a/lib/Analysis/VectorAnalysis.cpp
+++ b/lib/Analysis/VectorAnalysis.cpp
@@ -16,12 +16,17 @@
// =============================================================================
#include "mlir/Analysis/VectorAnalysis.h"
+#include "mlir/Analysis/LoopAnalysis.h"
+#include "mlir/IR/Builders.h"
#include "mlir/IR/BuiltinOps.h"
#include "mlir/IR/Statements.h"
#include "mlir/StandardOps/StandardOps.h"
#include "mlir/Support/Functional.h"
#include "mlir/Support/STLExtras.h"
+#include "llvm/ADT/DenseSet.h"
+#include "llvm/ADT/SetVector.h"
+
///
/// Implements Analysis functions specific to vectors which support
/// the vectorization and vectorization materialization passes.
@@ -29,6 +34,8 @@
using namespace mlir;
+using llvm::SetVector;
+
Optional<SmallVector<unsigned, 4>> mlir::shapeRatio(ArrayRef<int> superShape,
ArrayRef<int> subShape) {
if (superShape.size() < subShape.size()) {
@@ -76,18 +83,98 @@
return shapeRatio(superVectorType.getShape(), subVectorType.getShape());
}
-AffineMap mlir::makePermutationMap(MemRefType memrefType,
- VectorType vectorType) {
- unsigned memRefRank = memrefType.getRank();
- unsigned vectorRank = vectorType.getRank();
- assert(memRefRank >= vectorRank && "Broadcast not supported");
- unsigned offset = memRefRank - vectorRank;
- SmallVector<AffineExpr, 4> perm;
- perm.reserve(memRefRank);
- for (unsigned i = 0; i < vectorRank; ++i) {
- perm.push_back(getAffineDimExpr(offset + i, memrefType.getContext()));
+/// Constructs a permutation map from memref indices to vector dimension.
+///
+/// The implementation uses the knowledge of the mapping of enclosing loop to
+/// vector dimension. `enclosingLoopToVectorDim` carries this information as a
+/// map with:
+/// - keys representing "vectorized enclosing loops";
+/// - values representing the corresponding vector dimension.
+/// The algorithm traverses "vectorized enclosing loops" and extracts the
+/// at-most-one MemRef index that is invariant along said loop. This index is
+/// guaranteed to be at most one by construction: otherwise the MemRef is not
+/// vectorizable.
+/// If this invariant index is found, it is added to the permutation_map at the
+/// proper vector dimension.
+/// If no index is found to be invariant, 0 is added to the permutation_map and
+/// corresponds to a vector broadcast along that dimension.
+///
+/// Examples can be found in the documentation of `makePermutationMap`, in the
+/// header file.
+static AffineMap makePermutationMap(
+ MLIRContext *context,
+ llvm::iterator_range<Operation::operand_iterator> indices,
+ const DenseMap<ForStmt *, unsigned> &enclosingLoopToVectorDim) {
+ using functional::makePtrDynCaster;
+ using functional::map;
+ auto unwrappedIndices = map(makePtrDynCaster<SSAValue, MLValue>(), indices);
+ SmallVector<AffineExpr, 4> perm(enclosingLoopToVectorDim.size(),
+ getAffineConstantExpr(0, context));
+ for (auto kvp : enclosingLoopToVectorDim) {
+ assert(kvp.second < perm.size());
+ auto invariants = getInvariantAccesses(*kvp.first, unwrappedIndices);
+ unsigned numIndices = unwrappedIndices.size();
+ unsigned countInvariantIndices = 0;
+ for (unsigned dim = 0; dim < numIndices; ++dim) {
+ if (!invariants.count(unwrappedIndices[dim])) {
+ assert(perm[kvp.second] == getAffineConstantExpr(0, context) &&
+ "permutationMap already has an entry along dim");
+ perm[kvp.second] = getAffineDimExpr(dim, context);
+ } else {
+ ++countInvariantIndices;
+ }
+ }
+ assert((countInvariantIndices == numIndices ||
+ countInvariantIndices == numIndices - 1) &&
+ "Vectorization prerequisite violated: at most 1 index may be "
+ "invariant wrt a vectorized loop");
}
- return AffineMap::get(memRefRank, 0, perm, {});
+ return AffineMap::get(unwrappedIndices.size(), 0, perm, {});
+}
+
+/// Implementation detail that walks up the parents and records the ones with
+/// the specified type.
+/// TODO(ntv): could also be implemented as a collect parents followed by a
+/// filter and made available outside this file.
+template <typename T> static SetVector<T *> getParentsOfType(Statement *stmt) {
+ SetVector<T *> res;
+ auto *current = stmt;
+ while (auto *parent = current->getParentStmt()) {
+ auto *typedParent = dyn_cast<T>(parent);
+ if (typedParent) {
+ assert(res.count(typedParent) == 0 && "Already inserted");
+ res.insert(typedParent);
+ }
+ current = parent;
+ }
+ return res;
+}
+
+/// Returns the enclosing ForStmt, from closest to farthest.
+static SetVector<ForStmt *> getEnclosingForStmts(Statement *stmt) {
+ return getParentsOfType<ForStmt>(stmt);
+}
+
+AffineMap
+mlir::makePermutationMap(OperationStmt *opStmt,
+ const DenseMap<ForStmt *, unsigned> &loopToVectorDim) {
+ DenseMap<ForStmt *, unsigned> enclosingLoopToVectorDim;
+ auto enclosingLoops = getEnclosingForStmts(opStmt);
+ for (auto *forStmt : enclosingLoops) {
+ auto it = loopToVectorDim.find(forStmt);
+ if (it != loopToVectorDim.end()) {
+ enclosingLoopToVectorDim.insert(*it);
+ }
+ }
+
+ if (auto load = opStmt->dyn_cast<LoadOp>()) {
+ return ::makePermutationMap(opStmt->getContext(), load->getIndices(),
+ enclosingLoopToVectorDim);
+ }
+
+ auto store = opStmt->cast<StoreOp>();
+ return ::makePermutationMap(opStmt->getContext(), store->getIndices(),
+ enclosingLoopToVectorDim);
}
bool mlir::matcher::operatesOnStrictSuperVectors(const OperationStmt &opStmt,