Skip to content

Commit 3d006d1

Browse files
chsiggchsigg
committed
OpenXLA-specific changes
1 parent b01bb25 commit 3d006d1

File tree

37 files changed

+3567
-932
lines changed

37 files changed

+3567
-932
lines changed

‎BUILD

+934
Large diffs are not rendered by default.

‎lib/Analysis/AxisInfo.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -937,7 +937,7 @@ class ShROpAxisInfoVisitor final : public BinaryOpVisitorImpl<OpTy> {
937937
// Treat [2^n,2^n+1,...]'s divisibility as 1 instead of 2^n
938938
lhsDivisibility = 1;
939939
}
940-
return std::max<int64_t>(1, lhsDivisibility / (1 << shift));
940+
return std::max<int64_t>(1, lhsDivisibility / (int64_t(1) << shift));
941941
}
942942

943943
int64_t getConstancy(OpTy op, const AxisInfo &lhs, const AxisInfo &rhs,

‎lib/Conversion/TritonToTritonGPU/TritonGPUConversion.cpp

+12
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,12 @@ TritonGPUTypeConverter::TritonGPUTypeConverter(MLIRContext *context,
5757
addArgumentMaterialization([&](OpBuilder &builder,
5858
RankedTensorType tensorType, ValueRange inputs,
5959
Location loc) -> Value {
60+
// Allows partial TTIR to TTGIR conversion by materializing a conversion for
61+
// remaining arguments that have been converted to a new type.
62+
// We use this to rewrite triton_xla.sparse_dot in a separate pass after
63+
// 'convert-triton-to-tritongpu'.
64+
return builder.create<triton::gpu::ConvertLayoutOp>(loc, tensorType,
65+
inputs);
6066
llvm_unreachable("Argument rematerialization should not happen in Triton "
6167
"-> TritonGPU conversion");
6268
return {};
@@ -66,6 +72,12 @@ TritonGPUTypeConverter::TritonGPUTypeConverter(MLIRContext *context,
6672
// convert origValue to newValue
6773
addSourceMaterialization([&](OpBuilder &builder, RankedTensorType tensorType,
6874
ValueRange inputs, Location loc) -> Value {
75+
// Allows partial TTIR to TTGIR conversion by materializing a conversion for
76+
// remaining uses of values that have been converted to a new type.
77+
// We use this to rewrite triton_xla.sparse_dot in a separate pass after
78+
// 'convert-triton-to-tritongpu'.
79+
return builder.create<triton::gpu::ConvertLayoutOp>(loc, tensorType,
80+
inputs);
6981
llvm_unreachable("Source rematerialization should not happen in Triton -> "
7082
"TritonGPU Conversion");
7183
return {};

‎lib/Dialect/TritonGPU/IR/Ops.cpp

+9-4
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,11 @@ struct CanonicalizeConvertFromAlloc
108108
auto convert = op.getSrc().getDefiningOp<ConvertLayoutOp>();
109109
if (!convert)
110110
return failure();
111+
// LocalAllocOp lowering doesn't support going from DotOperandEncoding
112+
// to SharedEncoding, so we want to keep this layout conversion.
113+
if (mlir::isa<triton::gpu::DotOperandEncodingAttr>(
114+
convert.getSrc().getType().getEncoding()))
115+
return failure();
111116
rewriter.replaceOpWithNewOp<triton::gpu::LocalAllocOp>(
112117
op, op->getResult(0).getType(), convert.getSrc());
113118
return mlir::success();
@@ -170,13 +175,13 @@ struct CanonicalizeConvertFromConvert
170175
// heuristic to accommodate fused attention.
171176
auto srcType = op.getSrc().getType();
172177
auto dstType = op.getType();
173-
if (mlir::isa<DotOperandEncodingAttr>(dstType.getEncoding()) &&
174-
mlir::isa<NvidiaMmaEncodingAttr>(srcType.getEncoding()))
178+
if (mlir::isa_and_nonnull<DotOperandEncodingAttr>(dstType.getEncoding()) &&
179+
mlir::isa_and_nonnull<NvidiaMmaEncodingAttr>(srcType.getEncoding()))
175180
return failure();
176181

177182
// for hopper MMAv3
178-
if (mlir::isa<SharedEncodingAttr>(dstType.getEncoding()) &&
179-
mlir::isa<NvidiaMmaEncodingAttr>(srcType.getEncoding()) &&
183+
if (mlir::isa_and_nonnull<SharedEncodingAttr>(dstType.getEncoding()) &&
184+
mlir::isa_and_nonnull<NvidiaMmaEncodingAttr>(srcType.getEncoding()) &&
180185
llvm::any_of(op.getResult().getUsers(), [](Operation *dot) {
181186
return dot->hasTrait<OpTrait::DotLike>();
182187
})) {

‎lib/Dialect/TritonGPU/Transforms/AccelerateMatmul.cpp

+43-7
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,6 @@ namespace mlir {
2020
namespace triton {
2121
namespace gpu {
2222

23-
namespace {
24-
2523
// Get the highest version supported for the hardware and the dot.
2624
static int getMMAVersionSafe(int computeCapability, DotOp op) {
2725
// List supported mma version in order of preference.
@@ -44,8 +42,8 @@ static int getMMAVersionSafe(int computeCapability, DotOp op) {
4442
return 0;
4543
}
4644

47-
SmallVector<unsigned> warpsPerTileV2(DotOp dotOp, const ArrayRef<int64_t> shape,
48-
int numWarps) {
45+
SmallVector<unsigned>
46+
warpsPerTileV2(Operation *dotOp, const ArrayRef<int64_t> shape, int numWarps) {
4947
auto rank = shape.size();
5048
// Early exit for batched matmul
5149
if (rank == 3)
@@ -109,10 +107,10 @@ SmallVector<unsigned> warpsPerTileV2(DotOp dotOp, const ArrayRef<int64_t> shape,
109107
}
110108

111109
SmallVector<unsigned, 2>
112-
warpsPerTileV3(DotOp dotOp, const ArrayRef<int64_t> shape, int numWarps,
110+
warpsPerTileV3(Operation *dotOp, const ArrayRef<int64_t> shape, int numWarps,
113111
const SmallVector<unsigned, 3> &instrShape) {
114112
SetVector<Operation *> slices;
115-
mlir::getForwardSlice(dotOp.getResult(), &slices);
113+
mlir::getForwardSlice(dotOp->getResult(0), &slices);
116114
// Contains a chained dot. We prefer to assign warps to one axis
117115
// to facilitate use cases like flash attention, allowing reductions within
118116
// the same warp.
@@ -167,11 +165,26 @@ static Value getSharedMemoryMMAOperand(Value v, mlir::PatternRewriter &rewriter,
167165
auto newType = MemDescType::get(argType.getShape(), argType.getElementType(),
168166
newLayout, SharedMemorySpace);
169167
rewriter.setInsertionPointAfterValue(arg);
168+
169+
// LocalAllocOp lowering doesn't support going from DotOperandEncoding
170+
// to SharedEncoding.
171+
if (auto dotOpEnc = mlir::dyn_cast<DotOperandEncodingAttr>(
172+
argType.getEncoding())) {
173+
// Create a layout conversion from DotOperandEncoding to BlockedEncoding
174+
// then pass it to the LocalAllocOp.
175+
auto newArgType = RankedTensorType::get(
176+
argType.getShape(), argType.getElementType(), dotOpEnc.getParent());
177+
auto dotOperandToBlockedCvt =
178+
rewriter.create<ConvertLayoutOp>(arg.getLoc(), newArgType, arg);
179+
return rewriter.create<LocalAllocOp>(arg.getLoc(), newType,
180+
dotOperandToBlockedCvt);
181+
}
182+
170183
return rewriter.create<LocalAllocOp>(arg.getLoc(), newType, arg);
171184
}
172185

173186
SmallVector<unsigned, 3>
174-
getWarpsPerTile(DotOp dotOp, const ArrayRef<int64_t> shape, int version,
187+
getWarpsPerTile(Operation* dotOp, const ArrayRef<int64_t> shape, int version,
175188
int numWarps, const SmallVector<unsigned, 3> &instrShape) {
176189
switch (version) {
177190
case 2:
@@ -184,18 +197,32 @@ getWarpsPerTile(DotOp dotOp, const ArrayRef<int64_t> shape, int version,
184197
}
185198
}
186199

200+
// Move anonymous namespace down, so getWarpsPerTile is visible to the sparsity
201+
// extension.
202+
namespace {
203+
187204
class BlockedToMMA : public mlir::OpRewritePattern<DotOp> {
188205
int computeCapability;
189206
mutable llvm::DenseMap<Operation *, unsigned> dotOpInstNs;
190207

191208
static bool bwdFilter(Operation *op) {
209+
// Dot operand layout assignment to Predicates are not currently supported
210+
// during lowering from TritonGPU to LLVM in Triton for MMA cases. This
211+
// condition limits visibility of the original bit-width so that predicate
212+
// are not considered, hence, kwidth can never be = 32.
213+
if (isa<arith::UIToFPOp>(op)) {
214+
Type srcType = getElementTypeOrSelf(op->getOperand(0));
215+
if (srcType.isInteger(1))
216+
return false;
217+
}
192218
return op->getNumOperands() == 1 &&
193219
(isa<FpToFpOp, BitcastOp, ConvertLayoutOp>(op) ||
194220
isPureUnaryInlineAsm(op) ||
195221
op->getDialect()->getTypeID() ==
196222
mlir::TypeID::get<arith::ArithDialect>());
197223
}
198224

225+
public:
199226
// Finds the first different bitwidth in the chain of shape-preserving
200227
// unary ops that x depends on.
201228
// There are two primary scenarios:
@@ -813,6 +840,15 @@ class TritonGPUAccelerateMatmulPass
813840
}
814841
};
815842

843+
// Expose helper functions from BlockedToMMA to be reused for sparse matmul.
844+
int computeOrigBitWidth(Value x) {
845+
return BlockedToMMA::computeOrigBitWidth(x);
846+
}
847+
Value getSharedMemMMAOperand(Value v, mlir::PatternRewriter &rewriter,
848+
int opIdx, bool allowTranspose) {
849+
return getSharedMemoryMMAOperand(v, rewriter, opIdx, allowTranspose);
850+
}
851+
816852
} // namespace gpu
817853
} // namespace triton
818854
} // namespace mlir

‎lib/Dialect/TritonGPU/Transforms/Pipeliner/MatmulLoopPipeline.cpp

+8-1
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,7 @@ static int createAsyncCopy(scf::ForOp forOp, tt::LoadOp loadOp, Value alloc,
131131

132132
Value zero = builder.createWithStage<arith::ConstantIntOp>(
133133
forOp.getLoc(), stage, clusterId, 0, 32);
134+
134135
// Replace the load with insert/extract slice.
135136
builder.setInsertionPoint(loadOp);
136137
Location loc = loadOp.getLoc();
@@ -487,7 +488,8 @@ assignMemoryLayouts(scf::ForOp &forOp,
487488
}
488489
});
489490

490-
loadsToPipeline.insert(&op);
491+
// TODO: b/381421713 - Uncomment this once pipelining is fixed.
492+
// loadsToPipeline.insert(&op);
491493
LoadInfo loadInfo;
492494
for (auto use : users) {
493495
if (use->hasTrait<OpTrait::DotLike>()) {
@@ -527,6 +529,11 @@ assignMemoryLayouts(scf::ForOp &forOp,
527529
getBlockedEncoding(loadOp, axisInfoAnalysis);
528530
}
529531
}
532+
533+
// TODO: b/381421713 - Remove this once pipelining is fixed.
534+
if (!loadInfo.sharedEncoding) continue;
535+
loadsToPipeline.insert(&op);
536+
530537
loadToInfo[&op] = loadInfo;
531538
}
532539
// Make sure all loads in loadsToPipeline are in loadToInfo.

‎lib/Dialect/TritonGPU/Transforms/Prefetch.cpp

+24-2
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ Value Prefetcher::generatePrefetch(Value v, unsigned opIdx, bool isPrologue,
116116
// opIdx: 0 => a, 1 => b
117117
auto type = cast<triton::gpu::MemDescType>(v.getType());
118118
SmallVector<int64_t> shape{type.getShape().begin(), type.getShape().end()};
119-
SmallVector<int64_t> offset{0, 0};
119+
SmallVector<int64_t> offset(shape.size(), 0);
120120
Type elementType = type.getElementType();
121121

122122
// k => (prefetchWidth, k - prefetchWidth)
@@ -141,8 +141,14 @@ Value Prefetcher::generatePrefetch(Value v, unsigned opIdx, bool isPrologue,
141141
type.getMutableMemory(), type.getAllocShape()),
142142
v, offsetsVal);
143143

144+
// We need to assign kwidth to zero in the case where the parent layout is
145+
// Blocked, otherwise the verifier emits a failure. The parent layout is
146+
// Blocked only when Tensor Cores are disabled.
147+
int kwidth = dyn_cast<triton::gpu::BlockedEncodingAttr>(dotEncoding)
148+
? 0
149+
: prefetchWidth / 8;
144150
auto dotOperandEnc = triton::gpu::DotOperandEncodingAttr::get(
145-
builder.getContext(), opIdx, dotEncoding, prefetchWidth / 8);
151+
builder.getContext(), opIdx, dotEncoding, kwidth);
146152
Value prefetchSlice = builder.create<triton::gpu::LocalLoadOp>(
147153
v.getLoc(), RankedTensorType::get(shape, elementType, dotOperandEnc),
148154
newSmem);
@@ -191,6 +197,22 @@ LogicalResult Prefetcher::initialize() {
191197
break;
192198
if (!op->getResult(0).hasOneUse())
193199
break;
200+
// Similar to issues faced in HoistLayoutConversion pattern in
201+
// OptimizeDotOperands.cpp, we can't propagate through type casts from
202+
// predicates as they aren't supported in Triton when encoded with dot_op
203+
// layout.
204+
if (isa<arith::UIToFPOp>(op)) {
205+
Type srcType = getElementTypeOrSelf(op->getOperand(0));
206+
if (srcType.isInteger(1))
207+
break;
208+
}
209+
// Propagation through ExpandDims is currently not supported. This blindly
210+
// replaces the encoding with dot encoding & but ExpandDims requires a
211+
// SliceEncoding. This could be rewritten to support it somehow, but I
212+
// don't think it's trivial & it's currently crashing.
213+
if (isa<ExpandDimsOp>(op)) {
214+
break;
215+
}
194216
rets.push_back(op->getOperand(0));
195217
if (auto cvt = dyn_cast<triton::gpu::LocalLoadOp>(op)) {
196218
foundConvertFromShared = true;

‎lib/Dialect/TritonGPU/Transforms/Utility.cpp

+19-11
Original file line numberDiff line numberDiff line change
@@ -991,18 +991,26 @@ getSharedEncIfAllUsersAreDotEnc(Value val, bool &incompatible) {
991991
} else {
992992
if (!isa<ttg::LocalLoadOp, ttg::ConvertLayoutOp>(user))
993993
return std::nullopt;
994-
auto dotOpEnc = dyn_cast<ttg::DotOperandEncodingAttr>(
995-
cast<triton::gpu::TensorOrMemDesc>(user->getResult(0).getType())
996-
.getEncoding());
997-
if (!dotOpEnc)
994+
auto enc =
995+
cast<triton::gpu::TensorOrMemDesc>(user->getResult(0).getType()).getEncoding();
996+
if (isa<ttg::DotOperandEncodingAttr>(enc)) {
997+
auto srcTy = cast<triton::gpu::TensorOrMemDesc>(val.getType());
998+
auto CTALayout = ttg::getCTALayout(srcTy.getEncoding());
999+
auto order = ttg::getOrder(srcTy.getEncoding());
1000+
unsigned bitWidth = srcTy.getElementType().getIntOrFloatBitWidth();
1001+
tempAttr = ttg::SharedEncodingAttr::get(
1002+
val.getContext(), cast<ttg::DotOperandEncodingAttr>(enc),
1003+
srcTy.getShape(), order, CTALayout, bitWidth, /*needTrans=*/false);
1004+
} else if (enc.getAbstractAttribute().getName().str() ==
1005+
"triton.gpu.sparse_dot_meta_encoding") {
1006+
auto srcTy = cast<triton::gpu::TensorOrMemDesc>(val.getType());
1007+
tempAttr = ttg::SharedEncodingAttr::get(
1008+
val.getContext(), /*vec=*/1, /*perPhase=*/1, /*maxPhase=*/1,
1009+
ttg::getOrder(srcTy.getEncoding()),
1010+
ttg::getCTALayout(srcTy.getEncoding()));
1011+
} else {
9981012
return std::nullopt;
999-
auto srcTy = cast<triton::gpu::TensorOrMemDesc>(val.getType());
1000-
auto CTALayout = ttg::getCTALayout(srcTy.getEncoding());
1001-
auto order = ttg::getOrder(srcTy.getEncoding());
1002-
unsigned bitWidth = srcTy.getElementType().getIntOrFloatBitWidth();
1003-
tempAttr = ttg::SharedEncodingAttr::get(
1004-
val.getContext(), dotOpEnc, srcTy.getShape(), order, CTALayout,
1005-
bitWidth, /*needTrans=*/false);
1013+
}
10061014
}
10071015
// Check that the shared encodings needed by the users are compatible.
10081016
if (attr != nullptr && attr != tempAttr) {

‎lib/Dialect/TritonNvidiaGPU/Transforms/FenceInsertion.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ struct FenceInsertionPass
4444
return;
4545
ModuleOp mod = getOperation();
4646
mod.walk([&](Operation *op) {
47-
if (!isa<ttng::WarpGroupDotOp>(op))
47+
if (!op->hasTrait<OpTrait::DotLike>())
4848
return WalkResult::advance();
4949
OpBuilder builder(op);
5050
auto a = op->getOperand(0);

‎python/BUILD

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
# NOTE: Do not depend on any targets from this directory,
2+
# but use //third_party/py/triton instead.
3+
4+
load("@pybind11_bazel//:build_defs.bzl", "pybind_extension")
5+
6+
package(
7+
default_applicable_licenses = ["//:license"],
8+
default_visibility = [
9+
"//third_party/py/triton:__pkg__",
10+
"@triton//python:__subpackages__",
11+
],
12+
)
13+
14+
cc_library(
15+
name = "passes",
16+
hdrs = ["src/passes.h"],
17+
includes = ["src"],
18+
visibility = ["@triton//third_party:__subpackages__"],
19+
)
20+
21+
pybind_extension(
22+
name = "libtriton",
23+
srcs = [
24+
"src/interpreter.cc",
25+
"src/ir.cc",
26+
"src/llvm.cc",
27+
"src/main.cc",
28+
"src/passes.cc",
29+
],
30+
copts = ["-DTRITON_BACKENDS_TUPLE=(nvidia)"],
31+
deps = [
32+
":passes",
33+
"@llvm-project//llvm:Core",
34+
"@llvm-project//llvm:IPO",
35+
"@llvm-project//llvm:IRReader",
36+
"@llvm-project//llvm:InstCombine",
37+
"@llvm-project//llvm:Instrumentation",
38+
"@llvm-project//llvm:Linker",
39+
"@llvm-project//llvm:MC",
40+
"@llvm-project//llvm:Passes",
41+
"@llvm-project//llvm:Support",
42+
"@llvm-project//llvm:Target",
43+
"@llvm-project//mlir:BuiltinToLLVMIRTranslation",
44+
"@llvm-project//mlir:BytecodeWriter",
45+
"@llvm-project//mlir:ControlFlowDialect",
46+
"@llvm-project//mlir:ConversionPasses",
47+
"@llvm-project//mlir:IR",
48+
"@llvm-project//mlir:IndexDialect",
49+
"@llvm-project//mlir:LLVMDialect",
50+
"@llvm-project//mlir:LLVMIRTransforms",
51+
"@llvm-project//mlir:LLVMToLLVMIRTranslation",
52+
"@llvm-project//mlir:NVVMToLLVMIRTranslation",
53+
"@llvm-project//mlir:Parser",
54+
"@llvm-project//mlir:Pass",
55+
"@llvm-project//mlir:Support",
56+
"@llvm-project//mlir:ToLLVMIRTranslation",
57+
"@llvm-project//mlir:Transforms",
58+
"@llvm-project//mlir:UBDialect",
59+
"//:TritonAnalysis",
60+
"//:TritonDialects",
61+
"//:TritonGPUToLLVM",
62+
"//:TritonGPUTransforms",
63+
"//:TritonHSACO",
64+
"//:TritonLLVMIR",
65+
"//:TritonNvidiaGPUTransforms",
66+
"//:TritonPTX",
67+
"//:TritonToTritonGPU",
68+
"//:TritonTools",
69+
"//:TritonTransforms",
70+
"@triton//third_party/nvidia:triton_nvidia",
71+
"@triton//third_party/proton:ProtonIRDialect",
72+
],
73+
)
74+
75+
filegroup(
76+
name = "files",
77+
srcs = glob(
78+
include = ["triton/**/*.py"],
79+
),
80+
)

0 commit comments

Comments
 (0)