[BPF] do compile-once run-everywhere relocation for bitfields
A bpf specific clang intrinsic is introduced:
u32 __builtin_preserve_field_info(member_access, info_kind)
Depending on info_kind, different information will
be returned to the program. A relocation is also
recorded for this builtin so that bpf loader can
patch the instruction on the target host.
This clang intrinsic is used to get certain information
to facilitate struct/union member relocations.
The offset relocation is extended by 4 bytes to
include relocation kind.
Currently supported relocation kinds are
enum {
FIELD_BYTE_OFFSET = 0,
FIELD_BYTE_SIZE,
FIELD_EXISTENCE,
FIELD_SIGNEDNESS,
FIELD_LSHIFT_U64,
FIELD_RSHIFT_U64,
};
for __builtin_preserve_field_info. The old
access offset relocation is covered by
FIELD_BYTE_OFFSET = 0.
An example:
struct s {
int a;
int b1:9;
int b2:4;
};
enum {
FIELD_BYTE_OFFSET = 0,
FIELD_BYTE_SIZE,
FIELD_EXISTENCE,
FIELD_SIGNEDNESS,
FIELD_LSHIFT_U64,
FIELD_RSHIFT_U64,
};
void bpf_probe_read(void *, unsigned, const void *);
int field_read(struct s *arg) {
unsigned long long ull = 0;
unsigned offset = __builtin_preserve_field_info(arg->b2, FIELD_BYTE_OFFSET);
unsigned size = __builtin_preserve_field_info(arg->b2, FIELD_BYTE_SIZE);
#ifdef USE_PROBE_READ
bpf_probe_read(&ull, size, (const void *)arg + offset);
unsigned lshift = __builtin_preserve_field_info(arg->b2, FIELD_LSHIFT_U64);
#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
lshift = lshift + (size << 3) - 64;
#endif
#else
switch(size) {
case 1:
ull = *(unsigned char *)((void *)arg + offset); break;
case 2:
ull = *(unsigned short *)((void *)arg + offset); break;
case 4:
ull = *(unsigned int *)((void *)arg + offset); break;
case 8:
ull = *(unsigned long long *)((void *)arg + offset); break;
}
unsigned lshift = __builtin_preserve_field_info(arg->b2, FIELD_LSHIFT_U64);
#endif
ull <<= lshift;
if (__builtin_preserve_field_info(arg->b2, FIELD_SIGNEDNESS))
return (long long)ull >> __builtin_preserve_field_info(arg->b2, FIELD_RSHIFT_U64);
return ull >> __builtin_preserve_field_info(arg->b2, FIELD_RSHIFT_U64);
}
There is a minor overhead for bpf_probe_read() on big endian.
The code and relocation generated for field_read where bpf_probe_read() is
used to access argument data on little endian mode:
r3 = r1
r1 = 0
r1 = 4 <=== relocation (FIELD_BYTE_OFFSET)
r3 += r1
r1 = r10
r1 += -8
r2 = 4 <=== relocation (FIELD_BYTE_SIZE)
call bpf_probe_read
r2 = 51 <=== relocation (FIELD_LSHIFT_U64)
r1 = *(u64 *)(r10 - 8)
r1 <<= r2
r2 = 60 <=== relocation (FIELD_RSHIFT_U64)
r0 = r1
r0 >>= r2
r3 = 1 <=== relocation (FIELD_SIGNEDNESS)
if r3 == 0 goto LBB0_2
r1 s>>= r2
r0 = r1
LBB0_2:
exit
Compare to the above code between relocations FIELD_LSHIFT_U64 and
FIELD_LSHIFT_U64, the code with big endian mode has four more
instructions.
r1 = 41 <=== relocation (FIELD_LSHIFT_U64)
r6 += r1
r6 += -64
r6 <<= 32
r6 >>= 32
r1 = *(u64 *)(r10 - 8)
r1 <<= r6
r2 = 60 <=== relocation (FIELD_RSHIFT_U64)
The code and relocation generated when using direct load.
r2 = 0
r3 = 4
r4 = 4
if r4 s> 3 goto LBB0_3
if r4 == 1 goto LBB0_5
if r4 == 2 goto LBB0_6
goto LBB0_9
LBB0_6: # %sw.bb1
r1 += r3
r2 = *(u16 *)(r1 + 0)
goto LBB0_9
LBB0_3: # %entry
if r4 == 4 goto LBB0_7
if r4 == 8 goto LBB0_8
goto LBB0_9
LBB0_8: # %sw.bb9
r1 += r3
r2 = *(u64 *)(r1 + 0)
goto LBB0_9
LBB0_5: # %sw.bb
r1 += r3
r2 = *(u8 *)(r1 + 0)
goto LBB0_9
LBB0_7: # %sw.bb5
r1 += r3
r2 = *(u32 *)(r1 + 0)
LBB0_9: # %sw.epilog
r1 = 51
r2 <<= r1
r1 = 60
r0 = r2
r0 >>= r1
r3 = 1
if r3 == 0 goto LBB0_11
r2 s>>= r1
r0 = r2
LBB0_11: # %sw.epilog
exit
Considering verifier is able to do limited constant
propogation following branches. The following is the
code actually traversed.
r2 = 0
r3 = 4 <=== relocation
r4 = 4 <=== relocation
if r4 s> 3 goto LBB0_3
LBB0_3: # %entry
if r4 == 4 goto LBB0_7
LBB0_7: # %sw.bb5
r1 += r3
r2 = *(u32 *)(r1 + 0)
LBB0_9: # %sw.epilog
r1 = 51 <=== relocation
r2 <<= r1
r1 = 60 <=== relocation
r0 = r2
r0 >>= r1
r3 = 1
if r3 == 0 goto LBB0_11
r2 s>>= r1
r0 = r2
LBB0_11: # %sw.epilog
exit
For native load case, the load size is calculated to be the
same as the size of load width LLVM otherwise used to load
the value which is then used to extract the bitfield value.
Differential Revision: https://reviews.llvm.org/D67980
llvm-svn: 374099
diff --git a/llvm/lib/Target/BPF/BTFDebug.cpp b/llvm/lib/Target/BPF/BTFDebug.cpp
index 3ad3234..a8f8573 100644
--- a/llvm/lib/Target/BPF/BTFDebug.cpp
+++ b/llvm/lib/Target/BPF/BTFDebug.cpp
@@ -754,7 +754,7 @@
void BTFDebug::emitBTFExtSection() {
// Do not emit section if empty FuncInfoTable and LineInfoTable.
if (!FuncInfoTable.size() && !LineInfoTable.size() &&
- !OffsetRelocTable.size() && !ExternRelocTable.size())
+ !FieldRelocTable.size() && !ExternRelocTable.size())
return;
MCContext &Ctx = OS.getContext();
@@ -766,8 +766,8 @@
// Account for FuncInfo/LineInfo record size as well.
uint32_t FuncLen = 4, LineLen = 4;
- // Do not account for optional OffsetReloc/ExternReloc.
- uint32_t OffsetRelocLen = 0, ExternRelocLen = 0;
+ // Do not account for optional FieldReloc/ExternReloc.
+ uint32_t FieldRelocLen = 0, ExternRelocLen = 0;
for (const auto &FuncSec : FuncInfoTable) {
FuncLen += BTF::SecFuncInfoSize;
FuncLen += FuncSec.second.size() * BTF::BPFFuncInfoSize;
@@ -776,17 +776,17 @@
LineLen += BTF::SecLineInfoSize;
LineLen += LineSec.second.size() * BTF::BPFLineInfoSize;
}
- for (const auto &OffsetRelocSec : OffsetRelocTable) {
- OffsetRelocLen += BTF::SecOffsetRelocSize;
- OffsetRelocLen += OffsetRelocSec.second.size() * BTF::BPFOffsetRelocSize;
+ for (const auto &FieldRelocSec : FieldRelocTable) {
+ FieldRelocLen += BTF::SecFieldRelocSize;
+ FieldRelocLen += FieldRelocSec.second.size() * BTF::BPFFieldRelocSize;
}
for (const auto &ExternRelocSec : ExternRelocTable) {
ExternRelocLen += BTF::SecExternRelocSize;
ExternRelocLen += ExternRelocSec.second.size() * BTF::BPFExternRelocSize;
}
- if (OffsetRelocLen)
- OffsetRelocLen += 4;
+ if (FieldRelocLen)
+ FieldRelocLen += 4;
if (ExternRelocLen)
ExternRelocLen += 4;
@@ -795,8 +795,8 @@
OS.EmitIntValue(FuncLen, 4);
OS.EmitIntValue(LineLen, 4);
OS.EmitIntValue(FuncLen + LineLen, 4);
- OS.EmitIntValue(OffsetRelocLen, 4);
- OS.EmitIntValue(FuncLen + LineLen + OffsetRelocLen, 4);
+ OS.EmitIntValue(FieldRelocLen, 4);
+ OS.EmitIntValue(FuncLen + LineLen + FieldRelocLen, 4);
OS.EmitIntValue(ExternRelocLen, 4);
// Emit func_info table.
@@ -831,19 +831,20 @@
}
}
- // Emit offset reloc table.
- if (OffsetRelocLen) {
- OS.AddComment("OffsetReloc");
- OS.EmitIntValue(BTF::BPFOffsetRelocSize, 4);
- for (const auto &OffsetRelocSec : OffsetRelocTable) {
- OS.AddComment("Offset reloc section string offset=" +
- std::to_string(OffsetRelocSec.first));
- OS.EmitIntValue(OffsetRelocSec.first, 4);
- OS.EmitIntValue(OffsetRelocSec.second.size(), 4);
- for (const auto &OffsetRelocInfo : OffsetRelocSec.second) {
- Asm->EmitLabelReference(OffsetRelocInfo.Label, 4);
- OS.EmitIntValue(OffsetRelocInfo.TypeID, 4);
- OS.EmitIntValue(OffsetRelocInfo.OffsetNameOff, 4);
+ // Emit field reloc table.
+ if (FieldRelocLen) {
+ OS.AddComment("FieldReloc");
+ OS.EmitIntValue(BTF::BPFFieldRelocSize, 4);
+ for (const auto &FieldRelocSec : FieldRelocTable) {
+ OS.AddComment("Field reloc section string offset=" +
+ std::to_string(FieldRelocSec.first));
+ OS.EmitIntValue(FieldRelocSec.first, 4);
+ OS.EmitIntValue(FieldRelocSec.second.size(), 4);
+ for (const auto &FieldRelocInfo : FieldRelocSec.second) {
+ Asm->EmitLabelReference(FieldRelocInfo.Label, 4);
+ OS.EmitIntValue(FieldRelocInfo.TypeID, 4);
+ OS.EmitIntValue(FieldRelocInfo.OffsetNameOff, 4);
+ OS.EmitIntValue(FieldRelocInfo.RelocKind, 4);
}
}
}
@@ -958,23 +959,27 @@
return Id;
}
-/// Generate a struct member offset relocation.
-void BTFDebug::generateOffsetReloc(const MachineInstr *MI,
+/// Generate a struct member field relocation.
+void BTFDebug::generateFieldReloc(const MachineInstr *MI,
const MCSymbol *ORSym, DIType *RootTy,
StringRef AccessPattern) {
unsigned RootId = populateStructType(RootTy);
size_t FirstDollar = AccessPattern.find_first_of('$');
size_t FirstColon = AccessPattern.find_first_of(':');
+ size_t SecondColon = AccessPattern.find_first_of(':', FirstColon + 1);
StringRef IndexPattern = AccessPattern.substr(FirstDollar + 1);
- StringRef OffsetStr = AccessPattern.substr(FirstColon + 1,
- FirstDollar - FirstColon);
+ StringRef RelocKindStr = AccessPattern.substr(FirstColon + 1,
+ SecondColon - FirstColon);
+ StringRef PatchImmStr = AccessPattern.substr(SecondColon + 1,
+ FirstDollar - SecondColon);
- BTFOffsetReloc OffsetReloc;
- OffsetReloc.Label = ORSym;
- OffsetReloc.OffsetNameOff = addString(IndexPattern);
- OffsetReloc.TypeID = RootId;
- AccessOffsets[AccessPattern.str()] = std::stoi(OffsetStr);
- OffsetRelocTable[SecNameOff].push_back(OffsetReloc);
+ BTFFieldReloc FieldReloc;
+ FieldReloc.Label = ORSym;
+ FieldReloc.OffsetNameOff = addString(IndexPattern);
+ FieldReloc.TypeID = RootId;
+ FieldReloc.RelocKind = std::stoull(RelocKindStr);
+ PatchImms[AccessPattern.str()] = std::stoul(PatchImmStr);
+ FieldRelocTable[SecNameOff].push_back(FieldReloc);
}
void BTFDebug::processLDimm64(const MachineInstr *MI) {
@@ -982,7 +987,7 @@
// will generate an .BTF.ext record.
//
// If the insn is "r2 = LD_imm64 @__BTF_...",
- // add this insn into the .BTF.ext OffsetReloc subsection.
+ // add this insn into the .BTF.ext FieldReloc subsection.
// Relocation looks like:
// . SecName:
// . InstOffset
@@ -1013,7 +1018,7 @@
MDNode *MDN = GVar->getMetadata(LLVMContext::MD_preserve_access_index);
DIType *Ty = dyn_cast<DIType>(MDN);
- generateOffsetReloc(MI, ORSym, Ty, GVar->getName());
+ generateFieldReloc(MI, ORSym, Ty, GVar->getName());
} else if (GVar && !GVar->hasInitializer() && GVar->hasExternalLinkage() &&
GVar->getSection() == BPFCoreSharedInfo::PatchableExtSecName) {
MCSymbol *ORSym = OS.getContext().createTempSymbol();
@@ -1154,8 +1159,8 @@
const GlobalValue *GVal = MO.getGlobal();
auto *GVar = dyn_cast<GlobalVariable>(GVal);
if (GVar && GVar->hasAttribute(BPFCoreSharedInfo::AmaAttr)) {
- // Emit "mov ri, <imm>" for abstract member accesses.
- int64_t Imm = AccessOffsets[GVar->getName().str()];
+ // Emit "mov ri, <imm>" for patched immediate.
+ uint32_t Imm = PatchImms[GVar->getName().str()];
OutMI.setOpcode(BPF::MOV_ri);
OutMI.addOperand(MCOperand::createReg(MI->getOperand(0).getReg()));
OutMI.addOperand(MCOperand::createImm(Imm));