src/cc/BPF.cc - platform/external/bcc - Gitiles

 /*
  * Copyright (c) 2016 Facebook, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
  *
  * http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 #include <linux/bpf.h>
 #include <unistd.h>
 #include <cstdio>
 #include <cstring>
 #include <exception>
 #include <iostream>
 #include <memory>
 #include <sstream>
 #include <utility>
 #include <vector>

 #include "bcc_exception.h"
 #include "bcc_syms.h"
 #include "bpf_module.h"
 #include "common.h"
 #include "libbpf.h"
 #include "perf_reader.h"
 #include "table_storage.h"
 #include "usdt.h"

 #include "BPF.h"

 namespace ebpf {

 std::string uint_to_hex(uint64_t value) {
   std::stringstream ss;
   ss << std::hex << value;
   return ss.str();
 }

 std::string sanitize_str(std::string str, bool (*validator)(char),
                          char replacement = '_') {
   for (size_t i = 0; i < str.length(); i++)
     if (!validator(str[i]))
       str[i] = replacement;
   return str;
 }

 StatusTuple BPF::init(const std::string& bpf_program,
                       std::vector<std::string> cflags, std::vector<USDT> usdt) {
   std::string all_bpf_program;

   for (auto u : usdt) {
     if (!u.initialized_)
       TRY2(u.init());
     all_bpf_program += u.program_text_;
     usdt_.push_back(std::move(u));
   }

   auto flags_len = cflags.size();
   const char* flags[flags_len];
   for (size_t i = 0; i < flags_len; i++)
     flags[i] = cflags[i].c_str();

   all_bpf_program += bpf_program;
   if (bpf_module_->load_string(all_bpf_program, flags, flags_len) != 0)
     return StatusTuple(-1, "Unable to initialize BPF program");

   return StatusTuple(0);
 };

 BPF::~BPF() {
   auto res = detach_all();
   if (res.code() != 0)
     std::cerr << "Failed to detach all probes on destruction: " << std::endl
               << res.msg() << std::endl;
 }

 StatusTuple BPF::detach_all() {
   bool has_error = false;
   std::string error_msg;

   for (auto it : kprobes_) {
     auto res = detach_kprobe_event(it.first, it.second);
     if (res.code() != 0) {
       error_msg += "Failed to detach kprobe event " + it.first + ": ";
       error_msg += res.msg() + "\n";
       has_error = true;
     }
   }

   for (auto it : uprobes_) {
     auto res = detach_uprobe_event(it.first, it.second);
     if (res.code() != 0) {
       error_msg += "Failed to detach uprobe event " + it.first + ": ";
       error_msg += res.msg() + "\n";
       has_error = true;
     }
   }

   for (auto it : tracepoints_) {
     auto res = detach_tracepoint_event(it.first, it.second);
     if (res.code() != 0) {
       error_msg += "Failed to detach Tracepoint " + it.first + ": ";
       error_msg += res.msg() + "\n";
       has_error = true;
     }
   }

   for (auto it : perf_buffers_) {
     auto res = it.second->close_all_cpu();
     if (res.code() != 0) {
       error_msg += "Failed to close perf buffer " + it.first + ": ";
       error_msg += res.msg() + "\n";
       has_error = true;
     }
     delete it.second;
   }

   for (auto it : perf_events_) {
     auto res = detach_perf_event_all_cpu(it.second);
     if (res.code() != 0) {
       error_msg += res.msg() + "\n";
       has_error = true;
     }
   }

   for (auto it : funcs_) {
     int res = close(it.second);
     if (res != 0) {
       error_msg += "Failed to unload BPF program for " + it.first + ": ";
       error_msg += std::string(std::strerror(errno)) + "\n";
       has_error = true;
     }
   }

   if (has_error)
     return StatusTuple(-1, error_msg);
   else
     return StatusTuple(0);
 }

 StatusTuple BPF::attach_kprobe(const std::string& kernel_func,
                                const std::string& probe_func,
                                bpf_probe_attach_type attach_type,
                                pid_t pid, int cpu, int group_fd,
                                perf_reader_cb cb, void* cb_cookie) {
   std::string probe_event = get_kprobe_event(kernel_func, attach_type);
   if (kprobes_.find(probe_event) != kprobes_.end())
     return StatusTuple(-1, "kprobe %s already attached", probe_event.c_str());

   int probe_fd;
   TRY2(load_func(probe_func, BPF_PROG_TYPE_KPROBE, probe_fd));

   void* res =
       bpf_attach_kprobe(probe_fd, attach_type, probe_event.c_str(), kernel_func.c_str(),
                         pid, cpu, group_fd, cb, cb_cookie);

   if (!res) {
     TRY2(unload_func(probe_func));
     return StatusTuple(-1, "Unable to attach %skprobe for %s using %s",
                        attach_type_debug(attach_type).c_str(),
                        kernel_func.c_str(), probe_func.c_str());
   }

   open_probe_t p = {};
   p.reader_ptr = res;
   p.func = probe_func;
   kprobes_[probe_event] = std::move(p);
   return StatusTuple(0);
 }

 StatusTuple BPF::attach_uprobe(const std::string& binary_path,
                                const std::string& symbol,
                                const std::string& probe_func,
                                uint64_t symbol_addr,
                                bpf_probe_attach_type attach_type,
                                pid_t pid, int cpu, int group_fd,
                                perf_reader_cb cb, void* cb_cookie) {
   bcc_symbol sym = bcc_symbol();
   TRY2(check_binary_symbol(binary_path, symbol, symbol_addr, &sym));

   std::string probe_event =
       get_uprobe_event(sym.module, sym.offset, attach_type);
   if (uprobes_.find(probe_event) != uprobes_.end())
     return StatusTuple(-1, "uprobe %s already attached", probe_event.c_str());

   int probe_fd;
   TRY2(load_func(probe_func, BPF_PROG_TYPE_KPROBE, probe_fd));

   void* res =
       bpf_attach_uprobe(probe_fd, attach_type, probe_event.c_str(), binary_path.c_str(),
                         sym.offset, pid, cpu, group_fd, cb, cb_cookie);

   if (!res) {
     TRY2(unload_func(probe_func));
     return StatusTuple(
         -1,
         "Unable to attach %suprobe for binary %s symbol %s addr %lx using %s\n",
         attach_type_debug(attach_type).c_str(), binary_path.c_str(),
         symbol.c_str(), symbol_addr, probe_func.c_str());
   }

   open_probe_t p = {};
   p.reader_ptr = res;
   p.func = probe_func;
   uprobes_[probe_event] = std::move(p);
   return StatusTuple(0);
 }

 StatusTuple BPF::attach_usdt(const USDT& usdt, pid_t pid, int cpu,
                              int group_fd) {
   for (auto& u : usdt_)
     if (u == usdt) {
       bool failed = false;
       std::string err_msg;
       int cnt = 0;
       for (auto addr : u.addresses_) {
         auto res =
             attach_uprobe(u.binary_path_, std::string(), u.probe_func_, addr);
         if (res.code() != 0) {
           failed = true;
           err_msg += "USDT " + u.print_name() + " at " + std::to_string(addr);
           err_msg += ": " + res.msg() + "\n";
           break;
         }
         cnt++;
       }
       if (failed) {
         for (int i = 0; i < cnt; i++) {
           auto res =
               detach_uprobe(u.binary_path_, std::string(), u.addresses_[i]);
           err_msg += "During clean up: " + res.msg() + "\n";
         }
         return StatusTuple(-1, err_msg);
       } else
         return StatusTuple(0);
     }
   return StatusTuple(-1, "USDT %s not found", usdt.print_name().c_str());
 }

 StatusTuple BPF::attach_tracepoint(const std::string& tracepoint,
                                    const std::string& probe_func,
                                    pid_t pid, int cpu, int group_fd,
                                    perf_reader_cb cb, void* cb_cookie) {
   if (tracepoints_.find(tracepoint) != tracepoints_.end())
     return StatusTuple(-1, "Tracepoint %s already attached",
                        tracepoint.c_str());

   auto pos = tracepoint.find(":");
   if ((pos == std::string::npos) || (pos != tracepoint.rfind(":")))
     return StatusTuple(-1, "Unable to parse Tracepoint %s", tracepoint.c_str());
   std::string tp_category = tracepoint.substr(0, pos);
   std::string tp_name = tracepoint.substr(pos + 1);

   int probe_fd;
   TRY2(load_func(probe_func, BPF_PROG_TYPE_TRACEPOINT, probe_fd));

   void* res =
       bpf_attach_tracepoint(probe_fd, tp_category.c_str(), tp_name.c_str(), pid,
                             cpu, group_fd, cb, cb_cookie);

   if (!res) {
     TRY2(unload_func(probe_func));
     return StatusTuple(-1, "Unable to attach Tracepoint %s using %s",
                        tracepoint.c_str(), probe_func.c_str());
   }

   open_probe_t p = {};
   p.reader_ptr = res;
   p.func = probe_func;
   tracepoints_[tracepoint] = std::move(p);
   return StatusTuple(0);
 }

 StatusTuple BPF::attach_perf_event(uint32_t ev_type, uint32_t ev_config,
                                    const std::string& probe_func,
                                    uint64_t sample_period, uint64_t sample_freq,
                                    pid_t pid, int cpu, int group_fd) {
   auto ev_pair = std::make_pair(ev_type, ev_config);
   if (perf_events_.find(ev_pair) != perf_events_.end())
     return StatusTuple(-1, "Perf event type %d config %d already attached",
                        ev_type, ev_config);

   int probe_fd;
   TRY2(load_func(probe_func, BPF_PROG_TYPE_PERF_EVENT, probe_fd));

   auto fds = new std::map<int, int>();
   std::vector<int> cpus;
   if (cpu >= 0)
     cpus.push_back(cpu);
   else
     cpus = get_online_cpus();
   for (int i: cpus) {
     int fd = bpf_attach_perf_event(probe_fd, ev_type, ev_config, sample_period,
                                    sample_freq, pid, i, group_fd);
     if (fd < 0) {
       for (auto it : *fds)
         close(it.second);
       delete fds;
       TRY2(unload_func(probe_func));
       return StatusTuple(-1, "Failed to attach perf event type %d config %d",
                          ev_type, ev_config);
     }
     fds->emplace(i, fd);
   }

   open_probe_t p = {};
   p.func = probe_func;
   p.per_cpu_fd = fds;
   perf_events_[ev_pair] = std::move(p);
   return StatusTuple(0);
 }

 StatusTuple BPF::detach_kprobe(const std::string& kernel_func,
                                bpf_probe_attach_type attach_type) {
   std::string event = get_kprobe_event(kernel_func, attach_type);

   auto it = kprobes_.find(event);
   if (it == kprobes_.end())
     return StatusTuple(-1, "No open %skprobe for %s",
                        attach_type_debug(attach_type).c_str(),
                        kernel_func.c_str());

   TRY2(detach_kprobe_event(it->first, it->second));
   kprobes_.erase(it);
   return StatusTuple(0);
 }

 StatusTuple BPF::detach_uprobe(const std::string& binary_path,
                                const std::string& symbol, uint64_t symbol_addr,
                                bpf_probe_attach_type attach_type) {
   bcc_symbol sym = bcc_symbol();
   TRY2(check_binary_symbol(binary_path, symbol, symbol_addr, &sym));

   std::string event = get_uprobe_event(sym.module, sym.offset, attach_type);
   auto it = uprobes_.find(event);
   if (it == uprobes_.end())
     return StatusTuple(-1, "No open %suprobe for binary %s symbol %s addr %lx",
                        attach_type_debug(attach_type).c_str(),
                        binary_path.c_str(), symbol.c_str(), symbol_addr);

   TRY2(detach_uprobe_event(it->first, it->second));
   uprobes_.erase(it);
   return StatusTuple(0);
 }

 StatusTuple BPF::detach_usdt(const USDT& usdt) {
   for (auto& u : usdt_)
     if (u == usdt) {
       bool failed = false;
       std::string err_msg;
       for (auto addr : u.addresses_) {
         auto res = detach_uprobe(u.binary_path_, std::string(), addr);
         if (res.code() != 0) {
           failed = true;
           err_msg += "USDT " + u.print_name() + " at " + std::to_string(addr);
           err_msg += ": " + res.msg() + "\n";
         }
       }
       if (failed)
         return StatusTuple(-1, err_msg);
       else
         return StatusTuple(0);
     }
   return StatusTuple(-1, "USDT %s not found", usdt.print_name().c_str());
 }

 StatusTuple BPF::detach_tracepoint(const std::string& tracepoint) {
   auto it = tracepoints_.find(tracepoint);
   if (it == tracepoints_.end())
     return StatusTuple(-1, "No open Tracepoint %s", tracepoint.c_str());

   TRY2(detach_tracepoint_event(it->first, it->second));
   tracepoints_.erase(it);
   return StatusTuple(0);
 }

 StatusTuple BPF::detach_perf_event(uint32_t ev_type, uint32_t ev_config) {
   auto it = perf_events_.find(std::make_pair(ev_type, ev_config));
   if (it == perf_events_.end())
     return StatusTuple(-1, "Perf Event type %d config %d not attached",
                        ev_type, ev_config);
   TRY2(detach_perf_event_all_cpu(it->second));
   perf_events_.erase(it);
   return StatusTuple(0);
 }

 StatusTuple BPF::open_perf_buffer(const std::string& name,
                                   perf_reader_raw_cb cb, void* cb_cookie,
                                   int page_cnt) {
   if (perf_buffers_.find(name) == perf_buffers_.end())
     perf_buffers_[name] = new BPFPerfBuffer(bpf_module_.get(), name);
   if ((page_cnt & (page_cnt - 1)) != 0)
     return StatusTuple(-1, "open_perf_buffer page_cnt must be a power of two");
   auto table = perf_buffers_[name];
   TRY2(table->open_all_cpu(cb, cb_cookie, page_cnt));
   return StatusTuple(0);
 }

 StatusTuple BPF::close_perf_buffer(const std::string& name) {
   auto it = perf_buffers_.find(name);
   if (it == perf_buffers_.end())
     return StatusTuple(-1, "Perf buffer for %s not open", name.c_str());
   TRY2(it->second->close_all_cpu());
   return StatusTuple(0);
 }

 void BPF::poll_perf_buffer(const std::string& name, int timeout) {
   auto it = perf_buffers_.find(name);
   if (it == perf_buffers_.end())
     return;
   it->second->poll(timeout);
 }

 StatusTuple BPF::load_func(const std::string& func_name,
                            bpf_prog_type type, int& fd) {
   if (funcs_.find(func_name) != funcs_.end()) {
     fd = funcs_[func_name];
     return StatusTuple(0);
   }

   uint8_t* func_start = bpf_module_->function_start(func_name);
   if (!func_start)
     return StatusTuple(-1, "Can't find start of function %s",
                        func_name.c_str());
   size_t func_size = bpf_module_->function_size(func_name);

   fd = bpf_prog_load(type, reinterpret_cast<struct bpf_insn*>(func_start),
                      func_size, bpf_module_->license(),
                      bpf_module_->kern_version(), nullptr,
                      0  // BPFModule will handle error printing
                      );

   if (fd < 0)
     return StatusTuple(-1, "Failed to load %s: %d", func_name.c_str(), fd);
   funcs_[func_name] = fd;
   return StatusTuple(0);
 }

 StatusTuple BPF::unload_func(const std::string& func_name) {
   auto it = funcs_.find(func_name);
   if (it == funcs_.end())
     return StatusTuple(0);

   int res = close(it->second);
   if (res != 0)
     return StatusTuple(-1, "Can't close FD for %s: %d", it->first.c_str(), res);

   funcs_.erase(it);
   return StatusTuple(0);
 }

 StatusTuple BPF::check_binary_symbol(const std::string& binary_path,
                                      const std::string& symbol,
                                      uint64_t symbol_addr, bcc_symbol* output) {
   int res = bcc_resolve_symname(binary_path.c_str(), symbol.c_str(),
                                 symbol_addr, 0, output);
   if (res < 0)
     return StatusTuple(
         -1, "Unable to find offset for binary %s symbol %s address %lx",
         binary_path.c_str(), symbol.c_str(), symbol_addr);
   return StatusTuple(0);
 }

 std::string BPF::get_kprobe_event(const std::string& kernel_func,
                                   bpf_probe_attach_type type) {
   std::string res = attach_type_prefix(type) + "_";
   res += sanitize_str(kernel_func, &BPF::kprobe_event_validator);
   return res;
 }

 std::string BPF::get_uprobe_event(const std::string& binary_path,
                                   uint64_t offset, bpf_probe_attach_type type) {
   std::string res = attach_type_prefix(type) + "_";
   res += sanitize_str(binary_path, &BPF::uprobe_path_validator);
   res += "_0x" + uint_to_hex(offset);
   return res;
 }

 StatusTuple BPF::detach_kprobe_event(const std::string& event,
                                      open_probe_t& attr) {
   if (attr.reader_ptr) {
     perf_reader_free(attr.reader_ptr);
     attr.reader_ptr = nullptr;
   }
   TRY2(unload_func(attr.func));
   if (bpf_detach_kprobe(event.c_str()) < 0)
     return StatusTuple(-1, "Unable to detach kprobe %s", event.c_str());
   return StatusTuple(0);
 }

 StatusTuple BPF::detach_uprobe_event(const std::string& event,
                                      open_probe_t& attr) {
   if (attr.reader_ptr) {
     perf_reader_free(attr.reader_ptr);
     attr.reader_ptr = nullptr;
   }
   TRY2(unload_func(attr.func));
   if (bpf_detach_uprobe(event.c_str()) < 0)
     return StatusTuple(-1, "Unable to detach uprobe %s", event.c_str());
   return StatusTuple(0);
 }

 StatusTuple BPF::detach_tracepoint_event(const std::string& tracepoint,
                                          open_probe_t& attr) {
   if (attr.reader_ptr) {
     perf_reader_free(attr.reader_ptr);
     attr.reader_ptr = nullptr;
   }
   TRY2(unload_func(attr.func));

   // TODO: bpf_detach_tracepoint currently does nothing.
   return StatusTuple(0);
 }

 StatusTuple BPF::detach_perf_event_all_cpu(open_probe_t& attr) {
   bool has_error = false;
   std::string err_msg;
   for (auto it : *attr.per_cpu_fd) {
     int res = close(it.second);
     if (res < 0) {
       has_error = true;
       err_msg += "Failed to close perf event FD " + std::to_string(it.second) +
                  " For CPU " + std::to_string(it.first) + ": ";
       err_msg += std::string(std::strerror(errno)) + "\n";
     }
   }
   delete attr.per_cpu_fd;
   TRY2(unload_func(attr.func));

   if (has_error)
     return StatusTuple(-1, err_msg);
   return StatusTuple(0);
 }

 StatusTuple USDT::init() {
   auto ctx =
       std::unique_ptr<::USDT::Context>(new ::USDT::Context(binary_path_));
   if (!ctx->loaded())
     return StatusTuple(-1, "Unable to load USDT " + print_name());
   auto probe = ctx->get(name_);
   if (probe == nullptr)
     return StatusTuple(-1, "Unable to find USDT " + print_name());

   if (!probe->enable(probe_func_))
     return StatusTuple(-1, "Failed to enable USDT " + print_name());
   std::ostringstream stream;
   if (!probe->usdt_getarg(stream))
     return StatusTuple(
         -1, "Unable to generate program text for USDT " + print_name());
   program_text_ = ::USDT::USDT_PROGRAM_HEADER + stream.str();

   for (size_t i = 0; i < probe->num_locations(); i++)
     addresses_.push_back(probe->address(i));

   initialized_ = true;
   return StatusTuple(0);
 }

 }  // namespace ebpf
	/*
	* Copyright (c) 2016 Facebook, Inc.
	*
	* Licensed under the Apache License, Version 2.0 (the "License");
	* you may not use this file except in compliance with the License.
	* You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	#include <linux/bpf.h>
	#include <unistd.h>
	#include <cstdio>
	#include <cstring>
	#include <exception>
	#include <iostream>
	#include <memory>
	#include <sstream>
	#include <utility>
	#include <vector>

	#include "bcc_exception.h"
	#include "bcc_syms.h"
	#include "bpf_module.h"
	#include "common.h"
	#include "libbpf.h"
	#include "perf_reader.h"
	#include "table_storage.h"
	#include "usdt.h"

	#include "BPF.h"

	namespace ebpf {

	std::string uint_to_hex(uint64_t value) {
	std::stringstream ss;
	ss << std::hex << value;
	return ss.str();
	}

	std::string sanitize_str(std::string str, bool (*validator)(char),
	char replacement = '_') {
	for (size_t i = 0; i < str.length(); i++)
	if (!validator(str[i]))
	str[i] = replacement;
	return str;
	}

	StatusTuple BPF::init(const std::string& bpf_program,
	std::vector<std::string> cflags, std::vector<USDT> usdt) {
	std::string all_bpf_program;

	for (auto u : usdt) {
	if (!u.initialized_)
	TRY2(u.init());
	all_bpf_program += u.program_text_;
	usdt_.push_back(std::move(u));
	}

	auto flags_len = cflags.size();
	const char* flags[flags_len];
	for (size_t i = 0; i < flags_len; i++)
	flags[i] = cflags[i].c_str();

	all_bpf_program += bpf_program;
	if (bpf_module_->load_string(all_bpf_program, flags, flags_len) != 0)
	return StatusTuple(-1, "Unable to initialize BPF program");

	return StatusTuple(0);
	};

	BPF::~BPF() {
	auto res = detach_all();
	if (res.code() != 0)
	std::cerr << "Failed to detach all probes on destruction: " << std::endl
	<< res.msg() << std::endl;
	}

	StatusTuple BPF::detach_all() {
	bool has_error = false;
	std::string error_msg;

	for (auto it : kprobes_) {
	auto res = detach_kprobe_event(it.first, it.second);
	if (res.code() != 0) {
	error_msg += "Failed to detach kprobe event " + it.first + ": ";
	error_msg += res.msg() + "\n";
	has_error = true;
	}
	}

	for (auto it : uprobes_) {
	auto res = detach_uprobe_event(it.first, it.second);
	if (res.code() != 0) {
	error_msg += "Failed to detach uprobe event " + it.first + ": ";
	error_msg += res.msg() + "\n";
	has_error = true;
	}
	}

	for (auto it : tracepoints_) {
	auto res = detach_tracepoint_event(it.first, it.second);
	if (res.code() != 0) {
	error_msg += "Failed to detach Tracepoint " + it.first + ": ";
	error_msg += res.msg() + "\n";
	has_error = true;
	}
	}

	for (auto it : perf_buffers_) {
	auto res = it.second->close_all_cpu();
	if (res.code() != 0) {
	error_msg += "Failed to close perf buffer " + it.first + ": ";
	error_msg += res.msg() + "\n";
	has_error = true;
	}
	delete it.second;
	}

	for (auto it : perf_events_) {
	auto res = detach_perf_event_all_cpu(it.second);
	if (res.code() != 0) {
	error_msg += res.msg() + "\n";
	has_error = true;
	}
	}

	for (auto it : funcs_) {
	int res = close(it.second);
	if (res != 0) {
	error_msg += "Failed to unload BPF program for " + it.first + ": ";
	error_msg += std::string(std::strerror(errno)) + "\n";
	has_error = true;
	}
	}

	if (has_error)
	return StatusTuple(-1, error_msg);
	else
	return StatusTuple(0);
	}

	StatusTuple BPF::attach_kprobe(const std::string& kernel_func,
	const std::string& probe_func,
	bpf_probe_attach_type attach_type,
	pid_t pid, int cpu, int group_fd,
	perf_reader_cb cb, void* cb_cookie) {
	std::string probe_event = get_kprobe_event(kernel_func, attach_type);
	if (kprobes_.find(probe_event) != kprobes_.end())
	return StatusTuple(-1, "kprobe %s already attached", probe_event.c_str());

	int probe_fd;
	TRY2(load_func(probe_func, BPF_PROG_TYPE_KPROBE, probe_fd));

	void* res =
	bpf_attach_kprobe(probe_fd, attach_type, probe_event.c_str(), kernel_func.c_str(),
	pid, cpu, group_fd, cb, cb_cookie);

	if (!res) {
	TRY2(unload_func(probe_func));
	return StatusTuple(-1, "Unable to attach %skprobe for %s using %s",
	attach_type_debug(attach_type).c_str(),
	kernel_func.c_str(), probe_func.c_str());
	}

	open_probe_t p = {};
	p.reader_ptr = res;
	p.func = probe_func;
	kprobes_[probe_event] = std::move(p);
	return StatusTuple(0);
	}

	StatusTuple BPF::attach_uprobe(const std::string& binary_path,
	const std::string& symbol,
	const std::string& probe_func,
	uint64_t symbol_addr,
	bpf_probe_attach_type attach_type,
	pid_t pid, int cpu, int group_fd,
	perf_reader_cb cb, void* cb_cookie) {
	bcc_symbol sym = bcc_symbol();
	TRY2(check_binary_symbol(binary_path, symbol, symbol_addr, &sym));

	std::string probe_event =
	get_uprobe_event(sym.module, sym.offset, attach_type);
	if (uprobes_.find(probe_event) != uprobes_.end())
	return StatusTuple(-1, "uprobe %s already attached", probe_event.c_str());

	int probe_fd;
	TRY2(load_func(probe_func, BPF_PROG_TYPE_KPROBE, probe_fd));

	void* res =
	bpf_attach_uprobe(probe_fd, attach_type, probe_event.c_str(), binary_path.c_str(),
	sym.offset, pid, cpu, group_fd, cb, cb_cookie);

	if (!res) {
	TRY2(unload_func(probe_func));
	return StatusTuple(
	-1,
	"Unable to attach %suprobe for binary %s symbol %s addr %lx using %s\n",
	attach_type_debug(attach_type).c_str(), binary_path.c_str(),
	symbol.c_str(), symbol_addr, probe_func.c_str());
	}

	open_probe_t p = {};
	p.reader_ptr = res;
	p.func = probe_func;
	uprobes_[probe_event] = std::move(p);
	return StatusTuple(0);
	}

	StatusTuple BPF::attach_usdt(const USDT& usdt, pid_t pid, int cpu,
	int group_fd) {
	for (auto& u : usdt_)
	if (u == usdt) {
	bool failed = false;
	std::string err_msg;
	int cnt = 0;
	for (auto addr : u.addresses_) {
	auto res =
	attach_uprobe(u.binary_path_, std::string(), u.probe_func_, addr);
	if (res.code() != 0) {
	failed = true;
	err_msg += "USDT " + u.print_name() + " at " + std::to_string(addr);
	err_msg += ": " + res.msg() + "\n";
	break;
	}
	cnt++;
	}
	if (failed) {
	for (int i = 0; i < cnt; i++) {
	auto res =
	detach_uprobe(u.binary_path_, std::string(), u.addresses_[i]);
	err_msg += "During clean up: " + res.msg() + "\n";
	}
	return StatusTuple(-1, err_msg);
	} else
	return StatusTuple(0);
	}
	return StatusTuple(-1, "USDT %s not found", usdt.print_name().c_str());
	}

	StatusTuple BPF::attach_tracepoint(const std::string& tracepoint,
	const std::string& probe_func,
	pid_t pid, int cpu, int group_fd,
	perf_reader_cb cb, void* cb_cookie) {
	if (tracepoints_.find(tracepoint) != tracepoints_.end())
	return StatusTuple(-1, "Tracepoint %s already attached",
	tracepoint.c_str());

	auto pos = tracepoint.find(":");
	if ((pos == std::string::npos) \|\| (pos != tracepoint.rfind(":")))
	return StatusTuple(-1, "Unable to parse Tracepoint %s", tracepoint.c_str());
	std::string tp_category = tracepoint.substr(0, pos);
	std::string tp_name = tracepoint.substr(pos + 1);

	int probe_fd;
	TRY2(load_func(probe_func, BPF_PROG_TYPE_TRACEPOINT, probe_fd));

	void* res =
	bpf_attach_tracepoint(probe_fd, tp_category.c_str(), tp_name.c_str(), pid,
	cpu, group_fd, cb, cb_cookie);

	if (!res) {
	TRY2(unload_func(probe_func));
	return StatusTuple(-1, "Unable to attach Tracepoint %s using %s",
	tracepoint.c_str(), probe_func.c_str());
	}

	open_probe_t p = {};
	p.reader_ptr = res;
	p.func = probe_func;
	tracepoints_[tracepoint] = std::move(p);
	return StatusTuple(0);
	}

	StatusTuple BPF::attach_perf_event(uint32_t ev_type, uint32_t ev_config,
	const std::string& probe_func,
	uint64_t sample_period, uint64_t sample_freq,
	pid_t pid, int cpu, int group_fd) {
	auto ev_pair = std::make_pair(ev_type, ev_config);
	if (perf_events_.find(ev_pair) != perf_events_.end())
	return StatusTuple(-1, "Perf event type %d config %d already attached",
	ev_type, ev_config);

	int probe_fd;
	TRY2(load_func(probe_func, BPF_PROG_TYPE_PERF_EVENT, probe_fd));

	auto fds = new std::map<int, int>();
	std::vector<int> cpus;
	if (cpu >= 0)
	cpus.push_back(cpu);
	else
	cpus = get_online_cpus();
	for (int i: cpus) {
	int fd = bpf_attach_perf_event(probe_fd, ev_type, ev_config, sample_period,
	sample_freq, pid, i, group_fd);
	if (fd < 0) {
	for (auto it : *fds)
	close(it.second);
	delete fds;
	TRY2(unload_func(probe_func));
	return StatusTuple(-1, "Failed to attach perf event type %d config %d",
	ev_type, ev_config);
	}
	fds->emplace(i, fd);
	}

	open_probe_t p = {};
	p.func = probe_func;
	p.per_cpu_fd = fds;
	perf_events_[ev_pair] = std::move(p);
	return StatusTuple(0);
	}

	StatusTuple BPF::detach_kprobe(const std::string& kernel_func,
	bpf_probe_attach_type attach_type) {
	std::string event = get_kprobe_event(kernel_func, attach_type);

	auto it = kprobes_.find(event);
	if (it == kprobes_.end())
	return StatusTuple(-1, "No open %skprobe for %s",
	attach_type_debug(attach_type).c_str(),
	kernel_func.c_str());

	TRY2(detach_kprobe_event(it->first, it->second));
	kprobes_.erase(it);
	return StatusTuple(0);
	}

	StatusTuple BPF::detach_uprobe(const std::string& binary_path,
	const std::string& symbol, uint64_t symbol_addr,
	bpf_probe_attach_type attach_type) {
	bcc_symbol sym = bcc_symbol();
	TRY2(check_binary_symbol(binary_path, symbol, symbol_addr, &sym));

	std::string event = get_uprobe_event(sym.module, sym.offset, attach_type);
	auto it = uprobes_.find(event);
	if (it == uprobes_.end())
	return StatusTuple(-1, "No open %suprobe for binary %s symbol %s addr %lx",
	attach_type_debug(attach_type).c_str(),
	binary_path.c_str(), symbol.c_str(), symbol_addr);

	TRY2(detach_uprobe_event(it->first, it->second));
	uprobes_.erase(it);
	return StatusTuple(0);
	}

	StatusTuple BPF::detach_usdt(const USDT& usdt) {
	for (auto& u : usdt_)
	if (u == usdt) {
	bool failed = false;
	std::string err_msg;
	for (auto addr : u.addresses_) {
	auto res = detach_uprobe(u.binary_path_, std::string(), addr);
	if (res.code() != 0) {
	failed = true;
	err_msg += "USDT " + u.print_name() + " at " + std::to_string(addr);
	err_msg += ": " + res.msg() + "\n";
	}
	}
	if (failed)
	return StatusTuple(-1, err_msg);
	else
	return StatusTuple(0);
	}
	return StatusTuple(-1, "USDT %s not found", usdt.print_name().c_str());
	}

	StatusTuple BPF::detach_tracepoint(const std::string& tracepoint) {
	auto it = tracepoints_.find(tracepoint);
	if (it == tracepoints_.end())
	return StatusTuple(-1, "No open Tracepoint %s", tracepoint.c_str());

	TRY2(detach_tracepoint_event(it->first, it->second));
	tracepoints_.erase(it);
	return StatusTuple(0);
	}

	StatusTuple BPF::detach_perf_event(uint32_t ev_type, uint32_t ev_config) {
	auto it = perf_events_.find(std::make_pair(ev_type, ev_config));
	if (it == perf_events_.end())
	return StatusTuple(-1, "Perf Event type %d config %d not attached",
	ev_type, ev_config);
	TRY2(detach_perf_event_all_cpu(it->second));
	perf_events_.erase(it);
	return StatusTuple(0);
	}

	StatusTuple BPF::open_perf_buffer(const std::string& name,
	perf_reader_raw_cb cb, void* cb_cookie,
	int page_cnt) {
	if (perf_buffers_.find(name) == perf_buffers_.end())
	perf_buffers_[name] = new BPFPerfBuffer(bpf_module_.get(), name);
	if ((page_cnt & (page_cnt - 1)) != 0)
	return StatusTuple(-1, "open_perf_buffer page_cnt must be a power of two");
	auto table = perf_buffers_[name];
	TRY2(table->open_all_cpu(cb, cb_cookie, page_cnt));
	return StatusTuple(0);
	}

	StatusTuple BPF::close_perf_buffer(const std::string& name) {
	auto it = perf_buffers_.find(name);
	if (it == perf_buffers_.end())
	return StatusTuple(-1, "Perf buffer for %s not open", name.c_str());
	TRY2(it->second->close_all_cpu());
	return StatusTuple(0);
	}

	void BPF::poll_perf_buffer(const std::string& name, int timeout) {
	auto it = perf_buffers_.find(name);
	if (it == perf_buffers_.end())
	return;
	it->second->poll(timeout);
	}

	StatusTuple BPF::load_func(const std::string& func_name,
	bpf_prog_type type, int& fd) {
	if (funcs_.find(func_name) != funcs_.end()) {
	fd = funcs_[func_name];
	return StatusTuple(0);
	}

	uint8_t* func_start = bpf_module_->function_start(func_name);
	if (!func_start)
	return StatusTuple(-1, "Can't find start of function %s",
	func_name.c_str());
	size_t func_size = bpf_module_->function_size(func_name);

	fd = bpf_prog_load(type, reinterpret_cast<struct bpf_insn*>(func_start),
	func_size, bpf_module_->license(),
	bpf_module_->kern_version(), nullptr,
	0 // BPFModule will handle error printing
	);

	if (fd < 0)
	return StatusTuple(-1, "Failed to load %s: %d", func_name.c_str(), fd);
	funcs_[func_name] = fd;
	return StatusTuple(0);
	}

	StatusTuple BPF::unload_func(const std::string& func_name) {
	auto it = funcs_.find(func_name);
	if (it == funcs_.end())
	return StatusTuple(0);

	int res = close(it->second);
	if (res != 0)
	return StatusTuple(-1, "Can't close FD for %s: %d", it->first.c_str(), res);

	funcs_.erase(it);
	return StatusTuple(0);
	}

	StatusTuple BPF::check_binary_symbol(const std::string& binary_path,
	const std::string& symbol,
	uint64_t symbol_addr, bcc_symbol* output) {
	int res = bcc_resolve_symname(binary_path.c_str(), symbol.c_str(),
	symbol_addr, 0, output);
	if (res < 0)
	return StatusTuple(
	-1, "Unable to find offset for binary %s symbol %s address %lx",
	binary_path.c_str(), symbol.c_str(), symbol_addr);
	return StatusTuple(0);
	}

	std::string BPF::get_kprobe_event(const std::string& kernel_func,
	bpf_probe_attach_type type) {
	std::string res = attach_type_prefix(type) + "_";
	res += sanitize_str(kernel_func, &BPF::kprobe_event_validator);
	return res;
	}

	std::string BPF::get_uprobe_event(const std::string& binary_path,
	uint64_t offset, bpf_probe_attach_type type) {
	std::string res = attach_type_prefix(type) + "_";
	res += sanitize_str(binary_path, &BPF::uprobe_path_validator);
	res += "_0x" + uint_to_hex(offset);
	return res;
	}

	StatusTuple BPF::detach_kprobe_event(const std::string& event,
	open_probe_t& attr) {
	if (attr.reader_ptr) {
	perf_reader_free(attr.reader_ptr);
	attr.reader_ptr = nullptr;
	}
	TRY2(unload_func(attr.func));
	if (bpf_detach_kprobe(event.c_str()) < 0)
	return StatusTuple(-1, "Unable to detach kprobe %s", event.c_str());
	return StatusTuple(0);
	}

	StatusTuple BPF::detach_uprobe_event(const std::string& event,
	open_probe_t& attr) {
	if (attr.reader_ptr) {
	perf_reader_free(attr.reader_ptr);
	attr.reader_ptr = nullptr;
	}
	TRY2(unload_func(attr.func));
	if (bpf_detach_uprobe(event.c_str()) < 0)
	return StatusTuple(-1, "Unable to detach uprobe %s", event.c_str());
	return StatusTuple(0);
	}

	StatusTuple BPF::detach_tracepoint_event(const std::string& tracepoint,
	open_probe_t& attr) {
	if (attr.reader_ptr) {
	perf_reader_free(attr.reader_ptr);
	attr.reader_ptr = nullptr;
	}
	TRY2(unload_func(attr.func));

	// TODO: bpf_detach_tracepoint currently does nothing.
	return StatusTuple(0);
	}

	StatusTuple BPF::detach_perf_event_all_cpu(open_probe_t& attr) {
	bool has_error = false;
	std::string err_msg;
	for (auto it : *attr.per_cpu_fd) {
	int res = close(it.second);
	if (res < 0) {
	has_error = true;
	err_msg += "Failed to close perf event FD " + std::to_string(it.second) +
	" For CPU " + std::to_string(it.first) + ": ";
	err_msg += std::string(std::strerror(errno)) + "\n";
	}
	}
	delete attr.per_cpu_fd;
	TRY2(unload_func(attr.func));

	if (has_error)
	return StatusTuple(-1, err_msg);
	return StatusTuple(0);
	}

	StatusTuple USDT::init() {
	auto ctx =
	std::unique_ptr<::USDT::Context>(new ::USDT::Context(binary_path_));
	if (!ctx->loaded())
	return StatusTuple(-1, "Unable to load USDT " + print_name());
	auto probe = ctx->get(name_);
	if (probe == nullptr)
	return StatusTuple(-1, "Unable to find USDT " + print_name());

	if (!probe->enable(probe_func_))
	return StatusTuple(-1, "Failed to enable USDT " + print_name());
	std::ostringstream stream;
	if (!probe->usdt_getarg(stream))
	return StatusTuple(
	-1, "Unable to generate program text for USDT " + print_name());
	program_text_ = ::USDT::USDT_PROGRAM_HEADER + stream.str();

	for (size_t i = 0; i < probe->num_locations(); i++)
	addresses_.push_back(probe->address(i));

	initialized_ = true;
	return StatusTuple(0);
	}

	} // namespace ebpf