[clang-doc] Implement reducer portion of the frontend framework
Implements a simple, in-memory reducer for the mapped output of the
initial tool. This creates a collection object for storing the
deduplicated infos on each declaration, and populates that from the
mapper output. The collection object is serialized to LLVM
bitstream. On reading each serialized output, it checks to see if a
merge is necessary and if so, merges the new info with the existing
info (prefering the existing one if conflicts exist).
For a more detailed overview of the tool, see the design document
on the mailing list:
http://lists.llvm.org/pipermail/cfe-dev/2017-December/056203.html
Differential Revision: https://reviews.llvm.org/D43341
llvm-svn: 333932
diff --git a/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp b/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
index 51c3aa9..7593429 100644
--- a/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
+++ b/clang-tools-extra/clang-doc/tool/ClangDocMain.cpp
@@ -18,7 +18,10 @@
//
//===----------------------------------------------------------------------===//
+#include "BitcodeReader.h"
+#include "BitcodeWriter.h"
#include "ClangDoc.h"
+#include "Representation.h"
#include "clang/AST/AST.h"
#include "clang/AST/Decl.h"
#include "clang/ASTMatchers/ASTMatchFinder.h"
@@ -30,6 +33,7 @@
#include "clang/Tooling/StandaloneExecution.h"
#include "clang/Tooling/Tooling.h"
#include "llvm/ADT/APFloat.h"
+#include "llvm/Support/Error.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Process.h"
@@ -54,14 +58,66 @@
llvm::cl::desc("Dump mapper results to bitcode file."),
llvm::cl::init(false), llvm::cl::cat(ClangDocCategory));
+static llvm::cl::opt<bool> DumpIntermediateResult(
+ "dump-intermediate",
+ llvm::cl::desc("Dump intermediate results to bitcode file."),
+ llvm::cl::init(false), llvm::cl::cat(ClangDocCategory));
+
+enum OutputFormatTy {
+ yaml,
+};
+
+static llvm::cl::opt<OutputFormatTy>
+ Format("format", llvm::cl::desc("Format for outputted docs."),
+ llvm::cl::values(clEnumVal(yaml, "Documentation in YAML format.")),
+ llvm::cl::init(yaml), llvm::cl::cat(ClangDocCategory));
+
static llvm::cl::opt<bool> DoxygenOnly(
"doxygen",
llvm::cl::desc("Use only doxygen-style comments to generate docs."),
llvm::cl::init(false), llvm::cl::cat(ClangDocCategory));
+bool CreateDirectory(const Twine &DirName, bool ClearDirectory = false) {
+ std::error_code OK;
+ llvm::SmallString<128> DocsRootPath;
+ if (ClearDirectory) {
+ std::error_code RemoveStatus = llvm::sys::fs::remove_directories(DirName);
+ if (RemoveStatus != OK) {
+ llvm::errs() << "Unable to remove existing documentation directory for "
+ << DirName << ".\n";
+ return true;
+ }
+ }
+ std::error_code DirectoryStatus = llvm::sys::fs::create_directories(DirName);
+ if (DirectoryStatus != OK) {
+ llvm::errs() << "Unable to create documentation directories.\n";
+ return true;
+ }
+ return false;
+}
+
+bool DumpResultToFile(const Twine &DirName, const Twine &FileName,
+ StringRef Buffer, bool ClearDirectory = false) {
+ std::error_code OK;
+ llvm::SmallString<128> IRRootPath;
+ llvm::sys::path::native(OutDirectory, IRRootPath);
+ llvm::sys::path::append(IRRootPath, DirName);
+ if (CreateDirectory(IRRootPath, ClearDirectory))
+ return true;
+ llvm::sys::path::append(IRRootPath, FileName);
+ std::error_code OutErrorInfo;
+ llvm::raw_fd_ostream OS(IRRootPath, OutErrorInfo, llvm::sys::fs::F_None);
+ if (OutErrorInfo != OK) {
+ llvm::errs() << "Error opening documentation file.\n";
+ return true;
+ }
+ OS << Buffer;
+ OS.close();
+ return false;
+}
+
int main(int argc, const char **argv) {
llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
- std::error_code OK;
auto Exec = clang::tooling::createExecutorFromCommandLineArgs(
argc, argv, ClangDocCategory);
@@ -80,34 +136,66 @@
// Mapping phase
llvm::outs() << "Mapping decls...\n";
- auto Err = Exec->get()->execute(doc::newMapperActionFactory(
- Exec->get()->getExecutionContext()),
- ArgAdjuster);
- if (Err)
+ auto Err = Exec->get()->execute(
+ doc::newMapperActionFactory(Exec->get()->getExecutionContext()),
+ ArgAdjuster);
+ if (Err) {
llvm::errs() << toString(std::move(Err)) << "\n";
+ return 1;
+ }
if (DumpMapperResult) {
- Exec->get()->getToolResults()->forEachResult([&](StringRef Key,
- StringRef Value) {
- SmallString<128> IRRootPath;
- llvm::sys::path::native(OutDirectory, IRRootPath);
- llvm::sys::path::append(IRRootPath, "bc");
- std::error_code DirectoryStatus =
- llvm::sys::fs::create_directories(IRRootPath);
- if (DirectoryStatus != OK) {
- llvm::errs() << "Unable to create documentation directories.\n";
- return;
+ bool Err = false;
+ Exec->get()->getToolResults()->forEachResult(
+ [&](StringRef Key, StringRef Value) {
+ Err = DumpResultToFile("bc", Key + ".bc", Value);
+ });
+ if (Err)
+ llvm::errs() << "Error dumping map results.\n";
+ return Err;
+ }
+
+ // Collect values into output by key.
+ llvm::outs() << "Collecting infos...\n";
+ llvm::StringMap<std::vector<std::unique_ptr<doc::Info>>> MapOutput;
+
+ // In ToolResults, the Key is the hashed USR and the value is the
+ // bitcode-encoded representation of the Info object.
+ Exec->get()->getToolResults()->forEachResult([&](StringRef Key,
+ StringRef Value) {
+ llvm::BitstreamCursor Stream(Value);
+ doc::ClangDocBitcodeReader Reader(Stream);
+ auto Infos = Reader.readBitcode();
+ for (auto &I : Infos) {
+ auto R =
+ MapOutput.try_emplace(Key, std::vector<std::unique_ptr<doc::Info>>());
+ R.first->second.emplace_back(std::move(I));
+ }
+ });
+
+ // Reducing phase
+ llvm::outs() << "Reducing " << MapOutput.size() << " infos...\n";
+ llvm::StringMap<std::unique_ptr<doc::Info>> ReduceOutput;
+ for (auto &Group : MapOutput) {
+ auto Reduced = doc::mergeInfos(Group.getValue());
+ if (!Reduced)
+ llvm::errs() << llvm::toString(Reduced.takeError());
+
+ if (DumpIntermediateResult) {
+ SmallString<4096> Buffer;
+ llvm::BitstreamWriter Stream(Buffer);
+ doc::ClangDocBitcodeWriter Writer(Stream);
+ Writer.dispatchInfoForWrite(Reduced.get().get());
+ if (DumpResultToFile("bc", Group.getKey() + ".bc", Buffer)) {
+ llvm::errs() << "Error writing " << Group.getKey() << " to file.\n";
+ continue;
}
- llvm::sys::path::append(IRRootPath, Key + ".bc");
- std::error_code OutErrorInfo;
- llvm::raw_fd_ostream OS(IRRootPath, OutErrorInfo, llvm::sys::fs::F_None);
- if (OutErrorInfo != OK) {
- llvm::errs() << "Error opening documentation file.\n";
- return;
- }
- OS << Value;
- OS.close();
- });
+ }
+
+ ReduceOutput.insert(
+ std::make_pair(Group.getKey(), std::move(Reduced.get())));
+
+ // FIXME: Add support for emitting different output formats.
}
return 0;