tp: add API for decompressing compressed trace packets
Also consolidate the multiple zlib usages around trace processor into
a single reusable class.
Bug: 150855819
Change-Id: Ie070256562585385a1537b062f7fb8aa664cede7
diff --git a/Android.bp b/Android.bp
index 7c7278f..6209809 100644
--- a/Android.bp
+++ b/Android.bp
@@ -6325,7 +6325,6 @@
srcs: [
"src/trace_processor/additional_modules.cc",
"src/trace_processor/ftrace_utils.cc",
- "src/trace_processor/gzip_trace_parser.cc",
"src/trace_processor/importers/ftrace/binder_tracker.cc",
"src/trace_processor/importers/ftrace/ftrace_descriptors.cc",
"src/trace_processor/importers/ftrace/ftrace_module_impl.cc",
@@ -6337,6 +6336,7 @@
"src/trace_processor/importers/fuchsia/fuchsia_trace_parser.cc",
"src/trace_processor/importers/fuchsia/fuchsia_trace_tokenizer.cc",
"src/trace_processor/importers/fuchsia/fuchsia_trace_utils.cc",
+ "src/trace_processor/importers/gzip/gzip_trace_parser.cc",
"src/trace_processor/importers/proto/android_probes_module.cc",
"src/trace_processor/importers/proto/android_probes_parser.cc",
"src/trace_processor/importers/proto/graphics_event_module.cc",
@@ -6368,6 +6368,7 @@
"src/trace_processor/global_args_tracker.cc",
"src/trace_processor/heap_profile_tracker.cc",
"src/trace_processor/importers/ftrace/ftrace_module.cc",
+ "src/trace_processor/importers/gzip/gzip_utils.cc",
"src/trace_processor/importers/ninja/ninja_log_parser.cc",
"src/trace_processor/importers/proto/args_table_utils.cc",
"src/trace_processor/importers/proto/packet_sequence_state.cc",
diff --git a/BUILD b/BUILD
index ff8586a..0516a6e 100644
--- a/BUILD
+++ b/BUILD
@@ -872,8 +872,6 @@
"src/trace_processor/additional_modules.cc",
"src/trace_processor/additional_modules.h",
"src/trace_processor/ftrace_utils.cc",
- "src/trace_processor/gzip_trace_parser.cc",
- "src/trace_processor/gzip_trace_parser.h",
"src/trace_processor/importers/ftrace/binder_tracker.cc",
"src/trace_processor/importers/ftrace/binder_tracker.h",
"src/trace_processor/importers/ftrace/ftrace_descriptors.cc",
@@ -894,6 +892,8 @@
"src/trace_processor/importers/fuchsia/fuchsia_trace_tokenizer.cc",
"src/trace_processor/importers/fuchsia/fuchsia_trace_tokenizer.h",
"src/trace_processor/importers/fuchsia/fuchsia_trace_utils.cc",
+ "src/trace_processor/importers/gzip/gzip_trace_parser.cc",
+ "src/trace_processor/importers/gzip/gzip_trace_parser.h",
"src/trace_processor/importers/json/json_trace_parser.cc",
"src/trace_processor/importers/json/json_trace_parser.h",
"src/trace_processor/importers/json/json_trace_tokenizer.cc",
@@ -965,6 +965,8 @@
"src/trace_processor/importers/ftrace/ftrace_module.h",
"src/trace_processor/importers/fuchsia/fuchsia_record.h",
"src/trace_processor/importers/fuchsia/fuchsia_trace_utils.h",
+ "src/trace_processor/importers/gzip/gzip_utils.cc",
+ "src/trace_processor/importers/gzip/gzip_utils.h",
"src/trace_processor/importers/ninja/ninja_log_parser.cc",
"src/trace_processor/importers/ninja/ninja_log_parser.h",
"src/trace_processor/importers/proto/args_table_utils.cc",
diff --git a/include/perfetto/protozero/field.h b/include/perfetto/protozero/field.h
index 69b2a15..98e5e1d 100644
--- a/include/perfetto/protozero/field.h
+++ b/include/perfetto/protozero/field.h
@@ -20,6 +20,7 @@
#include <stdint.h>
#include <string>
+#include <vector>
#include "perfetto/base/logging.h"
#include "perfetto/protozero/contiguous_memory_range.h"
@@ -184,9 +185,16 @@
// Serializes the field back into a proto-encoded byte stream and appends it
// to |dst|. |dst| is resized accordingly.
- void SerializeAndAppendTo(std::string* dst);
+ void SerializeAndAppendTo(std::string* dst) const;
+
+ // Serializes the field back into a proto-encoded byte stream and appends it
+ // to |dst|. |dst| is resized accordingly.
+ void SerializeAndAppendTo(std::vector<uint8_t>* dst) const;
private:
+ template <typename Container>
+ void SerializeAndAppendToInternal(Container* dst) const;
+
// Fields are deliberately not initialized to keep the class trivially
// constructible. It makes a large perf difference for ProtoDecoder.
diff --git a/include/perfetto/trace_processor/read_trace.h b/include/perfetto/trace_processor/read_trace.h
index ac2053d..3431e7d 100644
--- a/include/perfetto/trace_processor/read_trace.h
+++ b/include/perfetto/trace_processor/read_trace.h
@@ -18,6 +18,7 @@
#define INCLUDE_PERFETTO_TRACE_PROCESSOR_READ_TRACE_H_
#include <functional>
+#include <vector>
#include "perfetto/base/export.h"
#include "perfetto/trace_processor/status.h"
@@ -32,6 +33,10 @@
const char* filename,
const std::function<void(uint64_t parsed_size)>& progress_callback = {});
+util::Status PERFETTO_EXPORT DecompressTrace(const uint8_t* data,
+ size_t size,
+ std::vector<uint8_t>* output);
+
} // namespace trace_processor
} // namespace perfetto
diff --git a/src/perfetto_cmd/packet_writer_unittest.cc b/src/perfetto_cmd/packet_writer_unittest.cc
index c6beecb..fb64f50 100644
--- a/src/perfetto_cmd/packet_writer_unittest.cc
+++ b/src/perfetto_cmd/packet_writer_unittest.cc
@@ -42,16 +42,6 @@
using TracePacketProto = protos::gen::TracePacket;
-std::string RandomString(size_t size) {
- std::minstd_rand0 rnd(0);
- std::uniform_int_distribution<> dist(0, 255);
- std::string s;
- s.resize(size);
- for (size_t i = 0; i < s.size(); i++)
- s[i] = static_cast<char>(dist(rnd));
- return s;
-}
-
template <typename F>
TracePacket CreateTracePacket(F fill_function) {
TracePacketProto msg;
@@ -65,6 +55,16 @@
}
#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB)
+std::string RandomString(size_t size) {
+ std::minstd_rand0 rnd(0);
+ std::uniform_int_distribution<> dist(0, 255);
+ std::string s;
+ s.resize(size);
+ for (size_t i = 0; i < s.size(); i++)
+ s[i] = static_cast<char>(dist(rnd));
+ return s;
+}
+
std::string Decompress(const std::string& data) {
uint8_t out[1024];
diff --git a/src/protozero/field.cc b/src/protozero/field.cc
index c92f7ce..be16482 100644
--- a/src/protozero/field.cc
+++ b/src/protozero/field.cc
@@ -26,7 +26,8 @@
namespace protozero {
-void Field::SerializeAndAppendTo(std::string* dst) {
+template <typename Container>
+void Field::SerializeAndAppendToInternal(Container* dst) const {
namespace pu = proto_utils;
size_t initial_size = dst->size();
dst->resize(initial_size + pu::kMaxSimpleFieldEncodedSize + size_);
@@ -68,4 +69,12 @@
dst->resize(initial_size + written_size);
}
+void Field::SerializeAndAppendTo(std::string* dst) const {
+ SerializeAndAppendToInternal(dst);
+}
+
+void Field::SerializeAndAppendTo(std::vector<uint8_t>* dst) const {
+ SerializeAndAppendToInternal(dst);
+}
+
} // namespace protozero
diff --git a/src/trace_processor/BUILD.gn b/src/trace_processor/BUILD.gn
index 0d0c868..f1a526a 100644
--- a/src/trace_processor/BUILD.gn
+++ b/src/trace_processor/BUILD.gn
@@ -100,6 +100,8 @@
"importers/ftrace/ftrace_module.h",
"importers/fuchsia/fuchsia_record.h",
"importers/fuchsia/fuchsia_trace_utils.h",
+ "importers/gzip/gzip_utils.cc",
+ "importers/gzip/gzip_utils.h",
"importers/ninja/ninja_log_parser.cc",
"importers/ninja/ninja_log_parser.h",
"importers/proto/args_table_utils.cc",
@@ -210,6 +212,8 @@
"importers/fuchsia/fuchsia_trace_tokenizer.cc",
"importers/fuchsia/fuchsia_trace_tokenizer.h",
"importers/fuchsia/fuchsia_trace_utils.cc",
+ "importers/gzip/gzip_trace_parser.cc",
+ "importers/gzip/gzip_trace_parser.h",
"importers/proto/android_probes_module.cc",
"importers/proto/android_probes_module.h",
"importers/proto/android_probes_parser.cc",
@@ -271,13 +275,6 @@
]
deps += [ "../../gn:jsoncpp" ]
}
- if (enable_perfetto_zlib) {
- sources += [
- "gzip_trace_parser.cc",
- "gzip_trace_parser.h",
- ]
- deps += [ "../../gn:zlib" ]
- }
}
if (enable_perfetto_trace_processor_json) {
@@ -448,11 +445,15 @@
sources = []
deps = []
if (enable_perfetto_trace_processor_sqlite) {
- sources += [ "trace_database_integrationtest.cc" ]
+ sources += [
+ "read_trace_integrationtest.cc",
+ "trace_database_integrationtest.cc",
+ ]
deps += [
":lib",
"../../gn:default_deps",
"../../gn:gtest_and_gmock",
+ "../../protos/perfetto/trace:zero",
"../base",
"../base:test_support",
"sqlite",
diff --git a/src/trace_processor/gzip_trace_parser.cc b/src/trace_processor/gzip_trace_parser.cc
deleted file mode 100644
index 129f551..0000000
--- a/src/trace_processor/gzip_trace_parser.cc
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Copyright (C) 2019 The Android Open Source Project
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// For bazel build.
-#include "perfetto/base/build_config.h"
-#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB)
-
-#include "src/trace_processor/gzip_trace_parser.h"
-
-#include <string>
-
-#include <zlib.h>
-
-#include "perfetto/base/logging.h"
-#include "perfetto/ext/base/string_utils.h"
-#include "perfetto/ext/base/string_view.h"
-#include "src/trace_processor/forwarding_trace_parser.h"
-
-namespace perfetto {
-namespace trace_processor {
-
-GzipTraceParser::GzipTraceParser(TraceProcessorContext* context)
- : context_(context), z_stream_(new z_stream()) {
- z_stream_->zalloc = Z_NULL;
- z_stream_->zfree = Z_NULL;
- z_stream_->opaque = Z_NULL;
- inflateInit2(z_stream_.get(), 32 + 15);
-}
-
-GzipTraceParser::~GzipTraceParser() {
- // Ensure the call to inflateEnd to prevent leaks of internal state.
- inflateEnd(z_stream_.get());
-}
-
-util::Status GzipTraceParser::Parse(std::unique_ptr<uint8_t[]> data,
- size_t size) {
- uint8_t* start = data.get();
- size_t len = size;
-
- if (!inner_) {
- inner_.reset(new ForwardingTraceParser(context_));
-
- // .ctrace files begin with: "TRACE:\n" or "done. TRACE:\n" strip this if
- // present.
- base::StringView beginning(reinterpret_cast<char*>(start), size);
-
- static const char* kSystraceFileHeader = "TRACE:\n";
- size_t offset = Find(kSystraceFileHeader, beginning);
- if (offset != std::string::npos) {
- start += strlen(kSystraceFileHeader) + offset;
- len -= strlen(kSystraceFileHeader) + offset;
- }
- }
-
- z_stream_->next_in = start;
- z_stream_->avail_in = static_cast<uInt>(len);
-
- // Our default uncompressed buffer size is 32MB as it allows for good
- // throughput.
- constexpr size_t kUncompressedBufferSize = 32 * 1024 * 1024;
- int ret = Z_OK;
- for (; ret != Z_STREAM_END && z_stream_->avail_in != 0;) {
- std::unique_ptr<uint8_t[]> buffer(new uint8_t[kUncompressedBufferSize]);
- z_stream_->next_out = buffer.get();
- z_stream_->avail_out = static_cast<uInt>(kUncompressedBufferSize);
-
- ret = inflate(z_stream_.get(), Z_NO_FLUSH);
- switch (ret) {
- case Z_NEED_DICT:
- case Z_DATA_ERROR:
- case Z_MEM_ERROR:
- // Ignore inflateEnd error as we will error out anyway.
- inflateEnd(z_stream_.get());
- return util::ErrStatus("Error decompressing ctrace file");
- }
-
- size_t read = kUncompressedBufferSize - z_stream_->avail_out;
- util::Status status = inner_->Parse(std::move(buffer), read);
- if (!status.ok())
- return status;
- }
- if (ret == Z_STREAM_END) {
- ret = inflateEnd(z_stream_.get());
- if (ret == Z_STREAM_ERROR)
- return util::ErrStatus("Error finishing decompression");
- }
- return util::OkStatus();
-}
-
-void GzipTraceParser::NotifyEndOfFile() {}
-
-} // namespace trace_processor
-} // namespace perfetto
-
-#endif // PERFETTO_BUILDFLAG(PERFETTO_ZLIB)
diff --git a/src/trace_processor/importers/gzip/gzip_trace_parser.cc b/src/trace_processor/importers/gzip/gzip_trace_parser.cc
new file mode 100644
index 0000000..89e8c1d
--- /dev/null
+++ b/src/trace_processor/importers/gzip/gzip_trace_parser.cc
@@ -0,0 +1,81 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "src/trace_processor/importers/gzip/gzip_trace_parser.h"
+
+#include <string>
+
+#include "perfetto/base/logging.h"
+#include "perfetto/ext/base/string_utils.h"
+#include "perfetto/ext/base/string_view.h"
+#include "src/trace_processor/forwarding_trace_parser.h"
+
+namespace perfetto {
+namespace trace_processor {
+
+GzipTraceParser::GzipTraceParser(TraceProcessorContext* context)
+ : context_(context) {}
+
+GzipTraceParser::~GzipTraceParser() = default;
+
+util::Status GzipTraceParser::Parse(std::unique_ptr<uint8_t[]> data,
+ size_t size) {
+ uint8_t* start = data.get();
+ size_t len = size;
+
+ if (!inner_) {
+ inner_.reset(new ForwardingTraceParser(context_));
+
+ // .ctrace files begin with: "TRACE:\n" or "done. TRACE:\n" strip this if
+ // present.
+ base::StringView beginning(reinterpret_cast<char*>(start), size);
+
+ static const char* kSystraceFileHeader = "TRACE:\n";
+ size_t offset = Find(kSystraceFileHeader, beginning);
+ if (offset != std::string::npos) {
+ start += strlen(kSystraceFileHeader) + offset;
+ len -= strlen(kSystraceFileHeader) + offset;
+ }
+ }
+ decompressor_.SetInput(start, len);
+
+ // Our default uncompressed buffer size is 32MB as it allows for good
+ // throughput.
+ using ResultCode = GzipDecompressor::ResultCode;
+ constexpr size_t kUncompressedBufferSize = 32 * 1024 * 1024;
+
+ for (auto ret = ResultCode::kOk; ret != ResultCode::kEof;) {
+ std::unique_ptr<uint8_t[]> buffer(new uint8_t[kUncompressedBufferSize]);
+ auto result =
+ decompressor_.Decompress(buffer.get(), kUncompressedBufferSize);
+ ret = result.ret;
+ if (ret == ResultCode::kError || ret == ResultCode::kNoProgress)
+ return util::ErrStatus("Unable to decompress gzip/ctrace trace");
+ if (ret == ResultCode::kNeedsMoreInput)
+ break;
+
+ util::Status status =
+ inner_->Parse(std::move(buffer), result.bytes_written);
+ if (!status.ok())
+ return status;
+ }
+ return util::OkStatus();
+}
+
+void GzipTraceParser::NotifyEndOfFile() {}
+
+} // namespace trace_processor
+} // namespace perfetto
diff --git a/src/trace_processor/gzip_trace_parser.h b/src/trace_processor/importers/gzip/gzip_trace_parser.h
similarity index 80%
rename from src/trace_processor/gzip_trace_parser.h
rename to src/trace_processor/importers/gzip/gzip_trace_parser.h
index de70afb..c83416e 100644
--- a/src/trace_processor/gzip_trace_parser.h
+++ b/src/trace_processor/importers/gzip/gzip_trace_parser.h
@@ -14,12 +14,11 @@
* limitations under the License.
*/
-#ifndef SRC_TRACE_PROCESSOR_GZIP_TRACE_PARSER_H_
-#define SRC_TRACE_PROCESSOR_GZIP_TRACE_PARSER_H_
+#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_GZIP_GZIP_TRACE_PARSER_H_
+#define SRC_TRACE_PROCESSOR_IMPORTERS_GZIP_GZIP_TRACE_PARSER_H_
#include "src/trace_processor/chunked_trace_reader.h"
-
-struct z_stream_s;
+#include "src/trace_processor/importers/gzip/gzip_utils.h"
namespace perfetto {
namespace trace_processor {
@@ -37,11 +36,11 @@
private:
TraceProcessorContext* const context_;
- std::unique_ptr<z_stream_s> z_stream_;
+ GzipDecompressor decompressor_;
std::unique_ptr<ChunkedTraceReader> inner_;
};
} // namespace trace_processor
} // namespace perfetto
-#endif // SRC_TRACE_PROCESSOR_GZIP_TRACE_PARSER_H_
+#endif // SRC_TRACE_PROCESSOR_IMPORTERS_GZIP_GZIP_TRACE_PARSER_H_
diff --git a/src/trace_processor/importers/gzip/gzip_utils.cc b/src/trace_processor/importers/gzip/gzip_utils.cc
new file mode 100644
index 0000000..c645198
--- /dev/null
+++ b/src/trace_processor/importers/gzip/gzip_utils.cc
@@ -0,0 +1,106 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "src/trace_processor/importers/gzip/gzip_utils.h"
+
+// For bazel build.
+#include "perfetto/base/build_config.h"
+
+#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB)
+#include <zlib.h>
+#else
+struct z_stream_s {};
+#endif
+
+namespace perfetto {
+namespace trace_processor {
+namespace gzip_utils {
+
+bool IsGzipSupported() {
+#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB)
+ return true;
+#else
+ return false;
+#endif
+}
+
+} // namespace gzip_utils
+
+#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB)
+GzipDecompressor::GzipDecompressor() : z_stream_(new z_stream()) {
+ z_stream_->zalloc = Z_NULL;
+ z_stream_->zfree = Z_NULL;
+ z_stream_->opaque = Z_NULL;
+ inflateInit2(z_stream_.get(), 32 + 15);
+}
+#else
+GzipDecompressor::GzipDecompressor() = default
+#endif
+
+GzipDecompressor::~GzipDecompressor() {
+#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB)
+ // Ensure the call to inflateEnd to prevent leaks of internal state.
+ inflateEnd(z_stream_.get());
+#endif
+}
+
+void GzipDecompressor::Reset() {
+#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB)
+ inflateReset(z_stream_.get());
+#endif
+}
+
+void GzipDecompressor::SetInput(const uint8_t* data, size_t size) {
+#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB)
+ // This const_cast is not harmfull as zlib will not modify the data in this
+ // pointer. This is only necessary because of the build flags we use to be
+ // compatible with other embedders.
+ z_stream_->next_in = const_cast<uint8_t*>(data);
+ z_stream_->avail_in = static_cast<uInt>(size);
+#endif
+}
+
+GzipDecompressor::Result GzipDecompressor::Decompress(uint8_t* out,
+ size_t out_size) {
+#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB)
+ if (z_stream_->avail_in == 0)
+ return Result{ResultCode::kNeedsMoreInput, 0};
+
+ z_stream_->next_out = out;
+ z_stream_->avail_out = static_cast<uInt>(out_size);
+
+ int ret = inflate(z_stream_.get(), Z_NO_FLUSH);
+ switch (ret) {
+ case Z_NEED_DICT:
+ case Z_DATA_ERROR:
+ case Z_MEM_ERROR:
+ // Ignore inflateEnd error as we will error out anyway.
+ inflateEnd(z_stream_.get());
+ return Result{ResultCode::kError, 0};
+ case Z_STREAM_END:
+ return Result{ResultCode::kEof, out_size - z_stream_->avail_out};
+ case Z_BUF_ERROR:
+ return Result{ResultCode::kNoProgress, 0};
+ default:
+ return Result{ResultCode::kOk, out_size - z_stream_->avail_out};
+ }
+#else
+ return Result{ResultCode::kError, 0};
+#endif
+}
+
+} // namespace trace_processor
+} // namespace perfetto
diff --git a/src/trace_processor/importers/gzip/gzip_utils.h b/src/trace_processor/importers/gzip/gzip_utils.h
new file mode 100644
index 0000000..8bd3d46
--- /dev/null
+++ b/src/trace_processor/importers/gzip/gzip_utils.h
@@ -0,0 +1,72 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef SRC_TRACE_PROCESSOR_IMPORTERS_GZIP_GZIP_UTILS_H_
+#define SRC_TRACE_PROCESSOR_IMPORTERS_GZIP_GZIP_UTILS_H_
+
+#include <memory>
+
+struct z_stream_s;
+
+namespace perfetto {
+namespace trace_processor {
+
+namespace gzip_utils {
+
+// Returns whether gzip related functioanlity is supported with the current
+// build flags.
+bool IsGzipSupported();
+
+} // namespace gzip_utils
+
+class GzipDecompressor {
+ public:
+ enum class ResultCode {
+ kOk,
+ kEof,
+ kError,
+ kNoProgress,
+ kNeedsMoreInput,
+ };
+ struct Result {
+ // The return code of the decompression.
+ ResultCode ret;
+
+ // The amount of bytes written to output.
+ // Only valid if |ResultCode::kOk|.
+ size_t bytes_written;
+ };
+
+ GzipDecompressor();
+ ~GzipDecompressor();
+
+ // Sets the input pointer and size of the gzip stream to inflate.
+ void SetInput(const uint8_t* data, size_t size);
+
+ // Decompresses the input previously provided in |SetInput|.
+ Result Decompress(uint8_t* out, size_t out_size);
+
+ // Sets the state of the decompressor to reuse with other gzip streams.
+ void Reset();
+
+ private:
+ std::unique_ptr<z_stream_s> z_stream_;
+};
+
+} // namespace trace_processor
+} // namespace perfetto
+
+#endif // SRC_TRACE_PROCESSOR_IMPORTERS_GZIP_GZIP_UTILS_H_
diff --git a/src/trace_processor/importers/proto/proto_trace_tokenizer.cc b/src/trace_processor/importers/proto/proto_trace_tokenizer.cc
index 1918f53..4cb8a35 100644
--- a/src/trace_processor/importers/proto/proto_trace_tokenizer.cc
+++ b/src/trace_processor/importers/proto/proto_trace_tokenizer.cc
@@ -29,6 +29,7 @@
#include "src/trace_processor/clock_tracker.h"
#include "src/trace_processor/event_tracker.h"
#include "src/trace_processor/importers/ftrace/ftrace_module.h"
+#include "src/trace_processor/importers/gzip/gzip_utils.h"
#include "src/trace_processor/importers/proto/packet_sequence_state.h"
#include "src/trace_processor/importers/proto/proto_incremental_state.h"
#include "src/trace_processor/storage/stats.h"
@@ -42,10 +43,6 @@
#include "protos/perfetto/trace/trace.pbzero.h"
#include "protos/perfetto/trace/trace_packet.pbzero.h"
-#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB)
-#include <zlib.h>
-#endif
-
namespace perfetto {
namespace trace_processor {
@@ -57,36 +54,33 @@
constexpr uint8_t kTracePacketTag =
MakeTagLengthDelimited(protos::pbzero::Trace::kPacketFieldNumber);
-#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB)
-TraceBlobView Decompress(TraceBlobView input) {
+TraceBlobView Decompress(GzipDecompressor* decompressor, TraceBlobView input) {
+ PERFETTO_DCHECK(gzip_utils::IsGzipSupported());
+
uint8_t out[4096];
- std::string s;
- z_stream stream{};
- stream.next_in = const_cast<uint8_t*>(input.data());
- stream.avail_in = static_cast<unsigned int>(input.length());
+ std::vector<uint8_t> data;
+ data.reserve(input.length());
- if (inflateInit(&stream) != Z_OK)
- return TraceBlobView(nullptr, 0, 0);
+ // Ensure that the decompressor is able to cope with a new stream of data.
+ decompressor->Reset();
+ decompressor->SetInput(input.data(), input.length());
- int ret;
- do {
- stream.next_out = out;
- stream.avail_out = sizeof(out);
- ret = inflate(&stream, Z_NO_FLUSH);
- if (ret != Z_STREAM_END && ret != Z_OK) {
- inflateEnd(&stream);
+ using ResultCode = GzipDecompressor::ResultCode;
+ for (auto ret = ResultCode::kOk; ret != ResultCode::kEof;) {
+ auto res = decompressor->Decompress(out, base::ArraySize(out));
+ ret = res.ret;
+ if (ret == ResultCode::kError || ret == ResultCode::kNoProgress ||
+ ret == ResultCode::kNeedsMoreInput)
return TraceBlobView(nullptr, 0, 0);
- }
- s.append(reinterpret_cast<char*>(out), sizeof(out) - stream.avail_out);
- } while (ret != Z_STREAM_END);
- inflateEnd(&stream);
- std::unique_ptr<uint8_t[]> output(new uint8_t[s.size()]);
- memcpy(output.get(), s.data(), s.size());
- return TraceBlobView(std::move(output), 0, s.size());
+ data.insert(data.end(), out, out + res.bytes_written);
+ }
+
+ std::unique_ptr<uint8_t[]> output(new uint8_t[data.size()]);
+ memcpy(output.get(), data.data(), data.size());
+ return TraceBlobView(std::move(output), 0, data.size());
}
-#endif // PERFETTO_BUILDFLAG(PERFETTO_ZLIB)
} // namespace
@@ -315,11 +309,14 @@
}
if (decoder.has_compressed_packets()) {
-#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB)
+ if (!gzip_utils::IsGzipSupported())
+ return util::Status("Cannot decode compressed packets. Zlib not enabled");
+
protozero::ConstBytes field = decoder.compressed_packets();
const size_t field_off = packet.offset_of(field.data);
TraceBlobView compressed_packets = packet.slice(field_off, field.size);
- TraceBlobView packets = Decompress(std::move(compressed_packets));
+ TraceBlobView packets =
+ Decompress(&decompressor_, std::move(compressed_packets));
const uint8_t* start = packets.data();
const uint8_t* end = packets.data() + packets.length();
@@ -339,11 +336,7 @@
if (PERFETTO_UNLIKELY(!status.ok()))
return status;
}
-
return util::OkStatus();
-#else
- return util::Status("Cannot decode compressed packets. Zlib not enabled");
-#endif
}
// If we're not forcing a full sort and this is a write_into_file trace, then
diff --git a/src/trace_processor/importers/proto/proto_trace_tokenizer.h b/src/trace_processor/importers/proto/proto_trace_tokenizer.h
index b87b4e0..62ddb39 100644
--- a/src/trace_processor/importers/proto/proto_trace_tokenizer.h
+++ b/src/trace_processor/importers/proto/proto_trace_tokenizer.h
@@ -23,6 +23,7 @@
#include <vector>
#include "src/trace_processor/chunked_trace_reader.h"
+#include "src/trace_processor/importers/gzip/gzip_utils.h"
#include "src/trace_processor/importers/proto/proto_incremental_state.h"
#include "src/trace_processor/trace_blob_view.h"
@@ -92,6 +93,9 @@
// Stores incremental state and references to interned data, e.g. for track
// event protos.
std::unique_ptr<ProtoIncrementalState> incremental_state;
+
+ // Allows support for compressed trace packets.
+ GzipDecompressor decompressor_;
};
} // namespace trace_processor
diff --git a/src/trace_processor/read_trace.cc b/src/trace_processor/read_trace.cc
index 0d73cf9..ef3d19e 100644
--- a/src/trace_processor/read_trace.cc
+++ b/src/trace_processor/read_trace.cc
@@ -17,8 +17,14 @@
#include "perfetto/trace_processor/read_trace.h"
#include "perfetto/ext/base/scoped_file.h"
+#include "perfetto/ext/base/utils.h"
#include "perfetto/trace_processor/trace_processor.h"
+#include "src/trace_processor/importers/gzip/gzip_utils.h"
+
+#include "protos/perfetto/trace/trace.pbzero.h"
+#include "protos/perfetto/trace/trace_packet.pbzero.h"
+
#if PERFETTO_BUILDFLAG(PERFETTO_OS_LINUX) || \
PERFETTO_BUILDFLAG(PERFETTO_OS_MACOSX)
#define PERFETTO_HAS_AIO_H() 1
@@ -120,5 +126,42 @@
return util::OkStatus();
}
+util::Status DecompressTrace(const uint8_t* data,
+ size_t size,
+ std::vector<uint8_t>* output) {
+ if (!gzip_utils::IsGzipSupported()) {
+ return util::ErrStatus(
+ "Cannot decompress trace in build where zlib is disabled");
+ }
+
+ protos::pbzero::Trace::Decoder decoder(data, size);
+ GzipDecompressor decompressor;
+ for (auto it = decoder.packet(); it; ++it) {
+ protos::pbzero::TracePacket::Decoder packet(*it);
+ if (!packet.has_compressed_packets()) {
+ it->SerializeAndAppendTo(output);
+ continue;
+ }
+
+ // Make sure that to reset the stream between the gzip streams.
+ auto bytes = packet.compressed_packets();
+ decompressor.Reset();
+ decompressor.SetInput(bytes.data, bytes.size);
+
+ using ResultCode = GzipDecompressor::ResultCode;
+ uint8_t out[4096];
+ for (auto ret = ResultCode::kOk; ret != ResultCode::kEof;) {
+ auto res = decompressor.Decompress(out, base::ArraySize(out));
+ ret = res.ret;
+ if (ret == ResultCode::kError || ret == ResultCode::kNoProgress ||
+ ret == ResultCode::kNeedsMoreInput) {
+ return util::ErrStatus("Failed while decompressing stream");
+ }
+ output->insert(output->end(), out, out + res.bytes_written);
+ }
+ }
+ return util::OkStatus();
+}
+
} // namespace trace_processor
} // namespace perfetto
diff --git a/src/trace_processor/read_trace_integrationtest.cc b/src/trace_processor/read_trace_integrationtest.cc
new file mode 100644
index 0000000..5070e35
--- /dev/null
+++ b/src/trace_processor/read_trace_integrationtest.cc
@@ -0,0 +1,63 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "perfetto/ext/base/scoped_file.h"
+#include "perfetto/ext/base/utils.h"
+#include "perfetto/trace_processor/read_trace.h"
+
+#include "src/base/test/utils.h"
+#include "test/gtest_and_gmock.h"
+
+#include "protos/perfetto/trace/trace.pbzero.h"
+#include "protos/perfetto/trace/trace_packet.pbzero.h"
+
+namespace perfetto {
+namespace trace_processor {
+namespace {
+
+TEST(ReadTraceIntegrationTest, CompressedTrace) {
+ base::ScopedFstream f(fopen(
+ base::GetTestDataPath(std::string("test/data/compressed.pb")).c_str(),
+ "rb"));
+ std::vector<uint8_t> raw_trace;
+ while (!feof(*f)) {
+ uint8_t buf[4096];
+ auto rsize =
+ fread(reinterpret_cast<char*>(buf), 1, base::ArraySize(buf), *f);
+ raw_trace.insert(raw_trace.end(), buf, buf + rsize);
+ }
+
+ std::vector<uint8_t> decompressed;
+ decompressed.reserve(raw_trace.size());
+
+ util::Status status = trace_processor::DecompressTrace(
+ raw_trace.data(), raw_trace.size(), &decompressed);
+ ASSERT_TRUE(status.ok());
+
+ protos::pbzero::Trace::Decoder decoder(decompressed.data(),
+ decompressed.size());
+ uint32_t packet_count = 0;
+ for (auto it = decoder.packet(); it; ++it) {
+ protos::pbzero::TracePacket::Decoder packet(*it);
+ ASSERT_FALSE(packet.has_compressed_packets());
+ ++packet_count;
+ }
+ ASSERT_EQ(packet_count, 2412u);
+}
+
+} // namespace
+} // namespace trace_processor
+} // namespace perfetto
diff --git a/src/trace_processor/trace_processor_impl.cc b/src/trace_processor/trace_processor_impl.cc
index 94096fa..df6fceb 100644
--- a/src/trace_processor/trace_processor_impl.cc
+++ b/src/trace_processor/trace_processor_impl.cc
@@ -26,10 +26,10 @@
#include "src/trace_processor/additional_modules.h"
#include "src/trace_processor/experimental_counter_dur_generator.h"
#include "src/trace_processor/experimental_flamegraph_generator.h"
-#include "src/trace_processor/gzip_trace_parser.h"
#include "src/trace_processor/importers/ftrace/sched_event_tracker.h"
#include "src/trace_processor/importers/fuchsia/fuchsia_trace_parser.h"
#include "src/trace_processor/importers/fuchsia/fuchsia_trace_tokenizer.h"
+#include "src/trace_processor/importers/gzip/gzip_trace_parser.h"
#include "src/trace_processor/importers/json/json_trace_parser.h"
#include "src/trace_processor/importers/json/json_trace_tokenizer.h"
#include "src/trace_processor/importers/systrace/systrace_trace_parser.h"
@@ -457,9 +457,8 @@
context_.systrace_trace_parser.reset(new SystraceTraceParser(&context_));
-#if PERFETTO_BUILDFLAG(PERFETTO_ZLIB)
- context_.gzip_trace_parser.reset(new GzipTraceParser(&context_));
-#endif
+ if (gzip_utils::IsGzipSupported())
+ context_.gzip_trace_parser.reset(new GzipTraceParser(&context_));
#if PERFETTO_BUILDFLAG(PERFETTO_TP_JSON)
context_.json_trace_tokenizer.reset(new JsonTraceTokenizer(&context_));