Darin Petkov | 7a22d79 | 2010-11-08 14:10:00 -0800 | [diff] [blame] | 1 | // Copyright (c) 2010 The Chromium OS Authors. All rights reserved. |
| 2 | // Use of this source code is governed by a BSD-style license that can be |
| 3 | // found in the LICENSE file. |
| 4 | |
| 5 | #include "update_engine/full_update_generator.h" |
| 6 | |
| 7 | #include <inttypes.h> |
| 8 | #include <fcntl.h> |
| 9 | |
| 10 | #include <tr1/memory> |
| 11 | |
| 12 | #include <base/string_util.h> |
| 13 | |
| 14 | #include "update_engine/bzip.h" |
| 15 | #include "update_engine/utils.h" |
| 16 | |
| 17 | using std::deque; |
| 18 | using std::min; |
| 19 | using std::max; |
| 20 | using std::string; |
| 21 | using std::tr1::shared_ptr; |
| 22 | using std::vector; |
| 23 | |
| 24 | namespace chromeos_update_engine { |
| 25 | |
| 26 | namespace { |
| 27 | |
| 28 | // This class encapsulates a full update chunk processing thread. The processor |
| 29 | // reads a chunk of data from the input file descriptor and compresses it. The |
| 30 | // processor needs to be started through Start() then waited on through Wait(). |
| 31 | class ChunkProcessor { |
| 32 | public: |
| 33 | // Read a chunk of |size| bytes from |fd| starting at offset |offset|. |
| 34 | ChunkProcessor(int fd, off_t offset, size_t size) |
| 35 | : thread_(NULL), |
| 36 | fd_(fd), |
| 37 | offset_(offset), |
| 38 | buffer_in_(size) {} |
| 39 | ~ChunkProcessor() { Wait(); } |
| 40 | |
| 41 | off_t offset() const { return offset_; } |
| 42 | const vector<char>& buffer_in() const { return buffer_in_; } |
| 43 | const vector<char>& buffer_compressed() const { return buffer_compressed_; } |
| 44 | |
| 45 | // Starts the processor. Returns true on success, false on failure. |
| 46 | bool Start(); |
| 47 | |
| 48 | // Waits for the processor to complete. Returns true on success, false on |
| 49 | // failure. |
| 50 | bool Wait(); |
| 51 | |
| 52 | bool ShouldCompress() const { |
| 53 | return buffer_compressed_.size() < buffer_in_.size(); |
| 54 | } |
| 55 | |
| 56 | private: |
| 57 | // Reads the input data into |buffer_in_| and compresses it into |
| 58 | // |buffer_compressed_|. Returns true on success, false otherwise. |
| 59 | bool ReadAndCompress(); |
| 60 | static gpointer ReadAndCompressThread(gpointer data); |
| 61 | |
| 62 | GThread* thread_; |
| 63 | int fd_; |
| 64 | off_t offset_; |
| 65 | vector<char> buffer_in_; |
| 66 | vector<char> buffer_compressed_; |
| 67 | |
| 68 | DISALLOW_COPY_AND_ASSIGN(ChunkProcessor); |
| 69 | }; |
| 70 | |
| 71 | bool ChunkProcessor::Start() { |
| 72 | thread_ = g_thread_create(ReadAndCompressThread, this, TRUE, NULL); |
| 73 | TEST_AND_RETURN_FALSE(thread_ != NULL); |
| 74 | return true; |
| 75 | } |
| 76 | |
| 77 | bool ChunkProcessor::Wait() { |
| 78 | if (!thread_) { |
| 79 | return false; |
| 80 | } |
| 81 | gpointer result = g_thread_join(thread_); |
| 82 | thread_ = NULL; |
| 83 | TEST_AND_RETURN_FALSE(result == this); |
| 84 | return true; |
| 85 | } |
| 86 | |
| 87 | gpointer ChunkProcessor::ReadAndCompressThread(gpointer data) { |
| 88 | return |
| 89 | reinterpret_cast<ChunkProcessor*>(data)->ReadAndCompress() ? data : NULL; |
| 90 | } |
| 91 | |
| 92 | bool ChunkProcessor::ReadAndCompress() { |
| 93 | ssize_t bytes_read = -1; |
| 94 | TEST_AND_RETURN_FALSE(utils::PReadAll(fd_, |
| 95 | buffer_in_.data(), |
| 96 | buffer_in_.size(), |
| 97 | offset_, |
| 98 | &bytes_read)); |
| 99 | TEST_AND_RETURN_FALSE(bytes_read == static_cast<ssize_t>(buffer_in_.size())); |
| 100 | TEST_AND_RETURN_FALSE(BzipCompress(buffer_in_, &buffer_compressed_)); |
| 101 | return true; |
| 102 | } |
| 103 | |
| 104 | } // namespace |
| 105 | |
| 106 | bool FullUpdateGenerator::Run( |
| 107 | Graph* graph, |
| 108 | const std::string& new_kernel_part, |
| 109 | const std::string& new_image, |
| 110 | off_t image_size, |
| 111 | int fd, |
| 112 | off_t* data_file_size, |
| 113 | off_t chunk_size, |
| 114 | off_t block_size, |
| 115 | vector<DeltaArchiveManifest_InstallOperation>* kernel_ops, |
| 116 | std::vector<Vertex::Index>* final_order) { |
| 117 | TEST_AND_RETURN_FALSE(chunk_size > 0); |
| 118 | TEST_AND_RETURN_FALSE((chunk_size % block_size) == 0); |
| 119 | |
| 120 | size_t max_threads = max(sysconf(_SC_NPROCESSORS_ONLN), 4L); |
| 121 | LOG(INFO) << "Max threads: " << max_threads; |
| 122 | |
| 123 | // Get the sizes early in the function, so we can fail fast if the user |
| 124 | // passed us bad paths. |
| 125 | TEST_AND_RETURN_FALSE(image_size >= 0 && |
| 126 | image_size <= utils::FileSize(new_image)); |
| 127 | const off_t kernel_size = utils::FileSize(new_kernel_part); |
| 128 | TEST_AND_RETURN_FALSE(kernel_size >= 0); |
| 129 | |
| 130 | off_t part_sizes[] = { image_size, kernel_size }; |
| 131 | string paths[] = { new_image, new_kernel_part }; |
| 132 | |
| 133 | for (int partition = 0; partition < 2; ++partition) { |
| 134 | const string& path = paths[partition]; |
| 135 | LOG(INFO) << "compressing " << path; |
Darin Petkov | 7a22d79 | 2010-11-08 14:10:00 -0800 | [diff] [blame] | 136 | int in_fd = open(path.c_str(), O_RDONLY, 0); |
| 137 | TEST_AND_RETURN_FALSE(in_fd >= 0); |
| 138 | ScopedFdCloser in_fd_closer(&in_fd); |
Darin Petkov | 7a22d79 | 2010-11-08 14:10:00 -0800 | [diff] [blame] | 139 | deque<shared_ptr<ChunkProcessor> > threads; |
Darin Petkov | 3375ff5 | 2010-11-08 16:20:54 -0800 | [diff] [blame] | 140 | int last_progress_update = INT_MIN; |
Darin Petkov | 7a22d79 | 2010-11-08 14:10:00 -0800 | [diff] [blame] | 141 | off_t bytes_left = part_sizes[partition], counter = 0, offset = 0; |
| 142 | while (bytes_left > 0 || !threads.empty()) { |
| 143 | // Check and start new chunk processors if possible. |
| 144 | while (threads.size() < max_threads && bytes_left > 0) { |
| 145 | shared_ptr<ChunkProcessor> processor( |
| 146 | new ChunkProcessor(in_fd, offset, min(bytes_left, chunk_size))); |
| 147 | threads.push_back(processor); |
| 148 | TEST_AND_RETURN_FALSE(processor->Start()); |
| 149 | bytes_left -= chunk_size; |
| 150 | offset += chunk_size; |
| 151 | } |
| 152 | |
| 153 | // Need to wait for a chunk processor to complete and process its ouput |
| 154 | // before spawning new processors. |
| 155 | shared_ptr<ChunkProcessor> processor = threads.front(); |
| 156 | threads.pop_front(); |
| 157 | TEST_AND_RETURN_FALSE(processor->Wait()); |
| 158 | |
| 159 | DeltaArchiveManifest_InstallOperation* op = NULL; |
| 160 | if (partition == 0) { |
| 161 | graph->resize(graph->size() + 1); |
| 162 | graph->back().file_name = |
| 163 | StringPrintf("<rootfs-operation-%" PRIi64 ">", counter++); |
| 164 | op = &graph->back().op; |
| 165 | final_order->push_back(graph->size() - 1); |
| 166 | } else { |
| 167 | kernel_ops->resize(kernel_ops->size() + 1); |
| 168 | op = &kernel_ops->back(); |
| 169 | } |
| 170 | |
| 171 | const bool compress = processor->ShouldCompress(); |
| 172 | const vector<char>& use_buf = |
| 173 | compress ? processor->buffer_compressed() : processor->buffer_in(); |
| 174 | op->set_type(compress ? |
| 175 | DeltaArchiveManifest_InstallOperation_Type_REPLACE_BZ : |
| 176 | DeltaArchiveManifest_InstallOperation_Type_REPLACE); |
| 177 | op->set_data_offset(*data_file_size); |
| 178 | TEST_AND_RETURN_FALSE(utils::WriteAll(fd, &use_buf[0], use_buf.size())); |
| 179 | *data_file_size += use_buf.size(); |
| 180 | op->set_data_length(use_buf.size()); |
| 181 | Extent* dst_extent = op->add_dst_extents(); |
| 182 | dst_extent->set_start_block(processor->offset() / block_size); |
| 183 | dst_extent->set_num_blocks(chunk_size / block_size); |
| 184 | |
Darin Petkov | 3375ff5 | 2010-11-08 16:20:54 -0800 | [diff] [blame] | 185 | int progress = static_cast<int>( |
| 186 | (processor->offset() + processor->buffer_in().size()) * 100.0 / |
| 187 | part_sizes[partition]); |
| 188 | if (last_progress_update < progress && |
| 189 | (last_progress_update + 10 <= progress || progress == 100)) { |
| 190 | LOG(INFO) << progress << "% complete (output size: " |
| 191 | << *data_file_size << ")"; |
| 192 | last_progress_update = progress; |
| 193 | } |
Darin Petkov | 7a22d79 | 2010-11-08 14:10:00 -0800 | [diff] [blame] | 194 | } |
| 195 | } |
| 196 | |
| 197 | return true; |
| 198 | } |
| 199 | |
| 200 | } // namespace chromeos_update_engine |