Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 1 | #include <sys/epoll.h> |
| 2 | #include <sys/eventfd.h> |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 3 | #include <sys/poll.h> |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 4 | |
| 5 | #include <algorithm> |
| 6 | #include <atomic> |
| 7 | #include <thread> |
| 8 | |
Fan Xu | 74df490 | 2018-09-20 16:40:51 -0700 | [diff] [blame] | 9 | #include <log/log.h> |
Tianyu | 0c6b7ff | 2018-10-03 11:30:04 -0700 | [diff] [blame] | 10 | #include <private/dvr/bufferhub_rpc.h> |
Fan Xu | 74df490 | 2018-09-20 16:40:51 -0700 | [diff] [blame] | 11 | #include <private/dvr/consumer_channel.h> |
| 12 | #include <private/dvr/producer_channel.h> |
| 13 | #include <sync/sync.h> |
| 14 | #include <utils/Trace.h> |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 15 | |
| 16 | using android::pdx::BorrowedHandle; |
Corey Tabaka | cd52dd9 | 2017-04-07 18:03:57 -0700 | [diff] [blame] | 17 | using android::pdx::ErrorStatus; |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 18 | using android::pdx::Message; |
| 19 | using android::pdx::RemoteChannelHandle; |
Corey Tabaka | cd52dd9 | 2017-04-07 18:03:57 -0700 | [diff] [blame] | 20 | using android::pdx::Status; |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 21 | using android::pdx::rpc::BufferWrapper; |
| 22 | using android::pdx::rpc::DispatchRemoteMethod; |
| 23 | using android::pdx::rpc::WrapBuffer; |
| 24 | |
| 25 | namespace android { |
| 26 | namespace dvr { |
| 27 | |
Jiwen 'Steve' Cai | 0728fa9 | 2018-04-24 19:03:14 -0700 | [diff] [blame] | 28 | ProducerChannel::ProducerChannel(BufferHubService* service, int buffer_id, |
| 29 | int channel_id, IonBuffer buffer, |
| 30 | IonBuffer metadata_buffer, |
| 31 | size_t user_metadata_size, int* error) |
| 32 | : BufferHubChannel(service, buffer_id, channel_id, kProducerType), |
| 33 | buffer_(std::move(buffer)), |
| 34 | metadata_buffer_(std::move(metadata_buffer)), |
| 35 | user_metadata_size_(user_metadata_size), |
| 36 | metadata_buf_size_(BufferHubDefs::kMetadataHeaderSize + |
| 37 | user_metadata_size) { |
| 38 | if (!buffer_.IsValid()) { |
| 39 | ALOGE("ProducerChannel::ProducerChannel: Invalid buffer."); |
| 40 | *error = -EINVAL; |
| 41 | return; |
| 42 | } |
| 43 | if (!metadata_buffer_.IsValid()) { |
| 44 | ALOGE("ProducerChannel::ProducerChannel: Invalid metadata buffer."); |
| 45 | *error = -EINVAL; |
| 46 | return; |
| 47 | } |
| 48 | |
| 49 | *error = InitializeBuffer(); |
| 50 | } |
| 51 | |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 52 | ProducerChannel::ProducerChannel(BufferHubService* service, int channel_id, |
Corey Tabaka | cd52dd9 | 2017-04-07 18:03:57 -0700 | [diff] [blame] | 53 | uint32_t width, uint32_t height, |
Hendrik Wagenaar | 108e84f | 2017-05-07 22:19:17 -0700 | [diff] [blame] | 54 | uint32_t layer_count, uint32_t format, |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 55 | uint64_t usage, size_t user_metadata_size, |
Hendrik Wagenaar | 108e84f | 2017-05-07 22:19:17 -0700 | [diff] [blame] | 56 | int* error) |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 57 | : BufferHubChannel(service, channel_id, channel_id, kProducerType), |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 58 | user_metadata_size_(user_metadata_size), |
| 59 | metadata_buf_size_(BufferHubDefs::kMetadataHeaderSize + |
| 60 | user_metadata_size) { |
| 61 | if (int ret = buffer_.Alloc(width, height, layer_count, format, usage)) { |
Hendrik Wagenaar | 4d3590f | 2017-05-06 22:36:04 -0700 | [diff] [blame] | 62 | ALOGE("ProducerChannel::ProducerChannel: Failed to allocate buffer: %s", |
| 63 | strerror(-ret)); |
| 64 | *error = ret; |
| 65 | return; |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 66 | } |
| 67 | |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 68 | if (int ret = metadata_buffer_.Alloc(metadata_buf_size_, /*height=*/1, |
| 69 | /*layer_count=*/1, |
| 70 | BufferHubDefs::kMetadataFormat, |
| 71 | BufferHubDefs::kMetadataUsage)) { |
| 72 | ALOGE("ProducerChannel::ProducerChannel: Failed to allocate metadata: %s", |
| 73 | strerror(-ret)); |
| 74 | *error = ret; |
| 75 | return; |
| 76 | } |
| 77 | |
Jiwen 'Steve' Cai | 0728fa9 | 2018-04-24 19:03:14 -0700 | [diff] [blame] | 78 | *error = InitializeBuffer(); |
| 79 | } |
| 80 | |
| 81 | int ProducerChannel::InitializeBuffer() { |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 82 | void* metadata_ptr = nullptr; |
| 83 | if (int ret = metadata_buffer_.Lock(BufferHubDefs::kMetadataUsage, /*x=*/0, |
| 84 | /*y=*/0, metadata_buf_size_, |
| 85 | /*height=*/1, &metadata_ptr)) { |
| 86 | ALOGE("ProducerChannel::ProducerChannel: Failed to lock metadata."); |
Jiwen 'Steve' Cai | 0728fa9 | 2018-04-24 19:03:14 -0700 | [diff] [blame] | 87 | return ret; |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 88 | } |
| 89 | metadata_header_ = |
| 90 | reinterpret_cast<BufferHubDefs::MetadataHeader*>(metadata_ptr); |
| 91 | |
| 92 | // Using placement new here to reuse shared memory instead of new allocation |
| 93 | // and also initialize the value to zero. |
Tianyu Jiang | f377a76 | 2019-02-13 13:46:42 -0800 | [diff] [blame^] | 94 | buffer_state_ = new (&metadata_header_->bufferState) std::atomic<uint32_t>(0); |
| 95 | fence_state_ = new (&metadata_header_->fenceState) std::atomic<uint32_t>(0); |
Tianyu | 6705349 | 2018-10-04 14:00:22 -0700 | [diff] [blame] | 96 | active_clients_bit_mask_ = |
Tianyu Jiang | f377a76 | 2019-02-13 13:46:42 -0800 | [diff] [blame^] | 97 | new (&metadata_header_->activeClientsBitMask) std::atomic<uint32_t>(0); |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 98 | |
Tianyu Jiang | 63dd7c3 | 2018-10-30 18:35:06 -0700 | [diff] [blame] | 99 | // Producer channel is never created after consumer channel, and one buffer |
| 100 | // only have one fixed producer for now. Thus, it is correct to assume |
| 101 | // producer state bit is kFirstClientBitMask for now. |
| 102 | active_clients_bit_mask_->store(BufferHubDefs::kFirstClientBitMask, |
| 103 | std::memory_order_release); |
| 104 | |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 105 | acquire_fence_fd_.Reset(epoll_create1(EPOLL_CLOEXEC)); |
| 106 | release_fence_fd_.Reset(epoll_create1(EPOLL_CLOEXEC)); |
| 107 | if (!acquire_fence_fd_ || !release_fence_fd_) { |
| 108 | ALOGE("ProducerChannel::ProducerChannel: Failed to create shared fences."); |
Jiwen 'Steve' Cai | 0728fa9 | 2018-04-24 19:03:14 -0700 | [diff] [blame] | 109 | return -EIO; |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 110 | } |
| 111 | |
| 112 | dummy_fence_fd_.Reset(eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK)); |
| 113 | if (!dummy_fence_fd_) { |
| 114 | ALOGE("ProducerChannel::ProducerChannel: Failed to create dummy fences."); |
Jiwen 'Steve' Cai | 0728fa9 | 2018-04-24 19:03:14 -0700 | [diff] [blame] | 115 | return EIO; |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 116 | } |
| 117 | |
| 118 | epoll_event event; |
| 119 | event.events = 0; |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 120 | event.data.u32 = 0U; |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 121 | if (epoll_ctl(release_fence_fd_.Get(), EPOLL_CTL_ADD, dummy_fence_fd_.Get(), |
| 122 | &event) < 0) { |
| 123 | ALOGE( |
| 124 | "ProducerChannel::ProducerChannel: Failed to modify the shared " |
| 125 | "release fence to include the dummy fence: %s", |
| 126 | strerror(errno)); |
Jiwen 'Steve' Cai | 0728fa9 | 2018-04-24 19:03:14 -0700 | [diff] [blame] | 127 | return -EIO; |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 128 | } |
| 129 | |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 130 | // Success. |
Jiwen 'Steve' Cai | 0728fa9 | 2018-04-24 19:03:14 -0700 | [diff] [blame] | 131 | return 0; |
| 132 | } |
| 133 | |
| 134 | std::unique_ptr<ProducerChannel> ProducerChannel::Create( |
| 135 | BufferHubService* service, int buffer_id, int channel_id, IonBuffer buffer, |
| 136 | IonBuffer metadata_buffer, size_t user_metadata_size) { |
| 137 | int error = 0; |
| 138 | std::unique_ptr<ProducerChannel> producer(new ProducerChannel( |
| 139 | service, buffer_id, channel_id, std::move(buffer), |
| 140 | std::move(metadata_buffer), user_metadata_size, &error)); |
| 141 | |
| 142 | if (error < 0) |
| 143 | return nullptr; |
| 144 | else |
| 145 | return producer; |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 146 | } |
| 147 | |
Corey Tabaka | cd52dd9 | 2017-04-07 18:03:57 -0700 | [diff] [blame] | 148 | Status<std::shared_ptr<ProducerChannel>> ProducerChannel::Create( |
| 149 | BufferHubService* service, int channel_id, uint32_t width, uint32_t height, |
Hendrik Wagenaar | 108e84f | 2017-05-07 22:19:17 -0700 | [diff] [blame] | 150 | uint32_t layer_count, uint32_t format, uint64_t usage, |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 151 | size_t user_metadata_size) { |
Corey Tabaka | cd52dd9 | 2017-04-07 18:03:57 -0700 | [diff] [blame] | 152 | int error; |
Jiwen 'Steve' Cai | 0057fdd | 2017-05-02 11:21:18 -0700 | [diff] [blame] | 153 | std::shared_ptr<ProducerChannel> producer( |
Hendrik Wagenaar | 108e84f | 2017-05-07 22:19:17 -0700 | [diff] [blame] | 154 | new ProducerChannel(service, channel_id, width, height, layer_count, |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 155 | format, usage, user_metadata_size, &error)); |
Corey Tabaka | cd52dd9 | 2017-04-07 18:03:57 -0700 | [diff] [blame] | 156 | if (error < 0) |
| 157 | return ErrorStatus(-error); |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 158 | else |
Corey Tabaka | cd52dd9 | 2017-04-07 18:03:57 -0700 | [diff] [blame] | 159 | return {std::move(producer)}; |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 160 | } |
| 161 | |
| 162 | ProducerChannel::~ProducerChannel() { |
Corey Tabaka | 3079cb7 | 2017-01-19 15:07:26 -0800 | [diff] [blame] | 163 | ALOGD_IF(TRACE, |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 164 | "ProducerChannel::~ProducerChannel: channel_id=%d buffer_id=%d " |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 165 | "state=%" PRIx32 ".", |
Tianyu Jiang | e2cdec9 | 2018-10-30 17:40:58 -0700 | [diff] [blame] | 166 | channel_id(), buffer_id(), |
| 167 | buffer_state_->load(std::memory_order_acquire)); |
Jiwen 'Steve' Cai | 23c1a73 | 2018-03-12 12:16:47 -0700 | [diff] [blame] | 168 | for (auto consumer : consumer_channels_) { |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 169 | consumer->OnProducerClosed(); |
Jiwen 'Steve' Cai | 23c1a73 | 2018-03-12 12:16:47 -0700 | [diff] [blame] | 170 | } |
| 171 | Hangup(); |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 172 | } |
| 173 | |
| 174 | BufferHubChannel::BufferInfo ProducerChannel::GetBufferInfo() const { |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 175 | // Derive the mask of signaled buffers in this producer / consumer set. |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 176 | uint32_t signaled_mask = signaled() ? BufferHubDefs::kFirstClientBitMask : 0; |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 177 | for (const ConsumerChannel* consumer : consumer_channels_) { |
Tianyu Jiang | 83a991f | 2018-10-30 16:59:29 -0700 | [diff] [blame] | 178 | signaled_mask |= consumer->signaled() ? consumer->client_state_mask() : 0; |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 179 | } |
| 180 | |
Hendrik Wagenaar | 4d3590f | 2017-05-06 22:36:04 -0700 | [diff] [blame] | 181 | return BufferInfo(buffer_id(), consumer_channels_.size(), buffer_.width(), |
Hendrik Wagenaar | 108e84f | 2017-05-07 22:19:17 -0700 | [diff] [blame] | 182 | buffer_.height(), buffer_.layer_count(), buffer_.format(), |
Tianyu Jiang | 0cdd2a0 | 2018-11-15 10:20:45 -0800 | [diff] [blame] | 183 | buffer_.usage(), |
Tianyu Jiang | e2cdec9 | 2018-10-30 17:40:58 -0700 | [diff] [blame] | 184 | buffer_state_->load(std::memory_order_acquire), |
Tianyu Jiang | f377a76 | 2019-02-13 13:46:42 -0800 | [diff] [blame^] | 185 | signaled_mask, metadata_header_->queueIndex); |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 186 | } |
| 187 | |
| 188 | void ProducerChannel::HandleImpulse(Message& message) { |
| 189 | ATRACE_NAME("ProducerChannel::HandleImpulse"); |
| 190 | switch (message.GetOp()) { |
| 191 | case BufferHubRPC::ProducerGain::Opcode: |
| 192 | OnProducerGain(message); |
| 193 | break; |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 194 | case BufferHubRPC::ProducerPost::Opcode: |
| 195 | OnProducerPost(message, {}); |
| 196 | break; |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 197 | } |
| 198 | } |
| 199 | |
| 200 | bool ProducerChannel::HandleMessage(Message& message) { |
| 201 | ATRACE_NAME("ProducerChannel::HandleMessage"); |
| 202 | switch (message.GetOp()) { |
| 203 | case BufferHubRPC::GetBuffer::Opcode: |
| 204 | DispatchRemoteMethod<BufferHubRPC::GetBuffer>( |
| 205 | *this, &ProducerChannel::OnGetBuffer, message); |
| 206 | return true; |
| 207 | |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 208 | case BufferHubRPC::NewConsumer::Opcode: |
| 209 | DispatchRemoteMethod<BufferHubRPC::NewConsumer>( |
| 210 | *this, &ProducerChannel::OnNewConsumer, message); |
| 211 | return true; |
| 212 | |
| 213 | case BufferHubRPC::ProducerPost::Opcode: |
| 214 | DispatchRemoteMethod<BufferHubRPC::ProducerPost>( |
| 215 | *this, &ProducerChannel::OnProducerPost, message); |
| 216 | return true; |
| 217 | |
| 218 | case BufferHubRPC::ProducerGain::Opcode: |
| 219 | DispatchRemoteMethod<BufferHubRPC::ProducerGain>( |
| 220 | *this, &ProducerChannel::OnProducerGain, message); |
| 221 | return true; |
| 222 | |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 223 | default: |
| 224 | return false; |
| 225 | } |
| 226 | } |
| 227 | |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 228 | BufferDescription<BorrowedHandle> ProducerChannel::GetBuffer( |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 229 | uint32_t client_state_mask) { |
Jiwen 'Steve' Cai | 57ae3ee | 2018-05-03 17:51:52 -0700 | [diff] [blame] | 230 | return {buffer_, |
| 231 | metadata_buffer_, |
| 232 | buffer_id(), |
| 233 | channel_id(), |
Tianyu Jiang | 7e204b7 | 2018-10-26 15:39:18 -0700 | [diff] [blame] | 234 | client_state_mask, |
Jiwen 'Steve' Cai | 57ae3ee | 2018-05-03 17:51:52 -0700 | [diff] [blame] | 235 | acquire_fence_fd_.Borrow(), |
| 236 | release_fence_fd_.Borrow()}; |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 237 | } |
| 238 | |
| 239 | Status<BufferDescription<BorrowedHandle>> ProducerChannel::OnGetBuffer( |
Corey Tabaka | d53870c | 2017-07-06 18:04:27 -0700 | [diff] [blame] | 240 | Message& /*message*/) { |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 241 | ATRACE_NAME("ProducerChannel::OnGetBuffer"); |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 242 | ALOGD_IF(TRACE, "ProducerChannel::OnGetBuffer: buffer=%d, state=%" PRIx32 ".", |
Tianyu Jiang | e2cdec9 | 2018-10-30 17:40:58 -0700 | [diff] [blame] | 243 | buffer_id(), buffer_state_->load(std::memory_order_acquire)); |
Tianyu Jiang | 63dd7c3 | 2018-10-30 18:35:06 -0700 | [diff] [blame] | 244 | return {GetBuffer(BufferHubDefs::kFirstClientBitMask)}; |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 245 | } |
| 246 | |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 247 | Status<uint32_t> ProducerChannel::CreateConsumerStateMask() { |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 248 | // Try find the next consumer state bit which has not been claimed by any |
| 249 | // consumer yet. |
Tianyu | 6705349 | 2018-10-04 14:00:22 -0700 | [diff] [blame] | 250 | // memory_order_acquire is chosen here because all writes in other threads |
| 251 | // that release active_clients_bit_mask_ need to be visible here. |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 252 | uint32_t current_active_clients_bit_mask = |
Tianyu | 6705349 | 2018-10-04 14:00:22 -0700 | [diff] [blame] | 253 | active_clients_bit_mask_->load(std::memory_order_acquire); |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 254 | uint32_t consumer_state_mask = |
Tianyu Jiang | 727ede4 | 2019-02-01 11:44:51 -0800 | [diff] [blame] | 255 | BufferHubDefs::findNextAvailableClientStateMask( |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 256 | current_active_clients_bit_mask | orphaned_consumer_bit_mask_); |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 257 | if (consumer_state_mask == 0U) { |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 258 | ALOGE("%s: reached the maximum mumber of consumers per producer: 63.", |
| 259 | __FUNCTION__); |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 260 | return ErrorStatus(E2BIG); |
| 261 | } |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 262 | uint32_t updated_active_clients_bit_mask = |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 263 | current_active_clients_bit_mask | consumer_state_mask; |
Tianyu | 6705349 | 2018-10-04 14:00:22 -0700 | [diff] [blame] | 264 | // Set the updated value only if the current value stays the same as what was |
| 265 | // read before. If the comparison succeeds, update the value without |
| 266 | // reordering anything before or after this read-modify-write in the current |
| 267 | // thread, and the modification will be visible in other threads that acquire |
| 268 | // active_clients_bit_mask_. If the comparison fails, load the result of |
| 269 | // all writes from all threads to updated_active_clients_bit_mask. |
Tianyu Jiang | 50f7ad1 | 2018-11-08 11:13:35 -0800 | [diff] [blame] | 270 | // Keep on finding the next available slient state mask until succeed or out |
| 271 | // of memory. |
| 272 | while (!active_clients_bit_mask_->compare_exchange_weak( |
| 273 | current_active_clients_bit_mask, updated_active_clients_bit_mask, |
| 274 | std::memory_order_acq_rel, std::memory_order_acquire)) { |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 275 | ALOGE("%s: Current active clients bit mask is changed to %" PRIx32 |
| 276 | ", which was expected to be %" PRIx32 |
Tianyu Jiang | 50f7ad1 | 2018-11-08 11:13:35 -0800 | [diff] [blame] | 277 | ". Trying to generate a new client state mask to resolve race " |
| 278 | "condition.", |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 279 | __FUNCTION__, updated_active_clients_bit_mask, |
| 280 | current_active_clients_bit_mask); |
Tianyu Jiang | 727ede4 | 2019-02-01 11:44:51 -0800 | [diff] [blame] | 281 | consumer_state_mask = BufferHubDefs::findNextAvailableClientStateMask( |
Tianyu Jiang | 50f7ad1 | 2018-11-08 11:13:35 -0800 | [diff] [blame] | 282 | current_active_clients_bit_mask | orphaned_consumer_bit_mask_); |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 283 | if (consumer_state_mask == 0U) { |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 284 | ALOGE("%s: reached the maximum mumber of consumers per producer: %d.", |
| 285 | __FUNCTION__, (BufferHubDefs::kMaxNumberOfClients - 1)); |
Tianyu Jiang | 50f7ad1 | 2018-11-08 11:13:35 -0800 | [diff] [blame] | 286 | return ErrorStatus(E2BIG); |
| 287 | } |
| 288 | updated_active_clients_bit_mask = |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 289 | current_active_clients_bit_mask | consumer_state_mask; |
Tianyu | 6705349 | 2018-10-04 14:00:22 -0700 | [diff] [blame] | 290 | } |
| 291 | |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 292 | return {consumer_state_mask}; |
Tianyu Jiang | 50f7ad1 | 2018-11-08 11:13:35 -0800 | [diff] [blame] | 293 | } |
| 294 | |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 295 | void ProducerChannel::RemoveConsumerClientMask(uint32_t consumer_state_mask) { |
Tianyu Jiang | 50f7ad1 | 2018-11-08 11:13:35 -0800 | [diff] [blame] | 296 | // Clear up the buffer state and fence state in case there is already |
| 297 | // something there due to possible race condition between producer post and |
| 298 | // consumer failed to create channel. |
| 299 | buffer_state_->fetch_and(~consumer_state_mask, std::memory_order_release); |
| 300 | fence_state_->fetch_and(~consumer_state_mask, std::memory_order_release); |
| 301 | |
| 302 | // Restore the consumer state bit and make it visible in other threads that |
| 303 | // acquire the active_clients_bit_mask_. |
| 304 | active_clients_bit_mask_->fetch_and(~consumer_state_mask, |
| 305 | std::memory_order_release); |
| 306 | } |
| 307 | |
| 308 | Status<RemoteChannelHandle> ProducerChannel::CreateConsumer( |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 309 | Message& message, uint32_t consumer_state_mask) { |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 310 | ATRACE_NAME(__FUNCTION__); |
Jiwen 'Steve' Cai | 617ba80 | 2018-12-20 14:22:55 -0800 | [diff] [blame] | 311 | ALOGD("%s: buffer_id=%d", __FUNCTION__, buffer_id()); |
Tianyu Jiang | 50f7ad1 | 2018-11-08 11:13:35 -0800 | [diff] [blame] | 312 | |
| 313 | int channel_id; |
| 314 | auto status = message.PushChannel(0, nullptr, &channel_id); |
| 315 | if (!status) { |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 316 | ALOGE("%s: Failed to push consumer channel: %s", __FUNCTION__, |
| 317 | status.GetErrorMessage().c_str()); |
Tianyu Jiang | 50f7ad1 | 2018-11-08 11:13:35 -0800 | [diff] [blame] | 318 | RemoveConsumerClientMask(consumer_state_mask); |
| 319 | return ErrorStatus(ENOMEM); |
| 320 | } |
| 321 | |
| 322 | auto consumer = std::make_shared<ConsumerChannel>( |
| 323 | service(), buffer_id(), channel_id, consumer_state_mask, |
| 324 | shared_from_this()); |
Alex Vakulenko | f0a7bd0 | 2017-03-31 18:06:19 -0700 | [diff] [blame] | 325 | const auto channel_status = service()->SetChannel(channel_id, consumer); |
| 326 | if (!channel_status) { |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 327 | ALOGE("%s: failed to set new consumer channel: %s.", __FUNCTION__, |
| 328 | channel_status.GetErrorMessage().c_str()); |
Tianyu Jiang | 50f7ad1 | 2018-11-08 11:13:35 -0800 | [diff] [blame] | 329 | RemoveConsumerClientMask(consumer_state_mask); |
Corey Tabaka | cd52dd9 | 2017-04-07 18:03:57 -0700 | [diff] [blame] | 330 | return ErrorStatus(ENOMEM); |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 331 | } |
| 332 | |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 333 | uint32_t current_buffer_state = |
Tianyu Jiang | ca9201a | 2018-11-06 12:40:33 -0800 | [diff] [blame] | 334 | buffer_state_->load(std::memory_order_acquire); |
Tianyu Jiang | a8df5f3 | 2019-01-14 18:42:12 -0800 | [diff] [blame] | 335 | // Return the consumer channel handle without signal when adding the new |
| 336 | // consumer to a buffer that is available to producer (a.k.a a fully-released |
| 337 | // buffer) or a gained buffer. |
| 338 | if (current_buffer_state == 0U || |
Tianyu Jiang | 727ede4 | 2019-02-01 11:44:51 -0800 | [diff] [blame] | 339 | BufferHubDefs::isAnyClientGained(current_buffer_state)) { |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 340 | return {status.take()}; |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 341 | } |
| 342 | |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 343 | // Signal the new consumer when adding it to a posted producer. |
| 344 | bool update_buffer_state = true; |
Tianyu Jiang | 727ede4 | 2019-02-01 11:44:51 -0800 | [diff] [blame] | 345 | if (!BufferHubDefs::isClientPosted(current_buffer_state, |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 346 | consumer_state_mask)) { |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 347 | uint32_t updated_buffer_state = |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 348 | current_buffer_state ^ |
| 349 | (consumer_state_mask & BufferHubDefs::kHighBitsMask); |
| 350 | while (!buffer_state_->compare_exchange_weak( |
| 351 | current_buffer_state, updated_buffer_state, std::memory_order_acq_rel, |
| 352 | std::memory_order_acquire)) { |
| 353 | ALOGI( |
| 354 | "%s: Failed to post to the new consumer. " |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 355 | "Current buffer state was changed to %" PRIx32 |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 356 | " when trying to acquire the buffer and modify the buffer state to " |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 357 | "%" PRIx32 |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 358 | ". About to try again if the buffer is still not gained nor fully " |
| 359 | "released.", |
| 360 | __FUNCTION__, current_buffer_state, updated_buffer_state); |
Tianyu Jiang | a8df5f3 | 2019-01-14 18:42:12 -0800 | [diff] [blame] | 361 | if (current_buffer_state == 0U || |
Tianyu Jiang | 727ede4 | 2019-02-01 11:44:51 -0800 | [diff] [blame] | 362 | BufferHubDefs::isAnyClientGained(current_buffer_state)) { |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 363 | ALOGI("%s: buffer is gained or fully released, state=%" PRIx32 ".", |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 364 | __FUNCTION__, current_buffer_state); |
| 365 | update_buffer_state = false; |
| 366 | break; |
| 367 | } |
| 368 | updated_buffer_state = |
| 369 | current_buffer_state ^ |
| 370 | (consumer_state_mask & BufferHubDefs::kHighBitsMask); |
| 371 | } |
| 372 | } |
Tianyu Jiang | 727ede4 | 2019-02-01 11:44:51 -0800 | [diff] [blame] | 373 | if (update_buffer_state || BufferHubDefs::isClientPosted( |
Tianyu Jiang | e60a4ad | 2019-01-04 14:37:23 -0800 | [diff] [blame] | 374 | buffer_state_->load(std::memory_order_acquire), |
| 375 | consumer_state_mask)) { |
Tianyu Jiang | 0cdd2a0 | 2018-11-15 10:20:45 -0800 | [diff] [blame] | 376 | consumer->OnProducerPosted(); |
| 377 | } |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 378 | |
Corey Tabaka | cd52dd9 | 2017-04-07 18:03:57 -0700 | [diff] [blame] | 379 | return {status.take()}; |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 380 | } |
| 381 | |
Corey Tabaka | cd52dd9 | 2017-04-07 18:03:57 -0700 | [diff] [blame] | 382 | Status<RemoteChannelHandle> ProducerChannel::OnNewConsumer(Message& message) { |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 383 | ATRACE_NAME("ProducerChannel::OnNewConsumer"); |
| 384 | ALOGD_IF(TRACE, "ProducerChannel::OnNewConsumer: buffer_id=%d", buffer_id()); |
Tianyu Jiang | 50f7ad1 | 2018-11-08 11:13:35 -0800 | [diff] [blame] | 385 | auto status = CreateConsumerStateMask(); |
| 386 | if (!status.ok()) { |
| 387 | return status.error_status(); |
| 388 | } |
| 389 | return CreateConsumer(message, /*consumer_state_mask=*/status.get()); |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 390 | } |
| 391 | |
Tianyu | 0c6b7ff | 2018-10-03 11:30:04 -0700 | [diff] [blame] | 392 | Status<void> ProducerChannel::OnProducerPost(Message&, |
| 393 | LocalFence acquire_fence) { |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 394 | ATRACE_NAME("ProducerChannel::OnProducerPost"); |
Jiwen 'Steve' Cai | 617ba80 | 2018-12-20 14:22:55 -0800 | [diff] [blame] | 395 | ALOGD("ProducerChannel::OnProducerPost: buffer_id=%d, state=0x%x", |
| 396 | buffer_id(), buffer_state_->load(std::memory_order_acquire)); |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 397 | |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 398 | epoll_event event; |
| 399 | event.events = 0; |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 400 | event.data.u32 = 0U; |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 401 | int ret = epoll_ctl(release_fence_fd_.Get(), EPOLL_CTL_MOD, |
| 402 | dummy_fence_fd_.Get(), &event); |
| 403 | ALOGE_IF(ret < 0, |
Tianyu | 0c6b7ff | 2018-10-03 11:30:04 -0700 | [diff] [blame] | 404 | "ProducerChannel::OnProducerPost: Failed to modify the shared " |
| 405 | "release fence to include the dummy fence: %s", |
| 406 | strerror(errno)); |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 407 | |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 408 | eventfd_t dummy_fence_count = 0U; |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 409 | if (eventfd_read(dummy_fence_fd_.Get(), &dummy_fence_count) < 0) { |
| 410 | const int error = errno; |
| 411 | if (error != EAGAIN) { |
| 412 | ALOGE( |
| 413 | "ProducerChannel::ProducerChannel: Failed to read dummy fence, " |
| 414 | "error: %s", |
| 415 | strerror(error)); |
| 416 | return ErrorStatus(error); |
| 417 | } |
Corey Tabaka | 8a4e6a9 | 2017-04-20 13:42:02 -0700 | [diff] [blame] | 418 | } |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 419 | |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 420 | ALOGW_IF(dummy_fence_count > 0, |
| 421 | "ProducerChannel::ProducerChannel: %" PRIu64 |
| 422 | " dummy fence(s) was signaled during last release/gain cycle " |
| 423 | "buffer_id=%d.", |
| 424 | dummy_fence_count, buffer_id()); |
| 425 | |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 426 | post_fence_ = std::move(acquire_fence); |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 427 | |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 428 | // Signal any interested consumers. If there are none, the buffer will stay |
| 429 | // in posted state until a consumer comes online. This behavior guarantees |
| 430 | // that no frame is silently dropped. |
Tianyu Jiang | ae9668c | 2018-12-07 15:14:47 -0800 | [diff] [blame] | 431 | for (auto& consumer : consumer_channels_) { |
Tianyu Jiang | 0cdd2a0 | 2018-11-15 10:20:45 -0800 | [diff] [blame] | 432 | consumer->OnProducerPosted(); |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 433 | } |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 434 | |
Corey Tabaka | cd52dd9 | 2017-04-07 18:03:57 -0700 | [diff] [blame] | 435 | return {}; |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 436 | } |
| 437 | |
Corey Tabaka | d53870c | 2017-07-06 18:04:27 -0700 | [diff] [blame] | 438 | Status<LocalFence> ProducerChannel::OnProducerGain(Message& /*message*/) { |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 439 | ATRACE_NAME("ProducerChannel::OnGain"); |
Jiwen 'Steve' Cai | 617ba80 | 2018-12-20 14:22:55 -0800 | [diff] [blame] | 440 | ALOGW("ProducerChannel::OnGain: buffer_id=%d", buffer_id()); |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 441 | |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 442 | ClearAvailable(); |
Alex Vakulenko | 052f3ae | 2017-03-31 09:10:43 -0700 | [diff] [blame] | 443 | post_fence_.close(); |
Tianyu Jiang | ae9668c | 2018-12-07 15:14:47 -0800 | [diff] [blame] | 444 | for (auto& consumer : consumer_channels_) { |
| 445 | consumer->OnProducerGained(); |
| 446 | } |
Corey Tabaka | cd52dd9 | 2017-04-07 18:03:57 -0700 | [diff] [blame] | 447 | return {std::move(returned_fence_)}; |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 448 | } |
| 449 | |
Fan Xu | b0eec51 | 2018-10-30 11:33:15 -0700 | [diff] [blame] | 450 | // TODO(b/112338294) Keep here for reference. Remove it after new logic is |
| 451 | // written. |
| 452 | /* Status<RemoteChannelHandle> ProducerChannel::OnProducerDetach( |
Jiwen 'Steve' Cai | 23c1a73 | 2018-03-12 12:16:47 -0700 | [diff] [blame] | 453 | Message& message) { |
| 454 | ATRACE_NAME("ProducerChannel::OnProducerDetach"); |
| 455 | ALOGD_IF(TRACE, "ProducerChannel::OnProducerDetach: buffer_id=%d", |
| 456 | buffer_id()); |
| 457 | |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 458 | uint32_t buffer_state = buffer_state_->load(std::memory_order_acquire); |
Tianyu Jiang | 727ede4 | 2019-02-01 11:44:51 -0800 | [diff] [blame] | 459 | if (!BufferHubDefs::isClientGained( |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 460 | buffer_state, BufferHubDefs::kFirstClientStateMask)) { |
Jiwen 'Steve' Cai | 1c73024 | 2018-12-31 18:40:02 -0800 | [diff] [blame] | 461 | // Can only detach a ProducerBuffer when it's in gained state. |
Jiwen 'Steve' Cai | 23c1a73 | 2018-03-12 12:16:47 -0700 | [diff] [blame] | 462 | ALOGW( |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 463 | "ProducerChannel::OnProducerDetach: The buffer (id=%d, state=%" |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 464 | PRIx32 |
Jiwen 'Steve' Cai | 23c1a73 | 2018-03-12 12:16:47 -0700 | [diff] [blame] | 465 | ") is not in gained state.", |
| 466 | buffer_id(), buffer_state); |
| 467 | return {}; |
| 468 | } |
| 469 | |
| 470 | int channel_id; |
| 471 | auto status = message.PushChannel(0, nullptr, &channel_id); |
| 472 | if (!status) { |
| 473 | ALOGE( |
| 474 | "ProducerChannel::OnProducerDetach: Failed to push detached buffer " |
| 475 | "channel: %s", |
| 476 | status.GetErrorMessage().c_str()); |
| 477 | return ErrorStatus(ENOMEM); |
| 478 | } |
| 479 | |
| 480 | // Make sure we unlock the buffer. |
| 481 | if (int ret = metadata_buffer_.Unlock()) { |
| 482 | ALOGE("ProducerChannel::OnProducerDetach: Failed to unlock metadata."); |
| 483 | return ErrorStatus(-ret); |
| 484 | }; |
| 485 | |
Jiwen 'Steve' Cai | 9004b8c | 2018-10-03 18:52:23 -0700 | [diff] [blame] | 486 | std::unique_ptr<BufferChannel> channel = |
| 487 | BufferChannel::Create(service(), buffer_id(), channel_id, |
| 488 | std::move(buffer_), user_metadata_size_); |
Jiwen 'Steve' Cai | a8049a2 | 2018-03-28 15:14:02 -0700 | [diff] [blame] | 489 | if (!channel) { |
| 490 | ALOGE("ProducerChannel::OnProducerDetach: Invalid buffer."); |
| 491 | return ErrorStatus(EINVAL); |
| 492 | } |
Jiwen 'Steve' Cai | 23c1a73 | 2018-03-12 12:16:47 -0700 | [diff] [blame] | 493 | |
Jiwen 'Steve' Cai | a8049a2 | 2018-03-28 15:14:02 -0700 | [diff] [blame] | 494 | const auto channel_status = |
| 495 | service()->SetChannel(channel_id, std::move(channel)); |
Jiwen 'Steve' Cai | 23c1a73 | 2018-03-12 12:16:47 -0700 | [diff] [blame] | 496 | if (!channel_status) { |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 497 | // Technically, this should never fail, as we just pushed the channel. |
| 498 | // Note that LOG_FATAL will be stripped out in non-debug build. |
Jiwen 'Steve' Cai | 23c1a73 | 2018-03-12 12:16:47 -0700 | [diff] [blame] | 499 | LOG_FATAL( |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 500 | "ProducerChannel::OnProducerDetach: Failed to set new detached " |
| 501 | "buffer channel: %s.", channel_status.GetErrorMessage().c_str()); |
Jiwen 'Steve' Cai | 23c1a73 | 2018-03-12 12:16:47 -0700 | [diff] [blame] | 502 | } |
| 503 | |
| 504 | return status; |
Fan Xu | b0eec51 | 2018-10-30 11:33:15 -0700 | [diff] [blame] | 505 | } */ |
Jiwen 'Steve' Cai | 23c1a73 | 2018-03-12 12:16:47 -0700 | [diff] [blame] | 506 | |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 507 | Status<LocalFence> ProducerChannel::OnConsumerAcquire(Message& /*message*/) { |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 508 | ATRACE_NAME("ProducerChannel::OnConsumerAcquire"); |
| 509 | ALOGD_IF(TRACE, "ProducerChannel::OnConsumerAcquire: buffer_id=%d", |
| 510 | buffer_id()); |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 511 | |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 512 | // Return a borrowed fd to avoid unnecessary duplication of the underlying |
| 513 | // fd. Serialization just needs to read the handle. |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 514 | return {std::move(post_fence_)}; |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 515 | } |
| 516 | |
Corey Tabaka | cd52dd9 | 2017-04-07 18:03:57 -0700 | [diff] [blame] | 517 | Status<void> ProducerChannel::OnConsumerRelease(Message&, |
| 518 | LocalFence release_fence) { |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 519 | ATRACE_NAME("ProducerChannel::OnConsumerRelease"); |
| 520 | ALOGD_IF(TRACE, "ProducerChannel::OnConsumerRelease: buffer_id=%d", |
| 521 | buffer_id()); |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 522 | |
| 523 | // Attempt to merge the fences if necessary. |
| 524 | if (release_fence) { |
| 525 | if (returned_fence_) { |
Corey Tabaka | 3079cb7 | 2017-01-19 15:07:26 -0800 | [diff] [blame] | 526 | LocalFence merged_fence(sync_merge("bufferhub_merged", |
| 527 | returned_fence_.get_fd(), |
| 528 | release_fence.get_fd())); |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 529 | const int error = errno; |
| 530 | if (!merged_fence) { |
| 531 | ALOGE("ProducerChannel::OnConsumerRelease: Failed to merge fences: %s", |
| 532 | strerror(error)); |
Corey Tabaka | cd52dd9 | 2017-04-07 18:03:57 -0700 | [diff] [blame] | 533 | return ErrorStatus(error); |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 534 | } |
| 535 | returned_fence_ = std::move(merged_fence); |
| 536 | } else { |
| 537 | returned_fence_ = std::move(release_fence); |
| 538 | } |
| 539 | } |
| 540 | |
Tianyu Jiang | a8df5f3 | 2019-01-14 18:42:12 -0800 | [diff] [blame] | 541 | if (IsBufferReleasedByAllActiveClientsExceptForOrphans()) { |
Tianyu Jiang | e60a4ad | 2019-01-04 14:37:23 -0800 | [diff] [blame] | 542 | buffer_state_->store(0U); |
Tianyu Jiang | 0cdd2a0 | 2018-11-15 10:20:45 -0800 | [diff] [blame] | 543 | SignalAvailable(); |
| 544 | if (orphaned_consumer_bit_mask_) { |
| 545 | ALOGW( |
| 546 | "%s: orphaned buffer detected during the this acquire/release cycle: " |
Jiwen 'Steve' Cai | 617ba80 | 2018-12-20 14:22:55 -0800 | [diff] [blame] | 547 | "id=%d orphaned=0x%" PRIx32 " queue_index=%" PRId64 ".", |
Tianyu Jiang | 0cdd2a0 | 2018-11-15 10:20:45 -0800 | [diff] [blame] | 548 | __FUNCTION__, buffer_id(), orphaned_consumer_bit_mask_, |
Tianyu Jiang | f377a76 | 2019-02-13 13:46:42 -0800 | [diff] [blame^] | 549 | metadata_header_->queueIndex); |
Tianyu Jiang | 0cdd2a0 | 2018-11-15 10:20:45 -0800 | [diff] [blame] | 550 | orphaned_consumer_bit_mask_ = 0; |
| 551 | } |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 552 | } |
| 553 | |
Corey Tabaka | cd52dd9 | 2017-04-07 18:03:57 -0700 | [diff] [blame] | 554 | return {}; |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 555 | } |
| 556 | |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 557 | void ProducerChannel::OnConsumerOrphaned(const uint32_t& consumer_state_mask) { |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 558 | // Remember the ignored consumer so that newly added consumer won't be |
| 559 | // taking the same state mask as this orphaned consumer. |
| 560 | ALOGE_IF(orphaned_consumer_bit_mask_ & consumer_state_mask, |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 561 | "%s: Consumer (consumer_state_mask=%" PRIx32 |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 562 | ") is already orphaned.", |
| 563 | __FUNCTION__, consumer_state_mask); |
| 564 | orphaned_consumer_bit_mask_ |= consumer_state_mask; |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 565 | |
Tianyu Jiang | a8df5f3 | 2019-01-14 18:42:12 -0800 | [diff] [blame] | 566 | if (IsBufferReleasedByAllActiveClientsExceptForOrphans()) { |
Tianyu Jiang | e60a4ad | 2019-01-04 14:37:23 -0800 | [diff] [blame] | 567 | buffer_state_->store(0U); |
Tianyu Jiang | 0cdd2a0 | 2018-11-15 10:20:45 -0800 | [diff] [blame] | 568 | SignalAvailable(); |
| 569 | } |
| 570 | |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 571 | // Atomically clear the fence state bit as an orphaned consumer will never |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 572 | // signal a release fence. |
| 573 | fence_state_->fetch_and(~consumer_state_mask, std::memory_order_release); |
| 574 | |
| 575 | // Atomically set the buffer state of this consumer to released state. |
| 576 | buffer_state_->fetch_and(~consumer_state_mask, std::memory_order_release); |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 577 | |
| 578 | ALOGW( |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 579 | "%s: detected new orphaned consumer buffer_id=%d " |
Jiwen 'Steve' Cai | 617ba80 | 2018-12-20 14:22:55 -0800 | [diff] [blame] | 580 | "consumer_state_mask=%" PRIx32 " queue_index=%" PRId64 |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 581 | " buffer_state=%" PRIx32 " fence_state=%" PRIx32 ".", |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 582 | __FUNCTION__, buffer_id(), consumer_state_mask, |
Tianyu Jiang | f377a76 | 2019-02-13 13:46:42 -0800 | [diff] [blame^] | 583 | metadata_header_->queueIndex, |
Tianyu Jiang | e2cdec9 | 2018-10-30 17:40:58 -0700 | [diff] [blame] | 584 | buffer_state_->load(std::memory_order_acquire), |
| 585 | fence_state_->load(std::memory_order_acquire)); |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 586 | } |
| 587 | |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 588 | void ProducerChannel::AddConsumer(ConsumerChannel* channel) { |
| 589 | consumer_channels_.push_back(channel); |
| 590 | } |
| 591 | |
| 592 | void ProducerChannel::RemoveConsumer(ConsumerChannel* channel) { |
| 593 | consumer_channels_.erase( |
| 594 | std::find(consumer_channels_.begin(), consumer_channels_.end(), channel)); |
Tianyu | 6705349 | 2018-10-04 14:00:22 -0700 | [diff] [blame] | 595 | // Restore the consumer state bit and make it visible in other threads that |
| 596 | // acquire the active_clients_bit_mask_. |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 597 | uint32_t consumer_state_mask = channel->client_state_mask(); |
| 598 | uint32_t current_active_clients_bit_mask = |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 599 | active_clients_bit_mask_->load(std::memory_order_acquire); |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 600 | uint32_t updated_active_clients_bit_mask = |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 601 | current_active_clients_bit_mask & (~consumer_state_mask); |
| 602 | while (!active_clients_bit_mask_->compare_exchange_weak( |
| 603 | current_active_clients_bit_mask, updated_active_clients_bit_mask, |
| 604 | std::memory_order_acq_rel, std::memory_order_acquire)) { |
| 605 | ALOGI( |
| 606 | "%s: Failed to remove consumer state mask. Current active clients bit " |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 607 | "mask is changed to %" PRIx32 |
| 608 | " when trying to acquire and modify it to %" PRIx32 |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 609 | ". About to try again.", |
| 610 | __FUNCTION__, current_active_clients_bit_mask, |
| 611 | updated_active_clients_bit_mask); |
| 612 | updated_active_clients_bit_mask = |
| 613 | current_active_clients_bit_mask & (~consumer_state_mask); |
| 614 | } |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 615 | |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 616 | const uint32_t current_buffer_state = |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 617 | buffer_state_->load(std::memory_order_acquire); |
Tianyu Jiang | 727ede4 | 2019-02-01 11:44:51 -0800 | [diff] [blame] | 618 | if (BufferHubDefs::isClientPosted(current_buffer_state, |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 619 | consumer_state_mask) || |
Tianyu Jiang | 727ede4 | 2019-02-01 11:44:51 -0800 | [diff] [blame] | 620 | BufferHubDefs::isClientAcquired(current_buffer_state, |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 621 | consumer_state_mask)) { |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 622 | // The consumer client is being destoryed without releasing. This could |
| 623 | // happen in corner cases when the consumer crashes. Here we mark it |
| 624 | // orphaned before remove it from producer. |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 625 | OnConsumerOrphaned(consumer_state_mask); |
| 626 | return; |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 627 | } |
| 628 | |
Tianyu Jiang | 727ede4 | 2019-02-01 11:44:51 -0800 | [diff] [blame] | 629 | if (BufferHubDefs::isClientReleased(current_buffer_state, |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 630 | consumer_state_mask) || |
Tianyu Jiang | 727ede4 | 2019-02-01 11:44:51 -0800 | [diff] [blame] | 631 | BufferHubDefs::isAnyClientGained(current_buffer_state)) { |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 632 | // The consumer is being close while it is suppose to signal a release |
| 633 | // fence. Signal the dummy fence here. |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 634 | if (fence_state_->load(std::memory_order_acquire) & consumer_state_mask) { |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 635 | epoll_event event; |
| 636 | event.events = EPOLLIN; |
Tianyu Jiang | a99f911 | 2018-12-13 18:23:07 -0800 | [diff] [blame] | 637 | event.data.u32 = consumer_state_mask; |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 638 | if (epoll_ctl(release_fence_fd_.Get(), EPOLL_CTL_MOD, |
| 639 | dummy_fence_fd_.Get(), &event) < 0) { |
| 640 | ALOGE( |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 641 | "%s: Failed to modify the shared release fence to include the " |
| 642 | "dummy fence: %s", |
| 643 | __FUNCTION__, strerror(errno)); |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 644 | return; |
| 645 | } |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 646 | ALOGW("%s: signal dummy release fence buffer_id=%d", __FUNCTION__, |
| 647 | buffer_id()); |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 648 | eventfd_write(dummy_fence_fd_.Get(), 1); |
| 649 | } |
| 650 | } |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 651 | } |
| 652 | |
Tianyu | f669f6a | 2018-10-10 15:34:32 -0700 | [diff] [blame] | 653 | // Returns true if the given parameters match the underlying buffer |
| 654 | // parameters. |
Corey Tabaka | cd52dd9 | 2017-04-07 18:03:57 -0700 | [diff] [blame] | 655 | bool ProducerChannel::CheckParameters(uint32_t width, uint32_t height, |
Hendrik Wagenaar | 108e84f | 2017-05-07 22:19:17 -0700 | [diff] [blame] | 656 | uint32_t layer_count, uint32_t format, |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 657 | uint64_t usage, |
Tianyu Jiang | a8df5f3 | 2019-01-14 18:42:12 -0800 | [diff] [blame] | 658 | size_t user_metadata_size) const { |
Corey Tabaka | 52ea25c | 2017-09-13 18:02:48 -0700 | [diff] [blame] | 659 | return user_metadata_size == user_metadata_size_ && |
| 660 | buffer_.width() == width && buffer_.height() == height && |
| 661 | buffer_.layer_count() == layer_count && buffer_.format() == format && |
| 662 | buffer_.usage() == usage; |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 663 | } |
| 664 | |
Tianyu Jiang | a8df5f3 | 2019-01-14 18:42:12 -0800 | [diff] [blame] | 665 | bool ProducerChannel::IsBufferReleasedByAllActiveClientsExceptForOrphans() |
| 666 | const { |
| 667 | return (buffer_state_->load(std::memory_order_acquire) & |
| 668 | ~orphaned_consumer_bit_mask_ & |
| 669 | active_clients_bit_mask_->load(std::memory_order_acquire)) == 0U; |
| 670 | } |
| 671 | |
Alex Vakulenko | e4eec20 | 2017-01-27 14:41:04 -0800 | [diff] [blame] | 672 | } // namespace dvr |
| 673 | } // namespace android |