Add shared memory based buffer metadata
This CLs reduces BufferHub CPU consumption by adding asynchronous
state transition so that out-of-process VR composition can run on 2016
pixel devices smoothly. In addition, this CL addresses a couple corner
cases in the existing bufferhub logic, which fixes various blackscreen
issues.
1/ Tracks buffer transition states (gained, posted, acquired, released)
from the client side via atomic shared memory and adds
PostAsync/AcquireAsync/ReleaseAsync/GainAsync with metadata and
fence support.
2/ Adds dequeue order guarantee for buffers enqueued with
dvrWriteBufferQueuePostBuffer.
3/ Synchronous BuffeHub operations are still supported.
4/ Bump up the bufferhubd's soft limit of open file descriptor.
5/ Handle orphaned consumer in acquired state. This is a corner case
that consumer process goes aways (most likely due to a crash) leaving
buffer stuck in acquired state with inconsistent buffer state.
6/ Fixes a race condition for released buffer to be Gain'ed and
Acquire'd when a new consumer is created in released state.
7/ Improve silent consumer queue efficiency: Silent queues no longer
import buffers or receive signals about new buffers and they are
limited to only spawning other consumers and notifications about
producers hanging up.
8/ Modify PDX/UDS channel event signaling to work around epoll
behavior. PDX UDS uses a combination of an eventfd and an epoll set
to simulate the original PDX transport channel events. An odd
behavior discovered in the kernel implementation of epoll was found
that causes the epoll fd to "unsignal" itself whenever epoll_wait()
is called on it, regardless of whether it should still be
pending. This breaks the edge triggerd behavior in nested epoll sets
that channel events depend on. Since this is unlikely to ever be
fixed in the kernel we work around the behavior by using the epoll
set only as a logical OR of two eventfds and never calling
epoll_wait() on it. When polling is required we use regluar poll()
with the eventfds and data fd to avoid the bad behavior in
epoll_wait().
9/ Keep reading data after PDX hangup signal. UDS will signal hangup
when the other end of the socket closes. However, data could still be
in the kerenl buffer and should be consumed. Fix an issue where the
service misses an impulse sent right before the socket is closed.
Bug: 65455724
Bug: 65458354
Bug: 65458312
Bug: 64027135
Bug: 67424527
Test: libpdx_uds_tests
bufferhub_tests
buffer_hub_queue-test
buffer_hub_queue_producer-test
dvr_api-test
Change-Id: Id07db1f206ccf4e06f7ee3c671193334408971ca
diff --git a/libs/vr/libbufferhub/include/private/dvr/buffer_hub_client.h b/libs/vr/libbufferhub/include/private/dvr/buffer_hub_client.h
index be20e72..1186f93 100644
--- a/libs/vr/libbufferhub/include/private/dvr/buffer_hub_client.h
+++ b/libs/vr/libbufferhub/include/private/dvr/buffer_hub_client.h
@@ -11,6 +11,8 @@
#include <private/dvr/ion_buffer.h>
+#include "bufferhub_rpc.h"
+
namespace android {
namespace dvr {
@@ -75,6 +77,14 @@
}
}
+ std::vector<pdx::ClientChannel::EventSource> GetEventSources() const {
+ if (auto* client_channel = GetChannel()) {
+ return client_channel->GetEventSources();
+ } else {
+ return {};
+ }
+ }
+
native_handle_t* native_handle() const {
return const_cast<native_handle_t*>(buffer_.handle());
}
@@ -84,6 +94,10 @@
int id() const { return id_; }
+ // A state mask which is unique to a buffer hub client among all its siblings
+ // sharing the same concrete graphic buffer.
+ uint64_t buffer_state_bit() const { return buffer_state_bit_; }
+
// The following methods return settings of the first buffer. Currently,
// it is only possible to create multi-buffer BufferHubBuffers with the same
// settings.
@@ -98,6 +112,9 @@
uint64_t producer_usage() const { return buffer_.usage(); }
uint64_t consumer_usage() const { return buffer_.usage(); }
+ uint64_t GetQueueIndex() const { return metadata_header_->queue_index; }
+ void SetQueueIndex(uint64_t index) { metadata_header_->queue_index = index; }
+
protected:
explicit BufferHubBuffer(LocalChannelHandle channel);
explicit BufferHubBuffer(const std::string& endpoint_path);
@@ -106,6 +123,31 @@
// Initialization helper.
int ImportBuffer();
+ // Check invalid metadata operation. Returns 0 if requested metadata is valid.
+ int CheckMetadata(size_t user_metadata_size) const;
+
+ // Send out the new fence by updating the shared fence (shared_release_fence
+ // for producer and shared_acquire_fence for consumer). Note that during this
+ // should only be used in LocalPost() or LocalRelease, and the shared fence
+ // shouldn't be poll'ed by the other end.
+ int UpdateSharedFence(const LocalHandle& new_fence,
+ const LocalHandle& shared_fence);
+
+ // IonBuffer that is shared between bufferhubd, producer, and consumers.
+ size_t metadata_buf_size_{0};
+ size_t user_metadata_size_{0};
+ BufferHubDefs::MetadataHeader* metadata_header_{nullptr};
+ void* user_metadata_ptr_{nullptr};
+ std::atomic<uint64_t>* buffer_state_{nullptr};
+ std::atomic<uint64_t>* fence_state_{nullptr};
+
+ LocalHandle shared_acquire_fence_;
+ LocalHandle shared_release_fence_;
+
+ // A local fence fd that holds the ownership of the fence fd on Post (for
+ // producer) and Release (for consumer).
+ LocalHandle pending_fence_fd_;
+
private:
BufferHubBuffer(const BufferHubBuffer&) = delete;
void operator=(const BufferHubBuffer&) = delete;
@@ -114,8 +156,9 @@
// for logging and debugging purposes only and should not be used for lookup
// or any other functional purpose as a security precaution.
int id_;
-
+ uint64_t buffer_state_bit_{0ULL};
IonBuffer buffer_;
+ IonBuffer metadata_buffer_;
};
// This represents a writable buffer. Calling Post notifies all clients and
@@ -136,12 +179,17 @@
static std::unique_ptr<BufferProducer> Import(
Status<LocalChannelHandle> status);
+ // Asynchronously posts a buffer. The fence and metadata are passed to
+ // consumer via shared fd and shared memory.
+ int PostAsync(const DvrNativeBufferMetadata* meta,
+ const LocalHandle& ready_fence);
+
// Post this buffer, passing |ready_fence| to the consumers. The bytes in
// |meta| are passed unaltered to the consumers. The producer must not modify
// the buffer until it is re-gained.
// This returns zero or a negative unix error code.
int Post(const LocalHandle& ready_fence, const void* meta,
- size_t meta_size_bytes);
+ size_t user_metadata_size);
template <typename Meta,
typename = typename std::enable_if<std::is_void<Meta>::value>::type>
@@ -160,16 +208,15 @@
// is in the released state.
// This returns zero or a negative unix error code.
int Gain(LocalHandle* release_fence);
+ int GainAsync();
// Asynchronously marks a released buffer as gained. This method is similar to
// the synchronous version above, except that it does not wait for BufferHub
- // to acknowledge success or failure, nor does it transfer a release fence to
- // the client. This version may be used in situations where a release fence is
- // not needed. Because of the asynchronous nature of the underlying message,
- // no error is returned if this method is called when the buffer is in an
- // incorrect state. Returns zero if sending the message succeeded, or a
- // negative errno code otherwise.
- int GainAsync();
+ // to acknowledge success or failure. Because of the asynchronous nature of
+ // the underlying message, no error is returned if this method is called when
+ // the buffer is in an incorrect state. Returns zero if sending the message
+ // succeeded, or a negative errno code if local error check fails.
+ int GainAsync(DvrNativeBufferMetadata* out_meta, LocalHandle* out_fence);
// Attaches the producer to |name| so that it becomes a persistent buffer that
// may be retrieved by name at a later time. This may be used in cases where a
@@ -216,7 +263,7 @@
BufferProducer(const std::string& name, int user_id, int group_id,
uint32_t width, uint32_t height, uint32_t format,
uint64_t producer_usage, uint64_t consumer_usage,
- size_t meta_size_bytes);
+ size_t user_metadata_size);
// Constructs a blob (flat) buffer with the given usage flags.
BufferProducer(uint32_t usage, size_t size);
@@ -234,6 +281,11 @@
// Imports the given file handle to a producer channel, taking ownership.
explicit BufferProducer(LocalChannelHandle channel);
+
+ // Local state transition helpers.
+ int LocalGain(DvrNativeBufferMetadata* out_meta, LocalHandle* out_fence);
+ int LocalPost(const DvrNativeBufferMetadata* meta,
+ const LocalHandle& ready_fence);
};
// This is a connection to a producer buffer, which can be located in another
@@ -263,7 +315,7 @@
// are available. This call will only succeed if the buffer is in the posted
// state.
// Returns zero on success, or a negative errno code otherwise.
- int Acquire(LocalHandle* ready_fence, void* meta, size_t meta_size_bytes);
+ int Acquire(LocalHandle* ready_fence, void* meta, size_t user_metadata_size);
// Attempt to retrieve a post event from buffer hub. If successful,
// |ready_fence| is set to a fence to wait on until the buffer is ready. This
@@ -274,20 +326,22 @@
return Acquire(ready_fence, meta, sizeof(*meta));
}
+ // Asynchronously acquires a bufer.
+ int AcquireAsync(DvrNativeBufferMetadata* out_meta, LocalHandle* out_fence);
+
// This should be called after a successful Acquire call. If the fence is
// valid the fence determines the buffer usage, otherwise the buffer is
// released immediately.
// This returns zero or a negative unix error code.
int Release(const LocalHandle& release_fence);
+ int ReleaseAsync();
// Asynchronously releases a buffer. Similar to the synchronous version above,
- // except that it does not wait for BufferHub to reply with success or error,
- // nor does it transfer a release fence. This version may be used in
- // situations where a release fence is not needed. Because of the asynchronous
- // nature of the underlying message, no error is returned if this method is
- // called when the buffer is in an incorrect state. Returns zero if sending
- // the message succeeded, or a negative errno code otherwise.
- int ReleaseAsync();
+ // except that it does not wait for BufferHub to reply with success or error.
+ // The fence and metadata are passed to consumer via shared fd and shared
+ // memory.
+ int ReleaseAsync(const DvrNativeBufferMetadata* meta,
+ const LocalHandle& release_fence);
// May be called after or instead of Acquire to indicate that the consumer
// does not need to access the buffer this cycle. This returns zero or a
@@ -305,6 +359,11 @@
friend BASE;
explicit BufferConsumer(LocalChannelHandle channel);
+
+ // Local state transition helpers.
+ int LocalAcquire(DvrNativeBufferMetadata* out_meta, LocalHandle* out_fence);
+ int LocalRelease(const DvrNativeBufferMetadata* meta,
+ const LocalHandle& release_fence);
};
} // namespace dvr
diff --git a/libs/vr/libbufferhub/include/private/dvr/bufferhub_rpc.h b/libs/vr/libbufferhub/include/private/dvr/bufferhub_rpc.h
index ca0e0e0..f9fd42d 100644
--- a/libs/vr/libbufferhub/include/private/dvr/bufferhub_rpc.h
+++ b/libs/vr/libbufferhub/include/private/dvr/bufferhub_rpc.h
@@ -5,6 +5,7 @@
#include <gui/BufferQueueDefs.h>
#include <sys/types.h>
+#include <dvr/dvr_api.h>
#include <pdx/channel_handle.h>
#include <pdx/file_handle.h>
#include <pdx/rpc/remote_method.h>
@@ -14,6 +15,71 @@
namespace android {
namespace dvr {
+namespace BufferHubDefs {
+
+static constexpr uint32_t kMetadataFormat = HAL_PIXEL_FORMAT_BLOB;
+static constexpr uint32_t kMetadataUsage =
+ GRALLOC_USAGE_SW_READ_OFTEN | GRALLOC_USAGE_SW_WRITE_OFTEN;
+
+// Single producuer multiple (up to 63) consumers ownership signal.
+// 64-bit atomic unsigned int.
+//
+// MSB LSB
+// | |
+// v v
+// [P|C62|...|C1|C0]
+// Gain'ed state: [0|..|0|0] -> Exclusively Writable.
+// Post'ed state: [1|..|0|0]
+// Acquired'ed state: [1|..|X|X] -> At least one bit is set in lower 63 bits
+// Released'ed state: [0|..|X|X] -> At least one bit is set in lower 63 bits
+static constexpr uint64_t kProducerStateBit = 1ULL << 63;
+static constexpr uint64_t kConsumerStateMask = (1ULL << 63) - 1;
+
+static inline void ModifyBufferState(std::atomic<uint64_t>* buffer_state,
+ uint64_t clear_mask, uint64_t set_mask) {
+ uint64_t old_state;
+ uint64_t new_state;
+ do {
+ old_state = buffer_state->load();
+ new_state = (old_state & ~clear_mask) | set_mask;
+ } while (!buffer_state->compare_exchange_weak(old_state, new_state));
+}
+
+static inline bool IsBufferGained(uint64_t state) { return state == 0; }
+
+static inline bool IsBufferPosted(uint64_t state,
+ uint64_t consumer_bit = kConsumerStateMask) {
+ return (state & kProducerStateBit) && !(state & consumer_bit);
+}
+
+static inline bool IsBufferAcquired(uint64_t state) {
+ return (state & kProducerStateBit) && (state & kConsumerStateMask);
+}
+
+static inline bool IsBufferReleased(uint64_t state) {
+ return !(state & kProducerStateBit) && (state & kConsumerStateMask);
+}
+
+struct __attribute__((packed, aligned(8))) MetadataHeader {
+ // Internal data format, which can be updated as long as the size, padding and
+ // field alignment of the struct is consistent within the same ABI. As this
+ // part is subject for future updates, it's not stable cross Android version,
+ // so don't have it visible from outside of the Android platform (include Apps
+ // and vendor HAL).
+ std::atomic<uint64_t> buffer_state;
+ std::atomic<uint64_t> fence_state;
+ uint64_t queue_index;
+
+ // Public data format, which should be updated with caution. See more details
+ // in dvr_api.h
+ DvrNativeBufferMetadata metadata;
+};
+
+static_assert(sizeof(MetadataHeader) == 128, "Unexpected MetadataHeader size");
+static constexpr size_t kMetadataHeaderSize = sizeof(MetadataHeader);
+
+} // namespace BufferHubDefs
+
template <typename FileHandleType>
class NativeBufferHandle {
public:
@@ -93,6 +159,57 @@
void operator=(const NativeBufferHandle&) = delete;
};
+template <typename FileHandleType>
+class BufferDescription {
+ public:
+ BufferDescription() = default;
+ BufferDescription(const IonBuffer& buffer, const IonBuffer& metadata, int id,
+ uint64_t buffer_state_bit,
+ const FileHandleType& acquire_fence_fd,
+ const FileHandleType& release_fence_fd)
+ : id_(id),
+ buffer_state_bit_(buffer_state_bit),
+ buffer_(buffer, id),
+ metadata_(metadata, id),
+ acquire_fence_fd_(acquire_fence_fd.Borrow()),
+ release_fence_fd_(release_fence_fd.Borrow()) {}
+
+ BufferDescription(BufferDescription&& other) = default;
+ BufferDescription& operator=(BufferDescription&& other) = default;
+
+ // ID of the buffer client. All BufferHubBuffer clients derived from the same
+ // buffer in bufferhubd share the same buffer id.
+ int id() const { return id_; }
+ // State mask of the buffer client. Each BufferHubBuffer client backed by the
+ // same buffer channel has uniqued state bit among its siblings. For a
+ // producer buffer the bit must be kProducerStateBit; for a consumer the bit
+ // must be one of the kConsumerStateMask.
+ uint64_t buffer_state_bit() const { return buffer_state_bit_; }
+ FileHandleType take_acquire_fence() { return std::move(acquire_fence_fd_); }
+ FileHandleType take_release_fence() { return std::move(release_fence_fd_); }
+
+ int ImportBuffer(IonBuffer* buffer) { return buffer_.Import(buffer); }
+ int ImportMetadata(IonBuffer* metadata) { return metadata_.Import(metadata); }
+
+ private:
+ int id_{-1};
+ uint64_t buffer_state_bit_{0};
+ // Two IonBuffers: one for the graphic buffer and one for metadata.
+ NativeBufferHandle<FileHandleType> buffer_;
+ NativeBufferHandle<FileHandleType> metadata_;
+
+ // Pamameters for shared fences.
+ FileHandleType acquire_fence_fd_;
+ FileHandleType release_fence_fd_;
+
+ PDX_SERIALIZABLE_MEMBERS(BufferDescription<FileHandleType>, id_,
+ buffer_state_bit_, buffer_, metadata_,
+ acquire_fence_fd_, release_fence_fd_);
+
+ BufferDescription(const BufferDescription&) = delete;
+ void operator=(const BufferDescription&) = delete;
+};
+
using BorrowedNativeBufferHandle = NativeBufferHandle<pdx::BorrowedHandle>;
using LocalNativeBufferHandle = NativeBufferHandle<pdx::LocalHandle>;
@@ -149,11 +266,11 @@
// Size of the meta data associated with all the buffers allocated from the
// queue.
- size_t meta_size_bytes;
+ size_t user_metadata_size;
private:
PDX_SERIALIZABLE_MEMBERS(ProducerQueueConfig, is_async, default_width,
- default_height, default_format, meta_size_bytes);
+ default_height, default_format, user_metadata_size);
};
class ProducerQueueConfigBuilder {
@@ -161,7 +278,7 @@
// Build a ProducerQueueConfig object.
ProducerQueueConfig Build() {
return {is_async_, default_width_, default_height_, default_format_,
- meta_size_bytes_};
+ user_metadata_size_};
}
ProducerQueueConfigBuilder& SetIsAsync(bool is_async) {
@@ -186,12 +303,12 @@
template <typename Meta>
ProducerQueueConfigBuilder& SetMetadata() {
- meta_size_bytes_ = sizeof(Meta);
+ user_metadata_size_ = sizeof(Meta);
return *this;
}
- ProducerQueueConfigBuilder& SetMetadataSize(size_t meta_size_bytes) {
- meta_size_bytes_ = meta_size_bytes;
+ ProducerQueueConfigBuilder& SetMetadataSize(size_t user_metadata_size) {
+ user_metadata_size_ = user_metadata_size;
return *this;
}
@@ -200,7 +317,7 @@
uint32_t default_width_{1};
uint32_t default_height_{1};
uint32_t default_format_{1}; // PIXEL_FORMAT_RGBA_8888
- size_t meta_size_bytes_{0};
+ size_t user_metadata_size_{0};
};
// Explicit specializations of ProducerQueueConfigBuilder::Build for void
@@ -208,7 +325,7 @@
template <>
inline ProducerQueueConfigBuilder&
ProducerQueueConfigBuilder::SetMetadata<void>() {
- meta_size_bytes_ = 0;
+ user_metadata_size_ = 0;
return *this;
}
@@ -269,7 +386,6 @@
};
// Aliases.
- using MetaData = pdx::rpc::BufferWrapper<std::uint8_t*>;
using LocalChannelHandle = pdx::LocalChannelHandle;
using LocalHandle = pdx::LocalHandle;
using Void = pdx::rpc::Void;
@@ -277,25 +393,24 @@
// Methods.
PDX_REMOTE_METHOD(CreateBuffer, kOpCreateBuffer,
void(uint32_t width, uint32_t height, uint32_t format,
- uint64_t usage, size_t meta_size_bytes));
+ uint64_t usage, size_t user_metadata_size));
PDX_REMOTE_METHOD(CreatePersistentBuffer, kOpCreatePersistentBuffer,
void(const std::string& name, int user_id, int group_id,
uint32_t width, uint32_t height, uint32_t format,
- uint64_t usage, size_t meta_size_bytes));
+ uint64_t usage, size_t user_metadata_size));
PDX_REMOTE_METHOD(GetPersistentBuffer, kOpGetPersistentBuffer,
void(const std::string& name));
PDX_REMOTE_METHOD(GetBuffer, kOpGetBuffer,
- NativeBufferHandle<LocalHandle>(Void));
+ BufferDescription<LocalHandle>(Void));
PDX_REMOTE_METHOD(NewConsumer, kOpNewConsumer, LocalChannelHandle(Void));
PDX_REMOTE_METHOD(ProducerMakePersistent, kOpProducerMakePersistent,
void(const std::string& name, int user_id, int group_id));
PDX_REMOTE_METHOD(ProducerRemovePersistence, kOpProducerRemovePersistence,
void(Void));
PDX_REMOTE_METHOD(ProducerPost, kOpProducerPost,
- void(LocalFence acquire_fence, MetaData));
+ void(LocalFence acquire_fence));
PDX_REMOTE_METHOD(ProducerGain, kOpProducerGain, LocalFence(Void));
- PDX_REMOTE_METHOD(ConsumerAcquire, kOpConsumerAcquire,
- std::pair<LocalFence, MetaData>(std::size_t metadata_size));
+ PDX_REMOTE_METHOD(ConsumerAcquire, kOpConsumerAcquire, LocalFence(Void));
PDX_REMOTE_METHOD(ConsumerRelease, kOpConsumerRelease,
void(LocalFence release_fence));
PDX_REMOTE_METHOD(ConsumerSetIgnore, kOpConsumerSetIgnore, void(bool ignore));
@@ -305,7 +420,7 @@
QueueInfo(const ProducerQueueConfig& producer_config,
const UsagePolicy& usage_policy));
PDX_REMOTE_METHOD(CreateConsumerQueue, kOpCreateConsumerQueue,
- LocalChannelHandle(Void));
+ LocalChannelHandle(bool silent_queue));
PDX_REMOTE_METHOD(GetQueueInfo, kOpGetQueueInfo, QueueInfo(Void));
PDX_REMOTE_METHOD(ProducerQueueAllocateBuffers,
kOpProducerQueueAllocateBuffers,