Fix PR22634 - std::allocator doesn't respect over-aligned types.
This patch fixes std::allocator, and more specifically, all users
of __libcpp_allocate and __libcpp_deallocate, to support over-aligned
types.
__libcpp_allocate/deallocate now take an alignment parameter, and when
the specified alignment is greater than that supported by malloc/new,
the aligned version of operator new is called (assuming it's available).
When aligned new isn't available, the old behavior has been kept, and the
alignment parameter is ignored.
This patch depends on recent changes to __builtin_operator_new/delete which
allow them to be used to call any regular new/delete operator. By using
__builtin_operator_new/delete when possible, the new/delete erasure optimization
is maintained.
git-svn-id: https://llvm.org/svn/llvm-project/libcxx/trunk@328180 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/include/__config b/include/__config
index f96d743..d150ede 100644
--- a/include/__config
+++ b/include/__config
@@ -955,6 +955,7 @@
# endif
#endif // defined(__APPLE__)
+
#if defined(__APPLE__) || defined(__FreeBSD__)
#define _LIBCPP_HAS_DEFAULTRUNELOCALE
#endif
diff --git a/include/__sso_allocator b/include/__sso_allocator
index 8147e75..4002736 100644
--- a/include/__sso_allocator
+++ b/include/__sso_allocator
@@ -55,14 +55,14 @@
__allocated_ = true;
return (pointer)&buf_;
}
- return static_cast<pointer>(_VSTD::__allocate(__n * sizeof(_Tp)));
+ return static_cast<pointer>(_VSTD::__libcpp_allocate(__n * sizeof(_Tp), __alignof(_Tp)));
}
_LIBCPP_INLINE_VISIBILITY void deallocate(pointer __p, size_type)
{
if (__p == (pointer)&buf_)
__allocated_ = false;
else
- _VSTD::__libcpp_deallocate(__p);
+ _VSTD::__libcpp_deallocate(__p, __alignof(_Tp));
}
_LIBCPP_INLINE_VISIBILITY size_type max_size() const throw() {return size_type(~0) / sizeof(_Tp);}
diff --git a/include/memory b/include/memory
index ea52ba2..021bd13 100644
--- a/include/memory
+++ b/include/memory
@@ -1796,10 +1796,10 @@
if (__n > max_size())
__throw_length_error("allocator<T>::allocate(size_t n)"
" 'n' exceeds maximum supported size");
- return static_cast<pointer>(_VSTD::__allocate(__n * sizeof(_Tp)));
+ return static_cast<pointer>(_VSTD::__libcpp_allocate(__n * sizeof(_Tp), __alignof(_Tp)));
}
_LIBCPP_INLINE_VISIBILITY void deallocate(pointer __p, size_type) _NOEXCEPT
- {_VSTD::__libcpp_deallocate((void*)__p);}
+ {_VSTD::__libcpp_deallocate((void*)__p, __alignof(_Tp));}
_LIBCPP_INLINE_VISIBILITY size_type max_size() const _NOEXCEPT
{return size_type(~0) / sizeof(_Tp);}
#if !defined(_LIBCPP_HAS_NO_RVALUE_REFERENCES) && !defined(_LIBCPP_HAS_NO_VARIADICS)
@@ -1897,10 +1897,10 @@
if (__n > max_size())
__throw_length_error("allocator<const T>::allocate(size_t n)"
" 'n' exceeds maximum supported size");
- return static_cast<pointer>(_VSTD::__allocate(__n * sizeof(_Tp)));
+ return static_cast<pointer>(_VSTD::__libcpp_allocate(__n * sizeof(_Tp), __alignof(_Tp)));
}
_LIBCPP_INLINE_VISIBILITY void deallocate(pointer __p, size_type) _NOEXCEPT
- {_VSTD::__libcpp_deallocate((void*) const_cast<_Tp *>(__p));}
+ {_VSTD::__libcpp_deallocate((void*) const_cast<_Tp *>(__p), __alignof(_Tp));}
_LIBCPP_INLINE_VISIBILITY size_type max_size() const _NOEXCEPT
{return size_type(~0) / sizeof(_Tp);}
#if !defined(_LIBCPP_HAS_NO_RVALUE_REFERENCES) && !defined(_LIBCPP_HAS_NO_VARIADICS)
@@ -2016,12 +2016,7 @@
while (__n > 0)
{
#if !defined(_LIBCPP_HAS_NO_ALIGNED_ALLOCATION)
-#if defined(__STDCPP_DEFAULT_NEW_ALIGNMENT__)
- if (std::alignment_of<_Tp>::value > __STDCPP_DEFAULT_NEW_ALIGNMENT__)
-#else
- if (std::alignment_of<_Tp>::value >
- std::alignment_of<std::max_align_t>::value)
-#endif
+ if (__is_overaligned_for_new(__alignof(_Tp)))
{
std::align_val_t __al =
std::align_val_t(std::alignment_of<_Tp>::value);
@@ -2032,12 +2027,7 @@
__n * sizeof(_Tp), nothrow));
}
#else
-#if defined(__STDCPP_DEFAULT_NEW_ALIGNMENT__)
- if (std::alignment_of<_Tp>::value > __STDCPP_DEFAULT_NEW_ALIGNMENT__)
-#else
- if (std::alignment_of<_Tp>::value >
- std::alignment_of<std::max_align_t>::value)
-#endif
+ if (__is_overaligned_for_new(__alignof(_Tp)))
{
// Since aligned operator new is unavailable, return an empty
// buffer rather than one with invalid alignment.
@@ -2061,20 +2051,7 @@
inline _LIBCPP_INLINE_VISIBILITY
void return_temporary_buffer(_Tp* __p) _NOEXCEPT
{
-#if !defined(_LIBCPP_HAS_NO_ALIGNED_ALLOCATION)
-#if defined(__STDCPP_DEFAULT_NEW_ALIGNMENT__)
- if (std::alignment_of<_Tp>::value > __STDCPP_DEFAULT_NEW_ALIGNMENT__)
-#else
- if (std::alignment_of<_Tp>::value >
- std::alignment_of<std::max_align_t>::value)
-#endif
- {
- std::align_val_t __al = std::align_val_t(std::alignment_of<_Tp>::value);
- ::operator delete(__p, __al);
- return;
- }
-#endif
- ::operator delete(__p);
+ _VSTD::__libcpp_deallocate((void*)__p, __alignof(_Tp));
}
#if _LIBCPP_STD_VER <= 14 || defined(_LIBCPP_ENABLE_CXX17_REMOVED_AUTO_PTR)
diff --git a/include/new b/include/new
index 27c248c..e42ffad 100644
--- a/include/new
+++ b/include/new
@@ -89,6 +89,7 @@
#include <__config>
#include <exception>
+#include <type_traits>
#include <cstddef>
#ifdef _LIBCPP_NO_EXCEPTIONS
#include <cstdlib>
@@ -113,6 +114,14 @@
# define _LIBCPP_HAS_NO_ALIGNED_ALLOCATION
#endif
+
+#if !__has_builtin(__builtin_operator_new) || \
+ __has_builtin(__builtin_operator_new) < 201802L || \
+ defined(_LIBCPP_HAS_NO_ALIGNED_ALLOCATION) || \
+ !defined(__cpp_aligned_new) || __cpp_aligned_new < 201606
+#define _LIBCPP_HAS_NO_BUILTIN_ALIGNED_OPERATOR_NEW_DELETE
+#endif
+
namespace std // purposefully not using versioning namespace
{
@@ -223,7 +232,27 @@
_LIBCPP_BEGIN_NAMESPACE_STD
-inline _LIBCPP_INLINE_VISIBILITY void *__allocate(size_t __size) {
+_LIBCPP_CONSTEXPR inline _LIBCPP_INLINE_VISIBILITY bool __is_overaligned_for_new(size_t __align) _NOEXCEPT {
+#ifdef __STDCPP_DEFAULT_NEW_ALIGNMENT__
+ return __align > __STDCPP_DEFAULT_NEW_ALIGNMENT__;
+#else
+ return __align > alignment_of<max_align_t>::value;
+#endif
+}
+
+inline _LIBCPP_INLINE_VISIBILITY void *__libcpp_allocate(size_t __size, size_t __align) {
+#ifndef _LIBCPP_HAS_NO_ALIGNED_ALLOCATION
+ if (__is_overaligned_for_new(__align)) {
+ const align_val_t __align_val = static_cast<align_val_t>(__align);
+# ifdef _LIBCPP_HAS_NO_BUILTIN_ALIGNED_OPERATOR_NEW_DELETE
+ return ::operator new(__size, __align_val);
+# else
+ return __builtin_operator_new(__size, __align_val);
+# endif
+ }
+#else
+ ((void)__align);
+#endif
#ifdef _LIBCPP_HAS_NO_BUILTIN_OPERATOR_NEW_DELETE
return ::operator new(__size);
#else
@@ -231,11 +260,23 @@
#endif
}
-inline _LIBCPP_INLINE_VISIBILITY void __libcpp_deallocate(void *__ptr) {
-#ifdef _LIBCPP_HAS_NO_BUILTIN_OPERATOR_NEW_DELETE
- ::operator delete(__ptr);
+inline _LIBCPP_INLINE_VISIBILITY void __libcpp_deallocate(void* __ptr, size_t __align) {
+#ifndef _LIBCPP_HAS_NO_ALIGNED_ALLOCATION
+ if (__is_overaligned_for_new(__align)) {
+ const align_val_t __align_val = static_cast<align_val_t>(__align);
+# ifdef _LIBCPP_HAS_NO_BUILTIN_ALIGNED_OPERATOR_NEW_DELETE
+ return ::operator delete(__ptr, __align_val);
+# else
+ return __builtin_operator_delete(__ptr, __align_val);
+# endif
+ }
#else
- __builtin_operator_delete(__ptr);
+ ((void)__align);
+#endif
+#ifdef _LIBCPP_HAS_NO_BUILTIN_OPERATOR_NEW_DELETE
+ return ::operator delete(__ptr);
+#else
+ return __builtin_operator_delete(__ptr);
#endif
}
diff --git a/include/valarray b/include/valarray
index b495ccf..8d3892a 100644
--- a/include/valarray
+++ b/include/valarray
@@ -2738,7 +2738,8 @@
{
__r.__begin_ =
__r.__end_ =
- static_cast<result_type*>(_VSTD::__allocate(__n * sizeof(result_type)));
+ static_cast<result_type*>(
+ _VSTD::__libcpp_allocate(__n * sizeof(result_type), __alignof(result_type)));
for (size_t __i = 0; __i != __n; ++__r.__end_, ++__i)
::new (__r.__end_) result_type(__expr_[__i]);
}
@@ -2755,7 +2756,8 @@
{
if (__n)
{
- __begin_ = __end_ = static_cast<value_type*>(_VSTD::__allocate(__n * sizeof(value_type)));
+ __begin_ = __end_ = static_cast<value_type*>(
+ _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type)));
#ifndef _LIBCPP_NO_EXCEPTIONS
try
{
@@ -2789,7 +2791,8 @@
{
if (__n)
{
- __begin_ = __end_ = static_cast<value_type*>(_VSTD::__allocate(__n * sizeof(value_type)));
+ __begin_ = __end_ = static_cast<value_type*>(
+ _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type)));
#ifndef _LIBCPP_NO_EXCEPTIONS
try
{
@@ -2814,7 +2817,8 @@
{
if (__v.size())
{
- __begin_ = __end_ = static_cast<value_type*>(_VSTD::__allocate(__v.size() * sizeof(value_type)));
+ __begin_ = __end_ = static_cast<value_type*>(
+ _VSTD::__libcpp_allocate(__v.size() * sizeof(value_type), __alignof(value_type)));
#ifndef _LIBCPP_NO_EXCEPTIONS
try
{
@@ -2851,7 +2855,8 @@
size_t __n = __il.size();
if (__n)
{
- __begin_ = __end_ = static_cast<value_type*>(_VSTD::__allocate(__n * sizeof(value_type)));
+ __begin_ = __end_ = static_cast<value_type*>(
+_VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type)));
#ifndef _LIBCPP_NO_EXCEPTIONS
try
{
@@ -2879,7 +2884,8 @@
size_t __n = __sa.__size_;
if (__n)
{
- __begin_ = __end_ = static_cast<value_type*>(_VSTD::__allocate(__n * sizeof(value_type)));
+ __begin_ = __end_ = static_cast<value_type*>(
+ _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type)));
#ifndef _LIBCPP_NO_EXCEPTIONS
try
{
@@ -2905,7 +2911,8 @@
size_t __n = __ga.__1d_.size();
if (__n)
{
- __begin_ = __end_ = static_cast<value_type*>(_VSTD::__allocate(__n * sizeof(value_type)));
+ __begin_ = __end_ = static_cast<value_type*>(
+ _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type)));
#ifndef _LIBCPP_NO_EXCEPTIONS
try
{
@@ -2934,7 +2941,8 @@
size_t __n = __ma.__1d_.size();
if (__n)
{
- __begin_ = __end_ = static_cast<value_type*>(_VSTD::__allocate(__n * sizeof(value_type)));
+ __begin_ = __end_ = static_cast<value_type*>(
+ _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type)));
#ifndef _LIBCPP_NO_EXCEPTIONS
try
{
@@ -2963,7 +2971,8 @@
size_t __n = __ia.__1d_.size();
if (__n)
{
- __begin_ = __end_ = static_cast<value_type*>(_VSTD::__allocate(__n * sizeof(value_type)));
+ __begin_ = __end_ = static_cast<value_type*>(
+ _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type)));
#ifndef _LIBCPP_NO_EXCEPTIONS
try
{
@@ -2999,7 +3008,8 @@
if (size() != __n)
{
__clear();
- __begin_ = static_cast<value_type*>(_VSTD::__allocate(__n * sizeof(value_type)));
+ __begin_ = static_cast<value_type*>(
+ _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type)));
__end_ = __begin_ + __n;
_VSTD::uninitialized_copy(__f, __l, __begin_);
} else {
@@ -3254,7 +3264,8 @@
{
__r.__begin_ =
__r.__end_ =
- static_cast<value_type*>(_VSTD::__allocate(__n * sizeof(value_type)));
+ static_cast<value_type*>(
+ _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type)));
for (const value_type* __p = __begin_; __n; ++__r.__end_, ++__p, --__n)
::new (__r.__end_) value_type(+*__p);
}
@@ -3271,7 +3282,8 @@
{
__r.__begin_ =
__r.__end_ =
- static_cast<value_type*>(_VSTD::__allocate(__n * sizeof(value_type)));
+ static_cast<value_type*>(
+ _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type)));
for (const value_type* __p = __begin_; __n; ++__r.__end_, ++__p, --__n)
::new (__r.__end_) value_type(-*__p);
}
@@ -3288,7 +3300,8 @@
{
__r.__begin_ =
__r.__end_ =
- static_cast<value_type*>(_VSTD::__allocate(__n * sizeof(value_type)));
+ static_cast<value_type*>(
+ _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type)));
for (const value_type* __p = __begin_; __n; ++__r.__end_, ++__p, --__n)
::new (__r.__end_) value_type(~*__p);
}
@@ -3305,7 +3318,7 @@
{
__r.__begin_ =
__r.__end_ =
- static_cast<bool*>(_VSTD::__allocate(__n * sizeof(bool)));
+ static_cast<bool*>(_VSTD::__libcpp_allocate(__n * sizeof(bool), __alignof(bool)));
for (const value_type* __p = __begin_; __n; ++__r.__end_, ++__p, --__n)
::new (__r.__end_) bool(!*__p);
}
@@ -3625,7 +3638,8 @@
{
__r.__begin_ =
__r.__end_ =
- static_cast<value_type*>(_VSTD::__allocate(__n * sizeof(value_type)));
+ static_cast<value_type*>(
+ _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type)));
const value_type* __sb;
value_type* __tb;
value_type* __te;
@@ -3663,7 +3677,8 @@
{
__r.__begin_ =
__r.__end_ =
- static_cast<value_type*>(_VSTD::__allocate(__n * sizeof(value_type)));
+ static_cast<value_type*>(
+ _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type)));
__i %= static_cast<int>(__n);
const value_type* __m = __i >= 0 ? __begin_ + __i : __end_ + __i;
for (const value_type* __s = __m; __s != __end_; ++__r.__end_, ++__s)
@@ -3684,7 +3699,8 @@
{
__r.__begin_ =
__r.__end_ =
- static_cast<value_type*>(_VSTD::__allocate(__n * sizeof(value_type)));
+ static_cast<value_type*>(
+ _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type)));
for (const value_type* __p = __begin_; __n; ++__r.__end_, ++__p, --__n)
::new (__r.__end_) value_type(__f(*__p));
}
@@ -3701,7 +3717,8 @@
{
__r.__begin_ =
__r.__end_ =
- static_cast<value_type*>(_VSTD::__allocate(__n * sizeof(value_type)));
+ static_cast<value_type*>(
+ _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type)));
for (const value_type* __p = __begin_; __n; ++__r.__end_, ++__p, --__n)
::new (__r.__end_) value_type(__f(*__p));
}
@@ -3716,7 +3733,7 @@
{
while (__end_ != __begin_)
(--__end_)->~value_type();
- _VSTD::__libcpp_deallocate(__begin_);
+ _VSTD::__libcpp_deallocate(__begin_, __alignof(value_type));
__begin_ = __end_ = nullptr;
}
}
@@ -3728,7 +3745,8 @@
__clear();
if (__n)
{
- __begin_ = __end_ = static_cast<value_type*>(_VSTD::__allocate(__n * sizeof(value_type)));
+ __begin_ = __end_ = static_cast<value_type*>(
+ _VSTD::__libcpp_allocate(__n * sizeof(value_type), __alignof(value_type)));
#ifndef _LIBCPP_NO_EXCEPTIONS
try
{