Add witness_assert_depth[_to_rank]().
This makes it possible to make lock state assertions about precisely
which locks are held.
diff --git a/include/jemalloc/internal/private_symbols.txt b/include/jemalloc/internal/private_symbols.txt
index 745220e..2567f56 100644
--- a/include/jemalloc/internal/private_symbols.txt
+++ b/include/jemalloc/internal/private_symbols.txt
@@ -549,13 +549,15 @@
tsdn_null
tsdn_rtree_ctx
tsdn_tsd
+witness_assert_depth
+witness_assert_depth_to_rank
witness_assert_lockless
witness_assert_not_owner
witness_assert_owner
+witness_depth_error
witness_init
witness_lock
witness_lock_error
-witness_lockless_error
witness_not_owner_error
witness_owner
witness_owner_error
diff --git a/include/jemalloc/internal/witness_externs.h b/include/jemalloc/internal/witness_externs.h
index dcd987c..5d91fde 100644
--- a/include/jemalloc/internal/witness_externs.h
+++ b/include/jemalloc/internal/witness_externs.h
@@ -23,10 +23,12 @@
void witness_not_owner_error(const witness_t *witness);
#endif
#ifdef JEMALLOC_JET
-typedef void (witness_lockless_error_t)(const witness_list_t *);
-extern witness_lockless_error_t *witness_lockless_error;
+typedef void (witness_depth_error_t)(const witness_list_t *,
+ witness_rank_t rank_inclusive, unsigned depth);
+extern witness_depth_error_t *witness_depth_error;
#else
-void witness_lockless_error(const witness_list_t *witnesses);
+void witness_depth_error(const witness_list_t *witnesses,
+ witness_rank_t rank_inclusive, unsigned depth);
#endif
void witnesses_cleanup(tsd_t *tsd);
diff --git a/include/jemalloc/internal/witness_inlines.h b/include/jemalloc/internal/witness_inlines.h
index c2a2781..51f3f6e 100644
--- a/include/jemalloc/internal/witness_inlines.h
+++ b/include/jemalloc/internal/witness_inlines.h
@@ -5,6 +5,9 @@
bool witness_owner(tsd_t *tsd, const witness_t *witness);
void witness_assert_owner(tsdn_t *tsdn, const witness_t *witness);
void witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness);
+void witness_assert_depth_to_rank(tsdn_t *tsdn, witness_rank_t rank_inclusive,
+ unsigned depth);
+void witness_assert_depth(tsdn_t *tsdn, unsigned depth);
void witness_assert_lockless(tsdn_t *tsdn);
void witness_lock(tsdn_t *tsdn, witness_t *witness);
void witness_unlock(tsdn_t *tsdn, witness_t *witness);
@@ -78,8 +81,10 @@
}
JEMALLOC_INLINE void
-witness_assert_lockless(tsdn_t *tsdn) {
+witness_assert_depth_to_rank(tsdn_t *tsdn, witness_rank_t rank_inclusive,
+ unsigned depth) {
tsd_t *tsd;
+ unsigned d;
witness_list_t *witnesses;
witness_t *w;
@@ -92,11 +97,30 @@
}
tsd = tsdn_tsd(tsdn);
+ d = 0;
witnesses = tsd_witnessesp_get(tsd);
w = ql_last(witnesses, link);
if (w != NULL) {
- witness_lockless_error(witnesses);
+ ql_reverse_foreach(w, witnesses, link) {
+ if (w->rank < rank_inclusive) {
+ break;
+ }
+ d++;
+ }
}
+ if (d != depth) {
+ witness_depth_error(witnesses, rank_inclusive, depth);
+ }
+}
+
+JEMALLOC_INLINE void
+witness_assert_depth(tsdn_t *tsdn, unsigned depth) {
+ witness_assert_depth_to_rank(tsdn, WITNESS_RANK_MIN, depth);
+}
+
+JEMALLOC_INLINE void
+witness_assert_lockless(tsdn_t *tsdn) {
+ witness_assert_depth(tsdn, 0);
}
JEMALLOC_INLINE void
diff --git a/include/jemalloc/internal/witness_types.h b/include/jemalloc/internal/witness_types.h
index c2a73f2..f765d7b 100644
--- a/include/jemalloc/internal/witness_types.h
+++ b/include/jemalloc/internal/witness_types.h
@@ -13,6 +13,8 @@
*/
#define WITNESS_RANK_OMIT 0U
+#define WITNESS_RANK_MIN 1U
+
#define WITNESS_RANK_INIT 1U
#define WITNESS_RANK_CTL 1U
#define WITNESS_RANK_ARENAS 2U
diff --git a/src/witness.c b/src/witness.c
index 1c03457..034ea92 100644
--- a/src/witness.c
+++ b/src/witness.c
@@ -65,14 +65,16 @@
#endif
#ifdef JEMALLOC_JET
-#undef witness_lockless_error
-#define witness_lockless_error JEMALLOC_N(n_witness_lockless_error)
+#undef witness_depth_error
+#define witness_depth_error JEMALLOC_N(n_witness_depth_error)
#endif
void
-witness_lockless_error(const witness_list_t *witnesses) {
+witness_depth_error(const witness_list_t *witnesses,
+ witness_rank_t rank_inclusive, unsigned depth) {
witness_t *w;
- malloc_printf("<jemalloc>: Should not own any locks:");
+ malloc_printf("<jemalloc>: Should own %u lock%s of rank >= %u:", depth,
+ (depth != 1) ? "s" : "", rank_inclusive);
ql_foreach(w, witnesses, link) {
malloc_printf(" %s(%u)", w->name, w->rank);
}
@@ -80,10 +82,9 @@
abort();
}
#ifdef JEMALLOC_JET
-#undef witness_lockless_error
-#define witness_lockless_error JEMALLOC_N(witness_lockless_error)
-witness_lockless_error_t *witness_lockless_error =
- JEMALLOC_N(n_witness_lockless_error);
+#undef witness_depth_error
+#define witness_depth_error JEMALLOC_N(witness_depth_error)
+witness_depth_error_t *witness_depth_error = JEMALLOC_N(n_witness_depth_error);
#endif
void
diff --git a/test/unit/witness.c b/test/unit/witness.c
index c914e4b..de2e602 100644
--- a/test/unit/witness.c
+++ b/test/unit/witness.c
@@ -3,12 +3,12 @@
static witness_lock_error_t *witness_lock_error_orig;
static witness_owner_error_t *witness_owner_error_orig;
static witness_not_owner_error_t *witness_not_owner_error_orig;
-static witness_lockless_error_t *witness_lockless_error_orig;
+static witness_depth_error_t *witness_depth_error_orig;
static bool saw_lock_error;
static bool saw_owner_error;
static bool saw_not_owner_error;
-static bool saw_lockless_error;
+static bool saw_depth_error;
static void
witness_lock_error_intercept(const witness_list_t *witnesses,
@@ -27,8 +27,9 @@
}
static void
-witness_lockless_error_intercept(const witness_list_t *witnesses) {
- saw_lockless_error = true;
+witness_depth_error_intercept(const witness_list_t *witnesses,
+ witness_rank_t rank_inclusive, unsigned depth) {
+ saw_depth_error = true;
}
static int
@@ -61,21 +62,36 @@
tsdn = tsdn_fetch();
witness_assert_lockless(tsdn);
+ witness_assert_depth(tsdn, 0);
+ witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 0);
witness_init(&a, "a", 1, NULL, NULL);
witness_assert_not_owner(tsdn, &a);
witness_lock(tsdn, &a);
witness_assert_owner(tsdn, &a);
+ witness_assert_depth(tsdn, 1);
+ witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 1);
+ witness_assert_depth_to_rank(tsdn, (witness_rank_t)2U, 0);
witness_init(&b, "b", 2, NULL, NULL);
witness_assert_not_owner(tsdn, &b);
witness_lock(tsdn, &b);
witness_assert_owner(tsdn, &b);
+ witness_assert_depth(tsdn, 2);
+ witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 2);
+ witness_assert_depth_to_rank(tsdn, (witness_rank_t)2U, 1);
+ witness_assert_depth_to_rank(tsdn, (witness_rank_t)3U, 0);
witness_unlock(tsdn, &a);
+ witness_assert_depth(tsdn, 1);
+ witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 1);
+ witness_assert_depth_to_rank(tsdn, (witness_rank_t)2U, 1);
+ witness_assert_depth_to_rank(tsdn, (witness_rank_t)3U, 0);
witness_unlock(tsdn, &b);
witness_assert_lockless(tsdn);
+ witness_assert_depth(tsdn, 0);
+ witness_assert_depth_to_rank(tsdn, (witness_rank_t)1U, 0);
}
TEST_END
@@ -93,12 +109,15 @@
witness_assert_not_owner(tsdn, &a);
witness_lock(tsdn, &a);
witness_assert_owner(tsdn, &a);
+ witness_assert_depth(tsdn, 1);
witness_init(&b, "b", 1, witness_comp, &b);
witness_assert_not_owner(tsdn, &b);
witness_lock(tsdn, &b);
witness_assert_owner(tsdn, &b);
+ witness_assert_depth(tsdn, 2);
witness_unlock(tsdn, &b);
+ witness_assert_depth(tsdn, 1);
witness_lock_error_orig = witness_lock_error;
witness_lock_error = witness_lock_error_intercept;
@@ -110,6 +129,7 @@
witness_lock(tsdn, &c);
assert_true(saw_lock_error, "Expected witness lock error");
witness_unlock(tsdn, &c);
+ witness_assert_depth(tsdn, 1);
saw_lock_error = false;
@@ -119,6 +139,7 @@
witness_lock(tsdn, &d);
assert_true(saw_lock_error, "Expected witness lock error");
witness_unlock(tsdn, &d);
+ witness_assert_depth(tsdn, 1);
witness_unlock(tsdn, &a);
@@ -146,11 +167,13 @@
witness_init(&b, "b", 2, NULL, NULL);
witness_lock(tsdn, &b);
+ witness_assert_depth(tsdn, 1);
assert_false(saw_lock_error, "Unexpected witness lock error");
witness_lock(tsdn, &a);
assert_true(saw_lock_error, "Expected witness lock error");
witness_unlock(tsdn, &a);
+ witness_assert_depth(tsdn, 1);
witness_unlock(tsdn, &b);
witness_assert_lockless(tsdn);
@@ -222,34 +245,38 @@
}
TEST_END
-TEST_BEGIN(test_witness_lockful) {
+TEST_BEGIN(test_witness_depth) {
witness_t a;
tsdn_t *tsdn;
test_skip_if(!config_debug);
- witness_lockless_error_orig = witness_lockless_error;
- witness_lockless_error = witness_lockless_error_intercept;
- saw_lockless_error = false;
+ witness_depth_error_orig = witness_depth_error;
+ witness_depth_error = witness_depth_error_intercept;
+ saw_depth_error = false;
tsdn = tsdn_fetch();
witness_assert_lockless(tsdn);
+ witness_assert_depth(tsdn, 0);
witness_init(&a, "a", 1, NULL, NULL);
- assert_false(saw_lockless_error, "Unexpected lockless error");
+ assert_false(saw_depth_error, "Unexpected depth error");
witness_assert_lockless(tsdn);
+ witness_assert_depth(tsdn, 0);
witness_lock(tsdn, &a);
witness_assert_lockless(tsdn);
- assert_true(saw_lockless_error, "Expected lockless error");
+ witness_assert_depth(tsdn, 0);
+ assert_true(saw_depth_error, "Expected depth error");
witness_unlock(tsdn, &a);
witness_assert_lockless(tsdn);
+ witness_assert_depth(tsdn, 0);
- witness_lockless_error = witness_lockless_error_orig;
+ witness_depth_error = witness_depth_error_orig;
}
TEST_END
@@ -261,5 +288,5 @@
test_witness_reversal,
test_witness_recursive,
test_witness_unlock_not_owned,
- test_witness_lockful);
+ test_witness_depth);
}