ksm: distribute remove_mm_from_lists
Do some housekeeping in ksm.c, to help make the next patch easier
to understand: remove the function remove_mm_from_lists, distributing
its code to its callsites scan_get_next_rmap_item and __ksm_exit.
That turns out to be a win in scan_get_next_rmap_item: move its
remove_trailing_rmap_items and cursor advancement up, and it becomes
simpler than before. __ksm_exit becomes messier, but will change
again; and moving its remove_trailing_rmap_items up lets us strengthen
the unstable tree item's age condition in remove_rmap_item_from_tree.
Signed-off-by: Hugh Dickins <hugh.dickins@tiscali.co.uk>
Acked-by: Izik Eidus <ieidus@redhat.com>
Cc: Andrea Arcangeli <aarcange@redhat.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
diff --git a/mm/ksm.c b/mm/ksm.c
index d9e3cfc..7e4d255d 100644
--- a/mm/ksm.c
+++ b/mm/ksm.c
@@ -444,14 +444,9 @@
* But __ksm_exit has to be careful: do the rb_erase
* if it's interrupting a scan, and this rmap_item was
* inserted by this scan rather than left from before.
- *
- * Because of the case in which remove_mm_from_lists
- * increments seqnr before removing rmaps, unstable_nr
- * may even be 2 behind seqnr, but should never be
- * further behind. Yes, I did have trouble with this!
*/
age = (unsigned char)(ksm_scan.seqnr - rmap_item->address);
- BUG_ON(age > 2);
+ BUG_ON(age > 1);
if (!age)
rb_erase(&rmap_item->node, &root_unstable_tree);
ksm_pages_unshared--;
@@ -546,37 +541,6 @@
return err;
}
-static void remove_mm_from_lists(struct mm_struct *mm)
-{
- struct mm_slot *mm_slot;
-
- spin_lock(&ksm_mmlist_lock);
- mm_slot = get_mm_slot(mm);
-
- /*
- * This mm_slot is always at the scanning cursor when we're
- * called from scan_get_next_rmap_item; but it's a special
- * case when we're called from __ksm_exit.
- */
- if (ksm_scan.mm_slot == mm_slot) {
- ksm_scan.mm_slot = list_entry(
- mm_slot->mm_list.next, struct mm_slot, mm_list);
- ksm_scan.address = 0;
- ksm_scan.rmap_item = list_entry(
- &ksm_scan.mm_slot->rmap_list, struct rmap_item, link);
- if (ksm_scan.mm_slot == &ksm_mm_head)
- ksm_scan.seqnr++;
- }
-
- hlist_del(&mm_slot->link);
- list_del(&mm_slot->mm_list);
- spin_unlock(&ksm_mmlist_lock);
-
- remove_trailing_rmap_items(mm_slot, mm_slot->rmap_list.next);
- free_mm_slot(mm_slot);
- clear_bit(MMF_VM_MERGEABLE, &mm->flags);
-}
-
static u32 calc_checksum(struct page *page)
{
u32 checksum;
@@ -1241,33 +1205,31 @@
}
}
- if (!ksm_scan.address) {
- /*
- * We've completed a full scan of all vmas, holding mmap_sem
- * throughout, and found no VM_MERGEABLE: so do the same as
- * __ksm_exit does to remove this mm from all our lists now.
- */
- remove_mm_from_lists(mm);
- up_read(&mm->mmap_sem);
- slot = ksm_scan.mm_slot;
- if (slot != &ksm_mm_head)
- goto next_mm;
- return NULL;
- }
-
/*
* Nuke all the rmap_items that are above this current rmap:
* because there were no VM_MERGEABLE vmas with such addresses.
*/
remove_trailing_rmap_items(slot, ksm_scan.rmap_item->link.next);
- up_read(&mm->mmap_sem);
spin_lock(&ksm_mmlist_lock);
- slot = list_entry(slot->mm_list.next, struct mm_slot, mm_list);
- ksm_scan.mm_slot = slot;
+ ksm_scan.mm_slot = list_entry(slot->mm_list.next,
+ struct mm_slot, mm_list);
+ if (ksm_scan.address == 0) {
+ /*
+ * We've completed a full scan of all vmas, holding mmap_sem
+ * throughout, and found no VM_MERGEABLE: so do the same as
+ * __ksm_exit does to remove this mm from all our lists now.
+ */
+ hlist_del(&slot->link);
+ list_del(&slot->mm_list);
+ free_mm_slot(slot);
+ clear_bit(MMF_VM_MERGEABLE, &mm->flags);
+ }
spin_unlock(&ksm_mmlist_lock);
+ up_read(&mm->mmap_sem);
/* Repeat until we've completed scanning the whole list */
+ slot = ksm_scan.mm_slot;
if (slot != &ksm_mm_head)
goto next_mm;
@@ -1408,13 +1370,38 @@
void __ksm_exit(struct mm_struct *mm)
{
+ struct mm_slot *mm_slot;
+
/*
* This process is exiting: doesn't hold and doesn't need mmap_sem;
* but we do need to exclude ksmd and other exiters while we modify
* the various lists and trees.
*/
mutex_lock(&ksm_thread_mutex);
- remove_mm_from_lists(mm);
+ spin_lock(&ksm_mmlist_lock);
+ mm_slot = get_mm_slot(mm);
+ if (!list_empty(&mm_slot->rmap_list)) {
+ spin_unlock(&ksm_mmlist_lock);
+ remove_trailing_rmap_items(mm_slot, mm_slot->rmap_list.next);
+ spin_lock(&ksm_mmlist_lock);
+ }
+
+ if (ksm_scan.mm_slot == mm_slot) {
+ ksm_scan.mm_slot = list_entry(
+ mm_slot->mm_list.next, struct mm_slot, mm_list);
+ ksm_scan.address = 0;
+ ksm_scan.rmap_item = list_entry(
+ &ksm_scan.mm_slot->rmap_list, struct rmap_item, link);
+ if (ksm_scan.mm_slot == &ksm_mm_head)
+ ksm_scan.seqnr++;
+ }
+
+ hlist_del(&mm_slot->link);
+ list_del(&mm_slot->mm_list);
+ spin_unlock(&ksm_mmlist_lock);
+
+ free_mm_slot(mm_slot);
+ clear_bit(MMF_VM_MERGEABLE, &mm->flags);
mutex_unlock(&ksm_thread_mutex);
}