Chris Wilson | b47eb4a | 2010-08-07 11:01:23 +0100 | [diff] [blame] | 1 | /* |
| 2 | * Copyright © 2008-2010 Intel Corporation |
| 3 | * |
| 4 | * Permission is hereby granted, free of charge, to any person obtaining a |
| 5 | * copy of this software and associated documentation files (the "Software"), |
| 6 | * to deal in the Software without restriction, including without limitation |
| 7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| 8 | * and/or sell copies of the Software, and to permit persons to whom the |
| 9 | * Software is furnished to do so, subject to the following conditions: |
| 10 | * |
| 11 | * The above copyright notice and this permission notice (including the next |
| 12 | * paragraph) shall be included in all copies or substantial portions of the |
| 13 | * Software. |
| 14 | * |
| 15 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 16 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 17 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| 18 | * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 19 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
| 20 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
| 21 | * IN THE SOFTWARE. |
| 22 | * |
| 23 | * Authors: |
| 24 | * Eric Anholt <eric@anholt.net> |
| 25 | * Chris Wilson <chris@chris-wilson.co.uuk> |
| 26 | * |
| 27 | */ |
| 28 | |
David Howells | 760285e | 2012-10-02 18:01:07 +0100 | [diff] [blame] | 29 | #include <drm/drmP.h> |
Chris Wilson | b47eb4a | 2010-08-07 11:01:23 +0100 | [diff] [blame] | 30 | #include "i915_drv.h" |
David Howells | 760285e | 2012-10-02 18:01:07 +0100 | [diff] [blame] | 31 | #include <drm/i915_drm.h> |
Chris Wilson | db53a30 | 2011-02-03 11:57:46 +0000 | [diff] [blame] | 32 | #include "i915_trace.h" |
Chris Wilson | b47eb4a | 2010-08-07 11:01:23 +0100 | [diff] [blame] | 33 | |
Chris Wilson | cd377ea | 2010-08-07 11:01:24 +0100 | [diff] [blame] | 34 | static bool |
Chris Wilson | 05394f3 | 2010-11-08 19:18:58 +0000 | [diff] [blame] | 35 | mark_free(struct drm_i915_gem_object *obj, struct list_head *unwind) |
Chris Wilson | b47eb4a | 2010-08-07 11:01:23 +0100 | [diff] [blame] | 36 | { |
Chris Wilson | 1b50247 | 2012-04-24 15:47:30 +0100 | [diff] [blame] | 37 | if (obj->pin_count) |
| 38 | return false; |
| 39 | |
Chris Wilson | 432e58e | 2010-11-25 19:32:06 +0000 | [diff] [blame] | 40 | list_add(&obj->exec_list, unwind); |
Chris Wilson | 05394f3 | 2010-11-08 19:18:58 +0000 | [diff] [blame] | 41 | return drm_mm_scan_add_block(obj->gtt_space); |
Chris Wilson | b47eb4a | 2010-08-07 11:01:23 +0100 | [diff] [blame] | 42 | } |
| 43 | |
| 44 | int |
Daniel Vetter | a6e0aa4 | 2010-09-16 15:45:15 +0200 | [diff] [blame] | 45 | i915_gem_evict_something(struct drm_device *dev, int min_size, |
Chris Wilson | 42d6ab4 | 2012-07-26 11:49:32 +0100 | [diff] [blame] | 46 | unsigned alignment, unsigned cache_level, |
Chris Wilson | 86a1ee2 | 2012-08-11 15:41:04 +0100 | [diff] [blame] | 47 | bool mappable, bool nonblocking) |
Chris Wilson | b47eb4a | 2010-08-07 11:01:23 +0100 | [diff] [blame] | 48 | { |
| 49 | drm_i915_private_t *dev_priv = dev->dev_private; |
Chris Wilson | cd377ea | 2010-08-07 11:01:24 +0100 | [diff] [blame] | 50 | struct list_head eviction_list, unwind_list; |
Chris Wilson | 05394f3 | 2010-11-08 19:18:58 +0000 | [diff] [blame] | 51 | struct drm_i915_gem_object *obj; |
Chris Wilson | cd377ea | 2010-08-07 11:01:24 +0100 | [diff] [blame] | 52 | int ret = 0; |
Chris Wilson | b47eb4a | 2010-08-07 11:01:23 +0100 | [diff] [blame] | 53 | |
Chris Wilson | db53a30 | 2011-02-03 11:57:46 +0000 | [diff] [blame] | 54 | trace_i915_gem_evict(dev, min_size, alignment, mappable); |
| 55 | |
Chris Wilson | cd377ea | 2010-08-07 11:01:24 +0100 | [diff] [blame] | 56 | /* |
| 57 | * The goal is to evict objects and amalgamate space in LRU order. |
| 58 | * The oldest idle objects reside on the inactive list, which is in |
| 59 | * retirement order. The next objects to retire are those on the (per |
| 60 | * ring) active list that do not have an outstanding flush. Once the |
| 61 | * hardware reports completion (the seqno is updated after the |
| 62 | * batchbuffer has been finished) the clean buffer objects would |
| 63 | * be retired to the inactive list. Any dirty objects would be added |
| 64 | * to the tail of the flushing list. So after processing the clean |
| 65 | * active objects we need to emit a MI_FLUSH to retire the flushing |
| 66 | * list, hence the retirement order of the flushing list is in |
| 67 | * advance of the dirty objects on the active lists. |
| 68 | * |
| 69 | * The retirement sequence is thus: |
| 70 | * 1. Inactive objects (already retired) |
| 71 | * 2. Clean active objects |
| 72 | * 3. Flushing list |
| 73 | * 4. Dirty active objects. |
| 74 | * |
| 75 | * On each list, the oldest objects lie at the HEAD with the freshest |
| 76 | * object on the TAIL. |
| 77 | */ |
Chris Wilson | b47eb4a | 2010-08-07 11:01:23 +0100 | [diff] [blame] | 78 | |
Chris Wilson | cd377ea | 2010-08-07 11:01:24 +0100 | [diff] [blame] | 79 | INIT_LIST_HEAD(&unwind_list); |
Daniel Vetter | a6e0aa4 | 2010-09-16 15:45:15 +0200 | [diff] [blame] | 80 | if (mappable) |
Chris Wilson | 6b9d89b | 2012-07-10 11:15:23 +0100 | [diff] [blame] | 81 | drm_mm_init_scan_with_range(&dev_priv->mm.gtt_space, |
Chris Wilson | 42d6ab4 | 2012-07-26 11:49:32 +0100 | [diff] [blame] | 82 | min_size, alignment, cache_level, |
Ben Widawsky | 5d4545a | 2013-01-17 12:45:15 -0800 | [diff] [blame] | 83 | 0, dev_priv->gtt.mappable_end); |
Daniel Vetter | a6e0aa4 | 2010-09-16 15:45:15 +0200 | [diff] [blame] | 84 | else |
Chris Wilson | 6b9d89b | 2012-07-10 11:15:23 +0100 | [diff] [blame] | 85 | drm_mm_init_scan(&dev_priv->mm.gtt_space, |
Chris Wilson | 42d6ab4 | 2012-07-26 11:49:32 +0100 | [diff] [blame] | 86 | min_size, alignment, cache_level); |
Chris Wilson | b47eb4a | 2010-08-07 11:01:23 +0100 | [diff] [blame] | 87 | |
Chris Wilson | cd377ea | 2010-08-07 11:01:24 +0100 | [diff] [blame] | 88 | /* First see if there is a large enough contiguous idle region... */ |
Chris Wilson | 05394f3 | 2010-11-08 19:18:58 +0000 | [diff] [blame] | 89 | list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) { |
| 90 | if (mark_free(obj, &unwind_list)) |
Chris Wilson | cd377ea | 2010-08-07 11:01:24 +0100 | [diff] [blame] | 91 | goto found; |
Chris Wilson | b47eb4a | 2010-08-07 11:01:23 +0100 | [diff] [blame] | 92 | } |
Chris Wilson | cd377ea | 2010-08-07 11:01:24 +0100 | [diff] [blame] | 93 | |
Chris Wilson | 86a1ee2 | 2012-08-11 15:41:04 +0100 | [diff] [blame] | 94 | if (nonblocking) |
| 95 | goto none; |
| 96 | |
Chris Wilson | cd377ea | 2010-08-07 11:01:24 +0100 | [diff] [blame] | 97 | /* Now merge in the soon-to-be-expired objects... */ |
Chris Wilson | 05394f3 | 2010-11-08 19:18:58 +0000 | [diff] [blame] | 98 | list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) { |
Chris Wilson | 05394f3 | 2010-11-08 19:18:58 +0000 | [diff] [blame] | 99 | if (mark_free(obj, &unwind_list)) |
Chris Wilson | cd377ea | 2010-08-07 11:01:24 +0100 | [diff] [blame] | 100 | goto found; |
| 101 | } |
| 102 | |
Chris Wilson | 86a1ee2 | 2012-08-11 15:41:04 +0100 | [diff] [blame] | 103 | none: |
Chris Wilson | cd377ea | 2010-08-07 11:01:24 +0100 | [diff] [blame] | 104 | /* Nothing found, clean up and bail out! */ |
Chris Wilson | 092de6f | 2011-01-10 14:21:05 +0000 | [diff] [blame] | 105 | while (!list_empty(&unwind_list)) { |
| 106 | obj = list_first_entry(&unwind_list, |
| 107 | struct drm_i915_gem_object, |
| 108 | exec_list); |
| 109 | |
Chris Wilson | 05394f3 | 2010-11-08 19:18:58 +0000 | [diff] [blame] | 110 | ret = drm_mm_scan_remove_block(obj->gtt_space); |
Chris Wilson | cd377ea | 2010-08-07 11:01:24 +0100 | [diff] [blame] | 111 | BUG_ON(ret); |
Chris Wilson | 092de6f | 2011-01-10 14:21:05 +0000 | [diff] [blame] | 112 | |
| 113 | list_del_init(&obj->exec_list); |
Chris Wilson | cd377ea | 2010-08-07 11:01:24 +0100 | [diff] [blame] | 114 | } |
| 115 | |
| 116 | /* We expect the caller to unpin, evict all and try again, or give up. |
| 117 | * So calling i915_gem_evict_everything() is unnecessary. |
| 118 | */ |
| 119 | return -ENOSPC; |
| 120 | |
| 121 | found: |
Chris Wilson | e39a015 | 2010-09-29 22:23:05 +0100 | [diff] [blame] | 122 | /* drm_mm doesn't allow any other other operations while |
| 123 | * scanning, therefore store to be evicted objects on a |
| 124 | * temporary list. */ |
Chris Wilson | cd377ea | 2010-08-07 11:01:24 +0100 | [diff] [blame] | 125 | INIT_LIST_HEAD(&eviction_list); |
Chris Wilson | e39a015 | 2010-09-29 22:23:05 +0100 | [diff] [blame] | 126 | while (!list_empty(&unwind_list)) { |
Chris Wilson | 05394f3 | 2010-11-08 19:18:58 +0000 | [diff] [blame] | 127 | obj = list_first_entry(&unwind_list, |
| 128 | struct drm_i915_gem_object, |
Chris Wilson | 432e58e | 2010-11-25 19:32:06 +0000 | [diff] [blame] | 129 | exec_list); |
Chris Wilson | 05394f3 | 2010-11-08 19:18:58 +0000 | [diff] [blame] | 130 | if (drm_mm_scan_remove_block(obj->gtt_space)) { |
Chris Wilson | 432e58e | 2010-11-25 19:32:06 +0000 | [diff] [blame] | 131 | list_move(&obj->exec_list, &eviction_list); |
Chris Wilson | b670824 | 2012-02-24 00:27:21 +0000 | [diff] [blame] | 132 | drm_gem_object_reference(&obj->base); |
Chris Wilson | e39a015 | 2010-09-29 22:23:05 +0100 | [diff] [blame] | 133 | continue; |
| 134 | } |
Chris Wilson | 432e58e | 2010-11-25 19:32:06 +0000 | [diff] [blame] | 135 | list_del_init(&obj->exec_list); |
Chris Wilson | cd377ea | 2010-08-07 11:01:24 +0100 | [diff] [blame] | 136 | } |
| 137 | |
Chris Wilson | e39a015 | 2010-09-29 22:23:05 +0100 | [diff] [blame] | 138 | /* Unbinding will emit any required flushes */ |
| 139 | while (!list_empty(&eviction_list)) { |
Chris Wilson | 05394f3 | 2010-11-08 19:18:58 +0000 | [diff] [blame] | 140 | obj = list_first_entry(&eviction_list, |
| 141 | struct drm_i915_gem_object, |
Chris Wilson | 432e58e | 2010-11-25 19:32:06 +0000 | [diff] [blame] | 142 | exec_list); |
Chris Wilson | e39a015 | 2010-09-29 22:23:05 +0100 | [diff] [blame] | 143 | if (ret == 0) |
Chris Wilson | 05394f3 | 2010-11-08 19:18:58 +0000 | [diff] [blame] | 144 | ret = i915_gem_object_unbind(obj); |
Chris Wilson | 092de6f | 2011-01-10 14:21:05 +0000 | [diff] [blame] | 145 | |
Chris Wilson | 432e58e | 2010-11-25 19:32:06 +0000 | [diff] [blame] | 146 | list_del_init(&obj->exec_list); |
Chris Wilson | 05394f3 | 2010-11-08 19:18:58 +0000 | [diff] [blame] | 147 | drm_gem_object_unreference(&obj->base); |
Chris Wilson | e39a015 | 2010-09-29 22:23:05 +0100 | [diff] [blame] | 148 | } |
Chris Wilson | cd377ea | 2010-08-07 11:01:24 +0100 | [diff] [blame] | 149 | |
Chris Wilson | e39a015 | 2010-09-29 22:23:05 +0100 | [diff] [blame] | 150 | return ret; |
Chris Wilson | b47eb4a | 2010-08-07 11:01:23 +0100 | [diff] [blame] | 151 | } |
| 152 | |
| 153 | int |
Chris Wilson | 6c085a7 | 2012-08-20 11:40:46 +0200 | [diff] [blame] | 154 | i915_gem_evict_everything(struct drm_device *dev) |
Chris Wilson | b47eb4a | 2010-08-07 11:01:23 +0100 | [diff] [blame] | 155 | { |
| 156 | drm_i915_private_t *dev_priv = dev->dev_private; |
Chris Wilson | a39d7ef | 2012-04-24 18:22:52 +0100 | [diff] [blame] | 157 | struct drm_i915_gem_object *obj, *next; |
Chris Wilson | b47eb4a | 2010-08-07 11:01:23 +0100 | [diff] [blame] | 158 | bool lists_empty; |
Chris Wilson | b451951 | 2012-05-11 14:29:30 +0100 | [diff] [blame] | 159 | int ret; |
Chris Wilson | b47eb4a | 2010-08-07 11:01:23 +0100 | [diff] [blame] | 160 | |
Chris Wilson | b47eb4a | 2010-08-07 11:01:23 +0100 | [diff] [blame] | 161 | lists_empty = (list_empty(&dev_priv->mm.inactive_list) && |
Chris Wilson | 395b70b | 2010-10-28 21:28:46 +0100 | [diff] [blame] | 162 | list_empty(&dev_priv->mm.active_list)); |
Chris Wilson | b47eb4a | 2010-08-07 11:01:23 +0100 | [diff] [blame] | 163 | if (lists_empty) |
| 164 | return -ENOSPC; |
| 165 | |
Chris Wilson | 6c085a7 | 2012-08-20 11:40:46 +0200 | [diff] [blame] | 166 | trace_i915_gem_evict_everything(dev); |
Chris Wilson | db53a30 | 2011-02-03 11:57:46 +0000 | [diff] [blame] | 167 | |
Ben Widawsky | b2da9fe | 2012-04-26 16:02:58 -0700 | [diff] [blame] | 168 | /* The gpu_idle will flush everything in the write domain to the |
| 169 | * active list. Then we must move everything off the active list |
| 170 | * with retire requests. |
| 171 | */ |
Chris Wilson | b451951 | 2012-05-11 14:29:30 +0100 | [diff] [blame] | 172 | ret = i915_gpu_idle(dev); |
| 173 | if (ret) |
| 174 | return ret; |
Ben Widawsky | b2da9fe | 2012-04-26 16:02:58 -0700 | [diff] [blame] | 175 | |
| 176 | i915_gem_retire_requests(dev); |
| 177 | |
Chris Wilson | a39d7ef | 2012-04-24 18:22:52 +0100 | [diff] [blame] | 178 | /* Having flushed everything, unbind() should never raise an error */ |
Chris Wilson | 5eac3ab | 2010-10-31 08:49:47 +0000 | [diff] [blame] | 179 | list_for_each_entry_safe(obj, next, |
Chris Wilson | 6c085a7 | 2012-08-20 11:40:46 +0200 | [diff] [blame] | 180 | &dev_priv->mm.inactive_list, mm_list) |
| 181 | if (obj->pin_count == 0) |
| 182 | WARN_ON(i915_gem_object_unbind(obj)); |
Chris Wilson | b47eb4a | 2010-08-07 11:01:23 +0100 | [diff] [blame] | 183 | |
Chris Wilson | b451951 | 2012-05-11 14:29:30 +0100 | [diff] [blame] | 184 | return 0; |
Chris Wilson | b47eb4a | 2010-08-07 11:01:23 +0100 | [diff] [blame] | 185 | } |