The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (C) 2015 Etnaviv Project |
| 3 | * |
| 4 | * This program is free software; you can redistribute it and/or modify it |
| 5 | * under the terms of the GNU General Public License version 2 as published by |
| 6 | * the Free Software Foundation. |
| 7 | * |
| 8 | * This program is distributed in the hope that it will be useful, but WITHOUT |
| 9 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 10 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for |
| 11 | * more details. |
| 12 | * |
| 13 | * You should have received a copy of the GNU General Public License along with |
| 14 | * this program. If not, see <http://www.gnu.org/licenses/>. |
| 15 | */ |
| 16 | |
| 17 | #include <linux/component.h> |
| 18 | #include <linux/fence.h> |
| 19 | #include <linux/moduleparam.h> |
| 20 | #include <linux/of_device.h> |
| 21 | #include "etnaviv_dump.h" |
| 22 | #include "etnaviv_gpu.h" |
| 23 | #include "etnaviv_gem.h" |
| 24 | #include "etnaviv_mmu.h" |
| 25 | #include "etnaviv_iommu.h" |
| 26 | #include "etnaviv_iommu_v2.h" |
| 27 | #include "common.xml.h" |
| 28 | #include "state.xml.h" |
| 29 | #include "state_hi.xml.h" |
| 30 | #include "cmdstream.xml.h" |
| 31 | |
| 32 | static const struct platform_device_id gpu_ids[] = { |
| 33 | { .name = "etnaviv-gpu,2d" }, |
| 34 | { }, |
| 35 | }; |
| 36 | |
| 37 | static bool etnaviv_dump_core = true; |
| 38 | module_param_named(dump_core, etnaviv_dump_core, bool, 0600); |
| 39 | |
| 40 | /* |
| 41 | * Driver functions: |
| 42 | */ |
| 43 | |
| 44 | int etnaviv_gpu_get_param(struct etnaviv_gpu *gpu, u32 param, u64 *value) |
| 45 | { |
| 46 | switch (param) { |
| 47 | case ETNAVIV_PARAM_GPU_MODEL: |
| 48 | *value = gpu->identity.model; |
| 49 | break; |
| 50 | |
| 51 | case ETNAVIV_PARAM_GPU_REVISION: |
| 52 | *value = gpu->identity.revision; |
| 53 | break; |
| 54 | |
| 55 | case ETNAVIV_PARAM_GPU_FEATURES_0: |
| 56 | *value = gpu->identity.features; |
| 57 | break; |
| 58 | |
| 59 | case ETNAVIV_PARAM_GPU_FEATURES_1: |
| 60 | *value = gpu->identity.minor_features0; |
| 61 | break; |
| 62 | |
| 63 | case ETNAVIV_PARAM_GPU_FEATURES_2: |
| 64 | *value = gpu->identity.minor_features1; |
| 65 | break; |
| 66 | |
| 67 | case ETNAVIV_PARAM_GPU_FEATURES_3: |
| 68 | *value = gpu->identity.minor_features2; |
| 69 | break; |
| 70 | |
| 71 | case ETNAVIV_PARAM_GPU_FEATURES_4: |
| 72 | *value = gpu->identity.minor_features3; |
| 73 | break; |
| 74 | |
Russell King | 602eb48 | 2016-01-24 17:36:04 +0000 | [diff] [blame] | 75 | case ETNAVIV_PARAM_GPU_FEATURES_5: |
| 76 | *value = gpu->identity.minor_features4; |
| 77 | break; |
| 78 | |
| 79 | case ETNAVIV_PARAM_GPU_FEATURES_6: |
| 80 | *value = gpu->identity.minor_features5; |
| 81 | break; |
| 82 | |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 83 | case ETNAVIV_PARAM_GPU_STREAM_COUNT: |
| 84 | *value = gpu->identity.stream_count; |
| 85 | break; |
| 86 | |
| 87 | case ETNAVIV_PARAM_GPU_REGISTER_MAX: |
| 88 | *value = gpu->identity.register_max; |
| 89 | break; |
| 90 | |
| 91 | case ETNAVIV_PARAM_GPU_THREAD_COUNT: |
| 92 | *value = gpu->identity.thread_count; |
| 93 | break; |
| 94 | |
| 95 | case ETNAVIV_PARAM_GPU_VERTEX_CACHE_SIZE: |
| 96 | *value = gpu->identity.vertex_cache_size; |
| 97 | break; |
| 98 | |
| 99 | case ETNAVIV_PARAM_GPU_SHADER_CORE_COUNT: |
| 100 | *value = gpu->identity.shader_core_count; |
| 101 | break; |
| 102 | |
| 103 | case ETNAVIV_PARAM_GPU_PIXEL_PIPES: |
| 104 | *value = gpu->identity.pixel_pipes; |
| 105 | break; |
| 106 | |
| 107 | case ETNAVIV_PARAM_GPU_VERTEX_OUTPUT_BUFFER_SIZE: |
| 108 | *value = gpu->identity.vertex_output_buffer_size; |
| 109 | break; |
| 110 | |
| 111 | case ETNAVIV_PARAM_GPU_BUFFER_SIZE: |
| 112 | *value = gpu->identity.buffer_size; |
| 113 | break; |
| 114 | |
| 115 | case ETNAVIV_PARAM_GPU_INSTRUCTION_COUNT: |
| 116 | *value = gpu->identity.instruction_count; |
| 117 | break; |
| 118 | |
| 119 | case ETNAVIV_PARAM_GPU_NUM_CONSTANTS: |
| 120 | *value = gpu->identity.num_constants; |
| 121 | break; |
| 122 | |
Russell King | 602eb48 | 2016-01-24 17:36:04 +0000 | [diff] [blame] | 123 | case ETNAVIV_PARAM_GPU_NUM_VARYINGS: |
| 124 | *value = gpu->identity.varyings_count; |
| 125 | break; |
| 126 | |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 127 | default: |
| 128 | DBG("%s: invalid param: %u", dev_name(gpu->dev), param); |
| 129 | return -EINVAL; |
| 130 | } |
| 131 | |
| 132 | return 0; |
| 133 | } |
| 134 | |
Russell King | 472f79d | 2016-01-24 17:35:59 +0000 | [diff] [blame] | 135 | |
| 136 | #define etnaviv_is_model_rev(gpu, mod, rev) \ |
| 137 | ((gpu)->identity.model == chipModel_##mod && \ |
| 138 | (gpu)->identity.revision == rev) |
Russell King | 52f36ba | 2016-01-24 17:35:54 +0000 | [diff] [blame] | 139 | #define etnaviv_field(val, field) \ |
| 140 | (((val) & field##__MASK) >> field##__SHIFT) |
| 141 | |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 142 | static void etnaviv_hw_specs(struct etnaviv_gpu *gpu) |
| 143 | { |
| 144 | if (gpu->identity.minor_features0 & |
| 145 | chipMinorFeatures0_MORE_MINOR_FEATURES) { |
Russell King | 602eb48 | 2016-01-24 17:36:04 +0000 | [diff] [blame] | 146 | u32 specs[4]; |
| 147 | unsigned int streams; |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 148 | |
| 149 | specs[0] = gpu_read(gpu, VIVS_HI_CHIP_SPECS); |
| 150 | specs[1] = gpu_read(gpu, VIVS_HI_CHIP_SPECS_2); |
Russell King | 602eb48 | 2016-01-24 17:36:04 +0000 | [diff] [blame] | 151 | specs[2] = gpu_read(gpu, VIVS_HI_CHIP_SPECS_3); |
| 152 | specs[3] = gpu_read(gpu, VIVS_HI_CHIP_SPECS_4); |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 153 | |
Russell King | 52f36ba | 2016-01-24 17:35:54 +0000 | [diff] [blame] | 154 | gpu->identity.stream_count = etnaviv_field(specs[0], |
| 155 | VIVS_HI_CHIP_SPECS_STREAM_COUNT); |
| 156 | gpu->identity.register_max = etnaviv_field(specs[0], |
| 157 | VIVS_HI_CHIP_SPECS_REGISTER_MAX); |
| 158 | gpu->identity.thread_count = etnaviv_field(specs[0], |
| 159 | VIVS_HI_CHIP_SPECS_THREAD_COUNT); |
| 160 | gpu->identity.vertex_cache_size = etnaviv_field(specs[0], |
| 161 | VIVS_HI_CHIP_SPECS_VERTEX_CACHE_SIZE); |
| 162 | gpu->identity.shader_core_count = etnaviv_field(specs[0], |
| 163 | VIVS_HI_CHIP_SPECS_SHADER_CORE_COUNT); |
| 164 | gpu->identity.pixel_pipes = etnaviv_field(specs[0], |
| 165 | VIVS_HI_CHIP_SPECS_PIXEL_PIPES); |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 166 | gpu->identity.vertex_output_buffer_size = |
Russell King | 52f36ba | 2016-01-24 17:35:54 +0000 | [diff] [blame] | 167 | etnaviv_field(specs[0], |
| 168 | VIVS_HI_CHIP_SPECS_VERTEX_OUTPUT_BUFFER_SIZE); |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 169 | |
Russell King | 52f36ba | 2016-01-24 17:35:54 +0000 | [diff] [blame] | 170 | gpu->identity.buffer_size = etnaviv_field(specs[1], |
| 171 | VIVS_HI_CHIP_SPECS_2_BUFFER_SIZE); |
| 172 | gpu->identity.instruction_count = etnaviv_field(specs[1], |
| 173 | VIVS_HI_CHIP_SPECS_2_INSTRUCTION_COUNT); |
| 174 | gpu->identity.num_constants = etnaviv_field(specs[1], |
| 175 | VIVS_HI_CHIP_SPECS_2_NUM_CONSTANTS); |
Russell King | 602eb48 | 2016-01-24 17:36:04 +0000 | [diff] [blame] | 176 | |
| 177 | gpu->identity.varyings_count = etnaviv_field(specs[2], |
| 178 | VIVS_HI_CHIP_SPECS_3_VARYINGS_COUNT); |
| 179 | |
| 180 | /* This overrides the value from older register if non-zero */ |
| 181 | streams = etnaviv_field(specs[3], |
| 182 | VIVS_HI_CHIP_SPECS_4_STREAM_COUNT); |
| 183 | if (streams) |
| 184 | gpu->identity.stream_count = streams; |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 185 | } |
| 186 | |
| 187 | /* Fill in the stream count if not specified */ |
| 188 | if (gpu->identity.stream_count == 0) { |
| 189 | if (gpu->identity.model >= 0x1000) |
| 190 | gpu->identity.stream_count = 4; |
| 191 | else |
| 192 | gpu->identity.stream_count = 1; |
| 193 | } |
| 194 | |
| 195 | /* Convert the register max value */ |
| 196 | if (gpu->identity.register_max) |
| 197 | gpu->identity.register_max = 1 << gpu->identity.register_max; |
Russell King | 507f899 | 2016-01-24 17:35:48 +0000 | [diff] [blame] | 198 | else if (gpu->identity.model == chipModel_GC400) |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 199 | gpu->identity.register_max = 32; |
| 200 | else |
| 201 | gpu->identity.register_max = 64; |
| 202 | |
| 203 | /* Convert thread count */ |
| 204 | if (gpu->identity.thread_count) |
| 205 | gpu->identity.thread_count = 1 << gpu->identity.thread_count; |
Russell King | 507f899 | 2016-01-24 17:35:48 +0000 | [diff] [blame] | 206 | else if (gpu->identity.model == chipModel_GC400) |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 207 | gpu->identity.thread_count = 64; |
Russell King | 507f899 | 2016-01-24 17:35:48 +0000 | [diff] [blame] | 208 | else if (gpu->identity.model == chipModel_GC500 || |
| 209 | gpu->identity.model == chipModel_GC530) |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 210 | gpu->identity.thread_count = 128; |
| 211 | else |
| 212 | gpu->identity.thread_count = 256; |
| 213 | |
| 214 | if (gpu->identity.vertex_cache_size == 0) |
| 215 | gpu->identity.vertex_cache_size = 8; |
| 216 | |
| 217 | if (gpu->identity.shader_core_count == 0) { |
| 218 | if (gpu->identity.model >= 0x1000) |
| 219 | gpu->identity.shader_core_count = 2; |
| 220 | else |
| 221 | gpu->identity.shader_core_count = 1; |
| 222 | } |
| 223 | |
| 224 | if (gpu->identity.pixel_pipes == 0) |
| 225 | gpu->identity.pixel_pipes = 1; |
| 226 | |
| 227 | /* Convert virtex buffer size */ |
| 228 | if (gpu->identity.vertex_output_buffer_size) { |
| 229 | gpu->identity.vertex_output_buffer_size = |
| 230 | 1 << gpu->identity.vertex_output_buffer_size; |
Russell King | 507f899 | 2016-01-24 17:35:48 +0000 | [diff] [blame] | 231 | } else if (gpu->identity.model == chipModel_GC400) { |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 232 | if (gpu->identity.revision < 0x4000) |
| 233 | gpu->identity.vertex_output_buffer_size = 512; |
| 234 | else if (gpu->identity.revision < 0x4200) |
| 235 | gpu->identity.vertex_output_buffer_size = 256; |
| 236 | else |
| 237 | gpu->identity.vertex_output_buffer_size = 128; |
| 238 | } else { |
| 239 | gpu->identity.vertex_output_buffer_size = 512; |
| 240 | } |
| 241 | |
| 242 | switch (gpu->identity.instruction_count) { |
| 243 | case 0: |
Russell King | 472f79d | 2016-01-24 17:35:59 +0000 | [diff] [blame] | 244 | if (etnaviv_is_model_rev(gpu, GC2000, 0x5108) || |
Russell King | 507f899 | 2016-01-24 17:35:48 +0000 | [diff] [blame] | 245 | gpu->identity.model == chipModel_GC880) |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 246 | gpu->identity.instruction_count = 512; |
| 247 | else |
| 248 | gpu->identity.instruction_count = 256; |
| 249 | break; |
| 250 | |
| 251 | case 1: |
| 252 | gpu->identity.instruction_count = 1024; |
| 253 | break; |
| 254 | |
| 255 | case 2: |
| 256 | gpu->identity.instruction_count = 2048; |
| 257 | break; |
| 258 | |
| 259 | default: |
| 260 | gpu->identity.instruction_count = 256; |
| 261 | break; |
| 262 | } |
| 263 | |
| 264 | if (gpu->identity.num_constants == 0) |
| 265 | gpu->identity.num_constants = 168; |
Russell King | 602eb48 | 2016-01-24 17:36:04 +0000 | [diff] [blame] | 266 | |
| 267 | if (gpu->identity.varyings_count == 0) { |
| 268 | if (gpu->identity.minor_features1 & chipMinorFeatures1_HALTI0) |
| 269 | gpu->identity.varyings_count = 12; |
| 270 | else |
| 271 | gpu->identity.varyings_count = 8; |
| 272 | } |
| 273 | |
| 274 | /* |
| 275 | * For some cores, two varyings are consumed for position, so the |
| 276 | * maximum varying count needs to be reduced by one. |
| 277 | */ |
| 278 | if (etnaviv_is_model_rev(gpu, GC5000, 0x5434) || |
| 279 | etnaviv_is_model_rev(gpu, GC4000, 0x5222) || |
| 280 | etnaviv_is_model_rev(gpu, GC4000, 0x5245) || |
| 281 | etnaviv_is_model_rev(gpu, GC4000, 0x5208) || |
| 282 | etnaviv_is_model_rev(gpu, GC3000, 0x5435) || |
| 283 | etnaviv_is_model_rev(gpu, GC2200, 0x5244) || |
| 284 | etnaviv_is_model_rev(gpu, GC2100, 0x5108) || |
| 285 | etnaviv_is_model_rev(gpu, GC2000, 0x5108) || |
| 286 | etnaviv_is_model_rev(gpu, GC1500, 0x5246) || |
| 287 | etnaviv_is_model_rev(gpu, GC880, 0x5107) || |
| 288 | etnaviv_is_model_rev(gpu, GC880, 0x5106)) |
| 289 | gpu->identity.varyings_count -= 1; |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 290 | } |
| 291 | |
| 292 | static void etnaviv_hw_identify(struct etnaviv_gpu *gpu) |
| 293 | { |
| 294 | u32 chipIdentity; |
| 295 | |
| 296 | chipIdentity = gpu_read(gpu, VIVS_HI_CHIP_IDENTITY); |
| 297 | |
| 298 | /* Special case for older graphic cores. */ |
Russell King | 52f36ba | 2016-01-24 17:35:54 +0000 | [diff] [blame] | 299 | if (etnaviv_field(chipIdentity, VIVS_HI_CHIP_IDENTITY_FAMILY) == 0x01) { |
Russell King | 507f899 | 2016-01-24 17:35:48 +0000 | [diff] [blame] | 300 | gpu->identity.model = chipModel_GC500; |
Russell King | 52f36ba | 2016-01-24 17:35:54 +0000 | [diff] [blame] | 301 | gpu->identity.revision = etnaviv_field(chipIdentity, |
| 302 | VIVS_HI_CHIP_IDENTITY_REVISION); |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 303 | } else { |
| 304 | |
| 305 | gpu->identity.model = gpu_read(gpu, VIVS_HI_CHIP_MODEL); |
| 306 | gpu->identity.revision = gpu_read(gpu, VIVS_HI_CHIP_REV); |
| 307 | |
| 308 | /* |
| 309 | * !!!! HACK ALERT !!!! |
| 310 | * Because people change device IDs without letting software |
| 311 | * know about it - here is the hack to make it all look the |
| 312 | * same. Only for GC400 family. |
| 313 | */ |
| 314 | if ((gpu->identity.model & 0xff00) == 0x0400 && |
Russell King | 507f899 | 2016-01-24 17:35:48 +0000 | [diff] [blame] | 315 | gpu->identity.model != chipModel_GC420) { |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 316 | gpu->identity.model = gpu->identity.model & 0x0400; |
| 317 | } |
| 318 | |
| 319 | /* Another special case */ |
Russell King | 472f79d | 2016-01-24 17:35:59 +0000 | [diff] [blame] | 320 | if (etnaviv_is_model_rev(gpu, GC300, 0x2201)) { |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 321 | u32 chipDate = gpu_read(gpu, VIVS_HI_CHIP_DATE); |
| 322 | u32 chipTime = gpu_read(gpu, VIVS_HI_CHIP_TIME); |
| 323 | |
| 324 | if (chipDate == 0x20080814 && chipTime == 0x12051100) { |
| 325 | /* |
| 326 | * This IP has an ECO; put the correct |
| 327 | * revision in it. |
| 328 | */ |
| 329 | gpu->identity.revision = 0x1051; |
| 330 | } |
| 331 | } |
| 332 | } |
| 333 | |
| 334 | dev_info(gpu->dev, "model: GC%x, revision: %x\n", |
| 335 | gpu->identity.model, gpu->identity.revision); |
| 336 | |
| 337 | gpu->identity.features = gpu_read(gpu, VIVS_HI_CHIP_FEATURE); |
| 338 | |
| 339 | /* Disable fast clear on GC700. */ |
Russell King | 507f899 | 2016-01-24 17:35:48 +0000 | [diff] [blame] | 340 | if (gpu->identity.model == chipModel_GC700) |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 341 | gpu->identity.features &= ~chipFeatures_FAST_CLEAR; |
| 342 | |
Russell King | 507f899 | 2016-01-24 17:35:48 +0000 | [diff] [blame] | 343 | if ((gpu->identity.model == chipModel_GC500 && |
| 344 | gpu->identity.revision < 2) || |
| 345 | (gpu->identity.model == chipModel_GC300 && |
| 346 | gpu->identity.revision < 0x2000)) { |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 347 | |
| 348 | /* |
| 349 | * GC500 rev 1.x and GC300 rev < 2.0 doesn't have these |
| 350 | * registers. |
| 351 | */ |
| 352 | gpu->identity.minor_features0 = 0; |
| 353 | gpu->identity.minor_features1 = 0; |
| 354 | gpu->identity.minor_features2 = 0; |
| 355 | gpu->identity.minor_features3 = 0; |
Russell King | 602eb48 | 2016-01-24 17:36:04 +0000 | [diff] [blame] | 356 | gpu->identity.minor_features4 = 0; |
| 357 | gpu->identity.minor_features5 = 0; |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 358 | } else |
| 359 | gpu->identity.minor_features0 = |
| 360 | gpu_read(gpu, VIVS_HI_CHIP_MINOR_FEATURE_0); |
| 361 | |
| 362 | if (gpu->identity.minor_features0 & |
| 363 | chipMinorFeatures0_MORE_MINOR_FEATURES) { |
| 364 | gpu->identity.minor_features1 = |
| 365 | gpu_read(gpu, VIVS_HI_CHIP_MINOR_FEATURE_1); |
| 366 | gpu->identity.minor_features2 = |
| 367 | gpu_read(gpu, VIVS_HI_CHIP_MINOR_FEATURE_2); |
| 368 | gpu->identity.minor_features3 = |
| 369 | gpu_read(gpu, VIVS_HI_CHIP_MINOR_FEATURE_3); |
Russell King | 602eb48 | 2016-01-24 17:36:04 +0000 | [diff] [blame] | 370 | gpu->identity.minor_features4 = |
| 371 | gpu_read(gpu, VIVS_HI_CHIP_MINOR_FEATURE_4); |
| 372 | gpu->identity.minor_features5 = |
| 373 | gpu_read(gpu, VIVS_HI_CHIP_MINOR_FEATURE_5); |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 374 | } |
| 375 | |
| 376 | /* GC600 idle register reports zero bits where modules aren't present */ |
| 377 | if (gpu->identity.model == chipModel_GC600) { |
| 378 | gpu->idle_mask = VIVS_HI_IDLE_STATE_TX | |
| 379 | VIVS_HI_IDLE_STATE_RA | |
| 380 | VIVS_HI_IDLE_STATE_SE | |
| 381 | VIVS_HI_IDLE_STATE_PA | |
| 382 | VIVS_HI_IDLE_STATE_SH | |
| 383 | VIVS_HI_IDLE_STATE_PE | |
| 384 | VIVS_HI_IDLE_STATE_DE | |
| 385 | VIVS_HI_IDLE_STATE_FE; |
| 386 | } else { |
| 387 | gpu->idle_mask = ~VIVS_HI_IDLE_STATE_AXI_LP; |
| 388 | } |
| 389 | |
| 390 | etnaviv_hw_specs(gpu); |
| 391 | } |
| 392 | |
| 393 | static void etnaviv_gpu_load_clock(struct etnaviv_gpu *gpu, u32 clock) |
| 394 | { |
| 395 | gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, clock | |
| 396 | VIVS_HI_CLOCK_CONTROL_FSCALE_CMD_LOAD); |
| 397 | gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, clock); |
| 398 | } |
| 399 | |
| 400 | static int etnaviv_hw_reset(struct etnaviv_gpu *gpu) |
| 401 | { |
| 402 | u32 control, idle; |
| 403 | unsigned long timeout; |
| 404 | bool failed = true; |
| 405 | |
| 406 | /* TODO |
| 407 | * |
| 408 | * - clock gating |
| 409 | * - puls eater |
| 410 | * - what about VG? |
| 411 | */ |
| 412 | |
| 413 | /* We hope that the GPU resets in under one second */ |
| 414 | timeout = jiffies + msecs_to_jiffies(1000); |
| 415 | |
| 416 | while (time_is_after_jiffies(timeout)) { |
| 417 | control = VIVS_HI_CLOCK_CONTROL_DISABLE_DEBUG_REGISTERS | |
| 418 | VIVS_HI_CLOCK_CONTROL_FSCALE_VAL(0x40); |
| 419 | |
| 420 | /* enable clock */ |
| 421 | etnaviv_gpu_load_clock(gpu, control); |
| 422 | |
| 423 | /* Wait for stable clock. Vivante's code waited for 1ms */ |
| 424 | usleep_range(1000, 10000); |
| 425 | |
| 426 | /* isolate the GPU. */ |
| 427 | control |= VIVS_HI_CLOCK_CONTROL_ISOLATE_GPU; |
| 428 | gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, control); |
| 429 | |
| 430 | /* set soft reset. */ |
| 431 | control |= VIVS_HI_CLOCK_CONTROL_SOFT_RESET; |
| 432 | gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, control); |
| 433 | |
| 434 | /* wait for reset. */ |
| 435 | msleep(1); |
| 436 | |
| 437 | /* reset soft reset bit. */ |
| 438 | control &= ~VIVS_HI_CLOCK_CONTROL_SOFT_RESET; |
| 439 | gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, control); |
| 440 | |
| 441 | /* reset GPU isolation. */ |
| 442 | control &= ~VIVS_HI_CLOCK_CONTROL_ISOLATE_GPU; |
| 443 | gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, control); |
| 444 | |
| 445 | /* read idle register. */ |
| 446 | idle = gpu_read(gpu, VIVS_HI_IDLE_STATE); |
| 447 | |
| 448 | /* try reseting again if FE it not idle */ |
| 449 | if ((idle & VIVS_HI_IDLE_STATE_FE) == 0) { |
| 450 | dev_dbg(gpu->dev, "FE is not idle\n"); |
| 451 | continue; |
| 452 | } |
| 453 | |
| 454 | /* read reset register. */ |
| 455 | control = gpu_read(gpu, VIVS_HI_CLOCK_CONTROL); |
| 456 | |
| 457 | /* is the GPU idle? */ |
| 458 | if (((control & VIVS_HI_CLOCK_CONTROL_IDLE_3D) == 0) || |
| 459 | ((control & VIVS_HI_CLOCK_CONTROL_IDLE_2D) == 0)) { |
| 460 | dev_dbg(gpu->dev, "GPU is not idle\n"); |
| 461 | continue; |
| 462 | } |
| 463 | |
| 464 | failed = false; |
| 465 | break; |
| 466 | } |
| 467 | |
| 468 | if (failed) { |
| 469 | idle = gpu_read(gpu, VIVS_HI_IDLE_STATE); |
| 470 | control = gpu_read(gpu, VIVS_HI_CLOCK_CONTROL); |
| 471 | |
| 472 | dev_err(gpu->dev, "GPU failed to reset: FE %sidle, 3D %sidle, 2D %sidle\n", |
| 473 | idle & VIVS_HI_IDLE_STATE_FE ? "" : "not ", |
| 474 | control & VIVS_HI_CLOCK_CONTROL_IDLE_3D ? "" : "not ", |
| 475 | control & VIVS_HI_CLOCK_CONTROL_IDLE_2D ? "" : "not "); |
| 476 | |
| 477 | return -EBUSY; |
| 478 | } |
| 479 | |
| 480 | /* We rely on the GPU running, so program the clock */ |
| 481 | control = VIVS_HI_CLOCK_CONTROL_DISABLE_DEBUG_REGISTERS | |
| 482 | VIVS_HI_CLOCK_CONTROL_FSCALE_VAL(0x40); |
| 483 | |
| 484 | /* enable clock */ |
| 485 | etnaviv_gpu_load_clock(gpu, control); |
| 486 | |
| 487 | return 0; |
| 488 | } |
| 489 | |
Russell King | 7d0c6e7 | 2016-01-21 15:20:45 +0000 | [diff] [blame] | 490 | static void etnaviv_gpu_enable_mlcg(struct etnaviv_gpu *gpu) |
| 491 | { |
| 492 | u32 pmc, ppc; |
| 493 | |
| 494 | /* enable clock gating */ |
| 495 | ppc = gpu_read(gpu, VIVS_PM_POWER_CONTROLS); |
| 496 | ppc |= VIVS_PM_POWER_CONTROLS_ENABLE_MODULE_CLOCK_GATING; |
| 497 | |
| 498 | /* Disable stall module clock gating for 4.3.0.1 and 4.3.0.2 revs */ |
| 499 | if (gpu->identity.revision == 0x4301 || |
| 500 | gpu->identity.revision == 0x4302) |
| 501 | ppc |= VIVS_PM_POWER_CONTROLS_DISABLE_STALL_MODULE_CLOCK_GATING; |
| 502 | |
| 503 | gpu_write(gpu, VIVS_PM_POWER_CONTROLS, ppc); |
| 504 | |
| 505 | pmc = gpu_read(gpu, VIVS_PM_MODULE_CONTROLS); |
| 506 | |
| 507 | /* Disable PA clock gating for GC400+ except for GC420 */ |
| 508 | if (gpu->identity.model >= chipModel_GC400 && |
| 509 | gpu->identity.model != chipModel_GC420) |
| 510 | pmc |= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_PA; |
| 511 | |
| 512 | /* |
| 513 | * Disable PE clock gating on revs < 5.0.0.0 when HZ is |
| 514 | * present without a bug fix. |
| 515 | */ |
| 516 | if (gpu->identity.revision < 0x5000 && |
| 517 | gpu->identity.minor_features0 & chipMinorFeatures0_HZ && |
| 518 | !(gpu->identity.minor_features1 & |
| 519 | chipMinorFeatures1_DISABLE_PE_GATING)) |
| 520 | pmc |= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_PE; |
| 521 | |
| 522 | if (gpu->identity.revision < 0x5422) |
| 523 | pmc |= BIT(15); /* Unknown bit */ |
| 524 | |
| 525 | pmc |= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_RA_HZ; |
| 526 | pmc |= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_RA_EZ; |
| 527 | |
| 528 | gpu_write(gpu, VIVS_PM_MODULE_CONTROLS, pmc); |
| 529 | } |
| 530 | |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 531 | static void etnaviv_gpu_hw_init(struct etnaviv_gpu *gpu) |
| 532 | { |
| 533 | u16 prefetch; |
| 534 | |
Russell King | 472f79d | 2016-01-24 17:35:59 +0000 | [diff] [blame] | 535 | if ((etnaviv_is_model_rev(gpu, GC320, 0x5007) || |
| 536 | etnaviv_is_model_rev(gpu, GC320, 0x5220)) && |
| 537 | gpu_read(gpu, VIVS_HI_CHIP_TIME) != 0x2062400) { |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 538 | u32 mc_memory_debug; |
| 539 | |
| 540 | mc_memory_debug = gpu_read(gpu, VIVS_MC_DEBUG_MEMORY) & ~0xff; |
| 541 | |
| 542 | if (gpu->identity.revision == 0x5007) |
| 543 | mc_memory_debug |= 0x0c; |
| 544 | else |
| 545 | mc_memory_debug |= 0x08; |
| 546 | |
| 547 | gpu_write(gpu, VIVS_MC_DEBUG_MEMORY, mc_memory_debug); |
| 548 | } |
| 549 | |
Russell King | 7d0c6e7 | 2016-01-21 15:20:45 +0000 | [diff] [blame] | 550 | /* enable module-level clock gating */ |
| 551 | etnaviv_gpu_enable_mlcg(gpu); |
| 552 | |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 553 | /* |
| 554 | * Update GPU AXI cache atttribute to "cacheable, no allocate". |
| 555 | * This is necessary to prevent the iMX6 SoC locking up. |
| 556 | */ |
| 557 | gpu_write(gpu, VIVS_HI_AXI_CONFIG, |
| 558 | VIVS_HI_AXI_CONFIG_AWCACHE(2) | |
| 559 | VIVS_HI_AXI_CONFIG_ARCACHE(2)); |
| 560 | |
| 561 | /* GC2000 rev 5108 needs a special bus config */ |
Russell King | 472f79d | 2016-01-24 17:35:59 +0000 | [diff] [blame] | 562 | if (etnaviv_is_model_rev(gpu, GC2000, 0x5108)) { |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 563 | u32 bus_config = gpu_read(gpu, VIVS_MC_BUS_CONFIG); |
| 564 | bus_config &= ~(VIVS_MC_BUS_CONFIG_FE_BUS_CONFIG__MASK | |
| 565 | VIVS_MC_BUS_CONFIG_TX_BUS_CONFIG__MASK); |
| 566 | bus_config |= VIVS_MC_BUS_CONFIG_FE_BUS_CONFIG(1) | |
| 567 | VIVS_MC_BUS_CONFIG_TX_BUS_CONFIG(0); |
| 568 | gpu_write(gpu, VIVS_MC_BUS_CONFIG, bus_config); |
| 569 | } |
| 570 | |
| 571 | /* set base addresses */ |
| 572 | gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_RA, gpu->memory_base); |
| 573 | gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_FE, gpu->memory_base); |
| 574 | gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_TX, gpu->memory_base); |
| 575 | gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_PEZ, gpu->memory_base); |
| 576 | gpu_write(gpu, VIVS_MC_MEMORY_BASE_ADDR_PE, gpu->memory_base); |
| 577 | |
| 578 | /* setup the MMU page table pointers */ |
| 579 | etnaviv_iommu_domain_restore(gpu, gpu->mmu->domain); |
| 580 | |
| 581 | /* Start command processor */ |
| 582 | prefetch = etnaviv_buffer_init(gpu); |
| 583 | |
| 584 | gpu_write(gpu, VIVS_HI_INTR_ENBL, ~0U); |
| 585 | gpu_write(gpu, VIVS_FE_COMMAND_ADDRESS, |
| 586 | gpu->buffer->paddr - gpu->memory_base); |
| 587 | gpu_write(gpu, VIVS_FE_COMMAND_CONTROL, |
| 588 | VIVS_FE_COMMAND_CONTROL_ENABLE | |
| 589 | VIVS_FE_COMMAND_CONTROL_PREFETCH(prefetch)); |
| 590 | } |
| 591 | |
| 592 | int etnaviv_gpu_init(struct etnaviv_gpu *gpu) |
| 593 | { |
| 594 | int ret, i; |
| 595 | struct iommu_domain *iommu; |
| 596 | enum etnaviv_iommu_version version; |
| 597 | bool mmuv2; |
| 598 | |
| 599 | ret = pm_runtime_get_sync(gpu->dev); |
Lucas Stach | 1409df0 | 2016-06-17 12:29:02 +0200 | [diff] [blame] | 600 | if (ret < 0) { |
| 601 | dev_err(gpu->dev, "Failed to enable GPU power domain\n"); |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 602 | return ret; |
Lucas Stach | 1409df0 | 2016-06-17 12:29:02 +0200 | [diff] [blame] | 603 | } |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 604 | |
| 605 | etnaviv_hw_identify(gpu); |
| 606 | |
| 607 | if (gpu->identity.model == 0) { |
| 608 | dev_err(gpu->dev, "Unknown GPU model\n"); |
Russell King | f642776 | 2016-01-24 17:32:13 +0000 | [diff] [blame] | 609 | ret = -ENXIO; |
| 610 | goto fail; |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 611 | } |
| 612 | |
Russell King | b98c668 | 2016-01-21 15:19:59 +0000 | [diff] [blame] | 613 | /* Exclude VG cores with FE2.0 */ |
| 614 | if (gpu->identity.features & chipFeatures_PIPE_VG && |
| 615 | gpu->identity.features & chipFeatures_FE20) { |
| 616 | dev_info(gpu->dev, "Ignoring GPU with VG and FE2.0\n"); |
| 617 | ret = -ENXIO; |
| 618 | goto fail; |
| 619 | } |
| 620 | |
Lucas Stach | 2144fff | 2016-04-21 13:52:38 +0200 | [diff] [blame] | 621 | /* |
| 622 | * Set the GPU linear window to be at the end of the DMA window, where |
| 623 | * the CMA area is likely to reside. This ensures that we are able to |
| 624 | * map the command buffers while having the linear window overlap as |
| 625 | * much RAM as possible, so we can optimize mappings for other buffers. |
| 626 | * |
| 627 | * For 3D cores only do this if MC2.0 is present, as with MC1.0 it leads |
| 628 | * to different views of the memory on the individual engines. |
| 629 | */ |
| 630 | if (!(gpu->identity.features & chipFeatures_PIPE_3D) || |
| 631 | (gpu->identity.minor_features0 & chipMinorFeatures0_MC20)) { |
| 632 | u32 dma_mask = (u32)dma_get_required_mask(gpu->dev); |
| 633 | if (dma_mask < PHYS_OFFSET + SZ_2G) |
| 634 | gpu->memory_base = PHYS_OFFSET; |
| 635 | else |
| 636 | gpu->memory_base = dma_mask - SZ_2G + 1; |
| 637 | } |
| 638 | |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 639 | ret = etnaviv_hw_reset(gpu); |
Lucas Stach | 1409df0 | 2016-06-17 12:29:02 +0200 | [diff] [blame] | 640 | if (ret) { |
| 641 | dev_err(gpu->dev, "GPU reset failed\n"); |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 642 | goto fail; |
Lucas Stach | 1409df0 | 2016-06-17 12:29:02 +0200 | [diff] [blame] | 643 | } |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 644 | |
| 645 | /* Setup IOMMU.. eventually we will (I think) do this once per context |
| 646 | * and have separate page tables per context. For now, to keep things |
| 647 | * simple and to get something working, just use a single address space: |
| 648 | */ |
| 649 | mmuv2 = gpu->identity.minor_features1 & chipMinorFeatures1_MMU_VERSION; |
| 650 | dev_dbg(gpu->dev, "mmuv2: %d\n", mmuv2); |
| 651 | |
| 652 | if (!mmuv2) { |
| 653 | iommu = etnaviv_iommu_domain_alloc(gpu); |
| 654 | version = ETNAVIV_IOMMU_V1; |
| 655 | } else { |
| 656 | iommu = etnaviv_iommu_v2_domain_alloc(gpu); |
| 657 | version = ETNAVIV_IOMMU_V2; |
| 658 | } |
| 659 | |
| 660 | if (!iommu) { |
Lucas Stach | 1409df0 | 2016-06-17 12:29:02 +0200 | [diff] [blame] | 661 | dev_err(gpu->dev, "Failed to allocate GPU IOMMU domain\n"); |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 662 | ret = -ENOMEM; |
| 663 | goto fail; |
| 664 | } |
| 665 | |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 666 | gpu->mmu = etnaviv_iommu_new(gpu, iommu, version); |
| 667 | if (!gpu->mmu) { |
Lucas Stach | 1409df0 | 2016-06-17 12:29:02 +0200 | [diff] [blame] | 668 | dev_err(gpu->dev, "Failed to instantiate GPU IOMMU\n"); |
Lucas Stach | 45d16a6 | 2016-01-25 12:41:05 +0100 | [diff] [blame] | 669 | iommu_domain_free(iommu); |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 670 | ret = -ENOMEM; |
| 671 | goto fail; |
| 672 | } |
| 673 | |
| 674 | /* Create buffer: */ |
| 675 | gpu->buffer = etnaviv_gpu_cmdbuf_new(gpu, PAGE_SIZE, 0); |
| 676 | if (!gpu->buffer) { |
| 677 | ret = -ENOMEM; |
| 678 | dev_err(gpu->dev, "could not create command buffer\n"); |
Lucas Stach | 45d16a6 | 2016-01-25 12:41:05 +0100 | [diff] [blame] | 679 | goto destroy_iommu; |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 680 | } |
| 681 | if (gpu->buffer->paddr - gpu->memory_base > 0x80000000) { |
| 682 | ret = -EINVAL; |
| 683 | dev_err(gpu->dev, |
| 684 | "command buffer outside valid memory window\n"); |
| 685 | goto free_buffer; |
| 686 | } |
| 687 | |
| 688 | /* Setup event management */ |
| 689 | spin_lock_init(&gpu->event_spinlock); |
| 690 | init_completion(&gpu->event_free); |
| 691 | for (i = 0; i < ARRAY_SIZE(gpu->event); i++) { |
| 692 | gpu->event[i].used = false; |
| 693 | complete(&gpu->event_free); |
| 694 | } |
| 695 | |
| 696 | /* Now program the hardware */ |
| 697 | mutex_lock(&gpu->lock); |
| 698 | etnaviv_gpu_hw_init(gpu); |
Russell King | f608631 | 2016-01-21 15:20:19 +0000 | [diff] [blame] | 699 | gpu->exec_state = -1; |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 700 | mutex_unlock(&gpu->lock); |
| 701 | |
| 702 | pm_runtime_mark_last_busy(gpu->dev); |
| 703 | pm_runtime_put_autosuspend(gpu->dev); |
| 704 | |
| 705 | return 0; |
| 706 | |
| 707 | free_buffer: |
| 708 | etnaviv_gpu_cmdbuf_free(gpu->buffer); |
| 709 | gpu->buffer = NULL; |
Lucas Stach | 45d16a6 | 2016-01-25 12:41:05 +0100 | [diff] [blame] | 710 | destroy_iommu: |
| 711 | etnaviv_iommu_destroy(gpu->mmu); |
| 712 | gpu->mmu = NULL; |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 713 | fail: |
| 714 | pm_runtime_mark_last_busy(gpu->dev); |
| 715 | pm_runtime_put_autosuspend(gpu->dev); |
| 716 | |
| 717 | return ret; |
| 718 | } |
| 719 | |
| 720 | #ifdef CONFIG_DEBUG_FS |
| 721 | struct dma_debug { |
| 722 | u32 address[2]; |
| 723 | u32 state[2]; |
| 724 | }; |
| 725 | |
| 726 | static void verify_dma(struct etnaviv_gpu *gpu, struct dma_debug *debug) |
| 727 | { |
| 728 | u32 i; |
| 729 | |
| 730 | debug->address[0] = gpu_read(gpu, VIVS_FE_DMA_ADDRESS); |
| 731 | debug->state[0] = gpu_read(gpu, VIVS_FE_DMA_DEBUG_STATE); |
| 732 | |
| 733 | for (i = 0; i < 500; i++) { |
| 734 | debug->address[1] = gpu_read(gpu, VIVS_FE_DMA_ADDRESS); |
| 735 | debug->state[1] = gpu_read(gpu, VIVS_FE_DMA_DEBUG_STATE); |
| 736 | |
| 737 | if (debug->address[0] != debug->address[1]) |
| 738 | break; |
| 739 | |
| 740 | if (debug->state[0] != debug->state[1]) |
| 741 | break; |
| 742 | } |
| 743 | } |
| 744 | |
| 745 | int etnaviv_gpu_debugfs(struct etnaviv_gpu *gpu, struct seq_file *m) |
| 746 | { |
| 747 | struct dma_debug debug; |
| 748 | u32 dma_lo, dma_hi, axi, idle; |
| 749 | int ret; |
| 750 | |
| 751 | seq_printf(m, "%s Status:\n", dev_name(gpu->dev)); |
| 752 | |
| 753 | ret = pm_runtime_get_sync(gpu->dev); |
| 754 | if (ret < 0) |
| 755 | return ret; |
| 756 | |
| 757 | dma_lo = gpu_read(gpu, VIVS_FE_DMA_LOW); |
| 758 | dma_hi = gpu_read(gpu, VIVS_FE_DMA_HIGH); |
| 759 | axi = gpu_read(gpu, VIVS_HI_AXI_STATUS); |
| 760 | idle = gpu_read(gpu, VIVS_HI_IDLE_STATE); |
| 761 | |
| 762 | verify_dma(gpu, &debug); |
| 763 | |
| 764 | seq_puts(m, "\tfeatures\n"); |
| 765 | seq_printf(m, "\t minor_features0: 0x%08x\n", |
| 766 | gpu->identity.minor_features0); |
| 767 | seq_printf(m, "\t minor_features1: 0x%08x\n", |
| 768 | gpu->identity.minor_features1); |
| 769 | seq_printf(m, "\t minor_features2: 0x%08x\n", |
| 770 | gpu->identity.minor_features2); |
| 771 | seq_printf(m, "\t minor_features3: 0x%08x\n", |
| 772 | gpu->identity.minor_features3); |
Russell King | 602eb48 | 2016-01-24 17:36:04 +0000 | [diff] [blame] | 773 | seq_printf(m, "\t minor_features4: 0x%08x\n", |
| 774 | gpu->identity.minor_features4); |
| 775 | seq_printf(m, "\t minor_features5: 0x%08x\n", |
| 776 | gpu->identity.minor_features5); |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 777 | |
| 778 | seq_puts(m, "\tspecs\n"); |
| 779 | seq_printf(m, "\t stream_count: %d\n", |
| 780 | gpu->identity.stream_count); |
| 781 | seq_printf(m, "\t register_max: %d\n", |
| 782 | gpu->identity.register_max); |
| 783 | seq_printf(m, "\t thread_count: %d\n", |
| 784 | gpu->identity.thread_count); |
| 785 | seq_printf(m, "\t vertex_cache_size: %d\n", |
| 786 | gpu->identity.vertex_cache_size); |
| 787 | seq_printf(m, "\t shader_core_count: %d\n", |
| 788 | gpu->identity.shader_core_count); |
| 789 | seq_printf(m, "\t pixel_pipes: %d\n", |
| 790 | gpu->identity.pixel_pipes); |
| 791 | seq_printf(m, "\t vertex_output_buffer_size: %d\n", |
| 792 | gpu->identity.vertex_output_buffer_size); |
| 793 | seq_printf(m, "\t buffer_size: %d\n", |
| 794 | gpu->identity.buffer_size); |
| 795 | seq_printf(m, "\t instruction_count: %d\n", |
| 796 | gpu->identity.instruction_count); |
| 797 | seq_printf(m, "\t num_constants: %d\n", |
| 798 | gpu->identity.num_constants); |
Russell King | 602eb48 | 2016-01-24 17:36:04 +0000 | [diff] [blame] | 799 | seq_printf(m, "\t varyings_count: %d\n", |
| 800 | gpu->identity.varyings_count); |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 801 | |
| 802 | seq_printf(m, "\taxi: 0x%08x\n", axi); |
| 803 | seq_printf(m, "\tidle: 0x%08x\n", idle); |
| 804 | idle |= ~gpu->idle_mask & ~VIVS_HI_IDLE_STATE_AXI_LP; |
| 805 | if ((idle & VIVS_HI_IDLE_STATE_FE) == 0) |
| 806 | seq_puts(m, "\t FE is not idle\n"); |
| 807 | if ((idle & VIVS_HI_IDLE_STATE_DE) == 0) |
| 808 | seq_puts(m, "\t DE is not idle\n"); |
| 809 | if ((idle & VIVS_HI_IDLE_STATE_PE) == 0) |
| 810 | seq_puts(m, "\t PE is not idle\n"); |
| 811 | if ((idle & VIVS_HI_IDLE_STATE_SH) == 0) |
| 812 | seq_puts(m, "\t SH is not idle\n"); |
| 813 | if ((idle & VIVS_HI_IDLE_STATE_PA) == 0) |
| 814 | seq_puts(m, "\t PA is not idle\n"); |
| 815 | if ((idle & VIVS_HI_IDLE_STATE_SE) == 0) |
| 816 | seq_puts(m, "\t SE is not idle\n"); |
| 817 | if ((idle & VIVS_HI_IDLE_STATE_RA) == 0) |
| 818 | seq_puts(m, "\t RA is not idle\n"); |
| 819 | if ((idle & VIVS_HI_IDLE_STATE_TX) == 0) |
| 820 | seq_puts(m, "\t TX is not idle\n"); |
| 821 | if ((idle & VIVS_HI_IDLE_STATE_VG) == 0) |
| 822 | seq_puts(m, "\t VG is not idle\n"); |
| 823 | if ((idle & VIVS_HI_IDLE_STATE_IM) == 0) |
| 824 | seq_puts(m, "\t IM is not idle\n"); |
| 825 | if ((idle & VIVS_HI_IDLE_STATE_FP) == 0) |
| 826 | seq_puts(m, "\t FP is not idle\n"); |
| 827 | if ((idle & VIVS_HI_IDLE_STATE_TS) == 0) |
| 828 | seq_puts(m, "\t TS is not idle\n"); |
| 829 | if (idle & VIVS_HI_IDLE_STATE_AXI_LP) |
| 830 | seq_puts(m, "\t AXI low power mode\n"); |
| 831 | |
| 832 | if (gpu->identity.features & chipFeatures_DEBUG_MODE) { |
| 833 | u32 read0 = gpu_read(gpu, VIVS_MC_DEBUG_READ0); |
| 834 | u32 read1 = gpu_read(gpu, VIVS_MC_DEBUG_READ1); |
| 835 | u32 write = gpu_read(gpu, VIVS_MC_DEBUG_WRITE); |
| 836 | |
| 837 | seq_puts(m, "\tMC\n"); |
| 838 | seq_printf(m, "\t read0: 0x%08x\n", read0); |
| 839 | seq_printf(m, "\t read1: 0x%08x\n", read1); |
| 840 | seq_printf(m, "\t write: 0x%08x\n", write); |
| 841 | } |
| 842 | |
| 843 | seq_puts(m, "\tDMA "); |
| 844 | |
| 845 | if (debug.address[0] == debug.address[1] && |
| 846 | debug.state[0] == debug.state[1]) { |
| 847 | seq_puts(m, "seems to be stuck\n"); |
| 848 | } else if (debug.address[0] == debug.address[1]) { |
Masanari Iida | c01e015 | 2016-04-20 00:27:33 +0900 | [diff] [blame] | 849 | seq_puts(m, "address is constant\n"); |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 850 | } else { |
Masanari Iida | c01e015 | 2016-04-20 00:27:33 +0900 | [diff] [blame] | 851 | seq_puts(m, "is running\n"); |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 852 | } |
| 853 | |
| 854 | seq_printf(m, "\t address 0: 0x%08x\n", debug.address[0]); |
| 855 | seq_printf(m, "\t address 1: 0x%08x\n", debug.address[1]); |
| 856 | seq_printf(m, "\t state 0: 0x%08x\n", debug.state[0]); |
| 857 | seq_printf(m, "\t state 1: 0x%08x\n", debug.state[1]); |
| 858 | seq_printf(m, "\t last fetch 64 bit word: 0x%08x 0x%08x\n", |
| 859 | dma_lo, dma_hi); |
| 860 | |
| 861 | ret = 0; |
| 862 | |
| 863 | pm_runtime_mark_last_busy(gpu->dev); |
| 864 | pm_runtime_put_autosuspend(gpu->dev); |
| 865 | |
| 866 | return ret; |
| 867 | } |
| 868 | #endif |
| 869 | |
| 870 | /* |
| 871 | * Power Management: |
| 872 | */ |
| 873 | static int enable_clk(struct etnaviv_gpu *gpu) |
| 874 | { |
| 875 | if (gpu->clk_core) |
| 876 | clk_prepare_enable(gpu->clk_core); |
| 877 | if (gpu->clk_shader) |
| 878 | clk_prepare_enable(gpu->clk_shader); |
| 879 | |
| 880 | return 0; |
| 881 | } |
| 882 | |
| 883 | static int disable_clk(struct etnaviv_gpu *gpu) |
| 884 | { |
| 885 | if (gpu->clk_core) |
| 886 | clk_disable_unprepare(gpu->clk_core); |
| 887 | if (gpu->clk_shader) |
| 888 | clk_disable_unprepare(gpu->clk_shader); |
| 889 | |
| 890 | return 0; |
| 891 | } |
| 892 | |
| 893 | static int enable_axi(struct etnaviv_gpu *gpu) |
| 894 | { |
| 895 | if (gpu->clk_bus) |
| 896 | clk_prepare_enable(gpu->clk_bus); |
| 897 | |
| 898 | return 0; |
| 899 | } |
| 900 | |
| 901 | static int disable_axi(struct etnaviv_gpu *gpu) |
| 902 | { |
| 903 | if (gpu->clk_bus) |
| 904 | clk_disable_unprepare(gpu->clk_bus); |
| 905 | |
| 906 | return 0; |
| 907 | } |
| 908 | |
| 909 | /* |
| 910 | * Hangcheck detection for locked gpu: |
| 911 | */ |
| 912 | static void recover_worker(struct work_struct *work) |
| 913 | { |
| 914 | struct etnaviv_gpu *gpu = container_of(work, struct etnaviv_gpu, |
| 915 | recover_work); |
| 916 | unsigned long flags; |
| 917 | unsigned int i; |
| 918 | |
| 919 | dev_err(gpu->dev, "hangcheck recover!\n"); |
| 920 | |
| 921 | if (pm_runtime_get_sync(gpu->dev) < 0) |
| 922 | return; |
| 923 | |
| 924 | mutex_lock(&gpu->lock); |
| 925 | |
| 926 | /* Only catch the first event, or when manually re-armed */ |
| 927 | if (etnaviv_dump_core) { |
| 928 | etnaviv_core_dump(gpu); |
| 929 | etnaviv_dump_core = false; |
| 930 | } |
| 931 | |
| 932 | etnaviv_hw_reset(gpu); |
| 933 | |
| 934 | /* complete all events, the GPU won't do it after the reset */ |
| 935 | spin_lock_irqsave(&gpu->event_spinlock, flags); |
| 936 | for (i = 0; i < ARRAY_SIZE(gpu->event); i++) { |
| 937 | if (!gpu->event[i].used) |
| 938 | continue; |
| 939 | fence_signal(gpu->event[i].fence); |
| 940 | gpu->event[i].fence = NULL; |
| 941 | gpu->event[i].used = false; |
| 942 | complete(&gpu->event_free); |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 943 | } |
| 944 | spin_unlock_irqrestore(&gpu->event_spinlock, flags); |
| 945 | gpu->completed_fence = gpu->active_fence; |
| 946 | |
| 947 | etnaviv_gpu_hw_init(gpu); |
| 948 | gpu->switch_context = true; |
Russell King | f608631 | 2016-01-21 15:20:19 +0000 | [diff] [blame] | 949 | gpu->exec_state = -1; |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 950 | |
| 951 | mutex_unlock(&gpu->lock); |
| 952 | pm_runtime_mark_last_busy(gpu->dev); |
| 953 | pm_runtime_put_autosuspend(gpu->dev); |
| 954 | |
| 955 | /* Retire the buffer objects in a work */ |
| 956 | etnaviv_queue_work(gpu->drm, &gpu->retire_work); |
| 957 | } |
| 958 | |
| 959 | static void hangcheck_timer_reset(struct etnaviv_gpu *gpu) |
| 960 | { |
| 961 | DBG("%s", dev_name(gpu->dev)); |
| 962 | mod_timer(&gpu->hangcheck_timer, |
| 963 | round_jiffies_up(jiffies + DRM_ETNAVIV_HANGCHECK_JIFFIES)); |
| 964 | } |
| 965 | |
| 966 | static void hangcheck_handler(unsigned long data) |
| 967 | { |
| 968 | struct etnaviv_gpu *gpu = (struct etnaviv_gpu *)data; |
| 969 | u32 fence = gpu->completed_fence; |
| 970 | bool progress = false; |
| 971 | |
| 972 | if (fence != gpu->hangcheck_fence) { |
| 973 | gpu->hangcheck_fence = fence; |
| 974 | progress = true; |
| 975 | } |
| 976 | |
| 977 | if (!progress) { |
| 978 | u32 dma_addr = gpu_read(gpu, VIVS_FE_DMA_ADDRESS); |
| 979 | int change = dma_addr - gpu->hangcheck_dma_addr; |
| 980 | |
| 981 | if (change < 0 || change > 16) { |
| 982 | gpu->hangcheck_dma_addr = dma_addr; |
| 983 | progress = true; |
| 984 | } |
| 985 | } |
| 986 | |
| 987 | if (!progress && fence_after(gpu->active_fence, fence)) { |
| 988 | dev_err(gpu->dev, "hangcheck detected gpu lockup!\n"); |
| 989 | dev_err(gpu->dev, " completed fence: %u\n", fence); |
| 990 | dev_err(gpu->dev, " active fence: %u\n", |
| 991 | gpu->active_fence); |
| 992 | etnaviv_queue_work(gpu->drm, &gpu->recover_work); |
| 993 | } |
| 994 | |
| 995 | /* if still more pending work, reset the hangcheck timer: */ |
| 996 | if (fence_after(gpu->active_fence, gpu->hangcheck_fence)) |
| 997 | hangcheck_timer_reset(gpu); |
| 998 | } |
| 999 | |
| 1000 | static void hangcheck_disable(struct etnaviv_gpu *gpu) |
| 1001 | { |
| 1002 | del_timer_sync(&gpu->hangcheck_timer); |
| 1003 | cancel_work_sync(&gpu->recover_work); |
| 1004 | } |
| 1005 | |
| 1006 | /* fence object management */ |
| 1007 | struct etnaviv_fence { |
| 1008 | struct etnaviv_gpu *gpu; |
| 1009 | struct fence base; |
| 1010 | }; |
| 1011 | |
| 1012 | static inline struct etnaviv_fence *to_etnaviv_fence(struct fence *fence) |
| 1013 | { |
| 1014 | return container_of(fence, struct etnaviv_fence, base); |
| 1015 | } |
| 1016 | |
| 1017 | static const char *etnaviv_fence_get_driver_name(struct fence *fence) |
| 1018 | { |
| 1019 | return "etnaviv"; |
| 1020 | } |
| 1021 | |
| 1022 | static const char *etnaviv_fence_get_timeline_name(struct fence *fence) |
| 1023 | { |
| 1024 | struct etnaviv_fence *f = to_etnaviv_fence(fence); |
| 1025 | |
| 1026 | return dev_name(f->gpu->dev); |
| 1027 | } |
| 1028 | |
| 1029 | static bool etnaviv_fence_enable_signaling(struct fence *fence) |
| 1030 | { |
| 1031 | return true; |
| 1032 | } |
| 1033 | |
| 1034 | static bool etnaviv_fence_signaled(struct fence *fence) |
| 1035 | { |
| 1036 | struct etnaviv_fence *f = to_etnaviv_fence(fence); |
| 1037 | |
| 1038 | return fence_completed(f->gpu, f->base.seqno); |
| 1039 | } |
| 1040 | |
| 1041 | static void etnaviv_fence_release(struct fence *fence) |
| 1042 | { |
| 1043 | struct etnaviv_fence *f = to_etnaviv_fence(fence); |
| 1044 | |
| 1045 | kfree_rcu(f, base.rcu); |
| 1046 | } |
| 1047 | |
| 1048 | static const struct fence_ops etnaviv_fence_ops = { |
| 1049 | .get_driver_name = etnaviv_fence_get_driver_name, |
| 1050 | .get_timeline_name = etnaviv_fence_get_timeline_name, |
| 1051 | .enable_signaling = etnaviv_fence_enable_signaling, |
| 1052 | .signaled = etnaviv_fence_signaled, |
| 1053 | .wait = fence_default_wait, |
| 1054 | .release = etnaviv_fence_release, |
| 1055 | }; |
| 1056 | |
| 1057 | static struct fence *etnaviv_gpu_fence_alloc(struct etnaviv_gpu *gpu) |
| 1058 | { |
| 1059 | struct etnaviv_fence *f; |
| 1060 | |
| 1061 | f = kzalloc(sizeof(*f), GFP_KERNEL); |
| 1062 | if (!f) |
| 1063 | return NULL; |
| 1064 | |
| 1065 | f->gpu = gpu; |
| 1066 | |
| 1067 | fence_init(&f->base, &etnaviv_fence_ops, &gpu->fence_spinlock, |
| 1068 | gpu->fence_context, ++gpu->next_fence); |
| 1069 | |
| 1070 | return &f->base; |
| 1071 | } |
| 1072 | |
| 1073 | int etnaviv_gpu_fence_sync_obj(struct etnaviv_gem_object *etnaviv_obj, |
| 1074 | unsigned int context, bool exclusive) |
| 1075 | { |
| 1076 | struct reservation_object *robj = etnaviv_obj->resv; |
| 1077 | struct reservation_object_list *fobj; |
| 1078 | struct fence *fence; |
| 1079 | int i, ret; |
| 1080 | |
| 1081 | if (!exclusive) { |
| 1082 | ret = reservation_object_reserve_shared(robj); |
| 1083 | if (ret) |
| 1084 | return ret; |
| 1085 | } |
| 1086 | |
| 1087 | /* |
| 1088 | * If we have any shared fences, then the exclusive fence |
| 1089 | * should be ignored as it will already have been signalled. |
| 1090 | */ |
| 1091 | fobj = reservation_object_get_list(robj); |
| 1092 | if (!fobj || fobj->shared_count == 0) { |
| 1093 | /* Wait on any existing exclusive fence which isn't our own */ |
| 1094 | fence = reservation_object_get_excl(robj); |
| 1095 | if (fence && fence->context != context) { |
| 1096 | ret = fence_wait(fence, true); |
| 1097 | if (ret) |
| 1098 | return ret; |
| 1099 | } |
| 1100 | } |
| 1101 | |
| 1102 | if (!exclusive || !fobj) |
| 1103 | return 0; |
| 1104 | |
| 1105 | for (i = 0; i < fobj->shared_count; i++) { |
| 1106 | fence = rcu_dereference_protected(fobj->shared[i], |
| 1107 | reservation_object_held(robj)); |
| 1108 | if (fence->context != context) { |
| 1109 | ret = fence_wait(fence, true); |
| 1110 | if (ret) |
| 1111 | return ret; |
| 1112 | } |
| 1113 | } |
| 1114 | |
| 1115 | return 0; |
| 1116 | } |
| 1117 | |
| 1118 | /* |
| 1119 | * event management: |
| 1120 | */ |
| 1121 | |
| 1122 | static unsigned int event_alloc(struct etnaviv_gpu *gpu) |
| 1123 | { |
| 1124 | unsigned long ret, flags; |
| 1125 | unsigned int i, event = ~0U; |
| 1126 | |
| 1127 | ret = wait_for_completion_timeout(&gpu->event_free, |
| 1128 | msecs_to_jiffies(10 * 10000)); |
| 1129 | if (!ret) |
| 1130 | dev_err(gpu->dev, "wait_for_completion_timeout failed"); |
| 1131 | |
| 1132 | spin_lock_irqsave(&gpu->event_spinlock, flags); |
| 1133 | |
| 1134 | /* find first free event */ |
| 1135 | for (i = 0; i < ARRAY_SIZE(gpu->event); i++) { |
| 1136 | if (gpu->event[i].used == false) { |
| 1137 | gpu->event[i].used = true; |
| 1138 | event = i; |
| 1139 | break; |
| 1140 | } |
| 1141 | } |
| 1142 | |
| 1143 | spin_unlock_irqrestore(&gpu->event_spinlock, flags); |
| 1144 | |
| 1145 | return event; |
| 1146 | } |
| 1147 | |
| 1148 | static void event_free(struct etnaviv_gpu *gpu, unsigned int event) |
| 1149 | { |
| 1150 | unsigned long flags; |
| 1151 | |
| 1152 | spin_lock_irqsave(&gpu->event_spinlock, flags); |
| 1153 | |
| 1154 | if (gpu->event[event].used == false) { |
| 1155 | dev_warn(gpu->dev, "event %u is already marked as free", |
| 1156 | event); |
| 1157 | spin_unlock_irqrestore(&gpu->event_spinlock, flags); |
| 1158 | } else { |
| 1159 | gpu->event[event].used = false; |
| 1160 | spin_unlock_irqrestore(&gpu->event_spinlock, flags); |
| 1161 | |
| 1162 | complete(&gpu->event_free); |
| 1163 | } |
| 1164 | } |
| 1165 | |
| 1166 | /* |
| 1167 | * Cmdstream submission/retirement: |
| 1168 | */ |
| 1169 | |
| 1170 | struct etnaviv_cmdbuf *etnaviv_gpu_cmdbuf_new(struct etnaviv_gpu *gpu, u32 size, |
| 1171 | size_t nr_bos) |
| 1172 | { |
| 1173 | struct etnaviv_cmdbuf *cmdbuf; |
Russell King | b6325f4 | 2016-01-21 15:20:50 +0000 | [diff] [blame] | 1174 | size_t sz = size_vstruct(nr_bos, sizeof(cmdbuf->bo_map[0]), |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 1175 | sizeof(*cmdbuf)); |
| 1176 | |
| 1177 | cmdbuf = kzalloc(sz, GFP_KERNEL); |
| 1178 | if (!cmdbuf) |
| 1179 | return NULL; |
| 1180 | |
Luis R. Rodriguez | f6e4566 | 2016-01-22 18:34:22 -0800 | [diff] [blame] | 1181 | cmdbuf->vaddr = dma_alloc_wc(gpu->dev, size, &cmdbuf->paddr, |
| 1182 | GFP_KERNEL); |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 1183 | if (!cmdbuf->vaddr) { |
| 1184 | kfree(cmdbuf); |
| 1185 | return NULL; |
| 1186 | } |
| 1187 | |
| 1188 | cmdbuf->gpu = gpu; |
| 1189 | cmdbuf->size = size; |
| 1190 | |
| 1191 | return cmdbuf; |
| 1192 | } |
| 1193 | |
| 1194 | void etnaviv_gpu_cmdbuf_free(struct etnaviv_cmdbuf *cmdbuf) |
| 1195 | { |
Luis R. Rodriguez | f6e4566 | 2016-01-22 18:34:22 -0800 | [diff] [blame] | 1196 | dma_free_wc(cmdbuf->gpu->dev, cmdbuf->size, cmdbuf->vaddr, |
| 1197 | cmdbuf->paddr); |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 1198 | kfree(cmdbuf); |
| 1199 | } |
| 1200 | |
| 1201 | static void retire_worker(struct work_struct *work) |
| 1202 | { |
| 1203 | struct etnaviv_gpu *gpu = container_of(work, struct etnaviv_gpu, |
| 1204 | retire_work); |
| 1205 | u32 fence = gpu->completed_fence; |
| 1206 | struct etnaviv_cmdbuf *cmdbuf, *tmp; |
| 1207 | unsigned int i; |
| 1208 | |
| 1209 | mutex_lock(&gpu->lock); |
| 1210 | list_for_each_entry_safe(cmdbuf, tmp, &gpu->active_cmd_list, node) { |
| 1211 | if (!fence_is_signaled(cmdbuf->fence)) |
| 1212 | break; |
| 1213 | |
| 1214 | list_del(&cmdbuf->node); |
| 1215 | fence_put(cmdbuf->fence); |
| 1216 | |
| 1217 | for (i = 0; i < cmdbuf->nr_bos; i++) { |
Russell King | b6325f4 | 2016-01-21 15:20:50 +0000 | [diff] [blame] | 1218 | struct etnaviv_vram_mapping *mapping = cmdbuf->bo_map[i]; |
| 1219 | struct etnaviv_gem_object *etnaviv_obj = mapping->object; |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 1220 | |
| 1221 | atomic_dec(&etnaviv_obj->gpu_active); |
| 1222 | /* drop the refcount taken in etnaviv_gpu_submit */ |
Russell King | b6325f4 | 2016-01-21 15:20:50 +0000 | [diff] [blame] | 1223 | etnaviv_gem_mapping_unreference(mapping); |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 1224 | } |
| 1225 | |
| 1226 | etnaviv_gpu_cmdbuf_free(cmdbuf); |
Lucas Stach | d9fd0c7 | 2016-01-07 12:43:15 +0100 | [diff] [blame] | 1227 | /* |
| 1228 | * We need to balance the runtime PM count caused by |
| 1229 | * each submission. Upon submission, we increment |
| 1230 | * the runtime PM counter, and allocate one event. |
| 1231 | * So here, we put the runtime PM count for each |
| 1232 | * completed event. |
| 1233 | */ |
| 1234 | pm_runtime_put_autosuspend(gpu->dev); |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 1235 | } |
| 1236 | |
| 1237 | gpu->retired_fence = fence; |
| 1238 | |
| 1239 | mutex_unlock(&gpu->lock); |
| 1240 | |
| 1241 | wake_up_all(&gpu->fence_event); |
| 1242 | } |
| 1243 | |
| 1244 | int etnaviv_gpu_wait_fence_interruptible(struct etnaviv_gpu *gpu, |
| 1245 | u32 fence, struct timespec *timeout) |
| 1246 | { |
| 1247 | int ret; |
| 1248 | |
| 1249 | if (fence_after(fence, gpu->next_fence)) { |
| 1250 | DRM_ERROR("waiting on invalid fence: %u (of %u)\n", |
| 1251 | fence, gpu->next_fence); |
| 1252 | return -EINVAL; |
| 1253 | } |
| 1254 | |
| 1255 | if (!timeout) { |
| 1256 | /* No timeout was requested: just test for completion */ |
| 1257 | ret = fence_completed(gpu, fence) ? 0 : -EBUSY; |
| 1258 | } else { |
| 1259 | unsigned long remaining = etnaviv_timeout_to_jiffies(timeout); |
| 1260 | |
| 1261 | ret = wait_event_interruptible_timeout(gpu->fence_event, |
| 1262 | fence_completed(gpu, fence), |
| 1263 | remaining); |
| 1264 | if (ret == 0) { |
| 1265 | DBG("timeout waiting for fence: %u (retired: %u completed: %u)", |
| 1266 | fence, gpu->retired_fence, |
| 1267 | gpu->completed_fence); |
| 1268 | ret = -ETIMEDOUT; |
| 1269 | } else if (ret != -ERESTARTSYS) { |
| 1270 | ret = 0; |
| 1271 | } |
| 1272 | } |
| 1273 | |
| 1274 | return ret; |
| 1275 | } |
| 1276 | |
| 1277 | /* |
| 1278 | * Wait for an object to become inactive. This, on it's own, is not race |
| 1279 | * free: the object is moved by the retire worker off the active list, and |
| 1280 | * then the iova is put. Moreover, the object could be re-submitted just |
| 1281 | * after we notice that it's become inactive. |
| 1282 | * |
| 1283 | * Although the retirement happens under the gpu lock, we don't want to hold |
| 1284 | * that lock in this function while waiting. |
| 1285 | */ |
| 1286 | int etnaviv_gpu_wait_obj_inactive(struct etnaviv_gpu *gpu, |
| 1287 | struct etnaviv_gem_object *etnaviv_obj, struct timespec *timeout) |
| 1288 | { |
| 1289 | unsigned long remaining; |
| 1290 | long ret; |
| 1291 | |
| 1292 | if (!timeout) |
| 1293 | return !is_active(etnaviv_obj) ? 0 : -EBUSY; |
| 1294 | |
| 1295 | remaining = etnaviv_timeout_to_jiffies(timeout); |
| 1296 | |
| 1297 | ret = wait_event_interruptible_timeout(gpu->fence_event, |
| 1298 | !is_active(etnaviv_obj), |
| 1299 | remaining); |
| 1300 | if (ret > 0) { |
| 1301 | struct etnaviv_drm_private *priv = gpu->drm->dev_private; |
| 1302 | |
| 1303 | /* Synchronise with the retire worker */ |
| 1304 | flush_workqueue(priv->wq); |
| 1305 | return 0; |
| 1306 | } else if (ret == -ERESTARTSYS) { |
| 1307 | return -ERESTARTSYS; |
| 1308 | } else { |
| 1309 | return -ETIMEDOUT; |
| 1310 | } |
| 1311 | } |
| 1312 | |
| 1313 | int etnaviv_gpu_pm_get_sync(struct etnaviv_gpu *gpu) |
| 1314 | { |
| 1315 | return pm_runtime_get_sync(gpu->dev); |
| 1316 | } |
| 1317 | |
| 1318 | void etnaviv_gpu_pm_put(struct etnaviv_gpu *gpu) |
| 1319 | { |
| 1320 | pm_runtime_mark_last_busy(gpu->dev); |
| 1321 | pm_runtime_put_autosuspend(gpu->dev); |
| 1322 | } |
| 1323 | |
| 1324 | /* add bo's to gpu's ring, and kick gpu: */ |
| 1325 | int etnaviv_gpu_submit(struct etnaviv_gpu *gpu, |
| 1326 | struct etnaviv_gem_submit *submit, struct etnaviv_cmdbuf *cmdbuf) |
| 1327 | { |
| 1328 | struct fence *fence; |
| 1329 | unsigned int event, i; |
| 1330 | int ret; |
| 1331 | |
| 1332 | ret = etnaviv_gpu_pm_get_sync(gpu); |
| 1333 | if (ret < 0) |
| 1334 | return ret; |
| 1335 | |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 1336 | /* |
| 1337 | * TODO |
| 1338 | * |
| 1339 | * - flush |
| 1340 | * - data endian |
| 1341 | * - prefetch |
| 1342 | * |
| 1343 | */ |
| 1344 | |
| 1345 | event = event_alloc(gpu); |
| 1346 | if (unlikely(event == ~0U)) { |
| 1347 | DRM_ERROR("no free event\n"); |
| 1348 | ret = -EBUSY; |
Lucas Stach | d985349 | 2016-07-28 11:50:48 +0200 | [diff] [blame] | 1349 | goto out_pm_put; |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 1350 | } |
| 1351 | |
| 1352 | fence = etnaviv_gpu_fence_alloc(gpu); |
| 1353 | if (!fence) { |
| 1354 | event_free(gpu, event); |
| 1355 | ret = -ENOMEM; |
Lucas Stach | d985349 | 2016-07-28 11:50:48 +0200 | [diff] [blame] | 1356 | goto out_pm_put; |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 1357 | } |
| 1358 | |
Lucas Stach | d985349 | 2016-07-28 11:50:48 +0200 | [diff] [blame] | 1359 | mutex_lock(&gpu->lock); |
| 1360 | |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 1361 | gpu->event[event].fence = fence; |
| 1362 | submit->fence = fence->seqno; |
| 1363 | gpu->active_fence = submit->fence; |
| 1364 | |
| 1365 | if (gpu->lastctx != cmdbuf->ctx) { |
| 1366 | gpu->mmu->need_flush = true; |
| 1367 | gpu->switch_context = true; |
| 1368 | gpu->lastctx = cmdbuf->ctx; |
| 1369 | } |
| 1370 | |
| 1371 | etnaviv_buffer_queue(gpu, event, cmdbuf); |
| 1372 | |
| 1373 | cmdbuf->fence = fence; |
| 1374 | list_add_tail(&cmdbuf->node, &gpu->active_cmd_list); |
| 1375 | |
| 1376 | /* We're committed to adding this command buffer, hold a PM reference */ |
| 1377 | pm_runtime_get_noresume(gpu->dev); |
| 1378 | |
| 1379 | for (i = 0; i < submit->nr_bos; i++) { |
| 1380 | struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj; |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 1381 | |
Russell King | b6325f4 | 2016-01-21 15:20:50 +0000 | [diff] [blame] | 1382 | /* Each cmdbuf takes a refcount on the mapping */ |
| 1383 | etnaviv_gem_mapping_reference(submit->bos[i].mapping); |
| 1384 | cmdbuf->bo_map[i] = submit->bos[i].mapping; |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 1385 | atomic_inc(&etnaviv_obj->gpu_active); |
| 1386 | |
| 1387 | if (submit->bos[i].flags & ETNA_SUBMIT_BO_WRITE) |
| 1388 | reservation_object_add_excl_fence(etnaviv_obj->resv, |
| 1389 | fence); |
| 1390 | else |
| 1391 | reservation_object_add_shared_fence(etnaviv_obj->resv, |
| 1392 | fence); |
| 1393 | } |
| 1394 | cmdbuf->nr_bos = submit->nr_bos; |
| 1395 | hangcheck_timer_reset(gpu); |
| 1396 | ret = 0; |
| 1397 | |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 1398 | mutex_unlock(&gpu->lock); |
| 1399 | |
Lucas Stach | d985349 | 2016-07-28 11:50:48 +0200 | [diff] [blame] | 1400 | out_pm_put: |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 1401 | etnaviv_gpu_pm_put(gpu); |
| 1402 | |
| 1403 | return ret; |
| 1404 | } |
| 1405 | |
| 1406 | /* |
| 1407 | * Init/Cleanup: |
| 1408 | */ |
| 1409 | static irqreturn_t irq_handler(int irq, void *data) |
| 1410 | { |
| 1411 | struct etnaviv_gpu *gpu = data; |
| 1412 | irqreturn_t ret = IRQ_NONE; |
| 1413 | |
| 1414 | u32 intr = gpu_read(gpu, VIVS_HI_INTR_ACKNOWLEDGE); |
| 1415 | |
| 1416 | if (intr != 0) { |
| 1417 | int event; |
| 1418 | |
| 1419 | pm_runtime_mark_last_busy(gpu->dev); |
| 1420 | |
| 1421 | dev_dbg(gpu->dev, "intr 0x%08x\n", intr); |
| 1422 | |
| 1423 | if (intr & VIVS_HI_INTR_ACKNOWLEDGE_AXI_BUS_ERROR) { |
| 1424 | dev_err(gpu->dev, "AXI bus error\n"); |
| 1425 | intr &= ~VIVS_HI_INTR_ACKNOWLEDGE_AXI_BUS_ERROR; |
| 1426 | } |
| 1427 | |
| 1428 | while ((event = ffs(intr)) != 0) { |
| 1429 | struct fence *fence; |
| 1430 | |
| 1431 | event -= 1; |
| 1432 | |
| 1433 | intr &= ~(1 << event); |
| 1434 | |
| 1435 | dev_dbg(gpu->dev, "event %u\n", event); |
| 1436 | |
| 1437 | fence = gpu->event[event].fence; |
| 1438 | gpu->event[event].fence = NULL; |
| 1439 | fence_signal(fence); |
| 1440 | |
| 1441 | /* |
| 1442 | * Events can be processed out of order. Eg, |
| 1443 | * - allocate and queue event 0 |
| 1444 | * - allocate event 1 |
| 1445 | * - event 0 completes, we process it |
| 1446 | * - allocate and queue event 0 |
| 1447 | * - event 1 and event 0 complete |
| 1448 | * we can end up processing event 0 first, then 1. |
| 1449 | */ |
| 1450 | if (fence_after(fence->seqno, gpu->completed_fence)) |
| 1451 | gpu->completed_fence = fence->seqno; |
| 1452 | |
| 1453 | event_free(gpu, event); |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 1454 | } |
| 1455 | |
| 1456 | /* Retire the buffer objects in a work */ |
| 1457 | etnaviv_queue_work(gpu->drm, &gpu->retire_work); |
| 1458 | |
| 1459 | ret = IRQ_HANDLED; |
| 1460 | } |
| 1461 | |
| 1462 | return ret; |
| 1463 | } |
| 1464 | |
| 1465 | static int etnaviv_gpu_clk_enable(struct etnaviv_gpu *gpu) |
| 1466 | { |
| 1467 | int ret; |
| 1468 | |
| 1469 | ret = enable_clk(gpu); |
| 1470 | if (ret) |
| 1471 | return ret; |
| 1472 | |
| 1473 | ret = enable_axi(gpu); |
| 1474 | if (ret) { |
| 1475 | disable_clk(gpu); |
| 1476 | return ret; |
| 1477 | } |
| 1478 | |
| 1479 | return 0; |
| 1480 | } |
| 1481 | |
| 1482 | static int etnaviv_gpu_clk_disable(struct etnaviv_gpu *gpu) |
| 1483 | { |
| 1484 | int ret; |
| 1485 | |
| 1486 | ret = disable_axi(gpu); |
| 1487 | if (ret) |
| 1488 | return ret; |
| 1489 | |
| 1490 | ret = disable_clk(gpu); |
| 1491 | if (ret) |
| 1492 | return ret; |
| 1493 | |
| 1494 | return 0; |
| 1495 | } |
| 1496 | |
| 1497 | static int etnaviv_gpu_hw_suspend(struct etnaviv_gpu *gpu) |
| 1498 | { |
| 1499 | if (gpu->buffer) { |
| 1500 | unsigned long timeout; |
| 1501 | |
| 1502 | /* Replace the last WAIT with END */ |
| 1503 | etnaviv_buffer_end(gpu); |
| 1504 | |
| 1505 | /* |
| 1506 | * We know that only the FE is busy here, this should |
| 1507 | * happen quickly (as the WAIT is only 200 cycles). If |
| 1508 | * we fail, just warn and continue. |
| 1509 | */ |
| 1510 | timeout = jiffies + msecs_to_jiffies(100); |
| 1511 | do { |
| 1512 | u32 idle = gpu_read(gpu, VIVS_HI_IDLE_STATE); |
| 1513 | |
| 1514 | if ((idle & gpu->idle_mask) == gpu->idle_mask) |
| 1515 | break; |
| 1516 | |
| 1517 | if (time_is_before_jiffies(timeout)) { |
| 1518 | dev_warn(gpu->dev, |
| 1519 | "timed out waiting for idle: idle=0x%x\n", |
| 1520 | idle); |
| 1521 | break; |
| 1522 | } |
| 1523 | |
| 1524 | udelay(5); |
| 1525 | } while (1); |
| 1526 | } |
| 1527 | |
| 1528 | return etnaviv_gpu_clk_disable(gpu); |
| 1529 | } |
| 1530 | |
| 1531 | #ifdef CONFIG_PM |
| 1532 | static int etnaviv_gpu_hw_resume(struct etnaviv_gpu *gpu) |
| 1533 | { |
| 1534 | u32 clock; |
| 1535 | int ret; |
| 1536 | |
| 1537 | ret = mutex_lock_killable(&gpu->lock); |
| 1538 | if (ret) |
| 1539 | return ret; |
| 1540 | |
| 1541 | clock = VIVS_HI_CLOCK_CONTROL_DISABLE_DEBUG_REGISTERS | |
| 1542 | VIVS_HI_CLOCK_CONTROL_FSCALE_VAL(0x40); |
| 1543 | |
| 1544 | etnaviv_gpu_load_clock(gpu, clock); |
| 1545 | etnaviv_gpu_hw_init(gpu); |
| 1546 | |
| 1547 | gpu->switch_context = true; |
Russell King | f608631 | 2016-01-21 15:20:19 +0000 | [diff] [blame] | 1548 | gpu->exec_state = -1; |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 1549 | |
| 1550 | mutex_unlock(&gpu->lock); |
| 1551 | |
| 1552 | return 0; |
| 1553 | } |
| 1554 | #endif |
| 1555 | |
| 1556 | static int etnaviv_gpu_bind(struct device *dev, struct device *master, |
| 1557 | void *data) |
| 1558 | { |
| 1559 | struct drm_device *drm = data; |
| 1560 | struct etnaviv_drm_private *priv = drm->dev_private; |
| 1561 | struct etnaviv_gpu *gpu = dev_get_drvdata(dev); |
| 1562 | int ret; |
| 1563 | |
| 1564 | #ifdef CONFIG_PM |
| 1565 | ret = pm_runtime_get_sync(gpu->dev); |
| 1566 | #else |
| 1567 | ret = etnaviv_gpu_clk_enable(gpu); |
| 1568 | #endif |
| 1569 | if (ret < 0) |
| 1570 | return ret; |
| 1571 | |
| 1572 | gpu->drm = drm; |
| 1573 | gpu->fence_context = fence_context_alloc(1); |
| 1574 | spin_lock_init(&gpu->fence_spinlock); |
| 1575 | |
| 1576 | INIT_LIST_HEAD(&gpu->active_cmd_list); |
| 1577 | INIT_WORK(&gpu->retire_work, retire_worker); |
| 1578 | INIT_WORK(&gpu->recover_work, recover_worker); |
| 1579 | init_waitqueue_head(&gpu->fence_event); |
| 1580 | |
Lucas Stach | 946dd8d | 2016-03-23 18:24:45 +0100 | [diff] [blame] | 1581 | setup_deferrable_timer(&gpu->hangcheck_timer, hangcheck_handler, |
| 1582 | (unsigned long)gpu); |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 1583 | |
| 1584 | priv->gpu[priv->num_gpus++] = gpu; |
| 1585 | |
| 1586 | pm_runtime_mark_last_busy(gpu->dev); |
| 1587 | pm_runtime_put_autosuspend(gpu->dev); |
| 1588 | |
| 1589 | return 0; |
| 1590 | } |
| 1591 | |
| 1592 | static void etnaviv_gpu_unbind(struct device *dev, struct device *master, |
| 1593 | void *data) |
| 1594 | { |
| 1595 | struct etnaviv_gpu *gpu = dev_get_drvdata(dev); |
| 1596 | |
| 1597 | DBG("%s", dev_name(gpu->dev)); |
| 1598 | |
| 1599 | hangcheck_disable(gpu); |
| 1600 | |
| 1601 | #ifdef CONFIG_PM |
| 1602 | pm_runtime_get_sync(gpu->dev); |
| 1603 | pm_runtime_put_sync_suspend(gpu->dev); |
| 1604 | #else |
| 1605 | etnaviv_gpu_hw_suspend(gpu); |
| 1606 | #endif |
| 1607 | |
| 1608 | if (gpu->buffer) { |
| 1609 | etnaviv_gpu_cmdbuf_free(gpu->buffer); |
| 1610 | gpu->buffer = NULL; |
| 1611 | } |
| 1612 | |
| 1613 | if (gpu->mmu) { |
| 1614 | etnaviv_iommu_destroy(gpu->mmu); |
| 1615 | gpu->mmu = NULL; |
| 1616 | } |
| 1617 | |
| 1618 | gpu->drm = NULL; |
| 1619 | } |
| 1620 | |
| 1621 | static const struct component_ops gpu_ops = { |
| 1622 | .bind = etnaviv_gpu_bind, |
| 1623 | .unbind = etnaviv_gpu_unbind, |
| 1624 | }; |
| 1625 | |
| 1626 | static const struct of_device_id etnaviv_gpu_match[] = { |
| 1627 | { |
| 1628 | .compatible = "vivante,gc" |
| 1629 | }, |
| 1630 | { /* sentinel */ } |
| 1631 | }; |
| 1632 | |
| 1633 | static int etnaviv_gpu_platform_probe(struct platform_device *pdev) |
| 1634 | { |
| 1635 | struct device *dev = &pdev->dev; |
| 1636 | struct etnaviv_gpu *gpu; |
| 1637 | int err = 0; |
| 1638 | |
| 1639 | gpu = devm_kzalloc(dev, sizeof(*gpu), GFP_KERNEL); |
| 1640 | if (!gpu) |
| 1641 | return -ENOMEM; |
| 1642 | |
| 1643 | gpu->dev = &pdev->dev; |
| 1644 | mutex_init(&gpu->lock); |
| 1645 | |
The etnaviv authors | a8c21a5 | 2015-12-03 18:21:29 +0100 | [diff] [blame] | 1646 | /* Map registers: */ |
| 1647 | gpu->mmio = etnaviv_ioremap(pdev, NULL, dev_name(gpu->dev)); |
| 1648 | if (IS_ERR(gpu->mmio)) |
| 1649 | return PTR_ERR(gpu->mmio); |
| 1650 | |
| 1651 | /* Get Interrupt: */ |
| 1652 | gpu->irq = platform_get_irq(pdev, 0); |
| 1653 | if (gpu->irq < 0) { |
| 1654 | err = gpu->irq; |
| 1655 | dev_err(dev, "failed to get irq: %d\n", err); |
| 1656 | goto fail; |
| 1657 | } |
| 1658 | |
| 1659 | err = devm_request_irq(&pdev->dev, gpu->irq, irq_handler, 0, |
| 1660 | dev_name(gpu->dev), gpu); |
| 1661 | if (err) { |
| 1662 | dev_err(dev, "failed to request IRQ%u: %d\n", gpu->irq, err); |
| 1663 | goto fail; |
| 1664 | } |
| 1665 | |
| 1666 | /* Get Clocks: */ |
| 1667 | gpu->clk_bus = devm_clk_get(&pdev->dev, "bus"); |
| 1668 | DBG("clk_bus: %p", gpu->clk_bus); |
| 1669 | if (IS_ERR(gpu->clk_bus)) |
| 1670 | gpu->clk_bus = NULL; |
| 1671 | |
| 1672 | gpu->clk_core = devm_clk_get(&pdev->dev, "core"); |
| 1673 | DBG("clk_core: %p", gpu->clk_core); |
| 1674 | if (IS_ERR(gpu->clk_core)) |
| 1675 | gpu->clk_core = NULL; |
| 1676 | |
| 1677 | gpu->clk_shader = devm_clk_get(&pdev->dev, "shader"); |
| 1678 | DBG("clk_shader: %p", gpu->clk_shader); |
| 1679 | if (IS_ERR(gpu->clk_shader)) |
| 1680 | gpu->clk_shader = NULL; |
| 1681 | |
| 1682 | /* TODO: figure out max mapped size */ |
| 1683 | dev_set_drvdata(dev, gpu); |
| 1684 | |
| 1685 | /* |
| 1686 | * We treat the device as initially suspended. The runtime PM |
| 1687 | * autosuspend delay is rather arbitary: no measurements have |
| 1688 | * yet been performed to determine an appropriate value. |
| 1689 | */ |
| 1690 | pm_runtime_use_autosuspend(gpu->dev); |
| 1691 | pm_runtime_set_autosuspend_delay(gpu->dev, 200); |
| 1692 | pm_runtime_enable(gpu->dev); |
| 1693 | |
| 1694 | err = component_add(&pdev->dev, &gpu_ops); |
| 1695 | if (err < 0) { |
| 1696 | dev_err(&pdev->dev, "failed to register component: %d\n", err); |
| 1697 | goto fail; |
| 1698 | } |
| 1699 | |
| 1700 | return 0; |
| 1701 | |
| 1702 | fail: |
| 1703 | return err; |
| 1704 | } |
| 1705 | |
| 1706 | static int etnaviv_gpu_platform_remove(struct platform_device *pdev) |
| 1707 | { |
| 1708 | component_del(&pdev->dev, &gpu_ops); |
| 1709 | pm_runtime_disable(&pdev->dev); |
| 1710 | return 0; |
| 1711 | } |
| 1712 | |
| 1713 | #ifdef CONFIG_PM |
| 1714 | static int etnaviv_gpu_rpm_suspend(struct device *dev) |
| 1715 | { |
| 1716 | struct etnaviv_gpu *gpu = dev_get_drvdata(dev); |
| 1717 | u32 idle, mask; |
| 1718 | |
| 1719 | /* If we have outstanding fences, we're not idle */ |
| 1720 | if (gpu->completed_fence != gpu->active_fence) |
| 1721 | return -EBUSY; |
| 1722 | |
| 1723 | /* Check whether the hardware (except FE) is idle */ |
| 1724 | mask = gpu->idle_mask & ~VIVS_HI_IDLE_STATE_FE; |
| 1725 | idle = gpu_read(gpu, VIVS_HI_IDLE_STATE) & mask; |
| 1726 | if (idle != mask) |
| 1727 | return -EBUSY; |
| 1728 | |
| 1729 | return etnaviv_gpu_hw_suspend(gpu); |
| 1730 | } |
| 1731 | |
| 1732 | static int etnaviv_gpu_rpm_resume(struct device *dev) |
| 1733 | { |
| 1734 | struct etnaviv_gpu *gpu = dev_get_drvdata(dev); |
| 1735 | int ret; |
| 1736 | |
| 1737 | ret = etnaviv_gpu_clk_enable(gpu); |
| 1738 | if (ret) |
| 1739 | return ret; |
| 1740 | |
| 1741 | /* Re-initialise the basic hardware state */ |
| 1742 | if (gpu->drm && gpu->buffer) { |
| 1743 | ret = etnaviv_gpu_hw_resume(gpu); |
| 1744 | if (ret) { |
| 1745 | etnaviv_gpu_clk_disable(gpu); |
| 1746 | return ret; |
| 1747 | } |
| 1748 | } |
| 1749 | |
| 1750 | return 0; |
| 1751 | } |
| 1752 | #endif |
| 1753 | |
| 1754 | static const struct dev_pm_ops etnaviv_gpu_pm_ops = { |
| 1755 | SET_RUNTIME_PM_OPS(etnaviv_gpu_rpm_suspend, etnaviv_gpu_rpm_resume, |
| 1756 | NULL) |
| 1757 | }; |
| 1758 | |
| 1759 | struct platform_driver etnaviv_gpu_driver = { |
| 1760 | .driver = { |
| 1761 | .name = "etnaviv-gpu", |
| 1762 | .owner = THIS_MODULE, |
| 1763 | .pm = &etnaviv_gpu_pm_ops, |
| 1764 | .of_match_table = etnaviv_gpu_match, |
| 1765 | }, |
| 1766 | .probe = etnaviv_gpu_platform_probe, |
| 1767 | .remove = etnaviv_gpu_platform_remove, |
| 1768 | .id_table = gpu_ids, |
| 1769 | }; |