Sam Ravnborg | a00736e | 2008-06-19 20:26:19 +0200 | [diff] [blame] | 1 | #ifndef _SPARC64_BACKOFF_H |
| 2 | #define _SPARC64_BACKOFF_H |
| 3 | |
David S. Miller | 187818c | 2012-10-28 13:04:47 -0700 | [diff] [blame] | 4 | /* The macros in this file implement an exponential backoff facility |
| 5 | * for atomic operations. |
| 6 | * |
| 7 | * When multiple threads compete on an atomic operation, it is |
| 8 | * possible for one thread to be continually denied a successful |
| 9 | * completion of the compare-and-swap instruction. Heavily |
| 10 | * threaded cpu implementations like Niagara can compound this |
| 11 | * problem even further. |
| 12 | * |
| 13 | * When an atomic operation fails and needs to be retried, we spin a |
| 14 | * certain number of times. At each subsequent failure of the same |
| 15 | * operation we double the spin count, realizing an exponential |
| 16 | * backoff. |
| 17 | * |
| 18 | * When we spin, we try to use an operation that will cause the |
| 19 | * current cpu strand to block, and therefore make the core fully |
| 20 | * available to any other other runnable strands. There are two |
| 21 | * options, based upon cpu capabilities. |
| 22 | * |
| 23 | * On all cpus prior to SPARC-T4 we do three dummy reads of the |
| 24 | * condition code register. Each read blocks the strand for something |
| 25 | * between 40 and 50 cpu cycles. |
| 26 | * |
| 27 | * For SPARC-T4 and later we have a special "pause" instruction |
| 28 | * available. This is implemented using writes to register %asr27. |
| 29 | * The cpu will block the number of cycles written into the register, |
| 30 | * unless a disrupting trap happens first. SPARC-T4 specifically |
| 31 | * implements pause with a granularity of 8 cycles. Each strand has |
| 32 | * an internal pause counter which decrements every 8 cycles. So the |
| 33 | * chip shifts the %asr27 value down by 3 bits, and writes the result |
| 34 | * into the pause counter. If a value smaller than 8 is written, the |
| 35 | * chip blocks for 1 cycle. |
| 36 | * |
| 37 | * To achieve the same amount of backoff as the three %ccr reads give |
| 38 | * on earlier chips, we shift the backoff value up by 7 bits. (Three |
| 39 | * %ccr reads block for about 128 cycles, 1 << 7 == 128) We write the |
| 40 | * whole amount we want to block into the pause register, rather than |
| 41 | * loop writing 128 each time. |
| 42 | */ |
| 43 | |
Sam Ravnborg | a00736e | 2008-06-19 20:26:19 +0200 | [diff] [blame] | 44 | #define BACKOFF_LIMIT (4 * 1024) |
| 45 | |
| 46 | #ifdef CONFIG_SMP |
| 47 | |
| 48 | #define BACKOFF_SETUP(reg) \ |
| 49 | mov 1, reg |
| 50 | |
David S. Miller | 0f58189 | 2010-08-18 22:53:26 -0700 | [diff] [blame] | 51 | #define BACKOFF_LABEL(spin_label, continue_label) \ |
| 52 | spin_label |
| 53 | |
David S. Miller | e9b9eb5 | 2012-10-27 23:00:41 -0700 | [diff] [blame] | 54 | #define BACKOFF_SPIN(reg, tmp, label) \ |
| 55 | mov reg, tmp; \ |
| 56 | 88: rd %ccr, %g0; \ |
| 57 | rd %ccr, %g0; \ |
| 58 | rd %ccr, %g0; \ |
David S. Miller | 187818c | 2012-10-28 13:04:47 -0700 | [diff] [blame] | 59 | .section .pause_3insn_patch,"ax";\ |
David S. Miller | e9b9eb5 | 2012-10-27 23:00:41 -0700 | [diff] [blame] | 60 | .word 88b; \ |
| 61 | sllx tmp, 7, tmp; \ |
| 62 | wr tmp, 0, %asr27; \ |
| 63 | clr tmp; \ |
| 64 | .previous; \ |
| 65 | brnz,pt tmp, 88b; \ |
| 66 | sub tmp, 1, tmp; \ |
| 67 | set BACKOFF_LIMIT, tmp; \ |
| 68 | cmp reg, tmp; \ |
| 69 | bg,pn %xcc, label; \ |
| 70 | nop; \ |
| 71 | ba,pt %xcc, label; \ |
| 72 | sllx reg, 1, reg; |
Sam Ravnborg | a00736e | 2008-06-19 20:26:19 +0200 | [diff] [blame] | 73 | |
| 74 | #else |
| 75 | |
| 76 | #define BACKOFF_SETUP(reg) |
David S. Miller | 0f58189 | 2010-08-18 22:53:26 -0700 | [diff] [blame] | 77 | |
| 78 | #define BACKOFF_LABEL(spin_label, continue_label) \ |
| 79 | continue_label |
| 80 | |
| 81 | #define BACKOFF_SPIN(reg, tmp, label) |
Sam Ravnborg | a00736e | 2008-06-19 20:26:19 +0200 | [diff] [blame] | 82 | |
| 83 | #endif |
| 84 | |
| 85 | #endif /* _SPARC64_BACKOFF_H */ |