Christoph Lameter | f6ac235 | 2006-06-30 01:55:32 -0700 | [diff] [blame] | 1 | #ifndef _LINUX_VMSTAT_H |
| 2 | #define _LINUX_VMSTAT_H |
| 3 | |
| 4 | #include <linux/types.h> |
| 5 | #include <linux/percpu.h> |
Christoph Lameter | 2244b95 | 2006-06-30 01:55:33 -0700 | [diff] [blame] | 6 | #include <linux/config.h> |
| 7 | #include <linux/mmzone.h> |
| 8 | #include <asm/atomic.h> |
Christoph Lameter | f6ac235 | 2006-06-30 01:55:32 -0700 | [diff] [blame] | 9 | |
Christoph Lameter | f8891e5 | 2006-06-30 01:55:45 -0700 | [diff] [blame] | 10 | #ifdef CONFIG_VM_EVENT_COUNTERS |
Christoph Lameter | f6ac235 | 2006-06-30 01:55:32 -0700 | [diff] [blame] | 11 | /* |
Christoph Lameter | f8891e5 | 2006-06-30 01:55:45 -0700 | [diff] [blame] | 12 | * Light weight per cpu counter implementation. |
Christoph Lameter | f6ac235 | 2006-06-30 01:55:32 -0700 | [diff] [blame] | 13 | * |
Christoph Lameter | f8891e5 | 2006-06-30 01:55:45 -0700 | [diff] [blame] | 14 | * Counters should only be incremented and no critical kernel component |
| 15 | * should rely on the counter values. |
| 16 | * |
| 17 | * Counters are handled completely inline. On many platforms the code |
| 18 | * generated will simply be the increment of a global address. |
Christoph Lameter | f6ac235 | 2006-06-30 01:55:32 -0700 | [diff] [blame] | 19 | */ |
Christoph Lameter | f6ac235 | 2006-06-30 01:55:32 -0700 | [diff] [blame] | 20 | |
Christoph Lameter | 27bf71c | 2006-09-25 23:31:15 -0700 | [diff] [blame] | 21 | #ifdef CONFIG_ZONE_DMA32 |
| 22 | #define DMA32_ZONE(xx) xx##_DMA32, |
| 23 | #else |
| 24 | #define DMA32_ZONE(xx) |
| 25 | #endif |
| 26 | |
| 27 | #ifdef CONFIG_HIGHMEM |
| 28 | #define HIGHMEM_ZONE(xx) , xx##_HIGH |
| 29 | #else |
| 30 | #define HIGHMEM_ZONE(xx) |
| 31 | #endif |
| 32 | |
| 33 | #define FOR_ALL_ZONES(xx) xx##_DMA, DMA32_ZONE(xx) xx##_NORMAL HIGHMEM_ZONE(xx) |
Christoph Lameter | f6ac235 | 2006-06-30 01:55:32 -0700 | [diff] [blame] | 34 | |
Christoph Lameter | f8891e5 | 2006-06-30 01:55:45 -0700 | [diff] [blame] | 35 | enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, |
| 36 | FOR_ALL_ZONES(PGALLOC), |
| 37 | PGFREE, PGACTIVATE, PGDEACTIVATE, |
| 38 | PGFAULT, PGMAJFAULT, |
| 39 | FOR_ALL_ZONES(PGREFILL), |
| 40 | FOR_ALL_ZONES(PGSTEAL), |
| 41 | FOR_ALL_ZONES(PGSCAN_KSWAPD), |
| 42 | FOR_ALL_ZONES(PGSCAN_DIRECT), |
| 43 | PGINODESTEAL, SLABS_SCANNED, KSWAPD_STEAL, KSWAPD_INODESTEAL, |
| 44 | PAGEOUTRUN, ALLOCSTALL, PGROTATED, |
| 45 | NR_VM_EVENT_ITEMS |
Christoph Lameter | f6ac235 | 2006-06-30 01:55:32 -0700 | [diff] [blame] | 46 | }; |
| 47 | |
Christoph Lameter | f8891e5 | 2006-06-30 01:55:45 -0700 | [diff] [blame] | 48 | struct vm_event_state { |
| 49 | unsigned long event[NR_VM_EVENT_ITEMS]; |
| 50 | }; |
Christoph Lameter | f6ac235 | 2006-06-30 01:55:32 -0700 | [diff] [blame] | 51 | |
Christoph Lameter | f8891e5 | 2006-06-30 01:55:45 -0700 | [diff] [blame] | 52 | DECLARE_PER_CPU(struct vm_event_state, vm_event_states); |
Christoph Lameter | f6ac235 | 2006-06-30 01:55:32 -0700 | [diff] [blame] | 53 | |
Christoph Lameter | f8891e5 | 2006-06-30 01:55:45 -0700 | [diff] [blame] | 54 | static inline void __count_vm_event(enum vm_event_item item) |
| 55 | { |
Jan Blunck | 38cbcdc | 2006-08-05 12:14:14 -0700 | [diff] [blame] | 56 | __get_cpu_var(vm_event_states).event[item]++; |
Christoph Lameter | f8891e5 | 2006-06-30 01:55:45 -0700 | [diff] [blame] | 57 | } |
Christoph Lameter | f6ac235 | 2006-06-30 01:55:32 -0700 | [diff] [blame] | 58 | |
Christoph Lameter | f8891e5 | 2006-06-30 01:55:45 -0700 | [diff] [blame] | 59 | static inline void count_vm_event(enum vm_event_item item) |
| 60 | { |
Jan Blunck | 38cbcdc | 2006-08-05 12:14:14 -0700 | [diff] [blame] | 61 | get_cpu_var(vm_event_states).event[item]++; |
Christoph Lameter | f8891e5 | 2006-06-30 01:55:45 -0700 | [diff] [blame] | 62 | put_cpu(); |
| 63 | } |
Christoph Lameter | f6ac235 | 2006-06-30 01:55:32 -0700 | [diff] [blame] | 64 | |
Christoph Lameter | f8891e5 | 2006-06-30 01:55:45 -0700 | [diff] [blame] | 65 | static inline void __count_vm_events(enum vm_event_item item, long delta) |
| 66 | { |
Jan Blunck | 38cbcdc | 2006-08-05 12:14:14 -0700 | [diff] [blame] | 67 | __get_cpu_var(vm_event_states).event[item] += delta; |
Christoph Lameter | f8891e5 | 2006-06-30 01:55:45 -0700 | [diff] [blame] | 68 | } |
Christoph Lameter | f6ac235 | 2006-06-30 01:55:32 -0700 | [diff] [blame] | 69 | |
Christoph Lameter | f8891e5 | 2006-06-30 01:55:45 -0700 | [diff] [blame] | 70 | static inline void count_vm_events(enum vm_event_item item, long delta) |
| 71 | { |
Jan Blunck | 38cbcdc | 2006-08-05 12:14:14 -0700 | [diff] [blame] | 72 | get_cpu_var(vm_event_states).event[item] += delta; |
Christoph Lameter | f8891e5 | 2006-06-30 01:55:45 -0700 | [diff] [blame] | 73 | put_cpu(); |
| 74 | } |
Christoph Lameter | f6ac235 | 2006-06-30 01:55:32 -0700 | [diff] [blame] | 75 | |
Christoph Lameter | f8891e5 | 2006-06-30 01:55:45 -0700 | [diff] [blame] | 76 | extern void all_vm_events(unsigned long *); |
| 77 | extern void vm_events_fold_cpu(int cpu); |
Christoph Lameter | f6ac235 | 2006-06-30 01:55:32 -0700 | [diff] [blame] | 78 | |
Christoph Lameter | f8891e5 | 2006-06-30 01:55:45 -0700 | [diff] [blame] | 79 | #else |
Christoph Lameter | f6ac235 | 2006-06-30 01:55:32 -0700 | [diff] [blame] | 80 | |
Christoph Lameter | f8891e5 | 2006-06-30 01:55:45 -0700 | [diff] [blame] | 81 | /* Disable counters */ |
| 82 | #define get_cpu_vm_events(e) 0L |
| 83 | #define count_vm_event(e) do { } while (0) |
| 84 | #define count_vm_events(e,d) do { } while (0) |
| 85 | #define __count_vm_event(e) do { } while (0) |
| 86 | #define __count_vm_events(e,d) do { } while (0) |
| 87 | #define vm_events_fold_cpu(x) do { } while (0) |
Christoph Lameter | f6ac235 | 2006-06-30 01:55:32 -0700 | [diff] [blame] | 88 | |
Christoph Lameter | f8891e5 | 2006-06-30 01:55:45 -0700 | [diff] [blame] | 89 | #endif /* CONFIG_VM_EVENT_COUNTERS */ |
| 90 | |
| 91 | #define __count_zone_vm_events(item, zone, delta) \ |
| 92 | __count_vm_events(item##_DMA + zone_idx(zone), delta) |
Christoph Lameter | f6ac235 | 2006-06-30 01:55:32 -0700 | [diff] [blame] | 93 | |
Christoph Lameter | 2244b95 | 2006-06-30 01:55:33 -0700 | [diff] [blame] | 94 | /* |
| 95 | * Zone based page accounting with per cpu differentials. |
| 96 | */ |
| 97 | extern atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS]; |
Christoph Lameter | f6ac235 | 2006-06-30 01:55:32 -0700 | [diff] [blame] | 98 | |
Christoph Lameter | 2244b95 | 2006-06-30 01:55:33 -0700 | [diff] [blame] | 99 | static inline void zone_page_state_add(long x, struct zone *zone, |
| 100 | enum zone_stat_item item) |
| 101 | { |
| 102 | atomic_long_add(x, &zone->vm_stat[item]); |
| 103 | atomic_long_add(x, &vm_stat[item]); |
| 104 | } |
| 105 | |
| 106 | static inline unsigned long global_page_state(enum zone_stat_item item) |
| 107 | { |
| 108 | long x = atomic_long_read(&vm_stat[item]); |
| 109 | #ifdef CONFIG_SMP |
| 110 | if (x < 0) |
| 111 | x = 0; |
| 112 | #endif |
| 113 | return x; |
| 114 | } |
| 115 | |
| 116 | static inline unsigned long zone_page_state(struct zone *zone, |
| 117 | enum zone_stat_item item) |
| 118 | { |
| 119 | long x = atomic_long_read(&zone->vm_stat[item]); |
| 120 | #ifdef CONFIG_SMP |
| 121 | if (x < 0) |
| 122 | x = 0; |
| 123 | #endif |
| 124 | return x; |
| 125 | } |
| 126 | |
| 127 | #ifdef CONFIG_NUMA |
| 128 | /* |
| 129 | * Determine the per node value of a stat item. This function |
| 130 | * is called frequently in a NUMA machine, so try to be as |
| 131 | * frugal as possible. |
| 132 | */ |
| 133 | static inline unsigned long node_page_state(int node, |
| 134 | enum zone_stat_item item) |
| 135 | { |
| 136 | struct zone *zones = NODE_DATA(node)->node_zones; |
| 137 | |
| 138 | return |
Christoph Lameter | fb0e794 | 2006-09-25 23:31:13 -0700 | [diff] [blame] | 139 | #ifdef CONFIG_ZONE_DMA32 |
Christoph Lameter | 2244b95 | 2006-06-30 01:55:33 -0700 | [diff] [blame] | 140 | zone_page_state(&zones[ZONE_DMA32], item) + |
| 141 | #endif |
| 142 | zone_page_state(&zones[ZONE_NORMAL], item) + |
Christoph Lameter | 2244b95 | 2006-06-30 01:55:33 -0700 | [diff] [blame] | 143 | #ifdef CONFIG_HIGHMEM |
| 144 | zone_page_state(&zones[ZONE_HIGHMEM], item) + |
| 145 | #endif |
| 146 | zone_page_state(&zones[ZONE_DMA], item); |
| 147 | } |
Christoph Lameter | ca889e6 | 2006-06-30 01:55:44 -0700 | [diff] [blame] | 148 | |
| 149 | extern void zone_statistics(struct zonelist *, struct zone *); |
| 150 | |
Christoph Lameter | 2244b95 | 2006-06-30 01:55:33 -0700 | [diff] [blame] | 151 | #else |
Christoph Lameter | ca889e6 | 2006-06-30 01:55:44 -0700 | [diff] [blame] | 152 | |
Christoph Lameter | 2244b95 | 2006-06-30 01:55:33 -0700 | [diff] [blame] | 153 | #define node_page_state(node, item) global_page_state(item) |
Christoph Lameter | ca889e6 | 2006-06-30 01:55:44 -0700 | [diff] [blame] | 154 | #define zone_statistics(_zl,_z) do { } while (0) |
| 155 | |
| 156 | #endif /* CONFIG_NUMA */ |
Christoph Lameter | 2244b95 | 2006-06-30 01:55:33 -0700 | [diff] [blame] | 157 | |
| 158 | #define __add_zone_page_state(__z, __i, __d) \ |
| 159 | __mod_zone_page_state(__z, __i, __d) |
| 160 | #define __sub_zone_page_state(__z, __i, __d) \ |
| 161 | __mod_zone_page_state(__z, __i,-(__d)) |
| 162 | |
| 163 | #define add_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, __d) |
| 164 | #define sub_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, -(__d)) |
| 165 | |
| 166 | static inline void zap_zone_vm_stats(struct zone *zone) |
| 167 | { |
| 168 | memset(zone->vm_stat, 0, sizeof(zone->vm_stat)); |
| 169 | } |
| 170 | |
Christoph Lameter | ca889e6 | 2006-06-30 01:55:44 -0700 | [diff] [blame] | 171 | extern void inc_zone_state(struct zone *, enum zone_stat_item); |
| 172 | |
Christoph Lameter | 2244b95 | 2006-06-30 01:55:33 -0700 | [diff] [blame] | 173 | #ifdef CONFIG_SMP |
| 174 | void __mod_zone_page_state(struct zone *, enum zone_stat_item item, int); |
| 175 | void __inc_zone_page_state(struct page *, enum zone_stat_item); |
| 176 | void __dec_zone_page_state(struct page *, enum zone_stat_item); |
| 177 | |
| 178 | void mod_zone_page_state(struct zone *, enum zone_stat_item, int); |
| 179 | void inc_zone_page_state(struct page *, enum zone_stat_item); |
| 180 | void dec_zone_page_state(struct page *, enum zone_stat_item); |
| 181 | |
| 182 | extern void inc_zone_state(struct zone *, enum zone_stat_item); |
| 183 | |
| 184 | void refresh_cpu_vm_stats(int); |
| 185 | void refresh_vm_stats(void); |
| 186 | |
| 187 | #else /* CONFIG_SMP */ |
| 188 | |
| 189 | /* |
| 190 | * We do not maintain differentials in a single processor configuration. |
| 191 | * The functions directly modify the zone and global counters. |
| 192 | */ |
| 193 | static inline void __mod_zone_page_state(struct zone *zone, |
| 194 | enum zone_stat_item item, int delta) |
| 195 | { |
| 196 | zone_page_state_add(delta, zone, item); |
| 197 | } |
| 198 | |
Christoph Lameter | 7f4599e | 2006-07-10 04:44:30 -0700 | [diff] [blame] | 199 | static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item) |
| 200 | { |
| 201 | atomic_long_inc(&zone->vm_stat[item]); |
| 202 | atomic_long_inc(&vm_stat[item]); |
| 203 | } |
| 204 | |
Christoph Lameter | 2244b95 | 2006-06-30 01:55:33 -0700 | [diff] [blame] | 205 | static inline void __inc_zone_page_state(struct page *page, |
| 206 | enum zone_stat_item item) |
| 207 | { |
Christoph Lameter | 7f4599e | 2006-07-10 04:44:30 -0700 | [diff] [blame] | 208 | __inc_zone_state(page_zone(page), item); |
Christoph Lameter | 2244b95 | 2006-06-30 01:55:33 -0700 | [diff] [blame] | 209 | } |
| 210 | |
| 211 | static inline void __dec_zone_page_state(struct page *page, |
| 212 | enum zone_stat_item item) |
| 213 | { |
| 214 | atomic_long_dec(&page_zone(page)->vm_stat[item]); |
| 215 | atomic_long_dec(&vm_stat[item]); |
| 216 | } |
| 217 | |
| 218 | /* |
| 219 | * We only use atomic operations to update counters. So there is no need to |
| 220 | * disable interrupts. |
| 221 | */ |
| 222 | #define inc_zone_page_state __inc_zone_page_state |
| 223 | #define dec_zone_page_state __dec_zone_page_state |
| 224 | #define mod_zone_page_state __mod_zone_page_state |
| 225 | |
| 226 | static inline void refresh_cpu_vm_stats(int cpu) { } |
| 227 | static inline void refresh_vm_stats(void) { } |
| 228 | #endif |
| 229 | |
| 230 | #endif /* _LINUX_VMSTAT_H */ |