sewardj | de4a1d0 | 2002-03-22 01:27:54 +0000 | [diff] [blame] | 1 | |
| 2 | /*--------------------------------------------------------------------*/ |
| 3 | /*--- Management of the translation table and cache. ---*/ |
njn | 8bddf58 | 2005-05-13 23:40:55 +0000 | [diff] [blame] | 4 | /*--- m_transtab.c ---*/ |
sewardj | de4a1d0 | 2002-03-22 01:27:54 +0000 | [diff] [blame] | 5 | /*--------------------------------------------------------------------*/ |
| 6 | |
| 7 | /* |
njn | b9c427c | 2004-12-01 14:14:42 +0000 | [diff] [blame] | 8 | This file is part of Valgrind, a dynamic binary instrumentation |
| 9 | framework. |
sewardj | de4a1d0 | 2002-03-22 01:27:54 +0000 | [diff] [blame] | 10 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 11 | Copyright (C) 2000-2005 Julian Seward |
sewardj | de4a1d0 | 2002-03-22 01:27:54 +0000 | [diff] [blame] | 12 | jseward@acm.org |
sewardj | de4a1d0 | 2002-03-22 01:27:54 +0000 | [diff] [blame] | 13 | |
| 14 | This program is free software; you can redistribute it and/or |
| 15 | modify it under the terms of the GNU General Public License as |
| 16 | published by the Free Software Foundation; either version 2 of the |
| 17 | License, or (at your option) any later version. |
| 18 | |
| 19 | This program is distributed in the hope that it will be useful, but |
| 20 | WITHOUT ANY WARRANTY; without even the implied warranty of |
| 21 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| 22 | General Public License for more details. |
| 23 | |
| 24 | You should have received a copy of the GNU General Public License |
| 25 | along with this program; if not, write to the Free Software |
| 26 | Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA |
| 27 | 02111-1307, USA. |
| 28 | |
njn25 | e49d8e7 | 2002-09-23 09:36:25 +0000 | [diff] [blame] | 29 | The GNU General Public License is contained in the file COPYING. |
sewardj | de4a1d0 | 2002-03-22 01:27:54 +0000 | [diff] [blame] | 30 | */ |
| 31 | |
njn | c7561b9 | 2005-06-19 01:24:32 +0000 | [diff] [blame] | 32 | #include "pub_core_basics.h" |
sewardj | 45f4e7c | 2005-09-27 19:20:21 +0000 | [diff] [blame] | 33 | #include "pub_core_debuglog.h" |
cerion | 1f0d814 | 2005-12-23 00:57:03 +0000 | [diff] [blame] | 34 | #include "pub_core_machine.h" // For VG(machine_get_VexArchInfo) |
njn | 97405b2 | 2005-06-02 03:39:33 +0000 | [diff] [blame] | 35 | #include "pub_core_libcbase.h" |
njn | 132bfcc | 2005-06-04 19:16:06 +0000 | [diff] [blame] | 36 | #include "pub_core_libcassert.h" |
njn | 36a20fa | 2005-06-03 03:08:39 +0000 | [diff] [blame] | 37 | #include "pub_core_libcprint.h" |
njn | 2024234 | 2005-05-16 23:31:24 +0000 | [diff] [blame] | 38 | #include "pub_core_options.h" |
sewardj | 10f08cf | 2005-06-29 10:16:14 +0000 | [diff] [blame] | 39 | #include "pub_core_tooliface.h" // For VG_(details).avg_translation_sizeB |
njn | 8bddf58 | 2005-05-13 23:40:55 +0000 | [diff] [blame] | 40 | #include "pub_core_transtab.h" |
sewardj | 45f4e7c | 2005-09-27 19:20:21 +0000 | [diff] [blame] | 41 | #include "pub_core_aspacemgr.h" |
| 42 | #include "pub_core_mallocfree.h" // VG_(out_of_memory_NORETURN) |
sewardj | de4a1d0 | 2002-03-22 01:27:54 +0000 | [diff] [blame] | 43 | |
sewardj | 18d7513 | 2002-05-16 11:06:21 +0000 | [diff] [blame] | 44 | /* #define DEBUG_TRANSTAB */ |
| 45 | |
sewardj | de4a1d0 | 2002-03-22 01:27:54 +0000 | [diff] [blame] | 46 | |
sewardj | 6c3769f | 2002-11-29 01:02:45 +0000 | [diff] [blame] | 47 | /*-------------------------------------------------------------*/ |
| 48 | /*--- Management of the FIFO-based translation table+cache. ---*/ |
| 49 | /*-------------------------------------------------------------*/ |
sewardj | de4a1d0 | 2002-03-22 01:27:54 +0000 | [diff] [blame] | 50 | |
sewardj | 6c3769f | 2002-11-29 01:02:45 +0000 | [diff] [blame] | 51 | /*------------------ CONSTANTS ------------------*/ |
sewardj | de4a1d0 | 2002-03-22 01:27:54 +0000 | [diff] [blame] | 52 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 53 | /* Number of sectors the TC is divided into. If you need a larger |
| 54 | overall translation cache, increase this value. */ |
| 55 | #define N_SECTORS 8 |
sewardj | de4a1d0 | 2002-03-22 01:27:54 +0000 | [diff] [blame] | 56 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 57 | /* Number of TC entries in each sector. This needs to be a prime |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 58 | number to work properly, it must be <= 65535 (so that a TT index |
| 59 | fits in a UShort, leaving room for 0xFFFF(EC2TTE_DELETED) to denote |
| 60 | 'deleted') and it is strongly recommended not to change this. |
| 61 | 65521 is the largest prime <= 65535. */ |
| 62 | #define N_TTES_PER_SECTOR /*30011*/ /*40009*/ 65521 |
sewardj | de4a1d0 | 2002-03-22 01:27:54 +0000 | [diff] [blame] | 63 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 64 | /* Because each sector contains a hash table of TTEntries, we need to |
| 65 | specify the maximum allowable loading, after which the sector is |
| 66 | deemed full. */ |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 67 | #define SECTOR_TT_LIMIT_PERCENT 80 |
sewardj | de4a1d0 | 2002-03-22 01:27:54 +0000 | [diff] [blame] | 68 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 69 | /* The sector is deemed full when this many entries are in it. */ |
| 70 | #define N_TTES_PER_SECTOR_USABLE \ |
| 71 | ((N_TTES_PER_SECTOR * SECTOR_TT_LIMIT_PERCENT) / 100) |
sewardj | de4a1d0 | 2002-03-22 01:27:54 +0000 | [diff] [blame] | 72 | |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 73 | /* Equivalence classes for fast address range deletion. There are 1 + |
| 74 | 2^ECLASS_WIDTH bins. The highest one, ECLASS_MISC, describes an |
| 75 | address range which does not fall cleanly within any specific bin. |
| 76 | Note that ECLASS_SHIFT + ECLASS_WIDTH must be < 32. */ |
| 77 | #define ECLASS_SHIFT 11 |
| 78 | #define ECLASS_WIDTH 8 |
| 79 | #define ECLASS_MISC (1 << ECLASS_WIDTH) |
| 80 | #define ECLASS_N (1 + ECLASS_MISC) |
| 81 | |
| 82 | #define EC2TTE_DELETED 0xFFFF /* 16-bit special value */ |
| 83 | |
sewardj | de4a1d0 | 2002-03-22 01:27:54 +0000 | [diff] [blame] | 84 | |
sewardj | 6c3769f | 2002-11-29 01:02:45 +0000 | [diff] [blame] | 85 | /*------------------ TYPES ------------------*/ |
| 86 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 87 | /* A translation-cache entry is two parts: |
| 88 | - The guest address of the first (entry) bb in the translation, |
| 89 | as a 64-bit word. |
| 90 | - One or more 64-bit words containing the code. |
| 91 | It is supposed to be 64-bit aligned. |
| 92 | */ |
| 93 | /* |
sewardj | 6c3769f | 2002-11-29 01:02:45 +0000 | [diff] [blame] | 94 | typedef |
| 95 | struct { |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 96 | Addr64 orig_addr; |
| 97 | ULong code[0]; |
| 98 | } |
| 99 | TCEntry; |
| 100 | */ |
| 101 | |
| 102 | /* A translation-table entry. This indicates precisely which areas of |
| 103 | guest code are included in the translation, and contains all other |
| 104 | auxiliary info too. */ |
| 105 | typedef |
| 106 | struct { |
| 107 | /* Profiling only: the count and weight (arbitrary meaning) for |
| 108 | this translation. Weight is a property of the translation |
| 109 | itself and computed once when the translation is created. |
| 110 | Count is an entry count for the translation and is |
| 111 | incremented by 1 every time the translation is used, if we |
| 112 | are profiling. */ |
| 113 | UInt count; |
| 114 | UShort weight; |
| 115 | |
| 116 | /* Status of the slot. Note, we need to be able to do lazy |
| 117 | deletion, hence the Deleted state. */ |
| 118 | enum { InUse, Deleted, Empty } status; |
| 119 | |
| 120 | /* Pointer to the corresponding TCEntry (must be in the same |
| 121 | sector!) */ |
| 122 | ULong* tce; |
| 123 | |
| 124 | /* This is the original guest address that purportedly is the |
| 125 | entry point of the translation. You might think that .entry |
| 126 | should be the same as .vge->base[0], and most of the time it |
| 127 | is. However, when doing redirections, that is not the case. |
| 128 | .vge must always correctly describe the guest code sections |
| 129 | from which this translation was made. However, .entry may or |
| 130 | may not be a lie, depending on whether or not we're doing |
| 131 | redirection. */ |
| 132 | Addr64 entry; |
| 133 | |
| 134 | /* This structure describes precisely what ranges of guest code |
| 135 | the translation covers, so we can decide whether or not to |
| 136 | delete it when translations of a given address range are |
| 137 | invalidated. */ |
| 138 | VexGuestExtents vge; |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 139 | |
| 140 | /* Address range summary info: these are pointers back to |
| 141 | eclass[] entries in the containing Sector. Those entries in |
| 142 | turn point back here -- the two structures are mutually |
| 143 | redundant but both necessary to make fast deletions work. |
| 144 | The eclass info is similar to, and derived from, this entry's |
| 145 | 'vge' field, but it is not the same */ |
| 146 | UShort n_tte2ec; // # tte2ec pointers (1 to 3) |
| 147 | UShort tte2ec_ec[3]; // for each, the eclass # |
| 148 | UInt tte2ec_ix[3]; // and the index within the eclass. |
| 149 | // for i in 0 .. n_tte2ec-1 |
| 150 | // sec->ec2tte[ tte2ec_ec[i] ][ tte2ec_ix[i] ] |
| 151 | // should be the index |
| 152 | // of this TTEntry in the containing Sector's tt array. |
sewardj | 6c3769f | 2002-11-29 01:02:45 +0000 | [diff] [blame] | 153 | } |
| 154 | TTEntry; |
| 155 | |
sewardj | 4ccf707 | 2004-11-28 16:58:05 +0000 | [diff] [blame] | 156 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 157 | /* Finally, a sector itself. Each sector contains an array of |
| 158 | TCEntries, which hold code, and an array of TTEntries, containing |
| 159 | all required administrative info. Profiling is supported using the |
| 160 | TTEntry .count and .weight fields, if required. Each sector is |
| 161 | independent in that no cross-sector references are allowed. |
sewardj | 4ccf707 | 2004-11-28 16:58:05 +0000 | [diff] [blame] | 162 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 163 | If the sector is not in use, all three pointers are NULL and |
| 164 | tt_n_inuse is zero. |
| 165 | */ |
| 166 | typedef |
| 167 | struct { |
| 168 | /* The TCEntry area. Size of this depends on the average |
| 169 | translation size. We try and size it so it becomes full |
| 170 | precisely when this sector's translation table (tt) reaches |
| 171 | its load limit (SECTOR_TT_LIMIT_PERCENT). */ |
| 172 | ULong* tc; |
sewardj | 4ccf707 | 2004-11-28 16:58:05 +0000 | [diff] [blame] | 173 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 174 | /* The TTEntry array. This is a fixed size, always containing |
| 175 | exactly N_TTES_PER_SECTOR entries. */ |
| 176 | TTEntry* tt; |
sewardj | 4ccf707 | 2004-11-28 16:58:05 +0000 | [diff] [blame] | 177 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 178 | /* This points to the current allocation point in tc. */ |
| 179 | ULong* tc_next; |
sewardj | 6c3769f | 2002-11-29 01:02:45 +0000 | [diff] [blame] | 180 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 181 | /* The count of tt entries with state InUse. */ |
| 182 | Int tt_n_inuse; |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 183 | |
| 184 | /* Expandable arrays of tt indices for each of the ECLASS_N |
| 185 | address range equivalence classes. These hold indices into |
| 186 | the containing sector's tt array, which in turn should point |
| 187 | back here. */ |
| 188 | Int ec2tte_size[ECLASS_N]; |
| 189 | Int ec2tte_used[ECLASS_N]; |
| 190 | UShort* ec2tte[ECLASS_N]; |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 191 | } |
| 192 | Sector; |
sewardj | de4a1d0 | 2002-03-22 01:27:54 +0000 | [diff] [blame] | 193 | |
sewardj | de4a1d0 | 2002-03-22 01:27:54 +0000 | [diff] [blame] | 194 | |
sewardj | 6c3769f | 2002-11-29 01:02:45 +0000 | [diff] [blame] | 195 | /*------------------ DECLS ------------------*/ |
| 196 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 197 | /* The root data structure is an array of sectors. The index of the |
| 198 | youngest sector is recorded, and new translations are put into that |
| 199 | sector. When it fills up, we move along to the next sector and |
| 200 | start to fill that up, wrapping around at the end of the array. |
| 201 | That way, once all N_TC_SECTORS have been bought into use for the |
| 202 | first time, and are full, we then re-use the oldest sector, |
| 203 | endlessly. |
sewardj | 6c3769f | 2002-11-29 01:02:45 +0000 | [diff] [blame] | 204 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 205 | When running, youngest sector should be between >= 0 and < |
| 206 | N_TC_SECTORS. The initial -1 value indicates the TT/TC system is |
| 207 | not yet initialised. |
| 208 | */ |
| 209 | static Sector sectors[N_SECTORS]; |
| 210 | static Int youngest_sector = -1; |
sewardj | 6c3769f | 2002-11-29 01:02:45 +0000 | [diff] [blame] | 211 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 212 | /* The number of ULongs in each TCEntry area. This is computed once |
| 213 | at startup and does not change. */ |
| 214 | static Int tc_sector_szQ; |
nethercote | 92e7b7f | 2004-08-07 17:52:25 +0000 | [diff] [blame] | 215 | |
| 216 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 217 | /* Fast helper for the TC. A direct-mapped cache which holds a |
sewardj | c54ee50 | 2004-11-29 19:46:47 +0000 | [diff] [blame] | 218 | pointer to a TC entry which may or may not be the correct one, but |
sewardj | de4a1d0 | 2002-03-22 01:27:54 +0000 | [diff] [blame] | 219 | which we hope usually is. This array is referred to directly from |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 220 | <arch>/dispatch.S. |
sewardj | 8aef119 | 2002-07-24 09:36:36 +0000 | [diff] [blame] | 221 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 222 | Entries in tt_fast may point to any valid TC entry, regardless of |
| 223 | which sector it's in. Consequently we must be very careful to |
| 224 | invalidate this cache when TC entries are changed or disappear. |
| 225 | |
| 226 | A special TCEntry -- bogus_tc_entry -- must be pointed at to cause |
| 227 | that cache entry to miss. This relies on the assumption that no |
| 228 | guest code actually has an address of 0x1. |
| 229 | */ |
| 230 | /*global*/ ULong* VG_(tt_fast)[VG_TT_FAST_SIZE]; |
| 231 | |
| 232 | static ULong bogus_tc_entry = (Addr64)1; |
sewardj | 22854b9 | 2002-11-30 14:00:47 +0000 | [diff] [blame] | 233 | |
| 234 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 235 | /* For profiling, we have a parallel array of pointers to .count |
| 236 | fields in TT entries. Again, these pointers must be invalidated |
| 237 | when translations disappear. A NULL pointer suffices to indicate |
| 238 | an unused slot. |
sewardj | 6c3769f | 2002-11-29 01:02:45 +0000 | [diff] [blame] | 239 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 240 | tt_fast and tt_fastN change together: if tt_fast[i] points to |
| 241 | bogus_tc_entry then the corresponding tt_fastN[i] must be null. If |
| 242 | tt_fast[i] points to some TC entry somewhere, then tt_fastN[i] |
| 243 | *must* point to the .count field of the corresponding TT entry. |
| 244 | |
| 245 | tt_fast and tt_fastN are referred to from assembly code |
| 246 | (dispatch.S). |
| 247 | */ |
| 248 | /*global*/ UInt* VG_(tt_fastN)[VG_TT_FAST_SIZE]; |
| 249 | |
| 250 | |
sewardj | 663a1bd | 2005-04-24 11:22:44 +0000 | [diff] [blame] | 251 | /* Make sure we're not used before initialisation. */ |
| 252 | static Bool init_done = False; |
| 253 | |
| 254 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 255 | /*------------------ STATS DECLS ------------------*/ |
| 256 | |
| 257 | /* Number of fast-cache updates and flushes done. */ |
| 258 | ULong n_fast_flushes = 0; |
| 259 | ULong n_fast_updates = 0; |
| 260 | |
| 261 | /* Number of full lookups done. */ |
| 262 | ULong n_full_lookups = 0; |
| 263 | ULong n_lookup_probes = 0; |
| 264 | |
sewardj | 26412bd | 2005-07-07 10:05:05 +0000 | [diff] [blame] | 265 | /* Number/osize/tsize of translations entered; also the number of |
| 266 | those for which self-checking was requested. */ |
| 267 | ULong n_in_count = 0; |
| 268 | ULong n_in_osize = 0; |
| 269 | ULong n_in_tsize = 0; |
| 270 | ULong n_in_sc_count = 0; |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 271 | |
| 272 | /* Number/osize of translations discarded due to lack of space. */ |
| 273 | ULong n_dump_count = 0; |
| 274 | ULong n_dump_osize = 0; |
| 275 | |
| 276 | /* Number/osize of translations discarded due to requests to do so. */ |
| 277 | ULong n_disc_count = 0; |
| 278 | ULong n_disc_osize = 0; |
| 279 | |
| 280 | |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 281 | /*-------------------------------------------------------------*/ |
| 282 | /*--- Address-range equivalence class stuff ---*/ |
| 283 | /*-------------------------------------------------------------*/ |
| 284 | |
| 285 | /* Return equivalence class number for a range. */ |
| 286 | |
| 287 | static Int range_to_eclass ( Addr64 start, UInt len ) |
| 288 | { |
| 289 | UInt mask = (1 << ECLASS_WIDTH) - 1; |
| 290 | UInt lo = (UInt)start; |
| 291 | UInt hi = lo + len - 1; |
| 292 | UInt loBits = (lo >> ECLASS_SHIFT) & mask; |
| 293 | UInt hiBits = (hi >> ECLASS_SHIFT) & mask; |
| 294 | if (loBits == hiBits) { |
| 295 | vg_assert(loBits < ECLASS_N-1); |
| 296 | return loBits; |
| 297 | } else { |
| 298 | return ECLASS_MISC; |
| 299 | } |
| 300 | } |
| 301 | |
| 302 | |
| 303 | /* Calculates the equivalence class numbers for any VexGuestExtent. |
| 304 | These are written in *eclasses, which must be big enough to hold 3 |
| 305 | Ints. The number written, between 1 and 3, is returned. The |
| 306 | eclasses are presented in order, and any duplicates are removed. |
| 307 | */ |
| 308 | |
| 309 | static |
| 310 | Int vexGuestExtents_to_eclasses ( /*OUT*/Int* eclasses, |
| 311 | VexGuestExtents* vge ) |
| 312 | { |
| 313 | # define SWAP(_lv1,_lv2) \ |
| 314 | do { Int t = _lv1; _lv1 = _lv2; _lv2 = t; } while (0) |
| 315 | |
| 316 | Int i, j, n_ec, r; |
| 317 | |
| 318 | vg_assert(vge->n_used >= 1 && vge->n_used <= 3); |
| 319 | |
| 320 | n_ec = 0; |
| 321 | for (i = 0; i < vge->n_used; i++) { |
| 322 | r = range_to_eclass( vge->base[i], (UInt)vge->len[i] ); |
| 323 | if (r == ECLASS_MISC) |
| 324 | goto bad; |
| 325 | /* only add if we haven't already seen it */ |
| 326 | for (j = 0; j < n_ec; j++) |
| 327 | if (eclasses[j] == r) |
| 328 | break; |
| 329 | if (j == n_ec) |
| 330 | eclasses[n_ec++] = r; |
| 331 | } |
| 332 | |
| 333 | if (n_ec == 1) |
| 334 | return 1; |
| 335 | |
| 336 | if (n_ec == 2) { |
| 337 | /* sort */ |
| 338 | if (eclasses[0] > eclasses[1]) |
| 339 | SWAP(eclasses[0], eclasses[1]); |
| 340 | return 2; |
| 341 | } |
| 342 | |
| 343 | if (n_ec == 3) { |
| 344 | /* sort */ |
| 345 | if (eclasses[0] > eclasses[2]) |
| 346 | SWAP(eclasses[0], eclasses[2]); |
| 347 | if (eclasses[0] > eclasses[1]) |
| 348 | SWAP(eclasses[0], eclasses[1]); |
| 349 | if (eclasses[1] > eclasses[2]) |
| 350 | SWAP(eclasses[1], eclasses[2]); |
| 351 | return 3; |
| 352 | } |
| 353 | |
| 354 | /* NOTREACHED */ |
| 355 | vg_assert(0); |
| 356 | |
| 357 | bad: |
| 358 | eclasses[0] = ECLASS_MISC; |
| 359 | return 1; |
| 360 | |
| 361 | # undef SWAP |
| 362 | } |
| 363 | |
| 364 | |
| 365 | /* Add tteno to the set of entries listed for equivalence class ec in |
| 366 | this sector. Returns used location in eclass array. */ |
| 367 | |
| 368 | static |
| 369 | UInt addEClassNo ( /*MOD*/Sector* sec, Int ec, UShort tteno ) |
| 370 | { |
| 371 | Int old_sz, new_sz, i, r; |
| 372 | UShort *old_ar, *new_ar; |
| 373 | |
| 374 | vg_assert(ec >= 0 && ec < ECLASS_N); |
| 375 | vg_assert(tteno < N_TTES_PER_SECTOR); |
| 376 | |
| 377 | if (0) VG_(printf)("ec %d gets %d\n", ec, (Int)tteno); |
| 378 | |
| 379 | if (sec->ec2tte_used[ec] >= sec->ec2tte_size[ec]) { |
| 380 | |
| 381 | vg_assert(sec->ec2tte_used[ec] == sec->ec2tte_size[ec]); |
| 382 | |
| 383 | old_sz = sec->ec2tte_size[ec]; |
| 384 | old_ar = sec->ec2tte[ec]; |
| 385 | new_sz = old_sz==0 ? 8 : old_sz<64 ? 2*old_sz : (3*old_sz)/2; |
| 386 | new_ar = VG_(arena_malloc)(VG_AR_TTAUX, new_sz * sizeof(UShort)); |
| 387 | for (i = 0; i < old_sz; i++) |
| 388 | new_ar[i] = old_ar[i]; |
| 389 | if (old_ar) |
| 390 | VG_(arena_free)(VG_AR_TTAUX, old_ar); |
| 391 | sec->ec2tte_size[ec] = new_sz; |
| 392 | sec->ec2tte[ec] = new_ar; |
| 393 | |
| 394 | if (0) VG_(printf)("expand ec %d to %d\n", ec, new_sz); |
| 395 | } |
| 396 | |
| 397 | /* Common case */ |
| 398 | r = sec->ec2tte_used[ec]++; |
| 399 | vg_assert(r >= 0 && r < sec->ec2tte_size[ec]); |
| 400 | sec->ec2tte[ec][r] = tteno; |
| 401 | return (UInt)r; |
| 402 | } |
| 403 | |
| 404 | |
| 405 | /* 'vge' is being added to 'sec' at TT entry 'tteno'. Add appropriate |
| 406 | eclass entries to 'sec'. */ |
| 407 | |
| 408 | static |
| 409 | void upd_eclasses_after_add ( /*MOD*/Sector* sec, Int tteno ) |
| 410 | { |
| 411 | Int i, r, eclasses[3]; |
| 412 | TTEntry* tte; |
| 413 | vg_assert(tteno >= 0 && tteno < N_TTES_PER_SECTOR); |
| 414 | |
| 415 | tte = &sec->tt[tteno]; |
| 416 | r = vexGuestExtents_to_eclasses( eclasses, &tte->vge ); |
| 417 | |
| 418 | vg_assert(r >= 1 && r <= 3); |
| 419 | tte->n_tte2ec = r; |
| 420 | |
| 421 | for (i = 0; i < r; i++) { |
| 422 | tte->tte2ec_ec[i] = eclasses[i]; |
| 423 | tte->tte2ec_ix[i] = addEClassNo( sec, eclasses[i], (UShort)tteno ); |
| 424 | } |
| 425 | } |
| 426 | |
| 427 | |
| 428 | /* Check the eclass info in 'sec' to ensure it is consistent. Returns |
| 429 | True if OK, False if something's not right. Expensive. */ |
| 430 | |
| 431 | static Bool sanity_check_eclasses_in_sector ( Sector* sec ) |
| 432 | { |
| 433 | # define BAD(_str) do { whassup = (_str); goto bad; } while (0) |
| 434 | |
| 435 | HChar* whassup = NULL; |
| 436 | Int i, j, k, n, ec_num, ec_idx; |
| 437 | TTEntry* tte; |
| 438 | UShort tteno; |
| 439 | ULong* tce; |
| 440 | |
| 441 | /* Basic checks on this sector */ |
| 442 | if (sec->tt_n_inuse < 0 || sec->tt_n_inuse > N_TTES_PER_SECTOR_USABLE) |
| 443 | BAD("invalid sec->tt_n_inuse"); |
| 444 | tce = sec->tc_next; |
| 445 | if (tce < &sec->tc[0] || tce > &sec->tc[tc_sector_szQ]) |
| 446 | BAD("sec->tc_next points outside tc"); |
| 447 | |
| 448 | /* For each eclass ... */ |
| 449 | for (i = 0; i < ECLASS_N; i++) { |
| 450 | if (sec->ec2tte_size[i] == 0 && sec->ec2tte[i] != NULL) |
| 451 | BAD("ec2tte_size/ec2tte mismatch(1)"); |
| 452 | if (sec->ec2tte_size[i] != 0 && sec->ec2tte[i] == NULL) |
| 453 | BAD("ec2tte_size/ec2tte mismatch(2)"); |
| 454 | if (sec->ec2tte_used[i] < 0 |
| 455 | || sec->ec2tte_used[i] > sec->ec2tte_size[i]) |
| 456 | BAD("implausible ec2tte_used"); |
| 457 | if (sec->ec2tte_used[i] == 0) |
| 458 | continue; |
| 459 | |
| 460 | /* For each tt reference in each eclass .. ensure the reference |
| 461 | is to a valid tt entry, and that the entry's address ranges |
| 462 | really include this eclass. */ |
| 463 | |
| 464 | for (j = 0; j < sec->ec2tte_used[i]; j++) { |
| 465 | tteno = sec->ec2tte[i][j]; |
| 466 | if (tteno == EC2TTE_DELETED) |
| 467 | continue; |
| 468 | if (tteno >= N_TTES_PER_SECTOR) |
| 469 | BAD("implausible tteno"); |
| 470 | tte = &sec->tt[tteno]; |
| 471 | if (tte->status != InUse) |
| 472 | BAD("tteno points to non-inuse tte"); |
| 473 | if (tte->n_tte2ec < 1 || tte->n_tte2ec > 3) |
| 474 | BAD("tte->n_tte2ec out of range"); |
| 475 | /* Exactly least one of tte->eclasses[0 .. tte->n_eclasses-1] |
| 476 | must equal i. Inspect tte's eclass info. */ |
| 477 | n = 0; |
| 478 | for (k = 0; k < tte->n_tte2ec; k++) { |
| 479 | if (k < tte->n_tte2ec-1 |
| 480 | && tte->tte2ec_ec[k] >= tte->tte2ec_ec[k+1]) |
| 481 | BAD("tte->tte2ec_ec[..] out of order"); |
| 482 | ec_num = tte->tte2ec_ec[k]; |
| 483 | if (ec_num < 0 || ec_num >= ECLASS_N) |
| 484 | BAD("tte->tte2ec_ec[..] out of range"); |
| 485 | if (ec_num != i) |
| 486 | continue; |
| 487 | ec_idx = tte->tte2ec_ix[k]; |
| 488 | if (ec_idx < 0 || ec_idx >= sec->ec2tte_used[i]) |
| 489 | BAD("tte->tte2ec_ix[..] out of range"); |
| 490 | if (ec_idx == j) |
| 491 | n++; |
| 492 | } |
| 493 | if (n != 1) |
| 494 | BAD("tteno does not point back at eclass"); |
| 495 | } |
| 496 | } |
| 497 | |
| 498 | /* That establishes that for each forward pointer from TTEntrys |
| 499 | there is a corresponding backward pointer from the eclass[] |
| 500 | arrays. However, it doesn't rule out the possibility of other, |
| 501 | bogus pointers in the eclass[] arrays. So do those similarly: |
| 502 | scan through them and check the TTEntryies they point at point |
| 503 | back. */ |
| 504 | |
| 505 | for (i = 0; i < N_TTES_PER_SECTOR_USABLE; i++) { |
| 506 | |
| 507 | tte = &sec->tt[i]; |
| 508 | if (tte->status == Empty || tte->status == Deleted) { |
| 509 | if (tte->n_tte2ec != 0) |
| 510 | BAD("tte->n_eclasses nonzero for unused tte"); |
| 511 | continue; |
| 512 | } |
| 513 | |
| 514 | vg_assert(tte->status == InUse); |
| 515 | |
| 516 | if (tte->n_tte2ec < 1 || tte->n_tte2ec > 3) |
| 517 | BAD("tte->n_eclasses out of range(2)"); |
| 518 | |
| 519 | for (j = 0; j < tte->n_tte2ec; j++) { |
| 520 | ec_num = tte->tte2ec_ec[j]; |
| 521 | if (ec_num < 0 || ec_num >= ECLASS_N) |
| 522 | BAD("tte->eclass[..] out of range"); |
| 523 | ec_idx = tte->tte2ec_ix[j]; |
| 524 | if (ec_idx < 0 || ec_idx >= sec->ec2tte_used[ec_num]) |
| 525 | BAD("tte->ec_idx[..] out of range(2)"); |
| 526 | if (sec->ec2tte[ec_num][ec_idx] != i) |
| 527 | BAD("ec2tte does not point back to tte"); |
| 528 | } |
| 529 | } |
| 530 | |
| 531 | return True; |
| 532 | |
| 533 | bad: |
| 534 | if (whassup) |
| 535 | VG_(debugLog)(0, "transtab", "eclass sanity fail: %s\n", whassup); |
| 536 | |
| 537 | # if 0 |
| 538 | VG_(printf)("eclass = %d\n", i); |
| 539 | VG_(printf)("tteno = %d\n", (Int)tteno); |
| 540 | switch (tte->status) { |
| 541 | case InUse: VG_(printf)("InUse\n"); break; |
| 542 | case Deleted: VG_(printf)("Deleted\n"); break; |
| 543 | case Empty: VG_(printf)("Empty\n"); break; |
| 544 | } |
| 545 | if (tte->status != Empty) { |
| 546 | for (k = 0; k < tte->vge.n_used; k++) |
| 547 | VG_(printf)("0x%llx %d\n", tte->vge.base[k], |
| 548 | (Int)tte->vge.len[k]); |
| 549 | } |
| 550 | # endif |
| 551 | |
| 552 | return False; |
| 553 | |
| 554 | # undef BAD |
| 555 | } |
| 556 | |
| 557 | |
| 558 | /* Sanity check absolutely everything. True == check passed. */ |
| 559 | |
| 560 | static Bool sanity_check_all_sectors ( void ) |
| 561 | { |
| 562 | Int sno; |
| 563 | Bool sane; |
| 564 | Sector* sec; |
| 565 | for (sno = 0; sno < N_SECTORS; sno++) { |
| 566 | sec = §ors[sno]; |
| 567 | if (sec->tc == NULL) |
| 568 | continue; |
| 569 | sane = sanity_check_eclasses_in_sector( sec ); |
| 570 | if (!sane) |
| 571 | return False; |
| 572 | } |
| 573 | return True; |
| 574 | } |
| 575 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 576 | |
| 577 | /*-------------------------------------------------------------*/ |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 578 | /*--- Add/find translations ---*/ |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 579 | /*-------------------------------------------------------------*/ |
| 580 | |
| 581 | static UInt vge_osize ( VexGuestExtents* vge ) |
sewardj | c0d8f68 | 2002-11-30 00:49:43 +0000 | [diff] [blame] | 582 | { |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 583 | UInt i, n = 0; |
| 584 | for (i = 0; i < vge->n_used; i++) |
| 585 | n += (UInt)vge->len[i]; |
| 586 | return n; |
sewardj | c0d8f68 | 2002-11-30 00:49:43 +0000 | [diff] [blame] | 587 | } |
| 588 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 589 | static Bool isValidSector ( Int sector ) |
sewardj | 6c3769f | 2002-11-29 01:02:45 +0000 | [diff] [blame] | 590 | { |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 591 | if (sector < 0 || sector >= N_SECTORS) |
| 592 | return False; |
| 593 | return True; |
| 594 | } |
| 595 | |
| 596 | static inline UInt HASH_TT ( Addr64 key ) |
| 597 | { |
| 598 | UInt kHi = (UInt)(key >> 32); |
| 599 | UInt kLo = (UInt)key; |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 600 | UInt k32 = kHi ^ kLo; |
| 601 | UInt ror = 7; |
| 602 | if (ror > 0) |
| 603 | k32 = (k32 >> ror) | (k32 << (32-ror)); |
| 604 | return k32 % N_TTES_PER_SECTOR; |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 605 | } |
| 606 | |
| 607 | static void setFastCacheEntry ( Addr64 key, ULong* tce, UInt* count ) |
| 608 | { |
sewardj | 3387dda | 2005-12-26 17:58:58 +0000 | [diff] [blame^] | 609 | UInt cno = (UInt)VG_TT_FAST_HASH(key); |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 610 | VG_(tt_fast)[cno] = tce; |
| 611 | VG_(tt_fastN)[cno] = count; |
| 612 | n_fast_updates++; |
| 613 | } |
| 614 | |
| 615 | static void invalidateFastCache ( void ) |
| 616 | { |
| 617 | UInt j; |
sewardj | 65e1939 | 2005-10-19 01:32:41 +0000 | [diff] [blame] | 618 | /* This loop is popular enough to make it worth unrolling a |
| 619 | bit, at least on ppc32. */ |
| 620 | vg_assert(VG_TT_FAST_SIZE > 0 && (VG_TT_FAST_SIZE % 4) == 0); |
| 621 | for (j = 0; j < VG_TT_FAST_SIZE; j += 4) { |
| 622 | VG_(tt_fast)[j+0] = &bogus_tc_entry; |
| 623 | VG_(tt_fast)[j+1] = &bogus_tc_entry; |
| 624 | VG_(tt_fast)[j+2] = &bogus_tc_entry; |
| 625 | VG_(tt_fast)[j+3] = &bogus_tc_entry; |
| 626 | VG_(tt_fastN)[j+0] = NULL; |
| 627 | VG_(tt_fastN)[j+1] = NULL; |
| 628 | VG_(tt_fastN)[j+2] = NULL; |
| 629 | VG_(tt_fastN)[j+3] = NULL; |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 630 | } |
sewardj | 65e1939 | 2005-10-19 01:32:41 +0000 | [diff] [blame] | 631 | vg_assert(j == VG_TT_FAST_SIZE); |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 632 | n_fast_flushes++; |
| 633 | } |
| 634 | |
| 635 | static void initialiseSector ( Int sno ) |
| 636 | { |
sewardj | 45f4e7c | 2005-09-27 19:20:21 +0000 | [diff] [blame] | 637 | Int i; |
| 638 | SysRes sres; |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 639 | Sector* sec; |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 640 | vg_assert(isValidSector(sno)); |
| 641 | |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 642 | sec = §ors[sno]; |
| 643 | |
| 644 | if (sec->tc == NULL) { |
| 645 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 646 | /* Sector has never been used before. Need to allocate tt and |
| 647 | tc. */ |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 648 | vg_assert(sec->tt == NULL); |
| 649 | vg_assert(sec->tc_next == NULL); |
| 650 | vg_assert(sec->tt_n_inuse == 0); |
| 651 | for (i = 0; i < ECLASS_N; i++) { |
| 652 | vg_assert(sec->ec2tte_size[i] == 0); |
| 653 | vg_assert(sec->ec2tte_used[i] == 0); |
| 654 | vg_assert(sec->ec2tte[i] == NULL); |
| 655 | } |
sewardj | 45f4e7c | 2005-09-27 19:20:21 +0000 | [diff] [blame] | 656 | |
| 657 | VG_(debugLog)(1,"transtab", "allocate sector %d\n", sno); |
| 658 | |
| 659 | sres = VG_(am_mmap_anon_float_valgrind)( 8 * tc_sector_szQ ); |
| 660 | if (sres.isError) { |
| 661 | VG_(out_of_memory_NORETURN)("initialiseSector(TC)", |
| 662 | 8 * tc_sector_szQ ); |
| 663 | /*NOTREACHED*/ |
| 664 | } |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 665 | sec->tc = (ULong*)sres.val; |
sewardj | 45f4e7c | 2005-09-27 19:20:21 +0000 | [diff] [blame] | 666 | |
| 667 | sres = VG_(am_mmap_anon_float_valgrind) |
| 668 | ( N_TTES_PER_SECTOR * sizeof(TTEntry) ); |
| 669 | if (sres.isError) { |
| 670 | VG_(out_of_memory_NORETURN)("initialiseSector(TT)", |
| 671 | N_TTES_PER_SECTOR * sizeof(TTEntry) ); |
| 672 | /*NOTREACHED*/ |
| 673 | } |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 674 | sec->tt = (TTEntry*)sres.val; |
| 675 | |
| 676 | for (i = 0; i < N_TTES_PER_SECTOR; i++) { |
| 677 | sec->tt[i].status = Empty; |
| 678 | sec->tt[i].n_tte2ec = 0; |
| 679 | } |
sewardj | 45f4e7c | 2005-09-27 19:20:21 +0000 | [diff] [blame] | 680 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 681 | if (VG_(clo_verbosity) > 2) |
| 682 | VG_(message)(Vg_DebugMsg, "TT/TC: initialise sector %d", sno); |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 683 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 684 | } else { |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 685 | |
| 686 | /* Sector has been used before. Dump the old contents. */ |
sewardj | 45f4e7c | 2005-09-27 19:20:21 +0000 | [diff] [blame] | 687 | VG_(debugLog)(1,"transtab", "recycle sector %d\n", sno); |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 688 | vg_assert(sec->tt != NULL); |
| 689 | vg_assert(sec->tc_next != NULL); |
| 690 | n_dump_count += sec->tt_n_inuse; |
| 691 | |
| 692 | /* Visit each just-about-to-be-abandoned translation. */ |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 693 | for (i = 0; i < N_TTES_PER_SECTOR; i++) { |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 694 | if (sec->tt[i].status == InUse) { |
| 695 | vg_assert(sec->tt[i].n_tte2ec >= 1); |
| 696 | vg_assert(sec->tt[i].n_tte2ec <= 3); |
| 697 | n_dump_osize += vge_osize(&sec->tt[i].vge); |
sewardj | 3786772 | 2005-10-12 10:51:01 +0000 | [diff] [blame] | 698 | /* Tell the tool too. */ |
| 699 | if (VG_(needs).basic_block_discards) { |
| 700 | VG_TDICT_CALL( tool_discard_basic_block_info, |
sewardj | 4ba057c | 2005-10-18 12:04:18 +0000 | [diff] [blame] | 701 | sec->tt[i].entry, |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 702 | sec->tt[i].vge ); |
sewardj | 3786772 | 2005-10-12 10:51:01 +0000 | [diff] [blame] | 703 | } |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 704 | } else { |
| 705 | vg_assert(sec->tt[i].n_tte2ec == 0); |
| 706 | } |
| 707 | sec->tt[i].status = Empty; |
| 708 | sec->tt[i].n_tte2ec = 0; |
| 709 | } |
| 710 | |
| 711 | /* Free up the eclass structures. */ |
| 712 | for (i = 0; i < ECLASS_N; i++) { |
| 713 | if (sec->ec2tte_size[i] == 0) { |
| 714 | vg_assert(sec->ec2tte_used[i] == 0); |
| 715 | vg_assert(sec->ec2tte[i] == NULL); |
| 716 | } else { |
| 717 | vg_assert(sec->ec2tte[i] != NULL); |
| 718 | VG_(arena_free)(VG_AR_TTAUX, sec->ec2tte[i]); |
| 719 | sec->ec2tte[i] = NULL; |
| 720 | sec->ec2tte_size[i] = 0; |
| 721 | sec->ec2tte_used[i] = 0; |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 722 | } |
| 723 | } |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 724 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 725 | if (VG_(clo_verbosity) > 2) |
| 726 | VG_(message)(Vg_DebugMsg, "TT/TC: recycle sector %d", sno); |
| 727 | } |
| 728 | |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 729 | sec->tc_next = sec->tc; |
| 730 | sec->tt_n_inuse = 0; |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 731 | |
| 732 | invalidateFastCache(); |
sewardj | 6c3769f | 2002-11-29 01:02:45 +0000 | [diff] [blame] | 733 | } |
| 734 | |
sewardj | 10f08cf | 2005-06-29 10:16:14 +0000 | [diff] [blame] | 735 | static void invalidate_icache ( void *ptr, Int nbytes ) |
cerion | 85665ca | 2005-06-20 15:51:07 +0000 | [diff] [blame] | 736 | { |
sewardj | 2c48c7b | 2005-11-29 13:05:56 +0000 | [diff] [blame] | 737 | # if defined(VGA_ppc32) || defined(VGA_ppc64) |
sewardj | 10f08cf | 2005-06-29 10:16:14 +0000 | [diff] [blame] | 738 | Addr startaddr = (Addr) ptr; |
| 739 | Addr endaddr = startaddr + nbytes; |
sewardj | e3826cf | 2005-11-13 00:30:22 +0000 | [diff] [blame] | 740 | Addr cls; |
sewardj | 10f08cf | 2005-06-29 10:16:14 +0000 | [diff] [blame] | 741 | Addr addr; |
sewardj | e3826cf | 2005-11-13 00:30:22 +0000 | [diff] [blame] | 742 | VexArchInfo vai; |
| 743 | |
| 744 | VG_(machine_get_VexArchInfo)( NULL, &vai ); |
cerion | 1f0d814 | 2005-12-23 00:57:03 +0000 | [diff] [blame] | 745 | cls = vai.ppc_cache_line_szB; |
sewardj | 10f08cf | 2005-06-29 10:16:14 +0000 | [diff] [blame] | 746 | |
sewardj | 2bf6ba5 | 2005-06-30 12:11:19 +0000 | [diff] [blame] | 747 | /* Stay sane .. */ |
| 748 | vg_assert(cls == 32 || cls == 128); |
cerion | 85665ca | 2005-06-20 15:51:07 +0000 | [diff] [blame] | 749 | |
| 750 | startaddr &= ~(cls - 1); |
| 751 | for (addr = startaddr; addr < endaddr; addr += cls) |
| 752 | asm volatile("dcbst 0,%0" : : "r" (addr)); |
| 753 | asm volatile("sync"); |
| 754 | for (addr = startaddr; addr < endaddr; addr += cls) |
| 755 | asm volatile("icbi 0,%0" : : "r" (addr)); |
| 756 | asm volatile("sync; isync"); |
sewardj | 10f08cf | 2005-06-29 10:16:14 +0000 | [diff] [blame] | 757 | |
| 758 | # elif defined(VGA_x86) |
| 759 | /* no need to do anything, hardware provides coherence */ |
| 760 | |
| 761 | # elif defined(VGA_amd64) |
| 762 | /* no need to do anything, hardware provides coherence */ |
| 763 | |
| 764 | # else |
| 765 | # error "Unknown ARCH" |
| 766 | # endif |
cerion | 85665ca | 2005-06-20 15:51:07 +0000 | [diff] [blame] | 767 | } |
cerion | 85665ca | 2005-06-20 15:51:07 +0000 | [diff] [blame] | 768 | |
sewardj | 6c3769f | 2002-11-29 01:02:45 +0000 | [diff] [blame] | 769 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 770 | /* Add a translation of vge to TT/TC. The translation is temporarily |
| 771 | in code[0 .. code_len-1]. |
| 772 | |
| 773 | pre: youngest_sector points to a valid (although possibly full) |
| 774 | sector. |
| 775 | */ |
njn | 8bddf58 | 2005-05-13 23:40:55 +0000 | [diff] [blame] | 776 | void VG_(add_to_transtab)( VexGuestExtents* vge, |
| 777 | Addr64 entry, |
| 778 | AddrH code, |
sewardj | 26412bd | 2005-07-07 10:05:05 +0000 | [diff] [blame] | 779 | UInt code_len, |
| 780 | Bool is_self_checking ) |
sewardj | 6c3769f | 2002-11-29 01:02:45 +0000 | [diff] [blame] | 781 | { |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 782 | Int tcAvailQ, reqdQ, y, i; |
| 783 | ULong *tce, *tce2; |
| 784 | UChar* srcP; |
| 785 | UChar* dstP; |
| 786 | |
sewardj | 663a1bd | 2005-04-24 11:22:44 +0000 | [diff] [blame] | 787 | vg_assert(init_done); |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 788 | vg_assert(vge->n_used >= 1 && vge->n_used <= 3); |
| 789 | vg_assert(code_len > 0 && code_len < 20000); |
| 790 | |
| 791 | if (0) |
njn | 8bddf58 | 2005-05-13 23:40:55 +0000 | [diff] [blame] | 792 | VG_(printf)("add_to_transtab(entry = 0x%llx, len = %d)\n", |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 793 | entry, code_len); |
| 794 | |
| 795 | n_in_count++; |
| 796 | n_in_tsize += code_len; |
| 797 | n_in_osize += vge_osize(vge); |
sewardj | 26412bd | 2005-07-07 10:05:05 +0000 | [diff] [blame] | 798 | if (is_self_checking) |
| 799 | n_in_sc_count++; |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 800 | |
| 801 | y = youngest_sector; |
| 802 | vg_assert(isValidSector(y)); |
| 803 | |
| 804 | if (sectors[y].tc == NULL) |
| 805 | initialiseSector(y); |
| 806 | |
| 807 | /* Try putting the translation in this sector. */ |
| 808 | reqdQ = 1 + ((code_len + 7) >> 3); |
| 809 | |
| 810 | /* Will it fit in tc? */ |
| 811 | tcAvailQ = ((ULong*)(§ors[y].tc[tc_sector_szQ])) |
| 812 | - ((ULong*)(sectors[y].tc_next)); |
| 813 | vg_assert(tcAvailQ >= 0); |
| 814 | vg_assert(tcAvailQ <= tc_sector_szQ); |
| 815 | |
| 816 | if (tcAvailQ < reqdQ |
| 817 | || sectors[y].tt_n_inuse >= N_TTES_PER_SECTOR_USABLE) { |
| 818 | /* No. So move on to the next sector. Either it's never been |
| 819 | used before, in which case it will get its tt/tc allocated |
| 820 | now, or it has been used before, in which case it is set to be |
| 821 | empty, hence throwing out the oldest sector. */ |
sewardj | a16ea0a | 2005-09-30 10:34:06 +0000 | [diff] [blame] | 822 | vg_assert(tc_sector_szQ > 0); |
| 823 | VG_(debugLog)(1,"transtab", |
| 824 | "declare sector %d full " |
| 825 | "(TT loading %2d%%, TC loading %2d%%)\n", |
| 826 | y, |
| 827 | (100 * sectors[y].tt_n_inuse) |
| 828 | / N_TTES_PER_SECTOR, |
| 829 | (100 * (tc_sector_szQ - tcAvailQ)) |
| 830 | / tc_sector_szQ); |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 831 | youngest_sector++; |
| 832 | if (youngest_sector >= N_SECTORS) |
| 833 | youngest_sector = 0; |
| 834 | y = youngest_sector; |
| 835 | initialiseSector(y); |
| 836 | } |
| 837 | |
| 838 | /* Be sure ... */ |
| 839 | tcAvailQ = ((ULong*)(§ors[y].tc[tc_sector_szQ])) |
| 840 | - ((ULong*)(sectors[y].tc_next)); |
| 841 | vg_assert(tcAvailQ >= 0); |
| 842 | vg_assert(tcAvailQ <= tc_sector_szQ); |
| 843 | vg_assert(tcAvailQ >= reqdQ); |
| 844 | vg_assert(sectors[y].tt_n_inuse < N_TTES_PER_SECTOR_USABLE); |
| 845 | vg_assert(sectors[y].tt_n_inuse >= 0); |
| 846 | |
| 847 | /* Copy into tc. */ |
| 848 | tce = sectors[y].tc_next; |
| 849 | vg_assert(tce >= §ors[y].tc[0]); |
| 850 | vg_assert(tce <= §ors[y].tc[tc_sector_szQ]); |
| 851 | |
| 852 | tce[0] = entry; |
| 853 | dstP = (UChar*)(&tce[1]); |
| 854 | srcP = (UChar*)code; |
| 855 | for (i = 0; i < code_len; i++) |
| 856 | dstP[i] = srcP[i]; |
| 857 | sectors[y].tc_next += reqdQ; |
| 858 | sectors[y].tt_n_inuse++; |
| 859 | |
cerion | 85665ca | 2005-06-20 15:51:07 +0000 | [diff] [blame] | 860 | invalidate_icache( dstP, code_len ); |
cerion | 85665ca | 2005-06-20 15:51:07 +0000 | [diff] [blame] | 861 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 862 | /* more paranoia */ |
| 863 | tce2 = sectors[y].tc_next; |
| 864 | vg_assert(tce2 >= §ors[y].tc[0]); |
| 865 | vg_assert(tce2 <= §ors[y].tc[tc_sector_szQ]); |
| 866 | |
| 867 | /* Find an empty tt slot, and use it. There must be such a slot |
| 868 | since tt is never allowed to get completely full. */ |
| 869 | i = HASH_TT(entry); |
| 870 | vg_assert(i >= 0 && i < N_TTES_PER_SECTOR); |
sewardj | 6c3769f | 2002-11-29 01:02:45 +0000 | [diff] [blame] | 871 | while (True) { |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 872 | if (sectors[y].tt[i].status == Empty |
| 873 | || sectors[y].tt[i].status == Deleted) |
sewardj | 6c3769f | 2002-11-29 01:02:45 +0000 | [diff] [blame] | 874 | break; |
| 875 | i++; |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 876 | if (i >= N_TTES_PER_SECTOR) |
sewardj | 6c3769f | 2002-11-29 01:02:45 +0000 | [diff] [blame] | 877 | i = 0; |
| 878 | } |
sewardj | 22854b9 | 2002-11-30 14:00:47 +0000 | [diff] [blame] | 879 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 880 | sectors[y].tt[i].status = InUse; |
| 881 | sectors[y].tt[i].tce = tce; |
| 882 | sectors[y].tt[i].count = 0; |
| 883 | sectors[y].tt[i].weight = 1; |
| 884 | sectors[y].tt[i].vge = *vge; |
| 885 | sectors[y].tt[i].entry = entry; |
| 886 | |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 887 | /* Update the fast-cache. */ |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 888 | setFastCacheEntry( entry, tce, §ors[y].tt[i].count ); |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 889 | |
| 890 | /* Note the eclass numbers for this translation. */ |
| 891 | upd_eclasses_after_add( §ors[y], i ); |
sewardj | 6c3769f | 2002-11-29 01:02:45 +0000 | [diff] [blame] | 892 | } |
| 893 | |
| 894 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 895 | /* Search for the translation of the given guest address. If |
| 896 | requested, a successful search can also cause the fast-caches to be |
| 897 | updated. |
sewardj | 6c3769f | 2002-11-29 01:02:45 +0000 | [diff] [blame] | 898 | */ |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 899 | Bool VG_(search_transtab) ( /*OUT*/AddrH* result, |
| 900 | Addr64 guest_addr, |
| 901 | Bool upd_cache ) |
sewardj | 6c3769f | 2002-11-29 01:02:45 +0000 | [diff] [blame] | 902 | { |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 903 | Int i, j, k, kstart, sno; |
sewardj | 663a1bd | 2005-04-24 11:22:44 +0000 | [diff] [blame] | 904 | |
| 905 | vg_assert(init_done); |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 906 | /* Find the initial probe point just once. It will be the same in |
| 907 | all sectors and avoids multiple expensive % operations. */ |
| 908 | n_full_lookups++; |
| 909 | k = -1; |
| 910 | kstart = HASH_TT(guest_addr); |
| 911 | vg_assert(kstart >= 0 && kstart < N_TTES_PER_SECTOR); |
sewardj | 6c3769f | 2002-11-29 01:02:45 +0000 | [diff] [blame] | 912 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 913 | /* Search in all the sectors. Although the order should not matter, |
| 914 | it might be most efficient to search in the order youngest to |
| 915 | oldest. */ |
| 916 | sno = youngest_sector; |
| 917 | for (i = 0; i < N_SECTORS; i++) { |
sewardj | 6c3769f | 2002-11-29 01:02:45 +0000 | [diff] [blame] | 918 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 919 | if (sectors[sno].tc == NULL) |
| 920 | goto notfound; /* sector not in use. */ |
sewardj | 6c3769f | 2002-11-29 01:02:45 +0000 | [diff] [blame] | 921 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 922 | k = kstart; |
| 923 | for (j = 0; j < N_TTES_PER_SECTOR; j++) { |
| 924 | n_lookup_probes++; |
| 925 | if (sectors[sno].tt[k].status == InUse |
| 926 | && sectors[sno].tt[k].entry == guest_addr) { |
| 927 | /* found it */ |
| 928 | if (upd_cache) |
| 929 | setFastCacheEntry( |
| 930 | guest_addr, sectors[sno].tt[k].tce, |
| 931 | §ors[sno].tt[k].count ); |
| 932 | if (result) |
| 933 | *result = sizeof(Addr64) + (AddrH)sectors[sno].tt[k].tce; |
| 934 | return True; |
| 935 | } |
| 936 | if (sectors[sno].tt[k].status == Empty) |
| 937 | break; /* not found in this sector */ |
| 938 | k++; |
| 939 | if (k == N_TTES_PER_SECTOR) |
| 940 | k = 0; |
sewardj | 6c3769f | 2002-11-29 01:02:45 +0000 | [diff] [blame] | 941 | } |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 942 | |
| 943 | /* If we fall off the end, all entries are InUse and not |
| 944 | matching, or Deleted. In any case we did not find it in this |
| 945 | sector. */ |
| 946 | |
| 947 | notfound: |
| 948 | /* move to the next oldest sector */ |
| 949 | sno = sno==0 ? (N_SECTORS-1) : (sno-1); |
sewardj | 6c3769f | 2002-11-29 01:02:45 +0000 | [diff] [blame] | 950 | } |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 951 | |
| 952 | /* Not found in any sector. */ |
| 953 | return False; |
sewardj | 6c3769f | 2002-11-29 01:02:45 +0000 | [diff] [blame] | 954 | } |
| 955 | |
| 956 | |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 957 | /*-------------------------------------------------------------*/ |
| 958 | /*--- Delete translations. ---*/ |
| 959 | /*-------------------------------------------------------------*/ |
| 960 | |
| 961 | /* Stuff for deleting translations which intersect with a given |
| 962 | address range. Unfortunately, to make this run at a reasonable |
| 963 | speed, it is complex. */ |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 964 | |
| 965 | static inline |
sewardj | a305450 | 2005-07-26 23:04:25 +0000 | [diff] [blame] | 966 | Bool overlap1 ( Addr64 s1, ULong r1, Addr64 s2, ULong r2 ) |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 967 | { |
sewardj | a305450 | 2005-07-26 23:04:25 +0000 | [diff] [blame] | 968 | Addr64 e1 = s1 + r1 - 1ULL; |
| 969 | Addr64 e2 = s2 + r2 - 1ULL; |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 970 | if (e1 < s2 || e2 < s1) |
| 971 | return False; |
| 972 | return True; |
| 973 | } |
| 974 | |
| 975 | static inline |
sewardj | a305450 | 2005-07-26 23:04:25 +0000 | [diff] [blame] | 976 | Bool overlaps ( Addr64 start, ULong range, VexGuestExtents* vge ) |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 977 | { |
| 978 | if (overlap1(start, range, vge->base[0], (UInt)vge->len[0])) |
| 979 | return True; |
| 980 | if (vge->n_used < 2) |
| 981 | return False; |
| 982 | if (overlap1(start, range, vge->base[1], (UInt)vge->len[1])) |
| 983 | return True; |
| 984 | if (vge->n_used < 3) |
| 985 | return False; |
| 986 | if (overlap1(start, range, vge->base[2], (UInt)vge->len[2])) |
| 987 | return True; |
| 988 | return False; |
| 989 | } |
| 990 | |
| 991 | |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 992 | /* Delete a tt entry, and update all the eclass data accordingly. */ |
| 993 | |
| 994 | static void delete_tte ( /*MOD*/Sector* sec, Int tteno ) |
| 995 | { |
| 996 | Int i, ec_num, ec_idx; |
| 997 | TTEntry* tte; |
| 998 | |
| 999 | vg_assert(tteno >= 0 && tteno < N_TTES_PER_SECTOR); |
| 1000 | tte = &sec->tt[tteno]; |
| 1001 | vg_assert(tte->status == InUse); |
| 1002 | vg_assert(tte->n_tte2ec >= 1 && tte->n_tte2ec <= 3); |
| 1003 | |
| 1004 | /* Deal with the ec-to-tte links first. */ |
| 1005 | for (i = 0; i < tte->n_tte2ec; i++) { |
| 1006 | ec_num = (Int)tte->tte2ec_ec[i]; |
| 1007 | ec_idx = tte->tte2ec_ix[i]; |
| 1008 | vg_assert(ec_num >= 0 && ec_num < ECLASS_N); |
| 1009 | vg_assert(ec_idx >= 0); |
| 1010 | vg_assert(ec_idx < sec->ec2tte_used[ec_num]); |
| 1011 | /* Assert that the two links point at each other. */ |
| 1012 | vg_assert(sec->ec2tte[ec_num][ec_idx] == (UShort)tteno); |
| 1013 | /* "delete" the pointer back to here. */ |
| 1014 | sec->ec2tte[ec_num][ec_idx] = EC2TTE_DELETED; |
| 1015 | } |
| 1016 | |
| 1017 | /* Now fix up this TTEntry. */ |
| 1018 | tte->status = Deleted; |
| 1019 | tte->n_tte2ec = 0; |
| 1020 | |
| 1021 | /* Stats .. */ |
| 1022 | sec->tt_n_inuse--; |
| 1023 | n_disc_count++; |
| 1024 | n_disc_osize += vge_osize(&tte->vge); |
| 1025 | |
| 1026 | /* Tell the tool too. */ |
| 1027 | if (VG_(needs).basic_block_discards) { |
| 1028 | VG_TDICT_CALL( tool_discard_basic_block_info, |
sewardj | 4ba057c | 2005-10-18 12:04:18 +0000 | [diff] [blame] | 1029 | tte->entry, |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 1030 | tte->vge ); |
| 1031 | } |
| 1032 | } |
| 1033 | |
| 1034 | |
| 1035 | /* Delete translations from sec which intersect specified range, but |
| 1036 | only consider translations in the specified eclass. */ |
| 1037 | |
| 1038 | static |
| 1039 | Bool delete_translations_in_sector_eclass ( /*MOD*/Sector* sec, |
| 1040 | Addr64 guest_start, ULong range, |
| 1041 | Int ec ) |
| 1042 | { |
| 1043 | Int i; |
| 1044 | UShort tteno; |
| 1045 | Bool anyDeld = False; |
| 1046 | TTEntry* tte; |
| 1047 | |
| 1048 | vg_assert(ec >= 0 && ec < ECLASS_N); |
| 1049 | |
| 1050 | for (i = 0; i < sec->ec2tte_used[ec]; i++) { |
| 1051 | |
| 1052 | tteno = sec->ec2tte[ec][i]; |
| 1053 | if (tteno == EC2TTE_DELETED) { |
| 1054 | /* already deleted */ |
| 1055 | continue; |
| 1056 | } |
| 1057 | |
| 1058 | vg_assert(tteno < N_TTES_PER_SECTOR); |
| 1059 | |
| 1060 | tte = &sec->tt[tteno]; |
| 1061 | vg_assert(tte->status == InUse); |
| 1062 | |
| 1063 | if (overlaps( guest_start, range, &tte->vge )) { |
| 1064 | anyDeld = True; |
| 1065 | delete_tte( sec, (Int)tteno ); |
| 1066 | } |
| 1067 | |
| 1068 | } |
| 1069 | |
| 1070 | return anyDeld; |
| 1071 | } |
| 1072 | |
| 1073 | |
| 1074 | /* Delete translations from sec which intersect specified range, the |
| 1075 | slow way, by inspecting all translations in sec. */ |
| 1076 | |
| 1077 | static |
| 1078 | Bool delete_translations_in_sector ( /*MOD*/Sector* sec, |
| 1079 | Addr64 guest_start, ULong range ) |
| 1080 | { |
| 1081 | Int i; |
| 1082 | Bool anyDeld = False; |
| 1083 | |
| 1084 | for (i = 0; i < N_TTES_PER_SECTOR; i++) { |
| 1085 | if (sec->tt[i].status == InUse |
| 1086 | && overlaps( guest_start, range, &sec->tt[i].vge )) { |
| 1087 | anyDeld = True; |
| 1088 | delete_tte( sec, i ); |
| 1089 | } |
| 1090 | } |
| 1091 | |
| 1092 | return anyDeld; |
| 1093 | } |
| 1094 | |
| 1095 | |
sewardj | 45f4e7c | 2005-09-27 19:20:21 +0000 | [diff] [blame] | 1096 | void VG_(discard_translations) ( Addr64 guest_start, ULong range, |
| 1097 | HChar* who ) |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1098 | { |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 1099 | Sector* sec; |
| 1100 | Int sno, ec; |
| 1101 | Bool anyDeleted = False; |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1102 | |
sewardj | 663a1bd | 2005-04-24 11:22:44 +0000 | [diff] [blame] | 1103 | vg_assert(init_done); |
| 1104 | |
sewardj | a16ea0a | 2005-09-30 10:34:06 +0000 | [diff] [blame] | 1105 | VG_(debugLog)(2, "transtab", |
sewardj | 45f4e7c | 2005-09-27 19:20:21 +0000 | [diff] [blame] | 1106 | "discard_translations(0x%llx, %lld) req by %s\n", |
| 1107 | guest_start, range, who ); |
| 1108 | |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 1109 | /* Pre-deletion sanity check */ |
| 1110 | if (VG_(clo_sanity_level >= 4)) { |
| 1111 | Bool sane = sanity_check_all_sectors(); |
| 1112 | vg_assert(sane); |
| 1113 | } |
| 1114 | |
| 1115 | if (range == 0) |
| 1116 | return; |
| 1117 | |
| 1118 | /* There are two different ways to do this. |
| 1119 | |
| 1120 | If the range fits within a single address-range equivalence |
| 1121 | class, as will be the case for a cache line sized invalidation, |
| 1122 | then we only have to inspect the set of translations listed in |
| 1123 | that equivalence class, and also in the "sin-bin" equivalence |
| 1124 | class ECLASS_MISC. |
| 1125 | |
| 1126 | Otherwise, the invalidation is of a larger range and probably |
| 1127 | results from munmap. In this case it's (probably!) faster just |
| 1128 | to inspect all translations, dump those we don't want, and |
| 1129 | regenerate the equivalence class information (since modifying it |
| 1130 | in-situ is even more expensive). |
| 1131 | */ |
| 1132 | |
| 1133 | /* First off, figure out if the range falls within a single class, |
| 1134 | and if so which one. */ |
| 1135 | |
| 1136 | ec = ECLASS_MISC; |
| 1137 | if (range < (1ULL << ECLASS_SHIFT)) |
| 1138 | ec = range_to_eclass( guest_start, (UInt)range ); |
| 1139 | |
| 1140 | /* if ec is ECLASS_MISC then we aren't looking at just a single |
| 1141 | class, so use the slow scheme. Else use the fast scheme, |
| 1142 | examining 'ec' and ECLASS_MISC. */ |
| 1143 | |
| 1144 | if (ec != ECLASS_MISC) { |
| 1145 | |
| 1146 | VG_(debugLog)(2, "transtab", |
| 1147 | " FAST, ec = %d\n", ec); |
| 1148 | |
| 1149 | /* Fast scheme */ |
| 1150 | vg_assert(ec >= 0 && ec < ECLASS_MISC); |
| 1151 | |
| 1152 | for (sno = 0; sno < N_SECTORS; sno++) { |
| 1153 | sec = §ors[sno]; |
| 1154 | if (sec->tc == NULL) |
| 1155 | continue; |
| 1156 | anyDeleted |= delete_translations_in_sector_eclass( |
| 1157 | sec, guest_start, range, ec ); |
| 1158 | anyDeleted |= delete_translations_in_sector_eclass( |
| 1159 | sec, guest_start, range, ECLASS_MISC ); |
| 1160 | } |
| 1161 | |
| 1162 | } else { |
| 1163 | |
| 1164 | /* slow scheme */ |
| 1165 | |
| 1166 | VG_(debugLog)(2, "transtab", |
| 1167 | " SLOW, ec = %d\n", ec); |
| 1168 | |
| 1169 | for (sno = 0; sno < N_SECTORS; sno++) { |
| 1170 | sec = §ors[sno]; |
| 1171 | if (sec->tc == NULL) |
| 1172 | continue; |
| 1173 | anyDeleted |= delete_translations_in_sector( |
| 1174 | sec, guest_start, range ); |
| 1175 | } |
| 1176 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1177 | } |
| 1178 | |
| 1179 | if (anyDeleted) |
| 1180 | invalidateFastCache(); |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 1181 | |
| 1182 | /* Post-deletion sanity check */ |
| 1183 | if (VG_(clo_sanity_level >= 4)) { |
| 1184 | Int i; |
| 1185 | TTEntry* tte; |
| 1186 | Bool sane = sanity_check_all_sectors(); |
| 1187 | vg_assert(sane); |
| 1188 | /* But now, also check the requested address range isn't |
| 1189 | present anywhere. */ |
| 1190 | for (sno = 0; sno < N_SECTORS; sno++) { |
| 1191 | sec = §ors[sno]; |
| 1192 | if (sec->tc == NULL) |
| 1193 | continue; |
| 1194 | for (i = 0; i < N_TTES_PER_SECTOR; i++) { |
| 1195 | tte = &sec->tt[i]; |
| 1196 | if (tte->status != InUse) |
| 1197 | continue; |
| 1198 | vg_assert(!overlaps( guest_start, range, &tte->vge )); |
| 1199 | } |
| 1200 | } |
| 1201 | } |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1202 | } |
| 1203 | |
| 1204 | |
| 1205 | /*------------------------------------------------------------*/ |
sewardj | de4a1d0 | 2002-03-22 01:27:54 +0000 | [diff] [blame] | 1206 | /*--- Initialisation. ---*/ |
| 1207 | /*------------------------------------------------------------*/ |
| 1208 | |
sewardj | c0d8f68 | 2002-11-30 00:49:43 +0000 | [diff] [blame] | 1209 | void VG_(init_tt_tc) ( void ) |
| 1210 | { |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 1211 | Int i, j, avg_codeszQ; |
sewardj | c0d8f68 | 2002-11-30 00:49:43 +0000 | [diff] [blame] | 1212 | |
sewardj | 663a1bd | 2005-04-24 11:22:44 +0000 | [diff] [blame] | 1213 | vg_assert(!init_done); |
| 1214 | init_done = True; |
| 1215 | |
sewardj | 4ccf707 | 2004-11-28 16:58:05 +0000 | [diff] [blame] | 1216 | /* Otherwise lots of things go wrong... */ |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1217 | vg_assert(sizeof(ULong) == 8); |
| 1218 | vg_assert(sizeof(Addr64) == 8); |
| 1219 | |
| 1220 | if (VG_(clo_verbosity) > 2) |
| 1221 | VG_(message)(Vg_DebugMsg, |
| 1222 | "TT/TC: VG_(init_tt_tc) " |
| 1223 | "(startup of code management)"); |
| 1224 | |
| 1225 | /* Figure out how big each tc area should be. */ |
njn | 43b9a8a | 2005-05-10 04:37:01 +0000 | [diff] [blame] | 1226 | avg_codeszQ = (VG_(details).avg_translation_sizeB + 7) / 8; |
| 1227 | tc_sector_szQ = N_TTES_PER_SECTOR_USABLE * (1 + avg_codeszQ); |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1228 | |
sewardj | c0d8f68 | 2002-11-30 00:49:43 +0000 | [diff] [blame] | 1229 | /* Ensure the calculated value is not way crazy. */ |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1230 | vg_assert(tc_sector_szQ >= 2 * N_TTES_PER_SECTOR_USABLE); |
| 1231 | vg_assert(tc_sector_szQ <= 50 * N_TTES_PER_SECTOR_USABLE); |
sewardj | c0d8f68 | 2002-11-30 00:49:43 +0000 | [diff] [blame] | 1232 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1233 | /* Initialise the sectors */ |
| 1234 | youngest_sector = 0; |
| 1235 | for (i = 0; i < N_SECTORS; i++) { |
| 1236 | sectors[i].tc = NULL; |
| 1237 | sectors[i].tt = NULL; |
| 1238 | sectors[i].tc_next = NULL; |
| 1239 | sectors[i].tt_n_inuse = 0; |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 1240 | for (j = 0; j < ECLASS_N; j++) { |
| 1241 | sectors[i].ec2tte_size[j] = 0; |
| 1242 | sectors[i].ec2tte_used[j] = 0; |
| 1243 | sectors[i].ec2tte[j] = NULL; |
| 1244 | } |
sewardj | c0d8f68 | 2002-11-30 00:49:43 +0000 | [diff] [blame] | 1245 | } |
sewardj | c0d8f68 | 2002-11-30 00:49:43 +0000 | [diff] [blame] | 1246 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1247 | /* and the fast caches. */ |
| 1248 | invalidateFastCache(); |
sewardj | c0d8f68 | 2002-11-30 00:49:43 +0000 | [diff] [blame] | 1249 | |
| 1250 | if (VG_(clo_verbosity) > 2) { |
| 1251 | VG_(message)(Vg_DebugMsg, |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1252 | "TT/TC: cache: %d sectors of %d bytes each = %d total", |
| 1253 | N_SECTORS, 8 * tc_sector_szQ, |
| 1254 | N_SECTORS * 8 * tc_sector_szQ ); |
sewardj | c0d8f68 | 2002-11-30 00:49:43 +0000 | [diff] [blame] | 1255 | VG_(message)(Vg_DebugMsg, |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1256 | "TT/TC: table: %d total entries, max occupancy %d (%d%%)", |
| 1257 | N_SECTORS * N_TTES_PER_SECTOR, |
| 1258 | N_SECTORS * N_TTES_PER_SECTOR_USABLE, |
| 1259 | SECTOR_TT_LIMIT_PERCENT ); |
| 1260 | } |
sewardj | 45f4e7c | 2005-09-27 19:20:21 +0000 | [diff] [blame] | 1261 | |
| 1262 | VG_(debugLog)(2, "transtab", |
| 1263 | "cache: %d sectors of %d bytes each = %d total\n", |
| 1264 | N_SECTORS, 8 * tc_sector_szQ, |
| 1265 | N_SECTORS * 8 * tc_sector_szQ ); |
| 1266 | VG_(debugLog)(2, "transtab", |
| 1267 | "table: %d total entries, max occupancy %d (%d%%)\n", |
| 1268 | N_SECTORS * N_TTES_PER_SECTOR, |
| 1269 | N_SECTORS * N_TTES_PER_SECTOR_USABLE, |
| 1270 | SECTOR_TT_LIMIT_PERCENT ); |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1271 | } |
| 1272 | |
| 1273 | |
| 1274 | /*------------------------------------------------------------*/ |
| 1275 | /*--- Printing out statistics. ---*/ |
| 1276 | /*------------------------------------------------------------*/ |
| 1277 | |
| 1278 | static ULong safe_idiv( ULong a, ULong b ) |
| 1279 | { |
| 1280 | return (b == 0 ? 0 : a / b); |
| 1281 | } |
| 1282 | |
| 1283 | UInt VG_(get_bbs_translated) ( void ) |
| 1284 | { |
| 1285 | return n_in_count; |
| 1286 | } |
| 1287 | |
| 1288 | void VG_(print_tt_tc_stats) ( void ) |
| 1289 | { |
| 1290 | VG_(message)(Vg_DebugMsg, |
njn | 0fd92f4 | 2005-10-06 03:32:42 +0000 | [diff] [blame] | 1291 | " tt/tc: %,llu tt lookups requiring %,llu probes", |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1292 | n_full_lookups, n_lookup_probes ); |
| 1293 | VG_(message)(Vg_DebugMsg, |
njn | 0fd92f4 | 2005-10-06 03:32:42 +0000 | [diff] [blame] | 1294 | " tt/tc: %,llu fast-cache updates, %,llu flushes", |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1295 | n_fast_updates, n_fast_flushes ); |
| 1296 | |
| 1297 | VG_(message)(Vg_DebugMsg, |
njn | 5096a39 | 2005-12-13 20:05:00 +0000 | [diff] [blame] | 1298 | " transtab: new %,lld " |
njn | 0fd92f4 | 2005-10-06 03:32:42 +0000 | [diff] [blame] | 1299 | "(%,llu -> %,llu; ratio %,llu:10) [%,llu scs]", |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1300 | n_in_count, n_in_osize, n_in_tsize, |
sewardj | 26412bd | 2005-07-07 10:05:05 +0000 | [diff] [blame] | 1301 | safe_idiv(10*n_in_tsize, n_in_osize), |
| 1302 | n_in_sc_count); |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1303 | VG_(message)(Vg_DebugMsg, |
njn | 5096a39 | 2005-12-13 20:05:00 +0000 | [diff] [blame] | 1304 | " transtab: dumped %,llu (%,llu -> ?" "?)", |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1305 | n_dump_count, n_dump_osize ); |
| 1306 | VG_(message)(Vg_DebugMsg, |
njn | 5096a39 | 2005-12-13 20:05:00 +0000 | [diff] [blame] | 1307 | " transtab: discarded %,llu (%,llu -> ?" "?)", |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1308 | n_disc_count, n_disc_osize ); |
sewardj | 6c1bbbb | 2005-10-18 02:30:42 +0000 | [diff] [blame] | 1309 | |
| 1310 | if (0) { |
| 1311 | Int i; |
| 1312 | VG_(printf)("\n"); |
| 1313 | for (i = 0; i < ECLASS_N; i++) { |
| 1314 | VG_(printf)(" %4d", sectors[0].ec2tte_used[i]); |
| 1315 | if (i % 16 == 15) |
| 1316 | VG_(printf)("\n"); |
| 1317 | } |
| 1318 | VG_(printf)("\n\n"); |
| 1319 | } |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1320 | } |
| 1321 | |
| 1322 | /*------------------------------------------------------------*/ |
| 1323 | /*--- Printing out of profiling results. ---*/ |
| 1324 | /*------------------------------------------------------------*/ |
| 1325 | |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1326 | static ULong score ( TTEntry* tte ) |
| 1327 | { |
| 1328 | return ((ULong)tte->weight) * ((ULong)tte->count); |
| 1329 | } |
| 1330 | |
njn | 2025cf9 | 2005-06-26 20:44:48 +0000 | [diff] [blame] | 1331 | ULong VG_(get_BB_profile) ( BBProfEntry tops[], UInt n_tops ) |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1332 | { |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1333 | Int sno, i, r, s; |
njn | 2025cf9 | 2005-06-26 20:44:48 +0000 | [diff] [blame] | 1334 | ULong score_total; |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1335 | |
| 1336 | /* First, compute the total weighted count, and find the top N |
njn | 2025cf9 | 2005-06-26 20:44:48 +0000 | [diff] [blame] | 1337 | ttes. tops contains pointers to the most-used n_tops blocks, in |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1338 | descending order (viz, tops[0] is the highest scorer). */ |
njn | 2025cf9 | 2005-06-26 20:44:48 +0000 | [diff] [blame] | 1339 | for (i = 0; i < n_tops; i++) { |
| 1340 | tops[i].addr = 0; |
| 1341 | tops[i].score = 0; |
| 1342 | } |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1343 | |
| 1344 | score_total = 0; |
| 1345 | |
| 1346 | for (sno = 0; sno < N_SECTORS; sno++) { |
| 1347 | if (sectors[sno].tc == NULL) |
| 1348 | continue; |
| 1349 | for (i = 0; i < N_TTES_PER_SECTOR; i++) { |
| 1350 | if (sectors[sno].tt[i].status != InUse) |
| 1351 | continue; |
| 1352 | score_total += score(§ors[sno].tt[i]); |
| 1353 | /* Find the rank for sectors[sno].tt[i]. */ |
njn | 2025cf9 | 2005-06-26 20:44:48 +0000 | [diff] [blame] | 1354 | r = n_tops-1; |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1355 | while (True) { |
| 1356 | if (r == -1) |
| 1357 | break; |
njn | 2025cf9 | 2005-06-26 20:44:48 +0000 | [diff] [blame] | 1358 | if (tops[r].addr == 0) { |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1359 | r--; |
| 1360 | continue; |
| 1361 | } |
njn | 2025cf9 | 2005-06-26 20:44:48 +0000 | [diff] [blame] | 1362 | if ( score(§ors[sno].tt[i]) > tops[r].score ) { |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1363 | r--; |
| 1364 | continue; |
| 1365 | } |
| 1366 | break; |
| 1367 | } |
| 1368 | r++; |
njn | 2025cf9 | 2005-06-26 20:44:48 +0000 | [diff] [blame] | 1369 | vg_assert(r >= 0 && r <= n_tops); |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1370 | /* This bb should be placed at r, and bbs above it shifted |
| 1371 | upwards one slot. */ |
njn | 2025cf9 | 2005-06-26 20:44:48 +0000 | [diff] [blame] | 1372 | if (r < n_tops) { |
| 1373 | for (s = n_tops-1; s > r; s--) |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1374 | tops[s] = tops[s-1]; |
njn | 2025cf9 | 2005-06-26 20:44:48 +0000 | [diff] [blame] | 1375 | tops[r].addr = sectors[sno].tt[i].entry; |
| 1376 | tops[r].score = score( §ors[sno].tt[i] ); |
sewardj | fa8ec11 | 2005-01-19 11:55:34 +0000 | [diff] [blame] | 1377 | } |
| 1378 | } |
sewardj | c0d8f68 | 2002-11-30 00:49:43 +0000 | [diff] [blame] | 1379 | } |
| 1380 | |
njn | 2025cf9 | 2005-06-26 20:44:48 +0000 | [diff] [blame] | 1381 | return score_total; |
sewardj | c0d8f68 | 2002-11-30 00:49:43 +0000 | [diff] [blame] | 1382 | } |
| 1383 | |
sewardj | de4a1d0 | 2002-03-22 01:27:54 +0000 | [diff] [blame] | 1384 | /*--------------------------------------------------------------------*/ |
njn | 8bddf58 | 2005-05-13 23:40:55 +0000 | [diff] [blame] | 1385 | /*--- end ---*/ |
sewardj | de4a1d0 | 2002-03-22 01:27:54 +0000 | [diff] [blame] | 1386 | /*--------------------------------------------------------------------*/ |