blob: 8521401bbd751406e6f2752e10868f63c7e3b48a [file] [log] [blame]
Doug Thompson2bc65412009-05-04 20:11:14 +02001#include "amd64_edac.h"
Andreas Herrmann23ac4ae2010-09-17 18:03:43 +02002#include <asm/amd_nb.h>
Doug Thompson2bc65412009-05-04 20:11:14 +02003
4static struct edac_pci_ctl_info *amd64_ctl_pci;
5
6static int report_gart_errors;
7module_param(report_gart_errors, int, 0644);
8
9/*
10 * Set by command line parameter. If BIOS has enabled the ECC, this override is
11 * cleared to prevent re-enabling the hardware by this driver.
12 */
13static int ecc_enable_override;
14module_param(ecc_enable_override, int, 0644);
15
Tejun Heoa29d8b82010-02-02 14:39:15 +090016static struct msr __percpu *msrs;
Borislav Petkov50542252009-12-11 18:14:40 +010017
Doug Thompson2bc65412009-05-04 20:11:14 +020018/* Lookup table for all possible MC control instances */
19struct amd64_pvt;
Borislav Petkov3011b202009-09-21 13:23:34 +020020static struct mem_ctl_info *mci_lookup[EDAC_MAX_NUMNODES];
21static struct amd64_pvt *pvt_lookup[EDAC_MAX_NUMNODES];
Doug Thompson2bc65412009-05-04 20:11:14 +020022
23/*
Borislav Petkov1433eb92009-10-21 13:44:36 +020024 * Address to DRAM bank mapping: see F2x80 for K8 and F2x[1,0]80 for Fam10 and
25 * later.
Borislav Petkovb70ef012009-06-25 19:32:38 +020026 */
Borislav Petkov1433eb92009-10-21 13:44:36 +020027static int ddr2_dbam_revCG[] = {
28 [0] = 32,
29 [1] = 64,
30 [2] = 128,
31 [3] = 256,
32 [4] = 512,
33 [5] = 1024,
34 [6] = 2048,
35};
36
37static int ddr2_dbam_revD[] = {
38 [0] = 32,
39 [1] = 64,
40 [2 ... 3] = 128,
41 [4] = 256,
42 [5] = 512,
43 [6] = 256,
44 [7] = 512,
45 [8 ... 9] = 1024,
46 [10] = 2048,
47};
48
49static int ddr2_dbam[] = { [0] = 128,
50 [1] = 256,
51 [2 ... 4] = 512,
52 [5 ... 6] = 1024,
53 [7 ... 8] = 2048,
54 [9 ... 10] = 4096,
55 [11] = 8192,
56};
57
58static int ddr3_dbam[] = { [0] = -1,
59 [1] = 256,
60 [2] = 512,
61 [3 ... 4] = -1,
62 [5 ... 6] = 1024,
63 [7 ... 8] = 2048,
64 [9 ... 10] = 4096,
65 [11] = 8192,
Borislav Petkovb70ef012009-06-25 19:32:38 +020066};
67
68/*
69 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
70 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
71 * or higher value'.
72 *
73 *FIXME: Produce a better mapping/linearisation.
74 */
75
76struct scrubrate scrubrates[] = {
77 { 0x01, 1600000000UL},
78 { 0x02, 800000000UL},
79 { 0x03, 400000000UL},
80 { 0x04, 200000000UL},
81 { 0x05, 100000000UL},
82 { 0x06, 50000000UL},
83 { 0x07, 25000000UL},
84 { 0x08, 12284069UL},
85 { 0x09, 6274509UL},
86 { 0x0A, 3121951UL},
87 { 0x0B, 1560975UL},
88 { 0x0C, 781440UL},
89 { 0x0D, 390720UL},
90 { 0x0E, 195300UL},
91 { 0x0F, 97650UL},
92 { 0x10, 48854UL},
93 { 0x11, 24427UL},
94 { 0x12, 12213UL},
95 { 0x13, 6101UL},
96 { 0x14, 3051UL},
97 { 0x15, 1523UL},
98 { 0x16, 761UL},
99 { 0x00, 0UL}, /* scrubbing off */
100};
101
102/*
Doug Thompson2bc65412009-05-04 20:11:14 +0200103 * Memory scrubber control interface. For K8, memory scrubbing is handled by
104 * hardware and can involve L2 cache, dcache as well as the main memory. With
105 * F10, this is extended to L3 cache scrubbing on CPU models sporting that
106 * functionality.
107 *
108 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
109 * (dram) over to cache lines. This is nasty, so we will use bandwidth in
110 * bytes/sec for the setting.
111 *
112 * Currently, we only do dram scrubbing. If the scrubbing is done in software on
113 * other archs, we might not have access to the caches directly.
114 */
115
116/*
117 * scan the scrub rate mapping table for a close or matching bandwidth value to
118 * issue. If requested is too big, then use last maximum value found.
119 */
120static int amd64_search_set_scrub_rate(struct pci_dev *ctl, u32 new_bw,
121 u32 min_scrubrate)
122{
123 u32 scrubval;
124 int i;
125
126 /*
127 * map the configured rate (new_bw) to a value specific to the AMD64
128 * memory controller and apply to register. Search for the first
129 * bandwidth entry that is greater or equal than the setting requested
130 * and program that. If at last entry, turn off DRAM scrubbing.
131 */
132 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
133 /*
134 * skip scrub rates which aren't recommended
135 * (see F10 BKDG, F3x58)
136 */
137 if (scrubrates[i].scrubval < min_scrubrate)
138 continue;
139
140 if (scrubrates[i].bandwidth <= new_bw)
141 break;
142
143 /*
144 * if no suitable bandwidth found, turn off DRAM scrubbing
145 * entirely by falling back to the last element in the
146 * scrubrates array.
147 */
148 }
149
150 scrubval = scrubrates[i].scrubval;
151 if (scrubval)
152 edac_printk(KERN_DEBUG, EDAC_MC,
153 "Setting scrub rate bandwidth: %u\n",
154 scrubrates[i].bandwidth);
155 else
156 edac_printk(KERN_DEBUG, EDAC_MC, "Turning scrubbing off.\n");
157
158 pci_write_bits32(ctl, K8_SCRCTRL, scrubval, 0x001F);
159
160 return 0;
161}
162
Borislav Petkoveba042a2010-05-25 18:21:07 +0200163static int amd64_set_scrub_rate(struct mem_ctl_info *mci, u32 bandwidth)
Doug Thompson2bc65412009-05-04 20:11:14 +0200164{
165 struct amd64_pvt *pvt = mci->pvt_info;
166 u32 min_scrubrate = 0x0;
167
168 switch (boot_cpu_data.x86) {
169 case 0xf:
170 min_scrubrate = K8_MIN_SCRUB_RATE_BITS;
171 break;
172 case 0x10:
173 min_scrubrate = F10_MIN_SCRUB_RATE_BITS;
174 break;
175 case 0x11:
176 min_scrubrate = F11_MIN_SCRUB_RATE_BITS;
177 break;
178
179 default:
180 amd64_printk(KERN_ERR, "Unsupported family!\n");
Borislav Petkovbc571172010-05-21 21:25:03 +0200181 return -EINVAL;
Doug Thompson2bc65412009-05-04 20:11:14 +0200182 }
Borislav Petkoveba042a2010-05-25 18:21:07 +0200183 return amd64_search_set_scrub_rate(pvt->misc_f3_ctl, bandwidth,
184 min_scrubrate);
Doug Thompson2bc65412009-05-04 20:11:14 +0200185}
186
187static int amd64_get_scrub_rate(struct mem_ctl_info *mci, u32 *bw)
188{
189 struct amd64_pvt *pvt = mci->pvt_info;
190 u32 scrubval = 0;
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +0200191 int status = -1, i;
Doug Thompson2bc65412009-05-04 20:11:14 +0200192
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +0200193 amd64_read_pci_cfg(pvt->misc_f3_ctl, K8_SCRCTRL, &scrubval);
Doug Thompson2bc65412009-05-04 20:11:14 +0200194
195 scrubval = scrubval & 0x001F;
196
197 edac_printk(KERN_DEBUG, EDAC_MC,
198 "pci-read, sdram scrub control value: %d \n", scrubval);
199
Roel Kluin926311f2010-01-11 20:58:21 +0100200 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
Doug Thompson2bc65412009-05-04 20:11:14 +0200201 if (scrubrates[i].scrubval == scrubval) {
202 *bw = scrubrates[i].bandwidth;
203 status = 0;
204 break;
205 }
206 }
207
208 return status;
209}
210
Doug Thompson67757632009-04-27 15:53:22 +0200211/* Map from a CSROW entry to the mask entry that operates on it */
212static inline u32 amd64_map_to_dcs_mask(struct amd64_pvt *pvt, int csrow)
213{
Borislav Petkov1433eb92009-10-21 13:44:36 +0200214 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F)
Borislav Petkov9d858bb2009-09-21 14:35:51 +0200215 return csrow;
216 else
217 return csrow >> 1;
Doug Thompson67757632009-04-27 15:53:22 +0200218}
219
220/* return the 'base' address the i'th CS entry of the 'dct' DRAM controller */
221static u32 amd64_get_dct_base(struct amd64_pvt *pvt, int dct, int csrow)
222{
223 if (dct == 0)
224 return pvt->dcsb0[csrow];
225 else
226 return pvt->dcsb1[csrow];
227}
228
229/*
230 * Return the 'mask' address the i'th CS entry. This function is needed because
231 * there number of DCSM registers on Rev E and prior vs Rev F and later is
232 * different.
233 */
234static u32 amd64_get_dct_mask(struct amd64_pvt *pvt, int dct, int csrow)
235{
236 if (dct == 0)
237 return pvt->dcsm0[amd64_map_to_dcs_mask(pvt, csrow)];
238 else
239 return pvt->dcsm1[amd64_map_to_dcs_mask(pvt, csrow)];
240}
241
242
243/*
244 * In *base and *limit, pass back the full 40-bit base and limit physical
245 * addresses for the node given by node_id. This information is obtained from
246 * DRAM Base (section 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers. The
247 * base and limit addresses are of type SysAddr, as defined at the start of
248 * section 3.4.4 (p. 70). They are the lowest and highest physical addresses
249 * in the address range they represent.
250 */
251static void amd64_get_base_and_limit(struct amd64_pvt *pvt, int node_id,
252 u64 *base, u64 *limit)
253{
254 *base = pvt->dram_base[node_id];
255 *limit = pvt->dram_limit[node_id];
256}
257
258/*
259 * Return 1 if the SysAddr given by sys_addr matches the base/limit associated
260 * with node_id
261 */
262static int amd64_base_limit_match(struct amd64_pvt *pvt,
263 u64 sys_addr, int node_id)
264{
265 u64 base, limit, addr;
266
267 amd64_get_base_and_limit(pvt, node_id, &base, &limit);
268
269 /* The K8 treats this as a 40-bit value. However, bits 63-40 will be
270 * all ones if the most significant implemented address bit is 1.
271 * Here we discard bits 63-40. See section 3.4.2 of AMD publication
272 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
273 * Application Programming.
274 */
275 addr = sys_addr & 0x000000ffffffffffull;
276
277 return (addr >= base) && (addr <= limit);
278}
279
280/*
281 * Attempt to map a SysAddr to a node. On success, return a pointer to the
282 * mem_ctl_info structure for the node that the SysAddr maps to.
283 *
284 * On failure, return NULL.
285 */
286static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
287 u64 sys_addr)
288{
289 struct amd64_pvt *pvt;
290 int node_id;
291 u32 intlv_en, bits;
292
293 /*
294 * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
295 * 3.4.4.2) registers to map the SysAddr to a node ID.
296 */
297 pvt = mci->pvt_info;
298
299 /*
300 * The value of this field should be the same for all DRAM Base
301 * registers. Therefore we arbitrarily choose to read it from the
302 * register for node 0.
303 */
304 intlv_en = pvt->dram_IntlvEn[0];
305
306 if (intlv_en == 0) {
Borislav Petkov8edc5442009-09-18 12:39:19 +0200307 for (node_id = 0; node_id < DRAM_REG_COUNT; node_id++) {
Doug Thompson67757632009-04-27 15:53:22 +0200308 if (amd64_base_limit_match(pvt, sys_addr, node_id))
Borislav Petkov8edc5442009-09-18 12:39:19 +0200309 goto found;
Doug Thompson67757632009-04-27 15:53:22 +0200310 }
Borislav Petkov8edc5442009-09-18 12:39:19 +0200311 goto err_no_match;
Doug Thompson67757632009-04-27 15:53:22 +0200312 }
313
Borislav Petkov72f158f2009-09-18 12:27:27 +0200314 if (unlikely((intlv_en != 0x01) &&
315 (intlv_en != 0x03) &&
316 (intlv_en != 0x07))) {
Doug Thompson67757632009-04-27 15:53:22 +0200317 amd64_printk(KERN_WARNING, "junk value of 0x%x extracted from "
318 "IntlvEn field of DRAM Base Register for node 0: "
Borislav Petkov72f158f2009-09-18 12:27:27 +0200319 "this probably indicates a BIOS bug.\n", intlv_en);
Doug Thompson67757632009-04-27 15:53:22 +0200320 return NULL;
321 }
322
323 bits = (((u32) sys_addr) >> 12) & intlv_en;
324
325 for (node_id = 0; ; ) {
Borislav Petkov8edc5442009-09-18 12:39:19 +0200326 if ((pvt->dram_IntlvSel[node_id] & intlv_en) == bits)
Doug Thompson67757632009-04-27 15:53:22 +0200327 break; /* intlv_sel field matches */
328
329 if (++node_id >= DRAM_REG_COUNT)
330 goto err_no_match;
331 }
332
333 /* sanity test for sys_addr */
334 if (unlikely(!amd64_base_limit_match(pvt, sys_addr, node_id))) {
335 amd64_printk(KERN_WARNING,
Borislav Petkov8edc5442009-09-18 12:39:19 +0200336 "%s(): sys_addr 0x%llx falls outside base/limit "
337 "address range for node %d with node interleaving "
338 "enabled.\n",
339 __func__, sys_addr, node_id);
Doug Thompson67757632009-04-27 15:53:22 +0200340 return NULL;
341 }
342
343found:
344 return edac_mc_find(node_id);
345
346err_no_match:
347 debugf2("sys_addr 0x%lx doesn't match any node\n",
348 (unsigned long)sys_addr);
349
350 return NULL;
351}
Doug Thompsone2ce7252009-04-27 15:57:12 +0200352
353/*
354 * Extract the DRAM CS base address from selected csrow register.
355 */
356static u64 base_from_dct_base(struct amd64_pvt *pvt, int csrow)
357{
358 return ((u64) (amd64_get_dct_base(pvt, 0, csrow) & pvt->dcsb_base)) <<
359 pvt->dcs_shift;
360}
361
362/*
363 * Extract the mask from the dcsb0[csrow] entry in a CPU revision-specific way.
364 */
365static u64 mask_from_dct_mask(struct amd64_pvt *pvt, int csrow)
366{
367 u64 dcsm_bits, other_bits;
368 u64 mask;
369
370 /* Extract bits from DRAM CS Mask. */
371 dcsm_bits = amd64_get_dct_mask(pvt, 0, csrow) & pvt->dcsm_mask;
372
373 other_bits = pvt->dcsm_mask;
374 other_bits = ~(other_bits << pvt->dcs_shift);
375
376 /*
377 * The extracted bits from DCSM belong in the spaces represented by
378 * the cleared bits in other_bits.
379 */
380 mask = (dcsm_bits << pvt->dcs_shift) | other_bits;
381
382 return mask;
383}
384
385/*
386 * @input_addr is an InputAddr associated with the node given by mci. Return the
387 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
388 */
389static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
390{
391 struct amd64_pvt *pvt;
392 int csrow;
393 u64 base, mask;
394
395 pvt = mci->pvt_info;
396
397 /*
398 * Here we use the DRAM CS Base and DRAM CS Mask registers. For each CS
399 * base/mask register pair, test the condition shown near the start of
400 * section 3.5.4 (p. 84, BKDG #26094, K8, revA-E).
401 */
Borislav Petkov9d858bb2009-09-21 14:35:51 +0200402 for (csrow = 0; csrow < pvt->cs_count; csrow++) {
Doug Thompsone2ce7252009-04-27 15:57:12 +0200403
404 /* This DRAM chip select is disabled on this node */
405 if ((pvt->dcsb0[csrow] & K8_DCSB_CS_ENABLE) == 0)
406 continue;
407
408 base = base_from_dct_base(pvt, csrow);
409 mask = ~mask_from_dct_mask(pvt, csrow);
410
411 if ((input_addr & mask) == (base & mask)) {
412 debugf2("InputAddr 0x%lx matches csrow %d (node %d)\n",
413 (unsigned long)input_addr, csrow,
414 pvt->mc_node_id);
415
416 return csrow;
417 }
418 }
419
420 debugf2("no matching csrow for InputAddr 0x%lx (MC node %d)\n",
421 (unsigned long)input_addr, pvt->mc_node_id);
422
423 return -1;
424}
425
426/*
427 * Return the base value defined by the DRAM Base register for the node
428 * represented by mci. This function returns the full 40-bit value despite the
429 * fact that the register only stores bits 39-24 of the value. See section
430 * 3.4.4.1 (BKDG #26094, K8, revA-E)
431 */
432static inline u64 get_dram_base(struct mem_ctl_info *mci)
433{
434 struct amd64_pvt *pvt = mci->pvt_info;
435
436 return pvt->dram_base[pvt->mc_node_id];
437}
438
439/*
440 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
441 * for the node represented by mci. Info is passed back in *hole_base,
442 * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if
443 * info is invalid. Info may be invalid for either of the following reasons:
444 *
445 * - The revision of the node is not E or greater. In this case, the DRAM Hole
446 * Address Register does not exist.
447 *
448 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
449 * indicating that its contents are not valid.
450 *
451 * The values passed back in *hole_base, *hole_offset, and *hole_size are
452 * complete 32-bit values despite the fact that the bitfields in the DHAR
453 * only represent bits 31-24 of the base and offset values.
454 */
455int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
456 u64 *hole_offset, u64 *hole_size)
457{
458 struct amd64_pvt *pvt = mci->pvt_info;
459 u64 base;
460
461 /* only revE and later have the DRAM Hole Address Register */
Borislav Petkov1433eb92009-10-21 13:44:36 +0200462 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_E) {
Doug Thompsone2ce7252009-04-27 15:57:12 +0200463 debugf1(" revision %d for node %d does not support DHAR\n",
464 pvt->ext_model, pvt->mc_node_id);
465 return 1;
466 }
467
468 /* only valid for Fam10h */
469 if (boot_cpu_data.x86 == 0x10 &&
470 (pvt->dhar & F10_DRAM_MEM_HOIST_VALID) == 0) {
471 debugf1(" Dram Memory Hoisting is DISABLED on this system\n");
472 return 1;
473 }
474
475 if ((pvt->dhar & DHAR_VALID) == 0) {
476 debugf1(" Dram Memory Hoisting is DISABLED on this node %d\n",
477 pvt->mc_node_id);
478 return 1;
479 }
480
481 /* This node has Memory Hoisting */
482
483 /* +------------------+--------------------+--------------------+-----
484 * | memory | DRAM hole | relocated |
485 * | [0, (x - 1)] | [x, 0xffffffff] | addresses from |
486 * | | | DRAM hole |
487 * | | | [0x100000000, |
488 * | | | (0x100000000+ |
489 * | | | (0xffffffff-x))] |
490 * +------------------+--------------------+--------------------+-----
491 *
492 * Above is a diagram of physical memory showing the DRAM hole and the
493 * relocated addresses from the DRAM hole. As shown, the DRAM hole
494 * starts at address x (the base address) and extends through address
495 * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the
496 * addresses in the hole so that they start at 0x100000000.
497 */
498
499 base = dhar_base(pvt->dhar);
500
501 *hole_base = base;
502 *hole_size = (0x1ull << 32) - base;
503
504 if (boot_cpu_data.x86 > 0xf)
505 *hole_offset = f10_dhar_offset(pvt->dhar);
506 else
507 *hole_offset = k8_dhar_offset(pvt->dhar);
508
509 debugf1(" DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
510 pvt->mc_node_id, (unsigned long)*hole_base,
511 (unsigned long)*hole_offset, (unsigned long)*hole_size);
512
513 return 0;
514}
515EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info);
516
Doug Thompson93c2df52009-05-04 20:46:50 +0200517/*
518 * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is
519 * assumed that sys_addr maps to the node given by mci.
520 *
521 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
522 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
523 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
524 * then it is also involved in translating a SysAddr to a DramAddr. Sections
525 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
526 * These parts of the documentation are unclear. I interpret them as follows:
527 *
528 * When node n receives a SysAddr, it processes the SysAddr as follows:
529 *
530 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
531 * Limit registers for node n. If the SysAddr is not within the range
532 * specified by the base and limit values, then node n ignores the Sysaddr
533 * (since it does not map to node n). Otherwise continue to step 2 below.
534 *
535 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
536 * disabled so skip to step 3 below. Otherwise see if the SysAddr is within
537 * the range of relocated addresses (starting at 0x100000000) from the DRAM
538 * hole. If not, skip to step 3 below. Else get the value of the
539 * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
540 * offset defined by this value from the SysAddr.
541 *
542 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
543 * Base register for node n. To obtain the DramAddr, subtract the base
544 * address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
545 */
546static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
547{
548 u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
549 int ret = 0;
550
551 dram_base = get_dram_base(mci);
552
553 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
554 &hole_size);
555 if (!ret) {
556 if ((sys_addr >= (1ull << 32)) &&
557 (sys_addr < ((1ull << 32) + hole_size))) {
558 /* use DHAR to translate SysAddr to DramAddr */
559 dram_addr = sys_addr - hole_offset;
560
561 debugf2("using DHAR to translate SysAddr 0x%lx to "
562 "DramAddr 0x%lx\n",
563 (unsigned long)sys_addr,
564 (unsigned long)dram_addr);
565
566 return dram_addr;
567 }
568 }
569
570 /*
571 * Translate the SysAddr to a DramAddr as shown near the start of
572 * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8
573 * only deals with 40-bit values. Therefore we discard bits 63-40 of
574 * sys_addr below. If bit 39 of sys_addr is 1 then the bits we
575 * discard are all 1s. Otherwise the bits we discard are all 0s. See
576 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
577 * Programmer's Manual Volume 1 Application Programming.
578 */
579 dram_addr = (sys_addr & 0xffffffffffull) - dram_base;
580
581 debugf2("using DRAM Base register to translate SysAddr 0x%lx to "
582 "DramAddr 0x%lx\n", (unsigned long)sys_addr,
583 (unsigned long)dram_addr);
584 return dram_addr;
585}
586
587/*
588 * @intlv_en is the value of the IntlvEn field from a DRAM Base register
589 * (section 3.4.4.1). Return the number of bits from a SysAddr that are used
590 * for node interleaving.
591 */
592static int num_node_interleave_bits(unsigned intlv_en)
593{
594 static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
595 int n;
596
597 BUG_ON(intlv_en > 7);
598 n = intlv_shift_table[intlv_en];
599 return n;
600}
601
602/* Translate the DramAddr given by @dram_addr to an InputAddr. */
603static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
604{
605 struct amd64_pvt *pvt;
606 int intlv_shift;
607 u64 input_addr;
608
609 pvt = mci->pvt_info;
610
611 /*
612 * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
613 * concerning translating a DramAddr to an InputAddr.
614 */
615 intlv_shift = num_node_interleave_bits(pvt->dram_IntlvEn[0]);
616 input_addr = ((dram_addr >> intlv_shift) & 0xffffff000ull) +
617 (dram_addr & 0xfff);
618
619 debugf2(" Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
620 intlv_shift, (unsigned long)dram_addr,
621 (unsigned long)input_addr);
622
623 return input_addr;
624}
625
626/*
627 * Translate the SysAddr represented by @sys_addr to an InputAddr. It is
628 * assumed that @sys_addr maps to the node given by mci.
629 */
630static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
631{
632 u64 input_addr;
633
634 input_addr =
635 dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
636
637 debugf2("SysAdddr 0x%lx translates to InputAddr 0x%lx\n",
638 (unsigned long)sys_addr, (unsigned long)input_addr);
639
640 return input_addr;
641}
642
643
644/*
645 * @input_addr is an InputAddr associated with the node represented by mci.
646 * Translate @input_addr to a DramAddr and return the result.
647 */
648static u64 input_addr_to_dram_addr(struct mem_ctl_info *mci, u64 input_addr)
649{
650 struct amd64_pvt *pvt;
651 int node_id, intlv_shift;
652 u64 bits, dram_addr;
653 u32 intlv_sel;
654
655 /*
656 * Near the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
657 * shows how to translate a DramAddr to an InputAddr. Here we reverse
658 * this procedure. When translating from a DramAddr to an InputAddr, the
659 * bits used for node interleaving are discarded. Here we recover these
660 * bits from the IntlvSel field of the DRAM Limit register (section
661 * 3.4.4.2) for the node that input_addr is associated with.
662 */
663 pvt = mci->pvt_info;
664 node_id = pvt->mc_node_id;
665 BUG_ON((node_id < 0) || (node_id > 7));
666
667 intlv_shift = num_node_interleave_bits(pvt->dram_IntlvEn[0]);
668
669 if (intlv_shift == 0) {
670 debugf1(" InputAddr 0x%lx translates to DramAddr of "
671 "same value\n", (unsigned long)input_addr);
672
673 return input_addr;
674 }
675
676 bits = ((input_addr & 0xffffff000ull) << intlv_shift) +
677 (input_addr & 0xfff);
678
679 intlv_sel = pvt->dram_IntlvSel[node_id] & ((1 << intlv_shift) - 1);
680 dram_addr = bits + (intlv_sel << 12);
681
682 debugf1("InputAddr 0x%lx translates to DramAddr 0x%lx "
683 "(%d node interleave bits)\n", (unsigned long)input_addr,
684 (unsigned long)dram_addr, intlv_shift);
685
686 return dram_addr;
687}
688
689/*
690 * @dram_addr is a DramAddr that maps to the node represented by mci. Convert
691 * @dram_addr to a SysAddr.
692 */
693static u64 dram_addr_to_sys_addr(struct mem_ctl_info *mci, u64 dram_addr)
694{
695 struct amd64_pvt *pvt = mci->pvt_info;
696 u64 hole_base, hole_offset, hole_size, base, limit, sys_addr;
697 int ret = 0;
698
699 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
700 &hole_size);
701 if (!ret) {
702 if ((dram_addr >= hole_base) &&
703 (dram_addr < (hole_base + hole_size))) {
704 sys_addr = dram_addr + hole_offset;
705
706 debugf1("using DHAR to translate DramAddr 0x%lx to "
707 "SysAddr 0x%lx\n", (unsigned long)dram_addr,
708 (unsigned long)sys_addr);
709
710 return sys_addr;
711 }
712 }
713
714 amd64_get_base_and_limit(pvt, pvt->mc_node_id, &base, &limit);
715 sys_addr = dram_addr + base;
716
717 /*
718 * The sys_addr we have computed up to this point is a 40-bit value
719 * because the k8 deals with 40-bit values. However, the value we are
720 * supposed to return is a full 64-bit physical address. The AMD
721 * x86-64 architecture specifies that the most significant implemented
722 * address bit through bit 63 of a physical address must be either all
723 * 0s or all 1s. Therefore we sign-extend the 40-bit sys_addr to a
724 * 64-bit value below. See section 3.4.2 of AMD publication 24592:
725 * AMD x86-64 Architecture Programmer's Manual Volume 1 Application
726 * Programming.
727 */
728 sys_addr |= ~((sys_addr & (1ull << 39)) - 1);
729
730 debugf1(" Node %d, DramAddr 0x%lx to SysAddr 0x%lx\n",
731 pvt->mc_node_id, (unsigned long)dram_addr,
732 (unsigned long)sys_addr);
733
734 return sys_addr;
735}
736
737/*
738 * @input_addr is an InputAddr associated with the node given by mci. Translate
739 * @input_addr to a SysAddr.
740 */
741static inline u64 input_addr_to_sys_addr(struct mem_ctl_info *mci,
742 u64 input_addr)
743{
744 return dram_addr_to_sys_addr(mci,
745 input_addr_to_dram_addr(mci, input_addr));
746}
747
748/*
749 * Find the minimum and maximum InputAddr values that map to the given @csrow.
750 * Pass back these values in *input_addr_min and *input_addr_max.
751 */
752static void find_csrow_limits(struct mem_ctl_info *mci, int csrow,
753 u64 *input_addr_min, u64 *input_addr_max)
754{
755 struct amd64_pvt *pvt;
756 u64 base, mask;
757
758 pvt = mci->pvt_info;
Borislav Petkov9d858bb2009-09-21 14:35:51 +0200759 BUG_ON((csrow < 0) || (csrow >= pvt->cs_count));
Doug Thompson93c2df52009-05-04 20:46:50 +0200760
761 base = base_from_dct_base(pvt, csrow);
762 mask = mask_from_dct_mask(pvt, csrow);
763
764 *input_addr_min = base & ~mask;
765 *input_addr_max = base | mask | pvt->dcs_mask_notused;
766}
767
Doug Thompson93c2df52009-05-04 20:46:50 +0200768/* Map the Error address to a PAGE and PAGE OFFSET. */
769static inline void error_address_to_page_and_offset(u64 error_address,
770 u32 *page, u32 *offset)
771{
772 *page = (u32) (error_address >> PAGE_SHIFT);
773 *offset = ((u32) error_address) & ~PAGE_MASK;
774}
775
776/*
777 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
778 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
779 * of a node that detected an ECC memory error. mci represents the node that
780 * the error address maps to (possibly different from the node that detected
781 * the error). Return the number of the csrow that sys_addr maps to, or -1 on
782 * error.
783 */
784static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
785{
786 int csrow;
787
788 csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
789
790 if (csrow == -1)
791 amd64_mc_printk(mci, KERN_ERR,
792 "Failed to translate InputAddr to csrow for "
793 "address 0x%lx\n", (unsigned long)sys_addr);
794 return csrow;
795}
Doug Thompsone2ce7252009-04-27 15:57:12 +0200796
Borislav Petkovbfc04ae2009-11-12 19:05:07 +0100797static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
Doug Thompson2da11652009-04-27 16:09:09 +0200798
Borislav Petkovad6a32e2010-03-09 12:46:00 +0100799static u16 extract_syndrome(struct err_regs *err)
800{
801 return ((err->nbsh >> 15) & 0xff) | ((err->nbsl >> 16) & 0xff00);
802}
803
Doug Thompson2da11652009-04-27 16:09:09 +0200804static void amd64_cpu_display_info(struct amd64_pvt *pvt)
805{
806 if (boot_cpu_data.x86 == 0x11)
807 edac_printk(KERN_DEBUG, EDAC_MC, "F11h CPU detected\n");
808 else if (boot_cpu_data.x86 == 0x10)
809 edac_printk(KERN_DEBUG, EDAC_MC, "F10h CPU detected\n");
810 else if (boot_cpu_data.x86 == 0xf)
811 edac_printk(KERN_DEBUG, EDAC_MC, "%s detected\n",
Borislav Petkov1433eb92009-10-21 13:44:36 +0200812 (pvt->ext_model >= K8_REV_F) ?
Doug Thompson2da11652009-04-27 16:09:09 +0200813 "Rev F or later" : "Rev E or earlier");
814 else
815 /* we'll hardly ever ever get here */
816 edac_printk(KERN_ERR, EDAC_MC, "Unknown cpu!\n");
817}
818
819/*
820 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
821 * are ECC capable.
822 */
823static enum edac_type amd64_determine_edac_cap(struct amd64_pvt *pvt)
824{
825 int bit;
Borislav Petkov584fcff2009-06-10 18:29:54 +0200826 enum dev_type edac_cap = EDAC_FLAG_NONE;
Doug Thompson2da11652009-04-27 16:09:09 +0200827
Borislav Petkov1433eb92009-10-21 13:44:36 +0200828 bit = (boot_cpu_data.x86 > 0xf || pvt->ext_model >= K8_REV_F)
Doug Thompson2da11652009-04-27 16:09:09 +0200829 ? 19
830 : 17;
831
Borislav Petkov584fcff2009-06-10 18:29:54 +0200832 if (pvt->dclr0 & BIT(bit))
Doug Thompson2da11652009-04-27 16:09:09 +0200833 edac_cap = EDAC_FLAG_SECDED;
834
835 return edac_cap;
836}
837
838
Borislav Petkov8566c4d2009-10-16 13:48:28 +0200839static void amd64_debug_display_dimm_sizes(int ctrl, struct amd64_pvt *pvt);
Doug Thompson2da11652009-04-27 16:09:09 +0200840
Borislav Petkov68798e12009-11-03 16:18:33 +0100841static void amd64_dump_dramcfg_low(u32 dclr, int chan)
842{
843 debugf1("F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
844
845 debugf1(" DIMM type: %sbuffered; all DIMMs support ECC: %s\n",
846 (dclr & BIT(16)) ? "un" : "",
847 (dclr & BIT(19)) ? "yes" : "no");
848
849 debugf1(" PAR/ERR parity: %s\n",
850 (dclr & BIT(8)) ? "enabled" : "disabled");
851
852 debugf1(" DCT 128bit mode width: %s\n",
853 (dclr & BIT(11)) ? "128b" : "64b");
854
855 debugf1(" x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
856 (dclr & BIT(12)) ? "yes" : "no",
857 (dclr & BIT(13)) ? "yes" : "no",
858 (dclr & BIT(14)) ? "yes" : "no",
859 (dclr & BIT(15)) ? "yes" : "no");
860}
861
Doug Thompson2da11652009-04-27 16:09:09 +0200862/* Display and decode various NB registers for debug purposes. */
863static void amd64_dump_misc_regs(struct amd64_pvt *pvt)
864{
865 int ganged;
866
Borislav Petkov68798e12009-11-03 16:18:33 +0100867 debugf1("F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
Doug Thompson2da11652009-04-27 16:09:09 +0200868
Borislav Petkov68798e12009-11-03 16:18:33 +0100869 debugf1(" NB two channel DRAM capable: %s\n",
870 (pvt->nbcap & K8_NBCAP_DCT_DUAL) ? "yes" : "no");
871
872 debugf1(" ECC capable: %s, ChipKill ECC capable: %s\n",
873 (pvt->nbcap & K8_NBCAP_SECDED) ? "yes" : "no",
874 (pvt->nbcap & K8_NBCAP_CHIPKILL) ? "yes" : "no");
875
876 amd64_dump_dramcfg_low(pvt->dclr0, 0);
Doug Thompson2da11652009-04-27 16:09:09 +0200877
Borislav Petkov8de1d912009-10-16 13:39:30 +0200878 debugf1("F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
Doug Thompson2da11652009-04-27 16:09:09 +0200879
Borislav Petkov8de1d912009-10-16 13:39:30 +0200880 debugf1("F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, "
881 "offset: 0x%08x\n",
882 pvt->dhar,
883 dhar_base(pvt->dhar),
884 (boot_cpu_data.x86 == 0xf) ? k8_dhar_offset(pvt->dhar)
885 : f10_dhar_offset(pvt->dhar));
Doug Thompson2da11652009-04-27 16:09:09 +0200886
Borislav Petkov8de1d912009-10-16 13:39:30 +0200887 debugf1(" DramHoleValid: %s\n",
888 (pvt->dhar & DHAR_VALID) ? "yes" : "no");
Doug Thompson2da11652009-04-27 16:09:09 +0200889
Borislav Petkov8de1d912009-10-16 13:39:30 +0200890 /* everything below this point is Fam10h and above */
Borislav Petkov8566c4d2009-10-16 13:48:28 +0200891 if (boot_cpu_data.x86 == 0xf) {
892 amd64_debug_display_dimm_sizes(0, pvt);
Doug Thompson2da11652009-04-27 16:09:09 +0200893 return;
Borislav Petkov8566c4d2009-10-16 13:48:28 +0200894 }
Doug Thompson2da11652009-04-27 16:09:09 +0200895
Borislav Petkovad6a32e2010-03-09 12:46:00 +0100896 amd64_printk(KERN_INFO, "using %s syndromes.\n",
897 ((pvt->syn_type == 8) ? "x8" : "x4"));
898
Borislav Petkov8de1d912009-10-16 13:39:30 +0200899 /* Only if NOT ganged does dclr1 have valid info */
Borislav Petkov68798e12009-11-03 16:18:33 +0100900 if (!dct_ganging_enabled(pvt))
901 amd64_dump_dramcfg_low(pvt->dclr1, 1);
Doug Thompson2da11652009-04-27 16:09:09 +0200902
903 /*
904 * Determine if ganged and then dump memory sizes for first controller,
905 * and if NOT ganged dump info for 2nd controller.
906 */
907 ganged = dct_ganging_enabled(pvt);
908
Borislav Petkov8566c4d2009-10-16 13:48:28 +0200909 amd64_debug_display_dimm_sizes(0, pvt);
Doug Thompson2da11652009-04-27 16:09:09 +0200910
911 if (!ganged)
Borislav Petkov8566c4d2009-10-16 13:48:28 +0200912 amd64_debug_display_dimm_sizes(1, pvt);
Doug Thompson2da11652009-04-27 16:09:09 +0200913}
914
915/* Read in both of DBAM registers */
916static void amd64_read_dbam_reg(struct amd64_pvt *pvt)
917{
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +0200918 amd64_read_pci_cfg(pvt->dram_f2_ctl, DBAM0, &pvt->dbam0);
Doug Thompson2da11652009-04-27 16:09:09 +0200919
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +0200920 if (boot_cpu_data.x86 >= 0x10)
921 amd64_read_pci_cfg(pvt->dram_f2_ctl, DBAM1, &pvt->dbam1);
Doug Thompson2da11652009-04-27 16:09:09 +0200922}
923
Doug Thompson94be4bf2009-04-27 16:12:00 +0200924/*
925 * NOTE: CPU Revision Dependent code: Rev E and Rev F
926 *
927 * Set the DCSB and DCSM mask values depending on the CPU revision value. Also
928 * set the shift factor for the DCSB and DCSM values.
929 *
930 * ->dcs_mask_notused, RevE:
931 *
932 * To find the max InputAddr for the csrow, start with the base address and set
933 * all bits that are "don't care" bits in the test at the start of section
934 * 3.5.4 (p. 84).
935 *
936 * The "don't care" bits are all set bits in the mask and all bits in the gaps
937 * between bit ranges [35:25] and [19:13]. The value REV_E_DCS_NOTUSED_BITS
938 * represents bits [24:20] and [12:0], which are all bits in the above-mentioned
939 * gaps.
940 *
941 * ->dcs_mask_notused, RevF and later:
942 *
943 * To find the max InputAddr for the csrow, start with the base address and set
944 * all bits that are "don't care" bits in the test at the start of NPT section
945 * 4.5.4 (p. 87).
946 *
947 * The "don't care" bits are all set bits in the mask and all bits in the gaps
948 * between bit ranges [36:27] and [21:13].
949 *
950 * The value REV_F_F1Xh_DCS_NOTUSED_BITS represents bits [26:22] and [12:0],
951 * which are all bits in the above-mentioned gaps.
952 */
953static void amd64_set_dct_base_and_mask(struct amd64_pvt *pvt)
954{
Borislav Petkov9d858bb2009-09-21 14:35:51 +0200955
Borislav Petkov1433eb92009-10-21 13:44:36 +0200956 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) {
Borislav Petkov9d858bb2009-09-21 14:35:51 +0200957 pvt->dcsb_base = REV_E_DCSB_BASE_BITS;
958 pvt->dcsm_mask = REV_E_DCSM_MASK_BITS;
959 pvt->dcs_mask_notused = REV_E_DCS_NOTUSED_BITS;
960 pvt->dcs_shift = REV_E_DCS_SHIFT;
961 pvt->cs_count = 8;
962 pvt->num_dcsm = 8;
963 } else {
Doug Thompson94be4bf2009-04-27 16:12:00 +0200964 pvt->dcsb_base = REV_F_F1Xh_DCSB_BASE_BITS;
965 pvt->dcsm_mask = REV_F_F1Xh_DCSM_MASK_BITS;
966 pvt->dcs_mask_notused = REV_F_F1Xh_DCS_NOTUSED_BITS;
967 pvt->dcs_shift = REV_F_F1Xh_DCS_SHIFT;
968
Borislav Petkov9d858bb2009-09-21 14:35:51 +0200969 if (boot_cpu_data.x86 == 0x11) {
970 pvt->cs_count = 4;
971 pvt->num_dcsm = 2;
972 } else {
973 pvt->cs_count = 8;
974 pvt->num_dcsm = 4;
Doug Thompson94be4bf2009-04-27 16:12:00 +0200975 }
Doug Thompson94be4bf2009-04-27 16:12:00 +0200976 }
977}
978
979/*
980 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask hw registers
981 */
982static void amd64_read_dct_base_mask(struct amd64_pvt *pvt)
983{
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +0200984 int cs, reg;
Doug Thompson94be4bf2009-04-27 16:12:00 +0200985
986 amd64_set_dct_base_and_mask(pvt);
987
Borislav Petkov9d858bb2009-09-21 14:35:51 +0200988 for (cs = 0; cs < pvt->cs_count; cs++) {
Doug Thompson94be4bf2009-04-27 16:12:00 +0200989 reg = K8_DCSB0 + (cs * 4);
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +0200990 if (!amd64_read_pci_cfg(pvt->dram_f2_ctl, reg, &pvt->dcsb0[cs]))
Doug Thompson94be4bf2009-04-27 16:12:00 +0200991 debugf0(" DCSB0[%d]=0x%08x reg: F2x%x\n",
992 cs, pvt->dcsb0[cs], reg);
993
994 /* If DCT are NOT ganged, then read in DCT1's base */
995 if (boot_cpu_data.x86 >= 0x10 && !dct_ganging_enabled(pvt)) {
996 reg = F10_DCSB1 + (cs * 4);
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +0200997 if (!amd64_read_pci_cfg(pvt->dram_f2_ctl, reg,
998 &pvt->dcsb1[cs]))
Doug Thompson94be4bf2009-04-27 16:12:00 +0200999 debugf0(" DCSB1[%d]=0x%08x reg: F2x%x\n",
1000 cs, pvt->dcsb1[cs], reg);
1001 } else {
1002 pvt->dcsb1[cs] = 0;
1003 }
1004 }
1005
1006 for (cs = 0; cs < pvt->num_dcsm; cs++) {
Wan Wei4afcd2d2009-07-27 14:34:15 +02001007 reg = K8_DCSM0 + (cs * 4);
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02001008 if (!amd64_read_pci_cfg(pvt->dram_f2_ctl, reg, &pvt->dcsm0[cs]))
Doug Thompson94be4bf2009-04-27 16:12:00 +02001009 debugf0(" DCSM0[%d]=0x%08x reg: F2x%x\n",
1010 cs, pvt->dcsm0[cs], reg);
1011
1012 /* If DCT are NOT ganged, then read in DCT1's mask */
1013 if (boot_cpu_data.x86 >= 0x10 && !dct_ganging_enabled(pvt)) {
1014 reg = F10_DCSM1 + (cs * 4);
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02001015 if (!amd64_read_pci_cfg(pvt->dram_f2_ctl, reg,
1016 &pvt->dcsm1[cs]))
Doug Thompson94be4bf2009-04-27 16:12:00 +02001017 debugf0(" DCSM1[%d]=0x%08x reg: F2x%x\n",
1018 cs, pvt->dcsm1[cs], reg);
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02001019 } else {
Doug Thompson94be4bf2009-04-27 16:12:00 +02001020 pvt->dcsm1[cs] = 0;
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02001021 }
Doug Thompson94be4bf2009-04-27 16:12:00 +02001022 }
1023}
1024
1025static enum mem_type amd64_determine_memory_type(struct amd64_pvt *pvt)
1026{
1027 enum mem_type type;
1028
Borislav Petkov1433eb92009-10-21 13:44:36 +02001029 if (boot_cpu_data.x86 >= 0x10 || pvt->ext_model >= K8_REV_F) {
Borislav Petkov6b4c0bd2009-11-12 15:37:57 +01001030 if (pvt->dchr0 & DDR3_MODE)
1031 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
1032 else
1033 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
Doug Thompson94be4bf2009-04-27 16:12:00 +02001034 } else {
Doug Thompson94be4bf2009-04-27 16:12:00 +02001035 type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
1036 }
1037
Borislav Petkov239642f2009-11-12 15:33:16 +01001038 debugf1(" Memory type is: %s\n", edac_mem_types[type]);
Doug Thompson94be4bf2009-04-27 16:12:00 +02001039
1040 return type;
1041}
1042
Doug Thompsonddff8762009-04-27 16:14:52 +02001043/*
1044 * Read the DRAM Configuration Low register. It differs between CG, D & E revs
1045 * and the later RevF memory controllers (DDR vs DDR2)
1046 *
1047 * Return:
1048 * number of memory channels in operation
1049 * Pass back:
1050 * contents of the DCL0_LOW register
1051 */
1052static int k8_early_channel_count(struct amd64_pvt *pvt)
1053{
1054 int flag, err = 0;
1055
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02001056 err = amd64_read_pci_cfg(pvt->dram_f2_ctl, F10_DCLR_0, &pvt->dclr0);
Doug Thompsonddff8762009-04-27 16:14:52 +02001057 if (err)
1058 return err;
1059
Borislav Petkov1433eb92009-10-21 13:44:36 +02001060 if ((boot_cpu_data.x86_model >> 4) >= K8_REV_F) {
Doug Thompsonddff8762009-04-27 16:14:52 +02001061 /* RevF (NPT) and later */
1062 flag = pvt->dclr0 & F10_WIDTH_128;
1063 } else {
1064 /* RevE and earlier */
1065 flag = pvt->dclr0 & REVE_WIDTH_128;
1066 }
1067
1068 /* not used */
1069 pvt->dclr1 = 0;
1070
1071 return (flag) ? 2 : 1;
1072}
1073
1074/* extract the ERROR ADDRESS for the K8 CPUs */
1075static u64 k8_get_error_address(struct mem_ctl_info *mci,
Borislav Petkovef44cc42009-07-23 14:45:48 +02001076 struct err_regs *info)
Doug Thompsonddff8762009-04-27 16:14:52 +02001077{
1078 return (((u64) (info->nbeah & 0xff)) << 32) +
1079 (info->nbeal & ~0x03);
1080}
1081
1082/*
1083 * Read the Base and Limit registers for K8 based Memory controllers; extract
1084 * fields from the 'raw' reg into separate data fields
1085 *
1086 * Isolates: BASE, LIMIT, IntlvEn, IntlvSel, RW_EN
1087 */
1088static void k8_read_dram_base_limit(struct amd64_pvt *pvt, int dram)
1089{
1090 u32 low;
1091 u32 off = dram << 3; /* 8 bytes between DRAM entries */
Doug Thompsonddff8762009-04-27 16:14:52 +02001092
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02001093 amd64_read_pci_cfg(pvt->addr_f1_ctl, K8_DRAM_BASE_LOW + off, &low);
Doug Thompsonddff8762009-04-27 16:14:52 +02001094
1095 /* Extract parts into separate data entries */
Borislav Petkov49978112009-10-12 17:23:03 +02001096 pvt->dram_base[dram] = ((u64) low & 0xFFFF0000) << 8;
Doug Thompsonddff8762009-04-27 16:14:52 +02001097 pvt->dram_IntlvEn[dram] = (low >> 8) & 0x7;
1098 pvt->dram_rw_en[dram] = (low & 0x3);
1099
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02001100 amd64_read_pci_cfg(pvt->addr_f1_ctl, K8_DRAM_LIMIT_LOW + off, &low);
Doug Thompsonddff8762009-04-27 16:14:52 +02001101
1102 /*
1103 * Extract parts into separate data entries. Limit is the HIGHEST memory
1104 * location of the region, so lower 24 bits need to be all ones
1105 */
Borislav Petkov49978112009-10-12 17:23:03 +02001106 pvt->dram_limit[dram] = (((u64) low & 0xFFFF0000) << 8) | 0x00FFFFFF;
Doug Thompsonddff8762009-04-27 16:14:52 +02001107 pvt->dram_IntlvSel[dram] = (low >> 8) & 0x7;
1108 pvt->dram_DstNode[dram] = (low & 0x7);
1109}
1110
1111static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci,
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001112 struct err_regs *err_info, u64 sys_addr)
Doug Thompsonddff8762009-04-27 16:14:52 +02001113{
1114 struct mem_ctl_info *src_mci;
Doug Thompsonddff8762009-04-27 16:14:52 +02001115 int channel, csrow;
1116 u32 page, offset;
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001117 u16 syndrome;
Doug Thompsonddff8762009-04-27 16:14:52 +02001118
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001119 syndrome = extract_syndrome(err_info);
Doug Thompsonddff8762009-04-27 16:14:52 +02001120
1121 /* CHIPKILL enabled */
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001122 if (err_info->nbcfg & K8_NBCFG_CHIPKILL) {
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001123 channel = get_channel_from_ecc_syndrome(mci, syndrome);
Doug Thompsonddff8762009-04-27 16:14:52 +02001124 if (channel < 0) {
1125 /*
1126 * Syndrome didn't map, so we don't know which of the
1127 * 2 DIMMs is in error. So we need to ID 'both' of them
1128 * as suspect.
1129 */
1130 amd64_mc_printk(mci, KERN_WARNING,
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001131 "unknown syndrome 0x%04x - possible "
1132 "error reporting race\n", syndrome);
Doug Thompsonddff8762009-04-27 16:14:52 +02001133 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1134 return;
1135 }
1136 } else {
1137 /*
1138 * non-chipkill ecc mode
1139 *
1140 * The k8 documentation is unclear about how to determine the
1141 * channel number when using non-chipkill memory. This method
1142 * was obtained from email communication with someone at AMD.
1143 * (Wish the email was placed in this comment - norsk)
1144 */
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001145 channel = ((sys_addr & BIT(3)) != 0);
Doug Thompsonddff8762009-04-27 16:14:52 +02001146 }
1147
1148 /*
1149 * Find out which node the error address belongs to. This may be
1150 * different from the node that detected the error.
1151 */
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001152 src_mci = find_mc_by_sys_addr(mci, sys_addr);
Keith Mannthey2cff18c2009-09-18 14:35:23 +02001153 if (!src_mci) {
Doug Thompsonddff8762009-04-27 16:14:52 +02001154 amd64_mc_printk(mci, KERN_ERR,
1155 "failed to map error address 0x%lx to a node\n",
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001156 (unsigned long)sys_addr);
Doug Thompsonddff8762009-04-27 16:14:52 +02001157 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1158 return;
1159 }
1160
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001161 /* Now map the sys_addr to a CSROW */
1162 csrow = sys_addr_to_csrow(src_mci, sys_addr);
Doug Thompsonddff8762009-04-27 16:14:52 +02001163 if (csrow < 0) {
1164 edac_mc_handle_ce_no_info(src_mci, EDAC_MOD_STR);
1165 } else {
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001166 error_address_to_page_and_offset(sys_addr, &page, &offset);
Doug Thompsonddff8762009-04-27 16:14:52 +02001167
1168 edac_mc_handle_ce(src_mci, page, offset, syndrome, csrow,
1169 channel, EDAC_MOD_STR);
1170 }
1171}
1172
Borislav Petkov1433eb92009-10-21 13:44:36 +02001173static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, int cs_mode)
Doug Thompsonddff8762009-04-27 16:14:52 +02001174{
Borislav Petkov1433eb92009-10-21 13:44:36 +02001175 int *dbam_map;
Doug Thompsonddff8762009-04-27 16:14:52 +02001176
Borislav Petkov1433eb92009-10-21 13:44:36 +02001177 if (pvt->ext_model >= K8_REV_F)
1178 dbam_map = ddr2_dbam;
1179 else if (pvt->ext_model >= K8_REV_D)
1180 dbam_map = ddr2_dbam_revD;
1181 else
1182 dbam_map = ddr2_dbam_revCG;
Doug Thompsonddff8762009-04-27 16:14:52 +02001183
Borislav Petkov1433eb92009-10-21 13:44:36 +02001184 return dbam_map[cs_mode];
Doug Thompsonddff8762009-04-27 16:14:52 +02001185}
1186
Doug Thompson1afd3c92009-04-27 16:16:50 +02001187/*
1188 * Get the number of DCT channels in use.
1189 *
1190 * Return:
1191 * number of Memory Channels in operation
1192 * Pass back:
1193 * contents of the DCL0_LOW register
1194 */
1195static int f10_early_channel_count(struct amd64_pvt *pvt)
1196{
Wan Wei57a30852009-08-07 17:04:49 +02001197 int dbams[] = { DBAM0, DBAM1 };
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02001198 int i, j, channels = 0;
Doug Thompson1afd3c92009-04-27 16:16:50 +02001199 u32 dbam;
Doug Thompsonddff8762009-04-27 16:14:52 +02001200
Doug Thompson1afd3c92009-04-27 16:16:50 +02001201 /* If we are in 128 bit mode, then we are using 2 channels */
1202 if (pvt->dclr0 & F10_WIDTH_128) {
Doug Thompson1afd3c92009-04-27 16:16:50 +02001203 channels = 2;
1204 return channels;
1205 }
1206
1207 /*
Borislav Petkovd16149e2009-10-16 19:55:49 +02001208 * Need to check if in unganged mode: In such, there are 2 channels,
1209 * but they are not in 128 bit mode and thus the above 'dclr0' status
1210 * bit will be OFF.
Doug Thompson1afd3c92009-04-27 16:16:50 +02001211 *
1212 * Need to check DCT0[0] and DCT1[0] to see if only one of them has
1213 * their CSEnable bit on. If so, then SINGLE DIMM case.
1214 */
Borislav Petkovd16149e2009-10-16 19:55:49 +02001215 debugf0("Data width is not 128 bits - need more decoding\n");
Doug Thompson1afd3c92009-04-27 16:16:50 +02001216
1217 /*
1218 * Check DRAM Bank Address Mapping values for each DIMM to see if there
1219 * is more than just one DIMM present in unganged mode. Need to check
1220 * both controllers since DIMMs can be placed in either one.
1221 */
Wan Wei57a30852009-08-07 17:04:49 +02001222 for (i = 0; i < ARRAY_SIZE(dbams); i++) {
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02001223 if (amd64_read_pci_cfg(pvt->dram_f2_ctl, dbams[i], &dbam))
Doug Thompson1afd3c92009-04-27 16:16:50 +02001224 goto err_reg;
1225
Wan Wei57a30852009-08-07 17:04:49 +02001226 for (j = 0; j < 4; j++) {
1227 if (DBAM_DIMM(j, dbam) > 0) {
1228 channels++;
1229 break;
1230 }
1231 }
Doug Thompson1afd3c92009-04-27 16:16:50 +02001232 }
1233
Borislav Petkovd16149e2009-10-16 19:55:49 +02001234 if (channels > 2)
1235 channels = 2;
1236
Borislav Petkov37da0452009-06-10 17:36:57 +02001237 debugf0("MCT channel count: %d\n", channels);
Doug Thompson1afd3c92009-04-27 16:16:50 +02001238
1239 return channels;
1240
1241err_reg:
1242 return -1;
1243
1244}
1245
Borislav Petkov1433eb92009-10-21 13:44:36 +02001246static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, int cs_mode)
Doug Thompson1afd3c92009-04-27 16:16:50 +02001247{
Borislav Petkov1433eb92009-10-21 13:44:36 +02001248 int *dbam_map;
1249
1250 if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
1251 dbam_map = ddr3_dbam;
1252 else
1253 dbam_map = ddr2_dbam;
1254
1255 return dbam_map[cs_mode];
Doug Thompson1afd3c92009-04-27 16:16:50 +02001256}
1257
1258/* Enable extended configuration access via 0xCF8 feature */
1259static void amd64_setup(struct amd64_pvt *pvt)
1260{
1261 u32 reg;
1262
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02001263 amd64_read_pci_cfg(pvt->misc_f3_ctl, F10_NB_CFG_HIGH, &reg);
Doug Thompson1afd3c92009-04-27 16:16:50 +02001264
1265 pvt->flags.cf8_extcfg = !!(reg & F10_NB_CFG_LOW_ENABLE_EXT_CFG);
1266 reg |= F10_NB_CFG_LOW_ENABLE_EXT_CFG;
1267 pci_write_config_dword(pvt->misc_f3_ctl, F10_NB_CFG_HIGH, reg);
1268}
1269
1270/* Restore the extended configuration access via 0xCF8 feature */
1271static void amd64_teardown(struct amd64_pvt *pvt)
1272{
1273 u32 reg;
1274
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02001275 amd64_read_pci_cfg(pvt->misc_f3_ctl, F10_NB_CFG_HIGH, &reg);
Doug Thompson1afd3c92009-04-27 16:16:50 +02001276
1277 reg &= ~F10_NB_CFG_LOW_ENABLE_EXT_CFG;
1278 if (pvt->flags.cf8_extcfg)
1279 reg |= F10_NB_CFG_LOW_ENABLE_EXT_CFG;
1280 pci_write_config_dword(pvt->misc_f3_ctl, F10_NB_CFG_HIGH, reg);
1281}
1282
1283static u64 f10_get_error_address(struct mem_ctl_info *mci,
Borislav Petkovef44cc42009-07-23 14:45:48 +02001284 struct err_regs *info)
Doug Thompson1afd3c92009-04-27 16:16:50 +02001285{
1286 return (((u64) (info->nbeah & 0xffff)) << 32) +
1287 (info->nbeal & ~0x01);
1288}
1289
1290/*
1291 * Read the Base and Limit registers for F10 based Memory controllers. Extract
1292 * fields from the 'raw' reg into separate data fields.
1293 *
1294 * Isolates: BASE, LIMIT, IntlvEn, IntlvSel, RW_EN.
1295 */
1296static void f10_read_dram_base_limit(struct amd64_pvt *pvt, int dram)
1297{
1298 u32 high_offset, low_offset, high_base, low_base, high_limit, low_limit;
1299
1300 low_offset = K8_DRAM_BASE_LOW + (dram << 3);
1301 high_offset = F10_DRAM_BASE_HIGH + (dram << 3);
1302
1303 /* read the 'raw' DRAM BASE Address register */
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02001304 amd64_read_pci_cfg(pvt->addr_f1_ctl, low_offset, &low_base);
Doug Thompson1afd3c92009-04-27 16:16:50 +02001305
1306 /* Read from the ECS data register */
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02001307 amd64_read_pci_cfg(pvt->addr_f1_ctl, high_offset, &high_base);
Doug Thompson1afd3c92009-04-27 16:16:50 +02001308
1309 /* Extract parts into separate data entries */
1310 pvt->dram_rw_en[dram] = (low_base & 0x3);
1311
1312 if (pvt->dram_rw_en[dram] == 0)
1313 return;
1314
1315 pvt->dram_IntlvEn[dram] = (low_base >> 8) & 0x7;
1316
Borislav Petkov66216a72009-09-22 16:48:37 +02001317 pvt->dram_base[dram] = (((u64)high_base & 0x000000FF) << 40) |
Borislav Petkov49978112009-10-12 17:23:03 +02001318 (((u64)low_base & 0xFFFF0000) << 8);
Doug Thompson1afd3c92009-04-27 16:16:50 +02001319
1320 low_offset = K8_DRAM_LIMIT_LOW + (dram << 3);
1321 high_offset = F10_DRAM_LIMIT_HIGH + (dram << 3);
1322
1323 /* read the 'raw' LIMIT registers */
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02001324 amd64_read_pci_cfg(pvt->addr_f1_ctl, low_offset, &low_limit);
Doug Thompson1afd3c92009-04-27 16:16:50 +02001325
1326 /* Read from the ECS data register for the HIGH portion */
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02001327 amd64_read_pci_cfg(pvt->addr_f1_ctl, high_offset, &high_limit);
Doug Thompson1afd3c92009-04-27 16:16:50 +02001328
Doug Thompson1afd3c92009-04-27 16:16:50 +02001329 pvt->dram_DstNode[dram] = (low_limit & 0x7);
1330 pvt->dram_IntlvSel[dram] = (low_limit >> 8) & 0x7;
1331
1332 /*
1333 * Extract address values and form a LIMIT address. Limit is the HIGHEST
1334 * memory location of the region, so low 24 bits need to be all ones.
1335 */
Borislav Petkov66216a72009-09-22 16:48:37 +02001336 pvt->dram_limit[dram] = (((u64)high_limit & 0x000000FF) << 40) |
Borislav Petkov49978112009-10-12 17:23:03 +02001337 (((u64) low_limit & 0xFFFF0000) << 8) |
Borislav Petkov66216a72009-09-22 16:48:37 +02001338 0x00FFFFFF;
Doug Thompson1afd3c92009-04-27 16:16:50 +02001339}
Doug Thompson6163b5d2009-04-27 16:20:17 +02001340
1341static void f10_read_dram_ctl_register(struct amd64_pvt *pvt)
1342{
Doug Thompson6163b5d2009-04-27 16:20:17 +02001343
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02001344 if (!amd64_read_pci_cfg(pvt->dram_f2_ctl, F10_DCTL_SEL_LOW,
1345 &pvt->dram_ctl_select_low)) {
Borislav Petkov72381bd2009-10-09 19:14:43 +02001346 debugf0("F2x110 (DCTL Sel. Low): 0x%08x, "
1347 "High range addresses at: 0x%x\n",
1348 pvt->dram_ctl_select_low,
1349 dct_sel_baseaddr(pvt));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001350
Borislav Petkov72381bd2009-10-09 19:14:43 +02001351 debugf0(" DCT mode: %s, All DCTs on: %s\n",
1352 (dct_ganging_enabled(pvt) ? "ganged" : "unganged"),
1353 (dct_dram_enabled(pvt) ? "yes" : "no"));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001354
Borislav Petkov72381bd2009-10-09 19:14:43 +02001355 if (!dct_ganging_enabled(pvt))
1356 debugf0(" Address range split per DCT: %s\n",
1357 (dct_high_range_enabled(pvt) ? "yes" : "no"));
1358
1359 debugf0(" DCT data interleave for ECC: %s, "
1360 "DRAM cleared since last warm reset: %s\n",
1361 (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
1362 (dct_memory_cleared(pvt) ? "yes" : "no"));
1363
1364 debugf0(" DCT channel interleave: %s, "
1365 "DCT interleave bits selector: 0x%x\n",
1366 (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
Doug Thompson6163b5d2009-04-27 16:20:17 +02001367 dct_sel_interleave_addr(pvt));
1368 }
1369
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02001370 amd64_read_pci_cfg(pvt->dram_f2_ctl, F10_DCTL_SEL_HIGH,
1371 &pvt->dram_ctl_select_high);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001372}
1373
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001374/*
1375 * determine channel based on the interleaving mode: F10h BKDG, 2.8.9 Memory
1376 * Interleaving Modes.
1377 */
Doug Thompson6163b5d2009-04-27 16:20:17 +02001378static u32 f10_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1379 int hi_range_sel, u32 intlv_en)
1380{
1381 u32 cs, temp, dct_sel_high = (pvt->dram_ctl_select_low >> 1) & 1;
1382
1383 if (dct_ganging_enabled(pvt))
1384 cs = 0;
1385 else if (hi_range_sel)
1386 cs = dct_sel_high;
1387 else if (dct_interleave_enabled(pvt)) {
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001388 /*
1389 * see F2x110[DctSelIntLvAddr] - channel interleave mode
1390 */
Doug Thompson6163b5d2009-04-27 16:20:17 +02001391 if (dct_sel_interleave_addr(pvt) == 0)
1392 cs = sys_addr >> 6 & 1;
1393 else if ((dct_sel_interleave_addr(pvt) >> 1) & 1) {
1394 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2;
1395
1396 if (dct_sel_interleave_addr(pvt) & 1)
1397 cs = (sys_addr >> 9 & 1) ^ temp;
1398 else
1399 cs = (sys_addr >> 6 & 1) ^ temp;
1400 } else if (intlv_en & 4)
1401 cs = sys_addr >> 15 & 1;
1402 else if (intlv_en & 2)
1403 cs = sys_addr >> 14 & 1;
1404 else if (intlv_en & 1)
1405 cs = sys_addr >> 13 & 1;
1406 else
1407 cs = sys_addr >> 12 & 1;
1408 } else if (dct_high_range_enabled(pvt) && !dct_ganging_enabled(pvt))
1409 cs = ~dct_sel_high & 1;
1410 else
1411 cs = 0;
1412
1413 return cs;
1414}
1415
1416static inline u32 f10_map_intlv_en_to_shift(u32 intlv_en)
1417{
1418 if (intlv_en == 1)
1419 return 1;
1420 else if (intlv_en == 3)
1421 return 2;
1422 else if (intlv_en == 7)
1423 return 3;
1424
1425 return 0;
1426}
1427
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001428/* See F10h BKDG, 2.8.10.2 DctSelBaseOffset Programming */
1429static inline u64 f10_get_base_addr_offset(u64 sys_addr, int hi_range_sel,
Doug Thompson6163b5d2009-04-27 16:20:17 +02001430 u32 dct_sel_base_addr,
1431 u64 dct_sel_base_off,
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001432 u32 hole_valid, u32 hole_off,
Doug Thompson6163b5d2009-04-27 16:20:17 +02001433 u64 dram_base)
1434{
1435 u64 chan_off;
1436
1437 if (hi_range_sel) {
Borislav Petkov9975a5f2010-03-08 18:29:35 +01001438 if (!(dct_sel_base_addr & 0xFFFF0000) &&
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001439 hole_valid && (sys_addr >= 0x100000000ULL))
Doug Thompson6163b5d2009-04-27 16:20:17 +02001440 chan_off = hole_off << 16;
1441 else
1442 chan_off = dct_sel_base_off;
1443 } else {
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001444 if (hole_valid && (sys_addr >= 0x100000000ULL))
Doug Thompson6163b5d2009-04-27 16:20:17 +02001445 chan_off = hole_off << 16;
1446 else
1447 chan_off = dram_base & 0xFFFFF8000000ULL;
1448 }
1449
1450 return (sys_addr & 0x0000FFFFFFFFFFC0ULL) -
1451 (chan_off & 0x0000FFFFFF800000ULL);
1452}
1453
1454/* Hack for the time being - Can we get this from BIOS?? */
1455#define CH0SPARE_RANK 0
1456#define CH1SPARE_RANK 1
1457
1458/*
1459 * checks if the csrow passed in is marked as SPARED, if so returns the new
1460 * spare row
1461 */
1462static inline int f10_process_possible_spare(int csrow,
1463 u32 cs, struct amd64_pvt *pvt)
1464{
1465 u32 swap_done;
1466 u32 bad_dram_cs;
1467
1468 /* Depending on channel, isolate respective SPARING info */
1469 if (cs) {
1470 swap_done = F10_ONLINE_SPARE_SWAPDONE1(pvt->online_spare);
1471 bad_dram_cs = F10_ONLINE_SPARE_BADDRAM_CS1(pvt->online_spare);
1472 if (swap_done && (csrow == bad_dram_cs))
1473 csrow = CH1SPARE_RANK;
1474 } else {
1475 swap_done = F10_ONLINE_SPARE_SWAPDONE0(pvt->online_spare);
1476 bad_dram_cs = F10_ONLINE_SPARE_BADDRAM_CS0(pvt->online_spare);
1477 if (swap_done && (csrow == bad_dram_cs))
1478 csrow = CH0SPARE_RANK;
1479 }
1480 return csrow;
1481}
1482
1483/*
1484 * Iterate over the DRAM DCT "base" and "mask" registers looking for a
1485 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
1486 *
1487 * Return:
1488 * -EINVAL: NOT FOUND
1489 * 0..csrow = Chip-Select Row
1490 */
1491static int f10_lookup_addr_in_dct(u32 in_addr, u32 nid, u32 cs)
1492{
1493 struct mem_ctl_info *mci;
1494 struct amd64_pvt *pvt;
1495 u32 cs_base, cs_mask;
1496 int cs_found = -EINVAL;
1497 int csrow;
1498
1499 mci = mci_lookup[nid];
1500 if (!mci)
1501 return cs_found;
1502
1503 pvt = mci->pvt_info;
1504
1505 debugf1("InputAddr=0x%x channelselect=%d\n", in_addr, cs);
1506
Borislav Petkov9d858bb2009-09-21 14:35:51 +02001507 for (csrow = 0; csrow < pvt->cs_count; csrow++) {
Doug Thompson6163b5d2009-04-27 16:20:17 +02001508
1509 cs_base = amd64_get_dct_base(pvt, cs, csrow);
1510 if (!(cs_base & K8_DCSB_CS_ENABLE))
1511 continue;
1512
1513 /*
1514 * We have an ENABLED CSROW, Isolate just the MASK bits of the
1515 * target: [28:19] and [13:5], which map to [36:27] and [21:13]
1516 * of the actual address.
1517 */
1518 cs_base &= REV_F_F1Xh_DCSB_BASE_BITS;
1519
1520 /*
1521 * Get the DCT Mask, and ENABLE the reserved bits: [18:16] and
1522 * [4:0] to become ON. Then mask off bits [28:0] ([36:8])
1523 */
1524 cs_mask = amd64_get_dct_mask(pvt, cs, csrow);
1525
1526 debugf1(" CSROW=%d CSBase=0x%x RAW CSMask=0x%x\n",
1527 csrow, cs_base, cs_mask);
1528
1529 cs_mask = (cs_mask | 0x0007C01F) & 0x1FFFFFFF;
1530
1531 debugf1(" Final CSMask=0x%x\n", cs_mask);
1532 debugf1(" (InputAddr & ~CSMask)=0x%x "
1533 "(CSBase & ~CSMask)=0x%x\n",
1534 (in_addr & ~cs_mask), (cs_base & ~cs_mask));
1535
1536 if ((in_addr & ~cs_mask) == (cs_base & ~cs_mask)) {
1537 cs_found = f10_process_possible_spare(csrow, cs, pvt);
1538
1539 debugf1(" MATCH csrow=%d\n", cs_found);
1540 break;
1541 }
1542 }
1543 return cs_found;
1544}
1545
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001546/* For a given @dram_range, check if @sys_addr falls within it. */
1547static int f10_match_to_this_node(struct amd64_pvt *pvt, int dram_range,
1548 u64 sys_addr, int *nid, int *chan_sel)
1549{
1550 int node_id, cs_found = -EINVAL, high_range = 0;
1551 u32 intlv_en, intlv_sel, intlv_shift, hole_off;
1552 u32 hole_valid, tmp, dct_sel_base, channel;
1553 u64 dram_base, chan_addr, dct_sel_base_off;
1554
1555 dram_base = pvt->dram_base[dram_range];
1556 intlv_en = pvt->dram_IntlvEn[dram_range];
1557
1558 node_id = pvt->dram_DstNode[dram_range];
1559 intlv_sel = pvt->dram_IntlvSel[dram_range];
1560
1561 debugf1("(dram=%d) Base=0x%llx SystemAddr= 0x%llx Limit=0x%llx\n",
1562 dram_range, dram_base, sys_addr, pvt->dram_limit[dram_range]);
1563
1564 /*
1565 * This assumes that one node's DHAR is the same as all the other
1566 * nodes' DHAR.
1567 */
1568 hole_off = (pvt->dhar & 0x0000FF80);
1569 hole_valid = (pvt->dhar & 0x1);
1570 dct_sel_base_off = (pvt->dram_ctl_select_high & 0xFFFFFC00) << 16;
1571
1572 debugf1(" HoleOffset=0x%x HoleValid=0x%x IntlvSel=0x%x\n",
1573 hole_off, hole_valid, intlv_sel);
1574
1575 if (intlv_en ||
1576 (intlv_sel != ((sys_addr >> 12) & intlv_en)))
1577 return -EINVAL;
1578
1579 dct_sel_base = dct_sel_baseaddr(pvt);
1580
1581 /*
1582 * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
1583 * select between DCT0 and DCT1.
1584 */
1585 if (dct_high_range_enabled(pvt) &&
1586 !dct_ganging_enabled(pvt) &&
1587 ((sys_addr >> 27) >= (dct_sel_base >> 11)))
1588 high_range = 1;
1589
1590 channel = f10_determine_channel(pvt, sys_addr, high_range, intlv_en);
1591
1592 chan_addr = f10_get_base_addr_offset(sys_addr, high_range, dct_sel_base,
1593 dct_sel_base_off, hole_valid,
1594 hole_off, dram_base);
1595
1596 intlv_shift = f10_map_intlv_en_to_shift(intlv_en);
1597
1598 /* remove Node ID (in case of memory interleaving) */
1599 tmp = chan_addr & 0xFC0;
1600
1601 chan_addr = ((chan_addr >> intlv_shift) & 0xFFFFFFFFF000ULL) | tmp;
1602
1603 /* remove channel interleave and hash */
1604 if (dct_interleave_enabled(pvt) &&
1605 !dct_high_range_enabled(pvt) &&
1606 !dct_ganging_enabled(pvt)) {
1607 if (dct_sel_interleave_addr(pvt) != 1)
1608 chan_addr = (chan_addr >> 1) & 0xFFFFFFFFFFFFFFC0ULL;
1609 else {
1610 tmp = chan_addr & 0xFC0;
1611 chan_addr = ((chan_addr & 0xFFFFFFFFFFFFC000ULL) >> 1)
1612 | tmp;
1613 }
1614 }
1615
1616 debugf1(" (ChannelAddrLong=0x%llx) >> 8 becomes InputAddr=0x%x\n",
1617 chan_addr, (u32)(chan_addr >> 8));
1618
1619 cs_found = f10_lookup_addr_in_dct(chan_addr >> 8, node_id, channel);
1620
1621 if (cs_found >= 0) {
1622 *nid = node_id;
1623 *chan_sel = channel;
1624 }
1625 return cs_found;
1626}
1627
1628static int f10_translate_sysaddr_to_cs(struct amd64_pvt *pvt, u64 sys_addr,
1629 int *node, int *chan_sel)
1630{
1631 int dram_range, cs_found = -EINVAL;
1632 u64 dram_base, dram_limit;
1633
1634 for (dram_range = 0; dram_range < DRAM_REG_COUNT; dram_range++) {
1635
1636 if (!pvt->dram_rw_en[dram_range])
1637 continue;
1638
1639 dram_base = pvt->dram_base[dram_range];
1640 dram_limit = pvt->dram_limit[dram_range];
1641
1642 if ((dram_base <= sys_addr) && (sys_addr <= dram_limit)) {
1643
1644 cs_found = f10_match_to_this_node(pvt, dram_range,
1645 sys_addr, node,
1646 chan_sel);
1647 if (cs_found >= 0)
1648 break;
1649 }
1650 }
1651 return cs_found;
1652}
1653
1654/*
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001655 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
1656 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001657 *
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001658 * The @sys_addr is usually an error address received from the hardware
1659 * (MCX_ADDR).
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001660 */
1661static void f10_map_sysaddr_to_csrow(struct mem_ctl_info *mci,
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001662 struct err_regs *err_info,
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001663 u64 sys_addr)
1664{
1665 struct amd64_pvt *pvt = mci->pvt_info;
1666 u32 page, offset;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001667 int nid, csrow, chan = 0;
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001668 u16 syndrome;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001669
1670 csrow = f10_translate_sysaddr_to_cs(pvt, sys_addr, &nid, &chan);
1671
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001672 if (csrow < 0) {
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001673 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001674 return;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001675 }
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001676
1677 error_address_to_page_and_offset(sys_addr, &page, &offset);
1678
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001679 syndrome = extract_syndrome(err_info);
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001680
1681 /*
1682 * We need the syndromes for channel detection only when we're
1683 * ganged. Otherwise @chan should already contain the channel at
1684 * this point.
1685 */
Borislav Petkov962b70a2010-08-03 16:51:28 +02001686 if (dct_ganging_enabled(pvt) && (pvt->nbcfg & K8_NBCFG_CHIPKILL))
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001687 chan = get_channel_from_ecc_syndrome(mci, syndrome);
1688
1689 if (chan >= 0)
1690 edac_mc_handle_ce(mci, page, offset, syndrome, csrow, chan,
1691 EDAC_MOD_STR);
1692 else
1693 /*
1694 * Channel unknown, report all channels on this CSROW as failed.
1695 */
1696 for (chan = 0; chan < mci->csrows[csrow].nr_channels; chan++)
1697 edac_mc_handle_ce(mci, page, offset, syndrome,
1698 csrow, chan, EDAC_MOD_STR);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001699}
1700
1701/*
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001702 * debug routine to display the memory sizes of all logical DIMMs and its
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001703 * CSROWs as well
1704 */
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001705static void amd64_debug_display_dimm_sizes(int ctrl, struct amd64_pvt *pvt)
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001706{
Borislav Petkov603adaf2009-12-21 14:52:53 +01001707 int dimm, size0, size1, factor = 0;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001708 u32 dbam;
1709 u32 *dcsb;
1710
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001711 if (boot_cpu_data.x86 == 0xf) {
Borislav Petkov603adaf2009-12-21 14:52:53 +01001712 if (pvt->dclr0 & F10_WIDTH_128)
1713 factor = 1;
1714
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001715 /* K8 families < revF not supported yet */
Borislav Petkov1433eb92009-10-21 13:44:36 +02001716 if (pvt->ext_model < K8_REV_F)
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001717 return;
1718 else
1719 WARN_ON(ctrl != 0);
1720 }
1721
1722 debugf1("F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n",
1723 ctrl, ctrl ? pvt->dbam1 : pvt->dbam0);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001724
1725 dbam = ctrl ? pvt->dbam1 : pvt->dbam0;
1726 dcsb = ctrl ? pvt->dcsb1 : pvt->dcsb0;
1727
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001728 edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
1729
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001730 /* Dump memory sizes for DIMM and its CSROWs */
1731 for (dimm = 0; dimm < 4; dimm++) {
1732
1733 size0 = 0;
1734 if (dcsb[dimm*2] & K8_DCSB_CS_ENABLE)
Borislav Petkov1433eb92009-10-21 13:44:36 +02001735 size0 = pvt->ops->dbam_to_cs(pvt, DBAM_DIMM(dimm, dbam));
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001736
1737 size1 = 0;
1738 if (dcsb[dimm*2 + 1] & K8_DCSB_CS_ENABLE)
Borislav Petkov1433eb92009-10-21 13:44:36 +02001739 size1 = pvt->ops->dbam_to_cs(pvt, DBAM_DIMM(dimm, dbam));
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001740
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001741 edac_printk(KERN_DEBUG, EDAC_MC, " %d: %5dMB %d: %5dMB\n",
Borislav Petkov603adaf2009-12-21 14:52:53 +01001742 dimm * 2, size0 << factor,
1743 dimm * 2 + 1, size1 << factor);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001744 }
1745}
1746
1747/*
Doug Thompson4d376072009-04-27 16:25:05 +02001748 * There currently are 3 types type of MC devices for AMD Athlon/Opterons
1749 * (as per PCI DEVICE_IDs):
1750 *
1751 * Family K8: That is the Athlon64 and Opteron CPUs. They all have the same PCI
1752 * DEVICE ID, even though there is differences between the different Revisions
1753 * (CG,D,E,F).
1754 *
1755 * Family F10h and F11h.
1756 *
1757 */
1758static struct amd64_family_type amd64_family_types[] = {
1759 [K8_CPUS] = {
1760 .ctl_name = "RevF",
1761 .addr_f1_ctl = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
1762 .misc_f3_ctl = PCI_DEVICE_ID_AMD_K8_NB_MISC,
1763 .ops = {
Borislav Petkov1433eb92009-10-21 13:44:36 +02001764 .early_channel_count = k8_early_channel_count,
1765 .get_error_address = k8_get_error_address,
1766 .read_dram_base_limit = k8_read_dram_base_limit,
1767 .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow,
1768 .dbam_to_cs = k8_dbam_to_chip_select,
Doug Thompson4d376072009-04-27 16:25:05 +02001769 }
1770 },
1771 [F10_CPUS] = {
1772 .ctl_name = "Family 10h",
1773 .addr_f1_ctl = PCI_DEVICE_ID_AMD_10H_NB_MAP,
1774 .misc_f3_ctl = PCI_DEVICE_ID_AMD_10H_NB_MISC,
1775 .ops = {
Borislav Petkov1433eb92009-10-21 13:44:36 +02001776 .early_channel_count = f10_early_channel_count,
1777 .get_error_address = f10_get_error_address,
1778 .read_dram_base_limit = f10_read_dram_base_limit,
1779 .read_dram_ctl_register = f10_read_dram_ctl_register,
1780 .map_sysaddr_to_csrow = f10_map_sysaddr_to_csrow,
1781 .dbam_to_cs = f10_dbam_to_chip_select,
Doug Thompson4d376072009-04-27 16:25:05 +02001782 }
1783 },
1784 [F11_CPUS] = {
1785 .ctl_name = "Family 11h",
1786 .addr_f1_ctl = PCI_DEVICE_ID_AMD_11H_NB_MAP,
1787 .misc_f3_ctl = PCI_DEVICE_ID_AMD_11H_NB_MISC,
1788 .ops = {
Borislav Petkov1433eb92009-10-21 13:44:36 +02001789 .early_channel_count = f10_early_channel_count,
1790 .get_error_address = f10_get_error_address,
1791 .read_dram_base_limit = f10_read_dram_base_limit,
1792 .read_dram_ctl_register = f10_read_dram_ctl_register,
1793 .map_sysaddr_to_csrow = f10_map_sysaddr_to_csrow,
1794 .dbam_to_cs = f10_dbam_to_chip_select,
Doug Thompson4d376072009-04-27 16:25:05 +02001795 }
1796 },
1797};
1798
1799static struct pci_dev *pci_get_related_function(unsigned int vendor,
1800 unsigned int device,
1801 struct pci_dev *related)
1802{
1803 struct pci_dev *dev = NULL;
1804
1805 dev = pci_get_device(vendor, device, dev);
1806 while (dev) {
1807 if ((dev->bus->number == related->bus->number) &&
1808 (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
1809 break;
1810 dev = pci_get_device(vendor, device, dev);
1811 }
1812
1813 return dev;
1814}
1815
Doug Thompsonb1289d62009-04-27 16:37:05 +02001816/*
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001817 * These are tables of eigenvectors (one per line) which can be used for the
1818 * construction of the syndrome tables. The modified syndrome search algorithm
1819 * uses those to find the symbol in error and thus the DIMM.
Doug Thompsonb1289d62009-04-27 16:37:05 +02001820 *
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001821 * Algorithm courtesy of Ross LaFetra from AMD.
Doug Thompsonb1289d62009-04-27 16:37:05 +02001822 */
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001823static u16 x4_vectors[] = {
1824 0x2f57, 0x1afe, 0x66cc, 0xdd88,
1825 0x11eb, 0x3396, 0x7f4c, 0xeac8,
1826 0x0001, 0x0002, 0x0004, 0x0008,
1827 0x1013, 0x3032, 0x4044, 0x8088,
1828 0x106b, 0x30d6, 0x70fc, 0xe0a8,
1829 0x4857, 0xc4fe, 0x13cc, 0x3288,
1830 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
1831 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
1832 0x15c1, 0x2a42, 0x89ac, 0x4758,
1833 0x2b03, 0x1602, 0x4f0c, 0xca08,
1834 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
1835 0x8ba7, 0x465e, 0x244c, 0x1cc8,
1836 0x2b87, 0x164e, 0x642c, 0xdc18,
1837 0x40b9, 0x80de, 0x1094, 0x20e8,
1838 0x27db, 0x1eb6, 0x9dac, 0x7b58,
1839 0x11c1, 0x2242, 0x84ac, 0x4c58,
1840 0x1be5, 0x2d7a, 0x5e34, 0xa718,
1841 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
1842 0x4c97, 0xc87e, 0x11fc, 0x33a8,
1843 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
1844 0x16b3, 0x3d62, 0x4f34, 0x8518,
1845 0x1e2f, 0x391a, 0x5cac, 0xf858,
1846 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
1847 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
1848 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
1849 0x4397, 0xc27e, 0x17fc, 0x3ea8,
1850 0x1617, 0x3d3e, 0x6464, 0xb8b8,
1851 0x23ff, 0x12aa, 0xab6c, 0x56d8,
1852 0x2dfb, 0x1ba6, 0x913c, 0x7328,
1853 0x185d, 0x2ca6, 0x7914, 0x9e28,
1854 0x171b, 0x3e36, 0x7d7c, 0xebe8,
1855 0x4199, 0x82ee, 0x19f4, 0x2e58,
1856 0x4807, 0xc40e, 0x130c, 0x3208,
1857 0x1905, 0x2e0a, 0x5804, 0xac08,
1858 0x213f, 0x132a, 0xadfc, 0x5ba8,
1859 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
Doug Thompsonb1289d62009-04-27 16:37:05 +02001860};
1861
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001862static u16 x8_vectors[] = {
1863 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
1864 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
1865 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
1866 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
1867 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
1868 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
1869 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
1870 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
1871 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
1872 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
1873 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
1874 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
1875 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
1876 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
1877 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
1878 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
1879 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
1880 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
1881 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
1882};
1883
1884static int decode_syndrome(u16 syndrome, u16 *vectors, int num_vecs,
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001885 int v_dim)
Doug Thompsonb1289d62009-04-27 16:37:05 +02001886{
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001887 unsigned int i, err_sym;
Doug Thompsonb1289d62009-04-27 16:37:05 +02001888
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001889 for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
1890 u16 s = syndrome;
1891 int v_idx = err_sym * v_dim;
1892 int v_end = (err_sym + 1) * v_dim;
Doug Thompsonb1289d62009-04-27 16:37:05 +02001893
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001894 /* walk over all 16 bits of the syndrome */
1895 for (i = 1; i < (1U << 16); i <<= 1) {
1896
1897 /* if bit is set in that eigenvector... */
1898 if (v_idx < v_end && vectors[v_idx] & i) {
1899 u16 ev_comp = vectors[v_idx++];
1900
1901 /* ... and bit set in the modified syndrome, */
1902 if (s & i) {
1903 /* remove it. */
1904 s ^= ev_comp;
1905
1906 if (!s)
1907 return err_sym;
1908 }
1909
1910 } else if (s & i)
1911 /* can't get to zero, move to next symbol */
1912 break;
1913 }
Doug Thompsonb1289d62009-04-27 16:37:05 +02001914 }
1915
1916 debugf0("syndrome(%x) not found\n", syndrome);
1917 return -1;
1918}
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001919
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001920static int map_err_sym_to_channel(int err_sym, int sym_size)
1921{
1922 if (sym_size == 4)
1923 switch (err_sym) {
1924 case 0x20:
1925 case 0x21:
1926 return 0;
1927 break;
1928 case 0x22:
1929 case 0x23:
1930 return 1;
1931 break;
1932 default:
1933 return err_sym >> 4;
1934 break;
1935 }
1936 /* x8 symbols */
1937 else
1938 switch (err_sym) {
1939 /* imaginary bits not in a DIMM */
1940 case 0x10:
1941 WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
1942 err_sym);
1943 return -1;
1944 break;
1945
1946 case 0x11:
1947 return 0;
1948 break;
1949 case 0x12:
1950 return 1;
1951 break;
1952 default:
1953 return err_sym >> 3;
1954 break;
1955 }
1956 return -1;
1957}
1958
1959static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
1960{
1961 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001962 int err_sym = -1;
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001963
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001964 if (pvt->syn_type == 8)
1965 err_sym = decode_syndrome(syndrome, x8_vectors,
1966 ARRAY_SIZE(x8_vectors),
1967 pvt->syn_type);
1968 else if (pvt->syn_type == 4)
1969 err_sym = decode_syndrome(syndrome, x4_vectors,
1970 ARRAY_SIZE(x4_vectors),
1971 pvt->syn_type);
1972 else {
1973 amd64_printk(KERN_WARNING, "%s: Illegal syndrome type: %u\n",
1974 __func__, pvt->syn_type);
1975 return err_sym;
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001976 }
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001977
1978 return map_err_sym_to_channel(err_sym, pvt->syn_type);
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001979}
1980
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001981/*
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001982 * Handle any Correctable Errors (CEs) that have occurred. Check for valid ERROR
1983 * ADDRESS and process.
1984 */
1985static void amd64_handle_ce(struct mem_ctl_info *mci,
Borislav Petkovef44cc42009-07-23 14:45:48 +02001986 struct err_regs *info)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001987{
1988 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001989 u64 sys_addr;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001990
1991 /* Ensure that the Error Address is VALID */
1992 if ((info->nbsh & K8_NBSH_VALID_ERROR_ADDR) == 0) {
1993 amd64_mc_printk(mci, KERN_ERR,
1994 "HW has no ERROR_ADDRESS available\n");
1995 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1996 return;
1997 }
1998
Borislav Petkov1f6bcee2009-11-13 14:02:57 +01001999 sys_addr = pvt->ops->get_error_address(mci, info);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002000
2001 amd64_mc_printk(mci, KERN_ERR,
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01002002 "CE ERROR_ADDRESS= 0x%llx\n", sys_addr);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002003
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01002004 pvt->ops->map_sysaddr_to_csrow(mci, info, sys_addr);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002005}
2006
2007/* Handle any Un-correctable Errors (UEs) */
2008static void amd64_handle_ue(struct mem_ctl_info *mci,
Borislav Petkovef44cc42009-07-23 14:45:48 +02002009 struct err_regs *info)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002010{
Borislav Petkov1f6bcee2009-11-13 14:02:57 +01002011 struct amd64_pvt *pvt = mci->pvt_info;
2012 struct mem_ctl_info *log_mci, *src_mci = NULL;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002013 int csrow;
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01002014 u64 sys_addr;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002015 u32 page, offset;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002016
2017 log_mci = mci;
2018
2019 if ((info->nbsh & K8_NBSH_VALID_ERROR_ADDR) == 0) {
2020 amd64_mc_printk(mci, KERN_CRIT,
2021 "HW has no ERROR_ADDRESS available\n");
2022 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
2023 return;
2024 }
2025
Borislav Petkov1f6bcee2009-11-13 14:02:57 +01002026 sys_addr = pvt->ops->get_error_address(mci, info);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002027
2028 /*
2029 * Find out which node the error address belongs to. This may be
2030 * different from the node that detected the error.
2031 */
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01002032 src_mci = find_mc_by_sys_addr(mci, sys_addr);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002033 if (!src_mci) {
2034 amd64_mc_printk(mci, KERN_CRIT,
2035 "ERROR ADDRESS (0x%lx) value NOT mapped to a MC\n",
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01002036 (unsigned long)sys_addr);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002037 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
2038 return;
2039 }
2040
2041 log_mci = src_mci;
2042
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01002043 csrow = sys_addr_to_csrow(log_mci, sys_addr);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002044 if (csrow < 0) {
2045 amd64_mc_printk(mci, KERN_CRIT,
2046 "ERROR_ADDRESS (0x%lx) value NOT mapped to 'csrow'\n",
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01002047 (unsigned long)sys_addr);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002048 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
2049 } else {
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01002050 error_address_to_page_and_offset(sys_addr, &page, &offset);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002051 edac_mc_handle_ue(log_mci, page, offset, csrow, EDAC_MOD_STR);
2052 }
2053}
2054
Borislav Petkov549d0422009-07-24 13:51:42 +02002055static inline void __amd64_decode_bus_error(struct mem_ctl_info *mci,
Borislav Petkovb69b29d2009-07-27 16:21:14 +02002056 struct err_regs *info)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002057{
Borislav Petkovb70ef012009-06-25 19:32:38 +02002058 u32 ec = ERROR_CODE(info->nbsl);
2059 u32 xec = EXT_ERROR_CODE(info->nbsl);
Borislav Petkov17adea02009-11-04 14:04:06 +01002060 int ecc_type = (info->nbsh >> 13) & 0x3;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002061
Borislav Petkovb70ef012009-06-25 19:32:38 +02002062 /* Bail early out if this was an 'observed' error */
2063 if (PP(ec) == K8_NBSL_PP_OBS)
2064 return;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002065
Borislav Petkovecaf5602009-07-23 16:32:01 +02002066 /* Do only ECC errors */
2067 if (xec && xec != F10_NBSL_EXT_ERR_ECC)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002068 return;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002069
Borislav Petkovecaf5602009-07-23 16:32:01 +02002070 if (ecc_type == 2)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002071 amd64_handle_ce(mci, info);
Borislav Petkovecaf5602009-07-23 16:32:01 +02002072 else if (ecc_type == 1)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002073 amd64_handle_ue(mci, info);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002074}
2075
Borislav Petkov7cfd4a82010-09-01 14:45:20 +02002076void amd64_decode_bus_error(int node_id, struct mce *m, u32 nbcfg)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002077{
Borislav Petkov549d0422009-07-24 13:51:42 +02002078 struct mem_ctl_info *mci = mci_lookup[node_id];
Borislav Petkov7cfd4a82010-09-01 14:45:20 +02002079 struct err_regs regs;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002080
Borislav Petkov7cfd4a82010-09-01 14:45:20 +02002081 regs.nbsl = (u32) m->status;
2082 regs.nbsh = (u32)(m->status >> 32);
2083 regs.nbeal = (u32) m->addr;
2084 regs.nbeah = (u32)(m->addr >> 32);
2085 regs.nbcfg = nbcfg;
2086
2087 __amd64_decode_bus_error(mci, &regs);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002088
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002089 /*
2090 * Check the UE bit of the NB status high register, if set generate some
2091 * logs. If NOT a GART error, then process the event as a NO-INFO event.
2092 * If it was a GART error, skip that process.
Borislav Petkov549d0422009-07-24 13:51:42 +02002093 *
2094 * FIXME: this should go somewhere else, if at all.
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002095 */
Borislav Petkov7cfd4a82010-09-01 14:45:20 +02002096 if (regs.nbsh & K8_NBSH_UC_ERR && !report_gart_errors)
Borislav Petkov5110dbd2009-06-25 19:51:04 +02002097 edac_mc_handle_ue_no_info(mci, "UE bit is set");
Borislav Petkov549d0422009-07-24 13:51:42 +02002098
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002099}
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002100
Doug Thompson0ec449e2009-04-27 19:41:25 +02002101/*
Doug Thompson0ec449e2009-04-27 19:41:25 +02002102 * Input:
2103 * 1) struct amd64_pvt which contains pvt->dram_f2_ctl pointer
2104 * 2) AMD Family index value
2105 *
2106 * Ouput:
2107 * Upon return of 0, the following filled in:
2108 *
2109 * struct pvt->addr_f1_ctl
2110 * struct pvt->misc_f3_ctl
2111 *
2112 * Filled in with related device funcitions of 'dram_f2_ctl'
2113 * These devices are "reserved" via the pci_get_device()
2114 *
2115 * Upon return of 1 (error status):
2116 *
2117 * Nothing reserved
2118 */
2119static int amd64_reserve_mc_sibling_devices(struct amd64_pvt *pvt, int mc_idx)
2120{
2121 const struct amd64_family_type *amd64_dev = &amd64_family_types[mc_idx];
2122
2123 /* Reserve the ADDRESS MAP Device */
2124 pvt->addr_f1_ctl = pci_get_related_function(pvt->dram_f2_ctl->vendor,
2125 amd64_dev->addr_f1_ctl,
2126 pvt->dram_f2_ctl);
2127
2128 if (!pvt->addr_f1_ctl) {
2129 amd64_printk(KERN_ERR, "error address map device not found: "
2130 "vendor %x device 0x%x (broken BIOS?)\n",
2131 PCI_VENDOR_ID_AMD, amd64_dev->addr_f1_ctl);
2132 return 1;
2133 }
2134
2135 /* Reserve the MISC Device */
2136 pvt->misc_f3_ctl = pci_get_related_function(pvt->dram_f2_ctl->vendor,
2137 amd64_dev->misc_f3_ctl,
2138 pvt->dram_f2_ctl);
2139
2140 if (!pvt->misc_f3_ctl) {
2141 pci_dev_put(pvt->addr_f1_ctl);
2142 pvt->addr_f1_ctl = NULL;
2143
2144 amd64_printk(KERN_ERR, "error miscellaneous device not found: "
2145 "vendor %x device 0x%x (broken BIOS?)\n",
2146 PCI_VENDOR_ID_AMD, amd64_dev->misc_f3_ctl);
2147 return 1;
2148 }
2149
2150 debugf1(" Addr Map device PCI Bus ID:\t%s\n",
2151 pci_name(pvt->addr_f1_ctl));
2152 debugf1(" DRAM MEM-CTL PCI Bus ID:\t%s\n",
2153 pci_name(pvt->dram_f2_ctl));
2154 debugf1(" Misc device PCI Bus ID:\t%s\n",
2155 pci_name(pvt->misc_f3_ctl));
2156
2157 return 0;
2158}
2159
2160static void amd64_free_mc_sibling_devices(struct amd64_pvt *pvt)
2161{
2162 pci_dev_put(pvt->addr_f1_ctl);
2163 pci_dev_put(pvt->misc_f3_ctl);
2164}
2165
2166/*
2167 * Retrieve the hardware registers of the memory controller (this includes the
2168 * 'Address Map' and 'Misc' device regs)
2169 */
2170static void amd64_read_mc_registers(struct amd64_pvt *pvt)
2171{
2172 u64 msr_val;
Borislav Petkovad6a32e2010-03-09 12:46:00 +01002173 u32 tmp;
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02002174 int dram;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002175
2176 /*
2177 * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
2178 * those are Read-As-Zero
2179 */
Borislav Petkove97f8bb2009-10-12 15:27:45 +02002180 rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
2181 debugf0(" TOP_MEM: 0x%016llx\n", pvt->top_mem);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002182
2183 /* check first whether TOP_MEM2 is enabled */
2184 rdmsrl(MSR_K8_SYSCFG, msr_val);
2185 if (msr_val & (1U << 21)) {
Borislav Petkove97f8bb2009-10-12 15:27:45 +02002186 rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
2187 debugf0(" TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002188 } else
2189 debugf0(" TOP_MEM2 disabled.\n");
2190
2191 amd64_cpu_display_info(pvt);
2192
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02002193 amd64_read_pci_cfg(pvt->misc_f3_ctl, K8_NBCAP, &pvt->nbcap);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002194
2195 if (pvt->ops->read_dram_ctl_register)
2196 pvt->ops->read_dram_ctl_register(pvt);
2197
2198 for (dram = 0; dram < DRAM_REG_COUNT; dram++) {
2199 /*
2200 * Call CPU specific READ function to get the DRAM Base and
2201 * Limit values from the DCT.
2202 */
2203 pvt->ops->read_dram_base_limit(pvt, dram);
2204
2205 /*
2206 * Only print out debug info on rows with both R and W Enabled.
2207 * Normal processing, compiler should optimize this whole 'if'
2208 * debug output block away.
2209 */
2210 if (pvt->dram_rw_en[dram] != 0) {
Borislav Petkove97f8bb2009-10-12 15:27:45 +02002211 debugf1(" DRAM-BASE[%d]: 0x%016llx "
2212 "DRAM-LIMIT: 0x%016llx\n",
Doug Thompson0ec449e2009-04-27 19:41:25 +02002213 dram,
Borislav Petkove97f8bb2009-10-12 15:27:45 +02002214 pvt->dram_base[dram],
2215 pvt->dram_limit[dram]);
2216
Doug Thompson0ec449e2009-04-27 19:41:25 +02002217 debugf1(" IntlvEn=%s %s %s "
2218 "IntlvSel=%d DstNode=%d\n",
2219 pvt->dram_IntlvEn[dram] ?
2220 "Enabled" : "Disabled",
2221 (pvt->dram_rw_en[dram] & 0x2) ? "W" : "!W",
2222 (pvt->dram_rw_en[dram] & 0x1) ? "R" : "!R",
2223 pvt->dram_IntlvSel[dram],
2224 pvt->dram_DstNode[dram]);
2225 }
2226 }
2227
2228 amd64_read_dct_base_mask(pvt);
2229
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02002230 amd64_read_pci_cfg(pvt->addr_f1_ctl, K8_DHAR, &pvt->dhar);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002231 amd64_read_dbam_reg(pvt);
2232
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02002233 amd64_read_pci_cfg(pvt->misc_f3_ctl,
2234 F10_ONLINE_SPARE, &pvt->online_spare);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002235
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02002236 amd64_read_pci_cfg(pvt->dram_f2_ctl, F10_DCLR_0, &pvt->dclr0);
2237 amd64_read_pci_cfg(pvt->dram_f2_ctl, F10_DCHR_0, &pvt->dchr0);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002238
Borislav Petkovad6a32e2010-03-09 12:46:00 +01002239 if (boot_cpu_data.x86 >= 0x10) {
2240 if (!dct_ganging_enabled(pvt)) {
2241 amd64_read_pci_cfg(pvt->dram_f2_ctl, F10_DCLR_1, &pvt->dclr1);
2242 amd64_read_pci_cfg(pvt->dram_f2_ctl, F10_DCHR_1, &pvt->dchr1);
2243 }
2244 amd64_read_pci_cfg(pvt->misc_f3_ctl, EXT_NB_MCA_CFG, &tmp);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002245 }
Borislav Petkovad6a32e2010-03-09 12:46:00 +01002246
2247 if (boot_cpu_data.x86 == 0x10 &&
2248 boot_cpu_data.x86_model > 7 &&
2249 /* F3x180[EccSymbolSize]=1 => x8 symbols */
2250 tmp & BIT(25))
2251 pvt->syn_type = 8;
2252 else
2253 pvt->syn_type = 4;
2254
Doug Thompson0ec449e2009-04-27 19:41:25 +02002255 amd64_dump_misc_regs(pvt);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002256}
2257
2258/*
2259 * NOTE: CPU Revision Dependent code
2260 *
2261 * Input:
Borislav Petkov9d858bb2009-09-21 14:35:51 +02002262 * @csrow_nr ChipSelect Row Number (0..pvt->cs_count-1)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002263 * k8 private pointer to -->
2264 * DRAM Bank Address mapping register
2265 * node_id
2266 * DCL register where dual_channel_active is
2267 *
2268 * The DBAM register consists of 4 sets of 4 bits each definitions:
2269 *
2270 * Bits: CSROWs
2271 * 0-3 CSROWs 0 and 1
2272 * 4-7 CSROWs 2 and 3
2273 * 8-11 CSROWs 4 and 5
2274 * 12-15 CSROWs 6 and 7
2275 *
2276 * Values range from: 0 to 15
2277 * The meaning of the values depends on CPU revision and dual-channel state,
2278 * see relevant BKDG more info.
2279 *
2280 * The memory controller provides for total of only 8 CSROWs in its current
2281 * architecture. Each "pair" of CSROWs normally represents just one DIMM in
2282 * single channel or two (2) DIMMs in dual channel mode.
2283 *
2284 * The following code logic collapses the various tables for CSROW based on CPU
2285 * revision.
2286 *
2287 * Returns:
2288 * The number of PAGE_SIZE pages on the specified CSROW number it
2289 * encompasses
2290 *
2291 */
2292static u32 amd64_csrow_nr_pages(int csrow_nr, struct amd64_pvt *pvt)
2293{
Borislav Petkov1433eb92009-10-21 13:44:36 +02002294 u32 cs_mode, nr_pages;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002295
2296 /*
2297 * The math on this doesn't look right on the surface because x/2*4 can
2298 * be simplified to x*2 but this expression makes use of the fact that
2299 * it is integral math where 1/2=0. This intermediate value becomes the
2300 * number of bits to shift the DBAM register to extract the proper CSROW
2301 * field.
2302 */
Borislav Petkov1433eb92009-10-21 13:44:36 +02002303 cs_mode = (pvt->dbam0 >> ((csrow_nr / 2) * 4)) & 0xF;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002304
Borislav Petkov1433eb92009-10-21 13:44:36 +02002305 nr_pages = pvt->ops->dbam_to_cs(pvt, cs_mode) << (20 - PAGE_SHIFT);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002306
2307 /*
2308 * If dual channel then double the memory size of single channel.
2309 * Channel count is 1 or 2
2310 */
2311 nr_pages <<= (pvt->channel_count - 1);
2312
Borislav Petkov1433eb92009-10-21 13:44:36 +02002313 debugf0(" (csrow=%d) DBAM map index= %d\n", csrow_nr, cs_mode);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002314 debugf0(" nr_pages= %u channel-count = %d\n",
2315 nr_pages, pvt->channel_count);
2316
2317 return nr_pages;
2318}
2319
2320/*
2321 * Initialize the array of csrow attribute instances, based on the values
2322 * from pci config hardware registers.
2323 */
2324static int amd64_init_csrows(struct mem_ctl_info *mci)
2325{
2326 struct csrow_info *csrow;
2327 struct amd64_pvt *pvt;
2328 u64 input_addr_min, input_addr_max, sys_addr;
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02002329 int i, empty = 1;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002330
2331 pvt = mci->pvt_info;
2332
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02002333 amd64_read_pci_cfg(pvt->misc_f3_ctl, K8_NBCFG, &pvt->nbcfg);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002334
2335 debugf0("NBCFG= 0x%x CHIPKILL= %s DRAM ECC= %s\n", pvt->nbcfg,
2336 (pvt->nbcfg & K8_NBCFG_CHIPKILL) ? "Enabled" : "Disabled",
2337 (pvt->nbcfg & K8_NBCFG_ECC_ENABLE) ? "Enabled" : "Disabled"
2338 );
2339
Borislav Petkov9d858bb2009-09-21 14:35:51 +02002340 for (i = 0; i < pvt->cs_count; i++) {
Doug Thompson0ec449e2009-04-27 19:41:25 +02002341 csrow = &mci->csrows[i];
2342
2343 if ((pvt->dcsb0[i] & K8_DCSB_CS_ENABLE) == 0) {
2344 debugf1("----CSROW %d EMPTY for node %d\n", i,
2345 pvt->mc_node_id);
2346 continue;
2347 }
2348
2349 debugf1("----CSROW %d VALID for MC node %d\n",
2350 i, pvt->mc_node_id);
2351
2352 empty = 0;
2353 csrow->nr_pages = amd64_csrow_nr_pages(i, pvt);
2354 find_csrow_limits(mci, i, &input_addr_min, &input_addr_max);
2355 sys_addr = input_addr_to_sys_addr(mci, input_addr_min);
2356 csrow->first_page = (u32) (sys_addr >> PAGE_SHIFT);
2357 sys_addr = input_addr_to_sys_addr(mci, input_addr_max);
2358 csrow->last_page = (u32) (sys_addr >> PAGE_SHIFT);
2359 csrow->page_mask = ~mask_from_dct_mask(pvt, i);
2360 /* 8 bytes of resolution */
2361
2362 csrow->mtype = amd64_determine_memory_type(pvt);
2363
2364 debugf1(" for MC node %d csrow %d:\n", pvt->mc_node_id, i);
2365 debugf1(" input_addr_min: 0x%lx input_addr_max: 0x%lx\n",
2366 (unsigned long)input_addr_min,
2367 (unsigned long)input_addr_max);
2368 debugf1(" sys_addr: 0x%lx page_mask: 0x%lx\n",
2369 (unsigned long)sys_addr, csrow->page_mask);
2370 debugf1(" nr_pages: %u first_page: 0x%lx "
2371 "last_page: 0x%lx\n",
2372 (unsigned)csrow->nr_pages,
2373 csrow->first_page, csrow->last_page);
2374
2375 /*
2376 * determine whether CHIPKILL or JUST ECC or NO ECC is operating
2377 */
2378 if (pvt->nbcfg & K8_NBCFG_ECC_ENABLE)
2379 csrow->edac_mode =
2380 (pvt->nbcfg & K8_NBCFG_CHIPKILL) ?
2381 EDAC_S4ECD4ED : EDAC_SECDED;
2382 else
2383 csrow->edac_mode = EDAC_NONE;
2384 }
2385
2386 return empty;
2387}
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002388
Borislav Petkov06724532009-09-16 13:05:46 +02002389/* get all cores on this DCT */
Rusty Russellba578cb2009-11-03 14:56:35 +10302390static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, int nid)
Doug Thompsonf9431992009-04-27 19:46:08 +02002391{
Borislav Petkov06724532009-09-16 13:05:46 +02002392 int cpu;
Doug Thompsonf9431992009-04-27 19:46:08 +02002393
Borislav Petkov06724532009-09-16 13:05:46 +02002394 for_each_online_cpu(cpu)
2395 if (amd_get_nb_id(cpu) == nid)
2396 cpumask_set_cpu(cpu, mask);
Doug Thompsonf9431992009-04-27 19:46:08 +02002397}
2398
2399/* check MCG_CTL on all the cpus on this node */
Borislav Petkov06724532009-09-16 13:05:46 +02002400static bool amd64_nb_mce_bank_enabled_on_node(int nid)
Doug Thompsonf9431992009-04-27 19:46:08 +02002401{
Rusty Russellba578cb2009-11-03 14:56:35 +10302402 cpumask_var_t mask;
Borislav Petkov50542252009-12-11 18:14:40 +01002403 int cpu, nbe;
Borislav Petkov06724532009-09-16 13:05:46 +02002404 bool ret = false;
Doug Thompsonf9431992009-04-27 19:46:08 +02002405
Rusty Russellba578cb2009-11-03 14:56:35 +10302406 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
2407 amd64_printk(KERN_WARNING, "%s: error allocating mask\n",
2408 __func__);
2409 return false;
2410 }
Borislav Petkov06724532009-09-16 13:05:46 +02002411
Rusty Russellba578cb2009-11-03 14:56:35 +10302412 get_cpus_on_this_dct_cpumask(mask, nid);
Borislav Petkov06724532009-09-16 13:05:46 +02002413
Rusty Russellba578cb2009-11-03 14:56:35 +10302414 rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
Borislav Petkov06724532009-09-16 13:05:46 +02002415
Rusty Russellba578cb2009-11-03 14:56:35 +10302416 for_each_cpu(cpu, mask) {
Borislav Petkov50542252009-12-11 18:14:40 +01002417 struct msr *reg = per_cpu_ptr(msrs, cpu);
2418 nbe = reg->l & K8_MSR_MCGCTL_NBE;
Borislav Petkov06724532009-09-16 13:05:46 +02002419
2420 debugf0("core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
Borislav Petkov50542252009-12-11 18:14:40 +01002421 cpu, reg->q,
Borislav Petkov06724532009-09-16 13:05:46 +02002422 (nbe ? "enabled" : "disabled"));
2423
2424 if (!nbe)
2425 goto out;
Borislav Petkov06724532009-09-16 13:05:46 +02002426 }
2427 ret = true;
2428
2429out:
Rusty Russellba578cb2009-11-03 14:56:35 +10302430 free_cpumask_var(mask);
Doug Thompsonf9431992009-04-27 19:46:08 +02002431 return ret;
2432}
2433
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002434static int amd64_toggle_ecc_err_reporting(struct amd64_pvt *pvt, bool on)
2435{
2436 cpumask_var_t cmask;
Borislav Petkov50542252009-12-11 18:14:40 +01002437 int cpu;
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002438
2439 if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
2440 amd64_printk(KERN_WARNING, "%s: error allocating mask\n",
2441 __func__);
2442 return false;
2443 }
2444
2445 get_cpus_on_this_dct_cpumask(cmask, pvt->mc_node_id);
2446
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002447 rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2448
2449 for_each_cpu(cpu, cmask) {
2450
Borislav Petkov50542252009-12-11 18:14:40 +01002451 struct msr *reg = per_cpu_ptr(msrs, cpu);
2452
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002453 if (on) {
Borislav Petkov50542252009-12-11 18:14:40 +01002454 if (reg->l & K8_MSR_MCGCTL_NBE)
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002455 pvt->flags.nb_mce_enable = 1;
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002456
Borislav Petkov50542252009-12-11 18:14:40 +01002457 reg->l |= K8_MSR_MCGCTL_NBE;
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002458 } else {
2459 /*
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002460 * Turn off NB MCE reporting only when it was off before
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002461 */
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002462 if (!pvt->flags.nb_mce_enable)
Borislav Petkov50542252009-12-11 18:14:40 +01002463 reg->l &= ~K8_MSR_MCGCTL_NBE;
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002464 }
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002465 }
2466 wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2467
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002468 free_cpumask_var(cmask);
2469
2470 return 0;
2471}
2472
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002473static void amd64_enable_ecc_error_reporting(struct mem_ctl_info *mci)
2474{
2475 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002476 u32 value, mask = K8_NBCTL_CECCEn | K8_NBCTL_UECCEn;
2477
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02002478 amd64_read_pci_cfg(pvt->misc_f3_ctl, K8_NBCTL, &value);
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002479
2480 /* turn on UECCn and CECCEn bits */
2481 pvt->old_nbctl = value & mask;
2482 pvt->nbctl_mcgctl_saved = 1;
2483
2484 value |= mask;
2485 pci_write_config_dword(pvt->misc_f3_ctl, K8_NBCTL, value);
2486
2487 if (amd64_toggle_ecc_err_reporting(pvt, ON))
2488 amd64_printk(KERN_WARNING, "Error enabling ECC reporting over "
2489 "MCGCTL!\n");
2490
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02002491 amd64_read_pci_cfg(pvt->misc_f3_ctl, K8_NBCFG, &value);
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002492
2493 debugf0("NBCFG(1)= 0x%x CHIPKILL= %s ECC_ENABLE= %s\n", value,
2494 (value & K8_NBCFG_CHIPKILL) ? "Enabled" : "Disabled",
2495 (value & K8_NBCFG_ECC_ENABLE) ? "Enabled" : "Disabled");
2496
2497 if (!(value & K8_NBCFG_ECC_ENABLE)) {
2498 amd64_printk(KERN_WARNING,
2499 "This node reports that DRAM ECC is "
2500 "currently Disabled; ENABLING now\n");
2501
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002502 pvt->flags.nb_ecc_prev = 0;
2503
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002504 /* Attempt to turn on DRAM ECC Enable */
2505 value |= K8_NBCFG_ECC_ENABLE;
2506 pci_write_config_dword(pvt->misc_f3_ctl, K8_NBCFG, value);
2507
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02002508 amd64_read_pci_cfg(pvt->misc_f3_ctl, K8_NBCFG, &value);
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002509
2510 if (!(value & K8_NBCFG_ECC_ENABLE)) {
2511 amd64_printk(KERN_WARNING,
2512 "Hardware rejects Enabling DRAM ECC checking\n"
2513 "Check memory DIMM configuration\n");
2514 } else {
2515 amd64_printk(KERN_DEBUG,
2516 "Hardware accepted DRAM ECC Enable\n");
2517 }
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002518 } else {
2519 pvt->flags.nb_ecc_prev = 1;
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002520 }
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002521
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002522 debugf0("NBCFG(2)= 0x%x CHIPKILL= %s ECC_ENABLE= %s\n", value,
2523 (value & K8_NBCFG_CHIPKILL) ? "Enabled" : "Disabled",
2524 (value & K8_NBCFG_ECC_ENABLE) ? "Enabled" : "Disabled");
2525
2526 pvt->ctl_error_info.nbcfg = value;
2527}
2528
2529static void amd64_restore_ecc_error_reporting(struct amd64_pvt *pvt)
2530{
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002531 u32 value, mask = K8_NBCTL_CECCEn | K8_NBCTL_UECCEn;
2532
2533 if (!pvt->nbctl_mcgctl_saved)
2534 return;
2535
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02002536 amd64_read_pci_cfg(pvt->misc_f3_ctl, K8_NBCTL, &value);
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002537 value &= ~mask;
2538 value |= pvt->old_nbctl;
2539
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002540 pci_write_config_dword(pvt->misc_f3_ctl, K8_NBCTL, value);
2541
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002542 /* restore previous BIOS DRAM ECC "off" setting which we force-enabled */
2543 if (!pvt->flags.nb_ecc_prev) {
2544 amd64_read_pci_cfg(pvt->misc_f3_ctl, K8_NBCFG, &value);
2545 value &= ~K8_NBCFG_ECC_ENABLE;
2546 pci_write_config_dword(pvt->misc_f3_ctl, K8_NBCFG, value);
2547 }
2548
2549 /* restore the NB Enable MCGCTL bit */
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002550 if (amd64_toggle_ecc_err_reporting(pvt, OFF))
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002551 amd64_printk(KERN_WARNING, "Error restoring NB MCGCTL settings!\n");
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002552}
2553
Doug Thompsonf9431992009-04-27 19:46:08 +02002554/*
2555 * EDAC requires that the BIOS have ECC enabled before taking over the
2556 * processing of ECC errors. This is because the BIOS can properly initialize
2557 * the memory system completely. A command line option allows to force-enable
2558 * hardware ECC later in amd64_enable_ecc_error_reporting().
2559 */
Borislav Petkovcab4d272010-02-11 17:15:57 +01002560static const char *ecc_msg =
2561 "ECC disabled in the BIOS or no ECC capability, module will not load.\n"
2562 " Either enable ECC checking or force module loading by setting "
2563 "'ecc_enable_override'.\n"
2564 " (Note that use of the override may cause unknown side effects.)\n";
Borislav Petkovbe3468e2009-08-05 15:47:22 +02002565
Doug Thompsonf9431992009-04-27 19:46:08 +02002566static int amd64_check_ecc_enabled(struct amd64_pvt *pvt)
2567{
2568 u32 value;
Borislav Petkov06724532009-09-16 13:05:46 +02002569 u8 ecc_enabled = 0;
2570 bool nb_mce_en = false;
Doug Thompsonf9431992009-04-27 19:46:08 +02002571
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02002572 amd64_read_pci_cfg(pvt->misc_f3_ctl, K8_NBCFG, &value);
Doug Thompsonf9431992009-04-27 19:46:08 +02002573
2574 ecc_enabled = !!(value & K8_NBCFG_ECC_ENABLE);
Borislav Petkovbe3468e2009-08-05 15:47:22 +02002575 if (!ecc_enabled)
Borislav Petkovcab4d272010-02-11 17:15:57 +01002576 amd64_printk(KERN_NOTICE, "This node reports that Memory ECC "
Borislav Petkovbe3468e2009-08-05 15:47:22 +02002577 "is currently disabled, set F3x%x[22] (%s).\n",
2578 K8_NBCFG, pci_name(pvt->misc_f3_ctl));
2579 else
2580 amd64_printk(KERN_INFO, "ECC is enabled by BIOS.\n");
Doug Thompsonf9431992009-04-27 19:46:08 +02002581
Borislav Petkov06724532009-09-16 13:05:46 +02002582 nb_mce_en = amd64_nb_mce_bank_enabled_on_node(pvt->mc_node_id);
2583 if (!nb_mce_en)
Borislav Petkovcab4d272010-02-11 17:15:57 +01002584 amd64_printk(KERN_NOTICE, "NB MCE bank disabled, set MSR "
Borislav Petkovbe3468e2009-08-05 15:47:22 +02002585 "0x%08x[4] on node %d to enable.\n",
2586 MSR_IA32_MCG_CTL, pvt->mc_node_id);
Doug Thompsonf9431992009-04-27 19:46:08 +02002587
Borislav Petkov06724532009-09-16 13:05:46 +02002588 if (!ecc_enabled || !nb_mce_en) {
Doug Thompsonf9431992009-04-27 19:46:08 +02002589 if (!ecc_enable_override) {
Borislav Petkovcab4d272010-02-11 17:15:57 +01002590 amd64_printk(KERN_NOTICE, "%s", ecc_msg);
Borislav Petkovbe3468e2009-08-05 15:47:22 +02002591 return -ENODEV;
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002592 } else {
2593 amd64_printk(KERN_WARNING, "Forcing ECC checking on!\n");
Borislav Petkovbe3468e2009-08-05 15:47:22 +02002594 }
Borislav Petkov43f5e682009-12-21 18:55:18 +01002595 }
Doug Thompsonf9431992009-04-27 19:46:08 +02002596
Borislav Petkovbe3468e2009-08-05 15:47:22 +02002597 return 0;
Doug Thompsonf9431992009-04-27 19:46:08 +02002598}
2599
Doug Thompson7d6034d2009-04-27 20:01:01 +02002600struct mcidev_sysfs_attribute sysfs_attrs[ARRAY_SIZE(amd64_dbg_attrs) +
2601 ARRAY_SIZE(amd64_inj_attrs) +
2602 1];
2603
2604struct mcidev_sysfs_attribute terminator = { .attr = { .name = NULL } };
2605
2606static void amd64_set_mc_sysfs_attributes(struct mem_ctl_info *mci)
2607{
2608 unsigned int i = 0, j = 0;
2609
2610 for (; i < ARRAY_SIZE(amd64_dbg_attrs); i++)
2611 sysfs_attrs[i] = amd64_dbg_attrs[i];
2612
2613 for (j = 0; j < ARRAY_SIZE(amd64_inj_attrs); j++, i++)
2614 sysfs_attrs[i] = amd64_inj_attrs[j];
2615
2616 sysfs_attrs[i] = terminator;
2617
2618 mci->mc_driver_sysfs_attributes = sysfs_attrs;
2619}
2620
2621static void amd64_setup_mci_misc_attributes(struct mem_ctl_info *mci)
2622{
2623 struct amd64_pvt *pvt = mci->pvt_info;
2624
2625 mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
2626 mci->edac_ctl_cap = EDAC_FLAG_NONE;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002627
2628 if (pvt->nbcap & K8_NBCAP_SECDED)
2629 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
2630
2631 if (pvt->nbcap & K8_NBCAP_CHIPKILL)
2632 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
2633
2634 mci->edac_cap = amd64_determine_edac_cap(pvt);
2635 mci->mod_name = EDAC_MOD_STR;
2636 mci->mod_ver = EDAC_AMD64_VERSION;
2637 mci->ctl_name = get_amd_family_name(pvt->mc_type_index);
2638 mci->dev_name = pci_name(pvt->dram_f2_ctl);
2639 mci->ctl_page_to_phys = NULL;
2640
Doug Thompson7d6034d2009-04-27 20:01:01 +02002641 /* memory scrubber interface */
2642 mci->set_sdram_scrub_rate = amd64_set_scrub_rate;
2643 mci->get_sdram_scrub_rate = amd64_get_scrub_rate;
2644}
2645
2646/*
2647 * Init stuff for this DRAM Controller device.
2648 *
2649 * Due to a hardware feature on Fam10h CPUs, the Enable Extended Configuration
2650 * Space feature MUST be enabled on ALL Processors prior to actually reading
2651 * from the ECS registers. Since the loading of the module can occur on any
2652 * 'core', and cores don't 'see' all the other processors ECS data when the
2653 * others are NOT enabled. Our solution is to first enable ECS access in this
2654 * routine on all processors, gather some data in a amd64_pvt structure and
2655 * later come back in a finish-setup function to perform that final
2656 * initialization. See also amd64_init_2nd_stage() for that.
2657 */
2658static int amd64_probe_one_instance(struct pci_dev *dram_f2_ctl,
2659 int mc_type_index)
2660{
2661 struct amd64_pvt *pvt = NULL;
2662 int err = 0, ret;
2663
2664 ret = -ENOMEM;
2665 pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
2666 if (!pvt)
2667 goto err_exit;
2668
Borislav Petkov37da0452009-06-10 17:36:57 +02002669 pvt->mc_node_id = get_node_id(dram_f2_ctl);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002670
2671 pvt->dram_f2_ctl = dram_f2_ctl;
2672 pvt->ext_model = boot_cpu_data.x86_model >> 4;
2673 pvt->mc_type_index = mc_type_index;
2674 pvt->ops = family_ops(mc_type_index);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002675
2676 /*
2677 * We have the dram_f2_ctl device as an argument, now go reserve its
2678 * sibling devices from the PCI system.
2679 */
2680 ret = -ENODEV;
2681 err = amd64_reserve_mc_sibling_devices(pvt, mc_type_index);
2682 if (err)
2683 goto err_free;
2684
2685 ret = -EINVAL;
2686 err = amd64_check_ecc_enabled(pvt);
2687 if (err)
2688 goto err_put;
2689
2690 /*
2691 * Key operation here: setup of HW prior to performing ops on it. Some
2692 * setup is required to access ECS data. After this is performed, the
2693 * 'teardown' function must be called upon error and normal exit paths.
2694 */
2695 if (boot_cpu_data.x86 >= 0x10)
2696 amd64_setup(pvt);
2697
2698 /*
2699 * Save the pointer to the private data for use in 2nd initialization
2700 * stage
2701 */
2702 pvt_lookup[pvt->mc_node_id] = pvt;
2703
2704 return 0;
2705
2706err_put:
2707 amd64_free_mc_sibling_devices(pvt);
2708
2709err_free:
2710 kfree(pvt);
2711
2712err_exit:
2713 return ret;
2714}
2715
2716/*
2717 * This is the finishing stage of the init code. Needs to be performed after all
2718 * MCs' hardware have been prepped for accessing extended config space.
2719 */
2720static int amd64_init_2nd_stage(struct amd64_pvt *pvt)
2721{
2722 int node_id = pvt->mc_node_id;
2723 struct mem_ctl_info *mci;
Andrew Morton18ba54a2009-12-07 19:04:23 +01002724 int ret = -ENODEV;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002725
2726 amd64_read_mc_registers(pvt);
2727
Doug Thompson7d6034d2009-04-27 20:01:01 +02002728 /*
2729 * We need to determine how many memory channels there are. Then use
2730 * that information for calculating the size of the dynamic instance
2731 * tables in the 'mci' structure
2732 */
2733 pvt->channel_count = pvt->ops->early_channel_count(pvt);
2734 if (pvt->channel_count < 0)
2735 goto err_exit;
2736
2737 ret = -ENOMEM;
Borislav Petkov9d858bb2009-09-21 14:35:51 +02002738 mci = edac_mc_alloc(0, pvt->cs_count, pvt->channel_count, node_id);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002739 if (!mci)
2740 goto err_exit;
2741
2742 mci->pvt_info = pvt;
2743
2744 mci->dev = &pvt->dram_f2_ctl->dev;
2745 amd64_setup_mci_misc_attributes(mci);
2746
2747 if (amd64_init_csrows(mci))
2748 mci->edac_cap = EDAC_FLAG_NONE;
2749
2750 amd64_enable_ecc_error_reporting(mci);
2751 amd64_set_mc_sysfs_attributes(mci);
2752
2753 ret = -ENODEV;
2754 if (edac_mc_add_mc(mci)) {
2755 debugf1("failed edac_mc_add_mc()\n");
2756 goto err_add_mc;
2757 }
2758
2759 mci_lookup[node_id] = mci;
2760 pvt_lookup[node_id] = NULL;
Borislav Petkov549d0422009-07-24 13:51:42 +02002761
2762 /* register stuff with EDAC MCE */
2763 if (report_gart_errors)
2764 amd_report_gart_errors(true);
2765
2766 amd_register_ecc_decoder(amd64_decode_bus_error);
2767
Doug Thompson7d6034d2009-04-27 20:01:01 +02002768 return 0;
2769
2770err_add_mc:
2771 edac_mc_free(mci);
2772
2773err_exit:
2774 debugf0("failure to init 2nd stage: ret=%d\n", ret);
2775
2776 amd64_restore_ecc_error_reporting(pvt);
2777
2778 if (boot_cpu_data.x86 > 0xf)
2779 amd64_teardown(pvt);
2780
2781 amd64_free_mc_sibling_devices(pvt);
2782
2783 kfree(pvt_lookup[pvt->mc_node_id]);
2784 pvt_lookup[node_id] = NULL;
2785
2786 return ret;
2787}
2788
2789
2790static int __devinit amd64_init_one_instance(struct pci_dev *pdev,
2791 const struct pci_device_id *mc_type)
2792{
2793 int ret = 0;
2794
Borislav Petkov37da0452009-06-10 17:36:57 +02002795 debugf0("(MC node=%d,mc_type='%s')\n", get_node_id(pdev),
Doug Thompson7d6034d2009-04-27 20:01:01 +02002796 get_amd_family_name(mc_type->driver_data));
2797
2798 ret = pci_enable_device(pdev);
2799 if (ret < 0)
2800 ret = -EIO;
2801 else
2802 ret = amd64_probe_one_instance(pdev, mc_type->driver_data);
2803
2804 if (ret < 0)
2805 debugf0("ret=%d\n", ret);
2806
2807 return ret;
2808}
2809
2810static void __devexit amd64_remove_one_instance(struct pci_dev *pdev)
2811{
2812 struct mem_ctl_info *mci;
2813 struct amd64_pvt *pvt;
2814
2815 /* Remove from EDAC CORE tracking list */
2816 mci = edac_mc_del_mc(&pdev->dev);
2817 if (!mci)
2818 return;
2819
2820 pvt = mci->pvt_info;
2821
2822 amd64_restore_ecc_error_reporting(pvt);
2823
2824 if (boot_cpu_data.x86 > 0xf)
2825 amd64_teardown(pvt);
2826
2827 amd64_free_mc_sibling_devices(pvt);
2828
Borislav Petkov549d0422009-07-24 13:51:42 +02002829 /* unregister from EDAC MCE */
2830 amd_report_gart_errors(false);
2831 amd_unregister_ecc_decoder(amd64_decode_bus_error);
2832
Doug Thompson7d6034d2009-04-27 20:01:01 +02002833 /* Free the EDAC CORE resources */
Borislav Petkov8f68ed92009-12-21 15:15:59 +01002834 mci->pvt_info = NULL;
2835 mci_lookup[pvt->mc_node_id] = NULL;
2836
2837 kfree(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002838 edac_mc_free(mci);
2839}
2840
2841/*
2842 * This table is part of the interface for loading drivers for PCI devices. The
2843 * PCI core identifies what devices are on a system during boot, and then
2844 * inquiry this table to see if this driver is for a given device found.
2845 */
2846static const struct pci_device_id amd64_pci_table[] __devinitdata = {
2847 {
2848 .vendor = PCI_VENDOR_ID_AMD,
2849 .device = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
2850 .subvendor = PCI_ANY_ID,
2851 .subdevice = PCI_ANY_ID,
2852 .class = 0,
2853 .class_mask = 0,
2854 .driver_data = K8_CPUS
2855 },
2856 {
2857 .vendor = PCI_VENDOR_ID_AMD,
2858 .device = PCI_DEVICE_ID_AMD_10H_NB_DRAM,
2859 .subvendor = PCI_ANY_ID,
2860 .subdevice = PCI_ANY_ID,
2861 .class = 0,
2862 .class_mask = 0,
2863 .driver_data = F10_CPUS
2864 },
2865 {
2866 .vendor = PCI_VENDOR_ID_AMD,
2867 .device = PCI_DEVICE_ID_AMD_11H_NB_DRAM,
2868 .subvendor = PCI_ANY_ID,
2869 .subdevice = PCI_ANY_ID,
2870 .class = 0,
2871 .class_mask = 0,
2872 .driver_data = F11_CPUS
2873 },
2874 {0, }
2875};
2876MODULE_DEVICE_TABLE(pci, amd64_pci_table);
2877
2878static struct pci_driver amd64_pci_driver = {
2879 .name = EDAC_MOD_STR,
2880 .probe = amd64_init_one_instance,
2881 .remove = __devexit_p(amd64_remove_one_instance),
2882 .id_table = amd64_pci_table,
2883};
2884
2885static void amd64_setup_pci_device(void)
2886{
2887 struct mem_ctl_info *mci;
2888 struct amd64_pvt *pvt;
2889
2890 if (amd64_ctl_pci)
2891 return;
2892
2893 mci = mci_lookup[0];
2894 if (mci) {
2895
2896 pvt = mci->pvt_info;
2897 amd64_ctl_pci =
2898 edac_pci_create_generic_ctl(&pvt->dram_f2_ctl->dev,
2899 EDAC_MOD_STR);
2900
2901 if (!amd64_ctl_pci) {
2902 pr_warning("%s(): Unable to create PCI control\n",
2903 __func__);
2904
2905 pr_warning("%s(): PCI error report via EDAC not set\n",
2906 __func__);
2907 }
2908 }
2909}
2910
2911static int __init amd64_edac_init(void)
2912{
2913 int nb, err = -ENODEV;
Borislav Petkov56b34b92009-12-21 18:13:01 +01002914 bool load_ok = false;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002915
2916 edac_printk(KERN_INFO, EDAC_MOD_STR, EDAC_AMD64_VERSION "\n");
2917
2918 opstate_init();
2919
2920 if (cache_k8_northbridges() < 0)
Borislav Petkov56b34b92009-12-21 18:13:01 +01002921 goto err_ret;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002922
Borislav Petkov50542252009-12-11 18:14:40 +01002923 msrs = msrs_alloc();
Borislav Petkov56b34b92009-12-21 18:13:01 +01002924 if (!msrs)
2925 goto err_ret;
Borislav Petkov50542252009-12-11 18:14:40 +01002926
Doug Thompson7d6034d2009-04-27 20:01:01 +02002927 err = pci_register_driver(&amd64_pci_driver);
2928 if (err)
Borislav Petkov56b34b92009-12-21 18:13:01 +01002929 goto err_pci;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002930
2931 /*
2932 * At this point, the array 'pvt_lookup[]' contains pointers to alloc'd
2933 * amd64_pvt structs. These will be used in the 2nd stage init function
2934 * to finish initialization of the MC instances.
2935 */
Borislav Petkov56b34b92009-12-21 18:13:01 +01002936 err = -ENODEV;
Andreas Herrmann900f9ac2010-09-17 18:02:54 +02002937 for (nb = 0; nb < k8_northbridges.num; nb++) {
Doug Thompson7d6034d2009-04-27 20:01:01 +02002938 if (!pvt_lookup[nb])
2939 continue;
2940
2941 err = amd64_init_2nd_stage(pvt_lookup[nb]);
2942 if (err)
Borislav Petkov37da0452009-06-10 17:36:57 +02002943 goto err_2nd_stage;
Borislav Petkov56b34b92009-12-21 18:13:01 +01002944
2945 load_ok = true;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002946 }
2947
Borislav Petkov56b34b92009-12-21 18:13:01 +01002948 if (load_ok) {
2949 amd64_setup_pci_device();
2950 return 0;
2951 }
Doug Thompson7d6034d2009-04-27 20:01:01 +02002952
Borislav Petkov37da0452009-06-10 17:36:57 +02002953err_2nd_stage:
Doug Thompson7d6034d2009-04-27 20:01:01 +02002954 pci_unregister_driver(&amd64_pci_driver);
Borislav Petkov56b34b92009-12-21 18:13:01 +01002955err_pci:
2956 msrs_free(msrs);
2957 msrs = NULL;
2958err_ret:
Doug Thompson7d6034d2009-04-27 20:01:01 +02002959 return err;
2960}
2961
2962static void __exit amd64_edac_exit(void)
2963{
2964 if (amd64_ctl_pci)
2965 edac_pci_release_generic_ctl(amd64_ctl_pci);
2966
2967 pci_unregister_driver(&amd64_pci_driver);
Borislav Petkov50542252009-12-11 18:14:40 +01002968
2969 msrs_free(msrs);
2970 msrs = NULL;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002971}
2972
2973module_init(amd64_edac_init);
2974module_exit(amd64_edac_exit);
2975
2976MODULE_LICENSE("GPL");
2977MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
2978 "Dave Peterson, Thayne Harbaugh");
2979MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
2980 EDAC_AMD64_VERSION);
2981
2982module_param(edac_op_state, int, 0444);
2983MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");