blob: 9a8bebcf6b177fa79e9635ecde71f05b2578e798 [file] [log] [blame]
Doug Thompson2bc65412009-05-04 20:11:14 +02001#include "amd64_edac.h"
Andreas Herrmann23ac4ae2010-09-17 18:03:43 +02002#include <asm/amd_nb.h>
Doug Thompson2bc65412009-05-04 20:11:14 +02003
4static struct edac_pci_ctl_info *amd64_ctl_pci;
5
6static int report_gart_errors;
7module_param(report_gart_errors, int, 0644);
8
9/*
10 * Set by command line parameter. If BIOS has enabled the ECC, this override is
11 * cleared to prevent re-enabling the hardware by this driver.
12 */
13static int ecc_enable_override;
14module_param(ecc_enable_override, int, 0644);
15
Tejun Heoa29d8b82010-02-02 14:39:15 +090016static struct msr __percpu *msrs;
Borislav Petkov50542252009-12-11 18:14:40 +010017
Borislav Petkov360b7f32010-10-15 19:25:38 +020018/*
19 * count successfully initialized driver instances for setup_pci_device()
20 */
21static atomic_t drv_instances = ATOMIC_INIT(0);
22
Borislav Petkovcc4d8862010-10-13 16:11:59 +020023/* Per-node driver instances */
24static struct mem_ctl_info **mcis;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +020025static struct ecc_settings **ecc_stngs;
Doug Thompson2bc65412009-05-04 20:11:14 +020026
27/*
Borislav Petkovb70ef012009-06-25 19:32:38 +020028 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
29 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
30 * or higher value'.
31 *
32 *FIXME: Produce a better mapping/linearisation.
33 */
Borislav Petkov39094442010-11-24 19:52:09 +010034struct scrubrate {
35 u32 scrubval; /* bit pattern for scrub rate */
36 u32 bandwidth; /* bandwidth consumed (bytes/sec) */
37} scrubrates[] = {
Borislav Petkovb70ef012009-06-25 19:32:38 +020038 { 0x01, 1600000000UL},
39 { 0x02, 800000000UL},
40 { 0x03, 400000000UL},
41 { 0x04, 200000000UL},
42 { 0x05, 100000000UL},
43 { 0x06, 50000000UL},
44 { 0x07, 25000000UL},
45 { 0x08, 12284069UL},
46 { 0x09, 6274509UL},
47 { 0x0A, 3121951UL},
48 { 0x0B, 1560975UL},
49 { 0x0C, 781440UL},
50 { 0x0D, 390720UL},
51 { 0x0E, 195300UL},
52 { 0x0F, 97650UL},
53 { 0x10, 48854UL},
54 { 0x11, 24427UL},
55 { 0x12, 12213UL},
56 { 0x13, 6101UL},
57 { 0x14, 3051UL},
58 { 0x15, 1523UL},
59 { 0x16, 761UL},
60 { 0x00, 0UL}, /* scrubbing off */
61};
62
Borislav Petkovb2b0c602010-10-08 18:32:29 +020063static int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
64 u32 *val, const char *func)
65{
66 int err = 0;
67
68 err = pci_read_config_dword(pdev, offset, val);
69 if (err)
70 amd64_warn("%s: error reading F%dx%03x.\n",
71 func, PCI_FUNC(pdev->devfn), offset);
72
73 return err;
74}
75
76int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
77 u32 val, const char *func)
78{
79 int err = 0;
80
81 err = pci_write_config_dword(pdev, offset, val);
82 if (err)
83 amd64_warn("%s: error writing to F%dx%03x.\n",
84 func, PCI_FUNC(pdev->devfn), offset);
85
86 return err;
87}
88
89/*
90 *
91 * Depending on the family, F2 DCT reads need special handling:
92 *
93 * K8: has a single DCT only
94 *
95 * F10h: each DCT has its own set of regs
96 * DCT0 -> F2x040..
97 * DCT1 -> F2x140..
98 *
99 * F15h: we select which DCT we access using F1x10C[DctCfgSel]
100 *
101 */
102static int k8_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
103 const char *func)
104{
105 if (addr >= 0x100)
106 return -EINVAL;
107
108 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
109}
110
111static int f10_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
112 const char *func)
113{
114 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
115}
116
117static int f15_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
118 const char *func)
119{
120 u32 reg = 0;
121 u8 dct = 0;
122
123 if (addr >= 0x140 && addr <= 0x1a0) {
124 dct = 1;
125 addr -= 0x100;
126 }
127
128 amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, &reg);
129 reg &= 0xfffffffe;
130 reg |= dct;
131 amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
132
133 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
134}
135
Borislav Petkovb70ef012009-06-25 19:32:38 +0200136/*
Doug Thompson2bc65412009-05-04 20:11:14 +0200137 * Memory scrubber control interface. For K8, memory scrubbing is handled by
138 * hardware and can involve L2 cache, dcache as well as the main memory. With
139 * F10, this is extended to L3 cache scrubbing on CPU models sporting that
140 * functionality.
141 *
142 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
143 * (dram) over to cache lines. This is nasty, so we will use bandwidth in
144 * bytes/sec for the setting.
145 *
146 * Currently, we only do dram scrubbing. If the scrubbing is done in software on
147 * other archs, we might not have access to the caches directly.
148 */
149
150/*
151 * scan the scrub rate mapping table for a close or matching bandwidth value to
152 * issue. If requested is too big, then use last maximum value found.
153 */
Borislav Petkov395ae782010-10-01 18:38:19 +0200154static int __amd64_set_scrub_rate(struct pci_dev *ctl, u32 new_bw, u32 min_rate)
Doug Thompson2bc65412009-05-04 20:11:14 +0200155{
156 u32 scrubval;
157 int i;
158
159 /*
160 * map the configured rate (new_bw) to a value specific to the AMD64
161 * memory controller and apply to register. Search for the first
162 * bandwidth entry that is greater or equal than the setting requested
163 * and program that. If at last entry, turn off DRAM scrubbing.
164 */
165 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
166 /*
167 * skip scrub rates which aren't recommended
168 * (see F10 BKDG, F3x58)
169 */
Borislav Petkov395ae782010-10-01 18:38:19 +0200170 if (scrubrates[i].scrubval < min_rate)
Doug Thompson2bc65412009-05-04 20:11:14 +0200171 continue;
172
173 if (scrubrates[i].bandwidth <= new_bw)
174 break;
175
176 /*
177 * if no suitable bandwidth found, turn off DRAM scrubbing
178 * entirely by falling back to the last element in the
179 * scrubrates array.
180 */
181 }
182
183 scrubval = scrubrates[i].scrubval;
Doug Thompson2bc65412009-05-04 20:11:14 +0200184
Borislav Petkov5980bb92011-01-07 16:26:49 +0100185 pci_write_bits32(ctl, SCRCTRL, scrubval, 0x001F);
Doug Thompson2bc65412009-05-04 20:11:14 +0200186
Borislav Petkov39094442010-11-24 19:52:09 +0100187 if (scrubval)
188 return scrubrates[i].bandwidth;
189
Doug Thompson2bc65412009-05-04 20:11:14 +0200190 return 0;
191}
192
Borislav Petkov395ae782010-10-01 18:38:19 +0200193static int amd64_set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
Doug Thompson2bc65412009-05-04 20:11:14 +0200194{
195 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkov87b3e0e2011-01-19 20:02:38 +0100196 u32 min_scrubrate = 0x5;
Doug Thompson2bc65412009-05-04 20:11:14 +0200197
Borislav Petkov87b3e0e2011-01-19 20:02:38 +0100198 if (boot_cpu_data.x86 == 0xf)
199 min_scrubrate = 0x0;
200
201 return __amd64_set_scrub_rate(pvt->F3, bw, min_scrubrate);
Doug Thompson2bc65412009-05-04 20:11:14 +0200202}
203
Borislav Petkov39094442010-11-24 19:52:09 +0100204static int amd64_get_scrub_rate(struct mem_ctl_info *mci)
Doug Thompson2bc65412009-05-04 20:11:14 +0200205{
206 struct amd64_pvt *pvt = mci->pvt_info;
207 u32 scrubval = 0;
Borislav Petkov39094442010-11-24 19:52:09 +0100208 int i, retval = -EINVAL;
Doug Thompson2bc65412009-05-04 20:11:14 +0200209
Borislav Petkov5980bb92011-01-07 16:26:49 +0100210 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
Doug Thompson2bc65412009-05-04 20:11:14 +0200211
212 scrubval = scrubval & 0x001F;
213
Roel Kluin926311f2010-01-11 20:58:21 +0100214 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
Doug Thompson2bc65412009-05-04 20:11:14 +0200215 if (scrubrates[i].scrubval == scrubval) {
Borislav Petkov39094442010-11-24 19:52:09 +0100216 retval = scrubrates[i].bandwidth;
Doug Thompson2bc65412009-05-04 20:11:14 +0200217 break;
218 }
219 }
Borislav Petkov39094442010-11-24 19:52:09 +0100220 return retval;
Doug Thompson2bc65412009-05-04 20:11:14 +0200221}
222
Doug Thompson67757632009-04-27 15:53:22 +0200223/*
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200224 * returns true if the SysAddr given by sys_addr matches the
225 * DRAM base/limit associated with node_id
Doug Thompson67757632009-04-27 15:53:22 +0200226 */
Borislav Petkovb487c332011-02-21 18:55:00 +0100227static bool amd64_base_limit_match(struct amd64_pvt *pvt, u64 sys_addr,
228 unsigned nid)
Doug Thompson67757632009-04-27 15:53:22 +0200229{
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200230 u64 addr;
Doug Thompson67757632009-04-27 15:53:22 +0200231
232 /* The K8 treats this as a 40-bit value. However, bits 63-40 will be
233 * all ones if the most significant implemented address bit is 1.
234 * Here we discard bits 63-40. See section 3.4.2 of AMD publication
235 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
236 * Application Programming.
237 */
238 addr = sys_addr & 0x000000ffffffffffull;
239
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200240 return ((addr >= get_dram_base(pvt, nid)) &&
241 (addr <= get_dram_limit(pvt, nid)));
Doug Thompson67757632009-04-27 15:53:22 +0200242}
243
244/*
245 * Attempt to map a SysAddr to a node. On success, return a pointer to the
246 * mem_ctl_info structure for the node that the SysAddr maps to.
247 *
248 * On failure, return NULL.
249 */
250static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
251 u64 sys_addr)
252{
253 struct amd64_pvt *pvt;
Borislav Petkovb487c332011-02-21 18:55:00 +0100254 unsigned node_id;
Doug Thompson67757632009-04-27 15:53:22 +0200255 u32 intlv_en, bits;
256
257 /*
258 * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
259 * 3.4.4.2) registers to map the SysAddr to a node ID.
260 */
261 pvt = mci->pvt_info;
262
263 /*
264 * The value of this field should be the same for all DRAM Base
265 * registers. Therefore we arbitrarily choose to read it from the
266 * register for node 0.
267 */
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200268 intlv_en = dram_intlv_en(pvt, 0);
Doug Thompson67757632009-04-27 15:53:22 +0200269
270 if (intlv_en == 0) {
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200271 for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
Doug Thompson67757632009-04-27 15:53:22 +0200272 if (amd64_base_limit_match(pvt, sys_addr, node_id))
Borislav Petkov8edc5442009-09-18 12:39:19 +0200273 goto found;
Doug Thompson67757632009-04-27 15:53:22 +0200274 }
Borislav Petkov8edc5442009-09-18 12:39:19 +0200275 goto err_no_match;
Doug Thompson67757632009-04-27 15:53:22 +0200276 }
277
Borislav Petkov72f158f2009-09-18 12:27:27 +0200278 if (unlikely((intlv_en != 0x01) &&
279 (intlv_en != 0x03) &&
280 (intlv_en != 0x07))) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200281 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
Doug Thompson67757632009-04-27 15:53:22 +0200282 return NULL;
283 }
284
285 bits = (((u32) sys_addr) >> 12) & intlv_en;
286
287 for (node_id = 0; ; ) {
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200288 if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
Doug Thompson67757632009-04-27 15:53:22 +0200289 break; /* intlv_sel field matches */
290
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200291 if (++node_id >= DRAM_RANGES)
Doug Thompson67757632009-04-27 15:53:22 +0200292 goto err_no_match;
293 }
294
295 /* sanity test for sys_addr */
296 if (unlikely(!amd64_base_limit_match(pvt, sys_addr, node_id))) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200297 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
298 "range for node %d with node interleaving enabled.\n",
299 __func__, sys_addr, node_id);
Doug Thompson67757632009-04-27 15:53:22 +0200300 return NULL;
301 }
302
303found:
Borislav Petkovb487c332011-02-21 18:55:00 +0100304 return edac_mc_find((int)node_id);
Doug Thompson67757632009-04-27 15:53:22 +0200305
306err_no_match:
307 debugf2("sys_addr 0x%lx doesn't match any node\n",
308 (unsigned long)sys_addr);
309
310 return NULL;
311}
Doug Thompsone2ce7252009-04-27 15:57:12 +0200312
313/*
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100314 * compute the CS base address of the @csrow on the DRAM controller @dct.
315 * For details see F2x[5C:40] in the processor's BKDG
Doug Thompsone2ce7252009-04-27 15:57:12 +0200316 */
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100317static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
318 u64 *base, u64 *mask)
Doug Thompsone2ce7252009-04-27 15:57:12 +0200319{
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100320 u64 csbase, csmask, base_bits, mask_bits;
321 u8 addr_shift;
322
323 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) {
324 csbase = pvt->csels[dct].csbases[csrow];
325 csmask = pvt->csels[dct].csmasks[csrow];
326 base_bits = GENMASK(21, 31) | GENMASK(9, 15);
327 mask_bits = GENMASK(21, 29) | GENMASK(9, 15);
328 addr_shift = 4;
329 } else {
330 csbase = pvt->csels[dct].csbases[csrow];
331 csmask = pvt->csels[dct].csmasks[csrow >> 1];
332 addr_shift = 8;
333
334 if (boot_cpu_data.x86 == 0x15)
335 base_bits = mask_bits = GENMASK(19,30) | GENMASK(5,13);
336 else
337 base_bits = mask_bits = GENMASK(19,28) | GENMASK(5,13);
338 }
339
340 *base = (csbase & base_bits) << addr_shift;
341
342 *mask = ~0ULL;
343 /* poke holes for the csmask */
344 *mask &= ~(mask_bits << addr_shift);
345 /* OR them in */
346 *mask |= (csmask & mask_bits) << addr_shift;
Doug Thompsone2ce7252009-04-27 15:57:12 +0200347}
348
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100349#define for_each_chip_select(i, dct, pvt) \
350 for (i = 0; i < pvt->csels[dct].b_cnt; i++)
Doug Thompsone2ce7252009-04-27 15:57:12 +0200351
Borislav Petkov614ec9d2011-01-13 18:02:22 +0100352#define chip_select_base(i, dct, pvt) \
353 pvt->csels[dct].csbases[i]
354
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100355#define for_each_chip_select_mask(i, dct, pvt) \
356 for (i = 0; i < pvt->csels[dct].m_cnt; i++)
Doug Thompsone2ce7252009-04-27 15:57:12 +0200357
358/*
359 * @input_addr is an InputAddr associated with the node given by mci. Return the
360 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
361 */
362static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
363{
364 struct amd64_pvt *pvt;
365 int csrow;
366 u64 base, mask;
367
368 pvt = mci->pvt_info;
369
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100370 for_each_chip_select(csrow, 0, pvt) {
371 if (!csrow_enabled(csrow, 0, pvt))
Doug Thompsone2ce7252009-04-27 15:57:12 +0200372 continue;
373
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100374 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
375
376 mask = ~mask;
Doug Thompsone2ce7252009-04-27 15:57:12 +0200377
378 if ((input_addr & mask) == (base & mask)) {
379 debugf2("InputAddr 0x%lx matches csrow %d (node %d)\n",
380 (unsigned long)input_addr, csrow,
381 pvt->mc_node_id);
382
383 return csrow;
384 }
385 }
Doug Thompsone2ce7252009-04-27 15:57:12 +0200386 debugf2("no matching csrow for InputAddr 0x%lx (MC node %d)\n",
387 (unsigned long)input_addr, pvt->mc_node_id);
388
389 return -1;
390}
391
392/*
Doug Thompsone2ce7252009-04-27 15:57:12 +0200393 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
394 * for the node represented by mci. Info is passed back in *hole_base,
395 * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if
396 * info is invalid. Info may be invalid for either of the following reasons:
397 *
398 * - The revision of the node is not E or greater. In this case, the DRAM Hole
399 * Address Register does not exist.
400 *
401 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
402 * indicating that its contents are not valid.
403 *
404 * The values passed back in *hole_base, *hole_offset, and *hole_size are
405 * complete 32-bit values despite the fact that the bitfields in the DHAR
406 * only represent bits 31-24 of the base and offset values.
407 */
408int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
409 u64 *hole_offset, u64 *hole_size)
410{
411 struct amd64_pvt *pvt = mci->pvt_info;
412 u64 base;
413
414 /* only revE and later have the DRAM Hole Address Register */
Borislav Petkov1433eb92009-10-21 13:44:36 +0200415 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_E) {
Doug Thompsone2ce7252009-04-27 15:57:12 +0200416 debugf1(" revision %d for node %d does not support DHAR\n",
417 pvt->ext_model, pvt->mc_node_id);
418 return 1;
419 }
420
Borislav Petkovbc21fa52010-11-11 17:29:13 +0100421 /* valid for Fam10h and above */
Borislav Petkovc8e518d2010-12-10 19:49:19 +0100422 if (boot_cpu_data.x86 >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
Doug Thompsone2ce7252009-04-27 15:57:12 +0200423 debugf1(" Dram Memory Hoisting is DISABLED on this system\n");
424 return 1;
425 }
426
Borislav Petkovc8e518d2010-12-10 19:49:19 +0100427 if (!dhar_valid(pvt)) {
Doug Thompsone2ce7252009-04-27 15:57:12 +0200428 debugf1(" Dram Memory Hoisting is DISABLED on this node %d\n",
429 pvt->mc_node_id);
430 return 1;
431 }
432
433 /* This node has Memory Hoisting */
434
435 /* +------------------+--------------------+--------------------+-----
436 * | memory | DRAM hole | relocated |
437 * | [0, (x - 1)] | [x, 0xffffffff] | addresses from |
438 * | | | DRAM hole |
439 * | | | [0x100000000, |
440 * | | | (0x100000000+ |
441 * | | | (0xffffffff-x))] |
442 * +------------------+--------------------+--------------------+-----
443 *
444 * Above is a diagram of physical memory showing the DRAM hole and the
445 * relocated addresses from the DRAM hole. As shown, the DRAM hole
446 * starts at address x (the base address) and extends through address
447 * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the
448 * addresses in the hole so that they start at 0x100000000.
449 */
450
Borislav Petkovbc21fa52010-11-11 17:29:13 +0100451 base = dhar_base(pvt);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200452
453 *hole_base = base;
454 *hole_size = (0x1ull << 32) - base;
455
456 if (boot_cpu_data.x86 > 0xf)
Borislav Petkovbc21fa52010-11-11 17:29:13 +0100457 *hole_offset = f10_dhar_offset(pvt);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200458 else
Borislav Petkovbc21fa52010-11-11 17:29:13 +0100459 *hole_offset = k8_dhar_offset(pvt);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200460
461 debugf1(" DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
462 pvt->mc_node_id, (unsigned long)*hole_base,
463 (unsigned long)*hole_offset, (unsigned long)*hole_size);
464
465 return 0;
466}
467EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info);
468
Doug Thompson93c2df52009-05-04 20:46:50 +0200469/*
470 * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is
471 * assumed that sys_addr maps to the node given by mci.
472 *
473 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
474 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
475 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
476 * then it is also involved in translating a SysAddr to a DramAddr. Sections
477 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
478 * These parts of the documentation are unclear. I interpret them as follows:
479 *
480 * When node n receives a SysAddr, it processes the SysAddr as follows:
481 *
482 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
483 * Limit registers for node n. If the SysAddr is not within the range
484 * specified by the base and limit values, then node n ignores the Sysaddr
485 * (since it does not map to node n). Otherwise continue to step 2 below.
486 *
487 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
488 * disabled so skip to step 3 below. Otherwise see if the SysAddr is within
489 * the range of relocated addresses (starting at 0x100000000) from the DRAM
490 * hole. If not, skip to step 3 below. Else get the value of the
491 * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
492 * offset defined by this value from the SysAddr.
493 *
494 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
495 * Base register for node n. To obtain the DramAddr, subtract the base
496 * address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
497 */
498static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
499{
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200500 struct amd64_pvt *pvt = mci->pvt_info;
Doug Thompson93c2df52009-05-04 20:46:50 +0200501 u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
502 int ret = 0;
503
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200504 dram_base = get_dram_base(pvt, pvt->mc_node_id);
Doug Thompson93c2df52009-05-04 20:46:50 +0200505
506 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
507 &hole_size);
508 if (!ret) {
509 if ((sys_addr >= (1ull << 32)) &&
510 (sys_addr < ((1ull << 32) + hole_size))) {
511 /* use DHAR to translate SysAddr to DramAddr */
512 dram_addr = sys_addr - hole_offset;
513
514 debugf2("using DHAR to translate SysAddr 0x%lx to "
515 "DramAddr 0x%lx\n",
516 (unsigned long)sys_addr,
517 (unsigned long)dram_addr);
518
519 return dram_addr;
520 }
521 }
522
523 /*
524 * Translate the SysAddr to a DramAddr as shown near the start of
525 * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8
526 * only deals with 40-bit values. Therefore we discard bits 63-40 of
527 * sys_addr below. If bit 39 of sys_addr is 1 then the bits we
528 * discard are all 1s. Otherwise the bits we discard are all 0s. See
529 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
530 * Programmer's Manual Volume 1 Application Programming.
531 */
Borislav Petkovf678b8c2010-12-13 19:21:07 +0100532 dram_addr = (sys_addr & GENMASK(0, 39)) - dram_base;
Doug Thompson93c2df52009-05-04 20:46:50 +0200533
534 debugf2("using DRAM Base register to translate SysAddr 0x%lx to "
535 "DramAddr 0x%lx\n", (unsigned long)sys_addr,
536 (unsigned long)dram_addr);
537 return dram_addr;
538}
539
540/*
541 * @intlv_en is the value of the IntlvEn field from a DRAM Base register
542 * (section 3.4.4.1). Return the number of bits from a SysAddr that are used
543 * for node interleaving.
544 */
545static int num_node_interleave_bits(unsigned intlv_en)
546{
547 static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
548 int n;
549
550 BUG_ON(intlv_en > 7);
551 n = intlv_shift_table[intlv_en];
552 return n;
553}
554
555/* Translate the DramAddr given by @dram_addr to an InputAddr. */
556static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
557{
558 struct amd64_pvt *pvt;
559 int intlv_shift;
560 u64 input_addr;
561
562 pvt = mci->pvt_info;
563
564 /*
565 * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
566 * concerning translating a DramAddr to an InputAddr.
567 */
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200568 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
Borislav Petkovf678b8c2010-12-13 19:21:07 +0100569 input_addr = ((dram_addr >> intlv_shift) & GENMASK(12, 35)) +
570 (dram_addr & 0xfff);
Doug Thompson93c2df52009-05-04 20:46:50 +0200571
572 debugf2(" Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
573 intlv_shift, (unsigned long)dram_addr,
574 (unsigned long)input_addr);
575
576 return input_addr;
577}
578
579/*
580 * Translate the SysAddr represented by @sys_addr to an InputAddr. It is
581 * assumed that @sys_addr maps to the node given by mci.
582 */
583static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
584{
585 u64 input_addr;
586
587 input_addr =
588 dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
589
590 debugf2("SysAdddr 0x%lx translates to InputAddr 0x%lx\n",
591 (unsigned long)sys_addr, (unsigned long)input_addr);
592
593 return input_addr;
594}
595
596
597/*
598 * @input_addr is an InputAddr associated with the node represented by mci.
599 * Translate @input_addr to a DramAddr and return the result.
600 */
601static u64 input_addr_to_dram_addr(struct mem_ctl_info *mci, u64 input_addr)
602{
603 struct amd64_pvt *pvt;
Borislav Petkovb487c332011-02-21 18:55:00 +0100604 unsigned node_id, intlv_shift;
Doug Thompson93c2df52009-05-04 20:46:50 +0200605 u64 bits, dram_addr;
606 u32 intlv_sel;
607
608 /*
609 * Near the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
610 * shows how to translate a DramAddr to an InputAddr. Here we reverse
611 * this procedure. When translating from a DramAddr to an InputAddr, the
612 * bits used for node interleaving are discarded. Here we recover these
613 * bits from the IntlvSel field of the DRAM Limit register (section
614 * 3.4.4.2) for the node that input_addr is associated with.
615 */
616 pvt = mci->pvt_info;
617 node_id = pvt->mc_node_id;
Borislav Petkovb487c332011-02-21 18:55:00 +0100618
619 BUG_ON(node_id > 7);
Doug Thompson93c2df52009-05-04 20:46:50 +0200620
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200621 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
Doug Thompson93c2df52009-05-04 20:46:50 +0200622 if (intlv_shift == 0) {
623 debugf1(" InputAddr 0x%lx translates to DramAddr of "
624 "same value\n", (unsigned long)input_addr);
625
626 return input_addr;
627 }
628
Borislav Petkovf678b8c2010-12-13 19:21:07 +0100629 bits = ((input_addr & GENMASK(12, 35)) << intlv_shift) +
630 (input_addr & 0xfff);
Doug Thompson93c2df52009-05-04 20:46:50 +0200631
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200632 intlv_sel = dram_intlv_sel(pvt, node_id) & ((1 << intlv_shift) - 1);
Doug Thompson93c2df52009-05-04 20:46:50 +0200633 dram_addr = bits + (intlv_sel << 12);
634
635 debugf1("InputAddr 0x%lx translates to DramAddr 0x%lx "
636 "(%d node interleave bits)\n", (unsigned long)input_addr,
637 (unsigned long)dram_addr, intlv_shift);
638
639 return dram_addr;
640}
641
642/*
643 * @dram_addr is a DramAddr that maps to the node represented by mci. Convert
644 * @dram_addr to a SysAddr.
645 */
646static u64 dram_addr_to_sys_addr(struct mem_ctl_info *mci, u64 dram_addr)
647{
648 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200649 u64 hole_base, hole_offset, hole_size, base, sys_addr;
Doug Thompson93c2df52009-05-04 20:46:50 +0200650 int ret = 0;
651
652 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
653 &hole_size);
654 if (!ret) {
655 if ((dram_addr >= hole_base) &&
656 (dram_addr < (hole_base + hole_size))) {
657 sys_addr = dram_addr + hole_offset;
658
659 debugf1("using DHAR to translate DramAddr 0x%lx to "
660 "SysAddr 0x%lx\n", (unsigned long)dram_addr,
661 (unsigned long)sys_addr);
662
663 return sys_addr;
664 }
665 }
666
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200667 base = get_dram_base(pvt, pvt->mc_node_id);
Doug Thompson93c2df52009-05-04 20:46:50 +0200668 sys_addr = dram_addr + base;
669
670 /*
671 * The sys_addr we have computed up to this point is a 40-bit value
672 * because the k8 deals with 40-bit values. However, the value we are
673 * supposed to return is a full 64-bit physical address. The AMD
674 * x86-64 architecture specifies that the most significant implemented
675 * address bit through bit 63 of a physical address must be either all
676 * 0s or all 1s. Therefore we sign-extend the 40-bit sys_addr to a
677 * 64-bit value below. See section 3.4.2 of AMD publication 24592:
678 * AMD x86-64 Architecture Programmer's Manual Volume 1 Application
679 * Programming.
680 */
681 sys_addr |= ~((sys_addr & (1ull << 39)) - 1);
682
683 debugf1(" Node %d, DramAddr 0x%lx to SysAddr 0x%lx\n",
684 pvt->mc_node_id, (unsigned long)dram_addr,
685 (unsigned long)sys_addr);
686
687 return sys_addr;
688}
689
690/*
691 * @input_addr is an InputAddr associated with the node given by mci. Translate
692 * @input_addr to a SysAddr.
693 */
694static inline u64 input_addr_to_sys_addr(struct mem_ctl_info *mci,
695 u64 input_addr)
696{
697 return dram_addr_to_sys_addr(mci,
698 input_addr_to_dram_addr(mci, input_addr));
699}
700
701/*
702 * Find the minimum and maximum InputAddr values that map to the given @csrow.
703 * Pass back these values in *input_addr_min and *input_addr_max.
704 */
705static void find_csrow_limits(struct mem_ctl_info *mci, int csrow,
706 u64 *input_addr_min, u64 *input_addr_max)
707{
708 struct amd64_pvt *pvt;
709 u64 base, mask;
710
711 pvt = mci->pvt_info;
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100712 BUG_ON((csrow < 0) || (csrow >= pvt->csels[0].b_cnt));
Doug Thompson93c2df52009-05-04 20:46:50 +0200713
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100714 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
Doug Thompson93c2df52009-05-04 20:46:50 +0200715
716 *input_addr_min = base & ~mask;
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100717 *input_addr_max = base | mask;
Doug Thompson93c2df52009-05-04 20:46:50 +0200718}
719
Doug Thompson93c2df52009-05-04 20:46:50 +0200720/* Map the Error address to a PAGE and PAGE OFFSET. */
721static inline void error_address_to_page_and_offset(u64 error_address,
722 u32 *page, u32 *offset)
723{
724 *page = (u32) (error_address >> PAGE_SHIFT);
725 *offset = ((u32) error_address) & ~PAGE_MASK;
726}
727
728/*
729 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
730 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
731 * of a node that detected an ECC memory error. mci represents the node that
732 * the error address maps to (possibly different from the node that detected
733 * the error). Return the number of the csrow that sys_addr maps to, or -1 on
734 * error.
735 */
736static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
737{
738 int csrow;
739
740 csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
741
742 if (csrow == -1)
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200743 amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
744 "address 0x%lx\n", (unsigned long)sys_addr);
Doug Thompson93c2df52009-05-04 20:46:50 +0200745 return csrow;
746}
Doug Thompsone2ce7252009-04-27 15:57:12 +0200747
Borislav Petkovbfc04ae2009-11-12 19:05:07 +0100748static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
Doug Thompson2da11652009-04-27 16:09:09 +0200749
Doug Thompson2da11652009-04-27 16:09:09 +0200750/*
751 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
752 * are ECC capable.
753 */
754static enum edac_type amd64_determine_edac_cap(struct amd64_pvt *pvt)
755{
Borislav Petkovcb328502010-12-22 14:28:24 +0100756 u8 bit;
Borislav Petkov584fcff2009-06-10 18:29:54 +0200757 enum dev_type edac_cap = EDAC_FLAG_NONE;
Doug Thompson2da11652009-04-27 16:09:09 +0200758
Borislav Petkov1433eb92009-10-21 13:44:36 +0200759 bit = (boot_cpu_data.x86 > 0xf || pvt->ext_model >= K8_REV_F)
Doug Thompson2da11652009-04-27 16:09:09 +0200760 ? 19
761 : 17;
762
Borislav Petkov584fcff2009-06-10 18:29:54 +0200763 if (pvt->dclr0 & BIT(bit))
Doug Thompson2da11652009-04-27 16:09:09 +0200764 edac_cap = EDAC_FLAG_SECDED;
765
766 return edac_cap;
767}
768
Borislav Petkov8c671752011-02-23 17:25:12 +0100769static void amd64_debug_display_dimm_sizes(struct amd64_pvt *, u8);
Doug Thompson2da11652009-04-27 16:09:09 +0200770
Borislav Petkov68798e12009-11-03 16:18:33 +0100771static void amd64_dump_dramcfg_low(u32 dclr, int chan)
772{
773 debugf1("F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
774
775 debugf1(" DIMM type: %sbuffered; all DIMMs support ECC: %s\n",
776 (dclr & BIT(16)) ? "un" : "",
777 (dclr & BIT(19)) ? "yes" : "no");
778
779 debugf1(" PAR/ERR parity: %s\n",
780 (dclr & BIT(8)) ? "enabled" : "disabled");
781
Borislav Petkovcb328502010-12-22 14:28:24 +0100782 if (boot_cpu_data.x86 == 0x10)
783 debugf1(" DCT 128bit mode width: %s\n",
784 (dclr & BIT(11)) ? "128b" : "64b");
Borislav Petkov68798e12009-11-03 16:18:33 +0100785
786 debugf1(" x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
787 (dclr & BIT(12)) ? "yes" : "no",
788 (dclr & BIT(13)) ? "yes" : "no",
789 (dclr & BIT(14)) ? "yes" : "no",
790 (dclr & BIT(15)) ? "yes" : "no");
791}
792
Doug Thompson2da11652009-04-27 16:09:09 +0200793/* Display and decode various NB registers for debug purposes. */
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200794static void dump_misc_regs(struct amd64_pvt *pvt)
Doug Thompson2da11652009-04-27 16:09:09 +0200795{
Borislav Petkov68798e12009-11-03 16:18:33 +0100796 debugf1("F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
Doug Thompson2da11652009-04-27 16:09:09 +0200797
Borislav Petkov68798e12009-11-03 16:18:33 +0100798 debugf1(" NB two channel DRAM capable: %s\n",
Borislav Petkov5980bb92011-01-07 16:26:49 +0100799 (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");
Borislav Petkov68798e12009-11-03 16:18:33 +0100800
801 debugf1(" ECC capable: %s, ChipKill ECC capable: %s\n",
Borislav Petkov5980bb92011-01-07 16:26:49 +0100802 (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
803 (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");
Borislav Petkov68798e12009-11-03 16:18:33 +0100804
805 amd64_dump_dramcfg_low(pvt->dclr0, 0);
Doug Thompson2da11652009-04-27 16:09:09 +0200806
Borislav Petkov8de1d912009-10-16 13:39:30 +0200807 debugf1("F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
Doug Thompson2da11652009-04-27 16:09:09 +0200808
Borislav Petkov8de1d912009-10-16 13:39:30 +0200809 debugf1("F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, "
810 "offset: 0x%08x\n",
Borislav Petkovbc21fa52010-11-11 17:29:13 +0100811 pvt->dhar, dhar_base(pvt),
812 (boot_cpu_data.x86 == 0xf) ? k8_dhar_offset(pvt)
813 : f10_dhar_offset(pvt));
Doug Thompson2da11652009-04-27 16:09:09 +0200814
Borislav Petkovc8e518d2010-12-10 19:49:19 +0100815 debugf1(" DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
Doug Thompson2da11652009-04-27 16:09:09 +0200816
Borislav Petkov8c671752011-02-23 17:25:12 +0100817 amd64_debug_display_dimm_sizes(pvt, 0);
Borislav Petkov4d796362011-02-03 15:59:57 +0100818
Borislav Petkov8de1d912009-10-16 13:39:30 +0200819 /* everything below this point is Fam10h and above */
Borislav Petkov4d796362011-02-03 15:59:57 +0100820 if (boot_cpu_data.x86 == 0xf)
Doug Thompson2da11652009-04-27 16:09:09 +0200821 return;
Borislav Petkov4d796362011-02-03 15:59:57 +0100822
Borislav Petkov8c671752011-02-23 17:25:12 +0100823 amd64_debug_display_dimm_sizes(pvt, 1);
Doug Thompson2da11652009-04-27 16:09:09 +0200824
Borislav Petkova3b7db02011-01-19 20:35:12 +0100825 amd64_info("using %s syndromes.\n", ((pvt->ecc_sym_sz == 8) ? "x8" : "x4"));
Borislav Petkovad6a32e2010-03-09 12:46:00 +0100826
Borislav Petkov8de1d912009-10-16 13:39:30 +0200827 /* Only if NOT ganged does dclr1 have valid info */
Borislav Petkov68798e12009-11-03 16:18:33 +0100828 if (!dct_ganging_enabled(pvt))
829 amd64_dump_dramcfg_low(pvt->dclr1, 1);
Doug Thompson2da11652009-04-27 16:09:09 +0200830}
831
Doug Thompson94be4bf2009-04-27 16:12:00 +0200832/*
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100833 * see BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
Doug Thompson94be4bf2009-04-27 16:12:00 +0200834 */
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100835static void prep_chip_selects(struct amd64_pvt *pvt)
Doug Thompson94be4bf2009-04-27 16:12:00 +0200836{
Borislav Petkov1433eb92009-10-21 13:44:36 +0200837 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) {
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100838 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
839 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
Borislav Petkov9d858bb2009-09-21 14:35:51 +0200840 } else {
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100841 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
842 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
Doug Thompson94be4bf2009-04-27 16:12:00 +0200843 }
844}
845
846/*
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100847 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
Doug Thompson94be4bf2009-04-27 16:12:00 +0200848 */
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200849static void read_dct_base_mask(struct amd64_pvt *pvt)
Doug Thompson94be4bf2009-04-27 16:12:00 +0200850{
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100851 int cs;
Doug Thompson94be4bf2009-04-27 16:12:00 +0200852
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100853 prep_chip_selects(pvt);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200854
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100855 for_each_chip_select(cs, 0, pvt) {
Borislav Petkov71d2a322011-02-21 19:37:24 +0100856 int reg0 = DCSB0 + (cs * 4);
857 int reg1 = DCSB1 + (cs * 4);
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100858 u32 *base0 = &pvt->csels[0].csbases[cs];
859 u32 *base1 = &pvt->csels[1].csbases[cs];
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200860
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100861 if (!amd64_read_dct_pci_cfg(pvt, reg0, base0))
Doug Thompson94be4bf2009-04-27 16:12:00 +0200862 debugf0(" DCSB0[%d]=0x%08x reg: F2x%x\n",
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100863 cs, *base0, reg0);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200864
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100865 if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt))
866 continue;
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200867
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100868 if (!amd64_read_dct_pci_cfg(pvt, reg1, base1))
869 debugf0(" DCSB1[%d]=0x%08x reg: F2x%x\n",
870 cs, *base1, reg1);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200871 }
872
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100873 for_each_chip_select_mask(cs, 0, pvt) {
Borislav Petkov71d2a322011-02-21 19:37:24 +0100874 int reg0 = DCSM0 + (cs * 4);
875 int reg1 = DCSM1 + (cs * 4);
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100876 u32 *mask0 = &pvt->csels[0].csmasks[cs];
877 u32 *mask1 = &pvt->csels[1].csmasks[cs];
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200878
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100879 if (!amd64_read_dct_pci_cfg(pvt, reg0, mask0))
Doug Thompson94be4bf2009-04-27 16:12:00 +0200880 debugf0(" DCSM0[%d]=0x%08x reg: F2x%x\n",
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100881 cs, *mask0, reg0);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200882
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100883 if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt))
884 continue;
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200885
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100886 if (!amd64_read_dct_pci_cfg(pvt, reg1, mask1))
887 debugf0(" DCSM1[%d]=0x%08x reg: F2x%x\n",
888 cs, *mask1, reg1);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200889 }
890}
891
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200892static enum mem_type amd64_determine_memory_type(struct amd64_pvt *pvt, int cs)
Doug Thompson94be4bf2009-04-27 16:12:00 +0200893{
894 enum mem_type type;
895
Borislav Petkovcb328502010-12-22 14:28:24 +0100896 /* F15h supports only DDR3 */
897 if (boot_cpu_data.x86 >= 0x15)
898 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
899 else if (boot_cpu_data.x86 == 0x10 || pvt->ext_model >= K8_REV_F) {
Borislav Petkov6b4c0bd2009-11-12 15:37:57 +0100900 if (pvt->dchr0 & DDR3_MODE)
901 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
902 else
903 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
Doug Thompson94be4bf2009-04-27 16:12:00 +0200904 } else {
Doug Thompson94be4bf2009-04-27 16:12:00 +0200905 type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
906 }
907
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200908 amd64_info("CS%d: %s\n", cs, edac_mem_types[type]);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200909
910 return type;
911}
912
Borislav Petkovcb328502010-12-22 14:28:24 +0100913/* Get the number of DCT channels the memory controller is using. */
Doug Thompsonddff8762009-04-27 16:14:52 +0200914static int k8_early_channel_count(struct amd64_pvt *pvt)
915{
Borislav Petkovcb328502010-12-22 14:28:24 +0100916 int flag;
Doug Thompsonddff8762009-04-27 16:14:52 +0200917
Borislav Petkov9f56da02010-10-01 19:44:53 +0200918 if (pvt->ext_model >= K8_REV_F)
Doug Thompsonddff8762009-04-27 16:14:52 +0200919 /* RevF (NPT) and later */
Borislav Petkov41d8bfa2011-01-18 19:16:08 +0100920 flag = pvt->dclr0 & WIDTH_128;
Borislav Petkov9f56da02010-10-01 19:44:53 +0200921 else
Doug Thompsonddff8762009-04-27 16:14:52 +0200922 /* RevE and earlier */
923 flag = pvt->dclr0 & REVE_WIDTH_128;
Doug Thompsonddff8762009-04-27 16:14:52 +0200924
925 /* not used */
926 pvt->dclr1 = 0;
927
928 return (flag) ? 2 : 1;
929}
930
Borislav Petkov70046622011-01-10 14:37:27 +0100931/* On F10h and later ErrAddr is MC4_ADDR[47:1] */
932static u64 get_error_address(struct mce *m)
Doug Thompsonddff8762009-04-27 16:14:52 +0200933{
Borislav Petkovc1ae6832011-03-30 15:42:10 +0200934 struct cpuinfo_x86 *c = &boot_cpu_data;
935 u64 addr;
Borislav Petkov70046622011-01-10 14:37:27 +0100936 u8 start_bit = 1;
937 u8 end_bit = 47;
938
Borislav Petkovc1ae6832011-03-30 15:42:10 +0200939 if (c->x86 == 0xf) {
Borislav Petkov70046622011-01-10 14:37:27 +0100940 start_bit = 3;
941 end_bit = 39;
942 }
943
Borislav Petkovc1ae6832011-03-30 15:42:10 +0200944 addr = m->addr & GENMASK(start_bit, end_bit);
945
946 /*
947 * Erratum 637 workaround
948 */
949 if (c->x86 == 0x15) {
950 struct amd64_pvt *pvt;
951 u64 cc6_base, tmp_addr;
952 u32 tmp;
953 u8 mce_nid, intlv_en;
954
955 if ((addr & GENMASK(24, 47)) >> 24 != 0x00fdf7)
956 return addr;
957
958 mce_nid = amd_get_nb_id(m->extcpu);
959 pvt = mcis[mce_nid]->pvt_info;
960
961 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp);
962 intlv_en = tmp >> 21 & 0x7;
963
964 /* add [47:27] + 3 trailing bits */
965 cc6_base = (tmp & GENMASK(0, 20)) << 3;
966
967 /* reverse and add DramIntlvEn */
968 cc6_base |= intlv_en ^ 0x7;
969
970 /* pin at [47:24] */
971 cc6_base <<= 24;
972
973 if (!intlv_en)
974 return cc6_base | (addr & GENMASK(0, 23));
975
976 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp);
977
978 /* faster log2 */
979 tmp_addr = (addr & GENMASK(12, 23)) << __fls(intlv_en + 1);
980
981 /* OR DramIntlvSel into bits [14:12] */
982 tmp_addr |= (tmp & GENMASK(21, 23)) >> 9;
983
984 /* add remaining [11:0] bits from original MC4_ADDR */
985 tmp_addr |= addr & GENMASK(0, 11);
986
987 return cc6_base | tmp_addr;
988 }
989
990 return addr;
Doug Thompsonddff8762009-04-27 16:14:52 +0200991}
992
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200993static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
Doug Thompsonddff8762009-04-27 16:14:52 +0200994{
Borislav Petkovf08e4572011-03-21 20:45:06 +0100995 struct cpuinfo_x86 *c = &boot_cpu_data;
Borislav Petkov71d2a322011-02-21 19:37:24 +0100996 int off = range << 3;
Doug Thompsonddff8762009-04-27 16:14:52 +0200997
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200998 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo);
999 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
Doug Thompsonddff8762009-04-27 16:14:52 +02001000
Borislav Petkovf08e4572011-03-21 20:45:06 +01001001 if (c->x86 == 0xf)
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001002 return;
Doug Thompsonddff8762009-04-27 16:14:52 +02001003
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001004 if (!dram_rw(pvt, range))
1005 return;
Doug Thompsonddff8762009-04-27 16:14:52 +02001006
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001007 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi);
1008 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
Borislav Petkovf08e4572011-03-21 20:45:06 +01001009
1010 /* Factor in CC6 save area by reading dst node's limit reg */
1011 if (c->x86 == 0x15) {
1012 struct pci_dev *f1 = NULL;
1013 u8 nid = dram_dst_node(pvt, range);
1014 u32 llim;
1015
1016 f1 = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(0x18 + nid, 1));
1017 if (WARN_ON(!f1))
1018 return;
1019
1020 amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim);
1021
1022 pvt->ranges[range].lim.lo &= GENMASK(0, 15);
1023
1024 /* {[39:27],111b} */
1025 pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16;
1026
1027 pvt->ranges[range].lim.hi &= GENMASK(0, 7);
1028
1029 /* [47:40] */
1030 pvt->ranges[range].lim.hi |= llim >> 13;
1031
1032 pci_dev_put(f1);
1033 }
Doug Thompsonddff8762009-04-27 16:14:52 +02001034}
1035
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001036static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
1037 u16 syndrome)
Doug Thompsonddff8762009-04-27 16:14:52 +02001038{
1039 struct mem_ctl_info *src_mci;
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001040 struct amd64_pvt *pvt = mci->pvt_info;
Doug Thompsonddff8762009-04-27 16:14:52 +02001041 int channel, csrow;
1042 u32 page, offset;
Doug Thompsonddff8762009-04-27 16:14:52 +02001043
1044 /* CHIPKILL enabled */
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001045 if (pvt->nbcfg & NBCFG_CHIPKILL) {
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001046 channel = get_channel_from_ecc_syndrome(mci, syndrome);
Doug Thompsonddff8762009-04-27 16:14:52 +02001047 if (channel < 0) {
1048 /*
1049 * Syndrome didn't map, so we don't know which of the
1050 * 2 DIMMs is in error. So we need to ID 'both' of them
1051 * as suspect.
1052 */
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001053 amd64_mc_warn(mci, "unknown syndrome 0x%04x - possible "
1054 "error reporting race\n", syndrome);
Doug Thompsonddff8762009-04-27 16:14:52 +02001055 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1056 return;
1057 }
1058 } else {
1059 /*
1060 * non-chipkill ecc mode
1061 *
1062 * The k8 documentation is unclear about how to determine the
1063 * channel number when using non-chipkill memory. This method
1064 * was obtained from email communication with someone at AMD.
1065 * (Wish the email was placed in this comment - norsk)
1066 */
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001067 channel = ((sys_addr & BIT(3)) != 0);
Doug Thompsonddff8762009-04-27 16:14:52 +02001068 }
1069
1070 /*
1071 * Find out which node the error address belongs to. This may be
1072 * different from the node that detected the error.
1073 */
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001074 src_mci = find_mc_by_sys_addr(mci, sys_addr);
Keith Mannthey2cff18c2009-09-18 14:35:23 +02001075 if (!src_mci) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001076 amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001077 (unsigned long)sys_addr);
Doug Thompsonddff8762009-04-27 16:14:52 +02001078 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1079 return;
1080 }
1081
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001082 /* Now map the sys_addr to a CSROW */
1083 csrow = sys_addr_to_csrow(src_mci, sys_addr);
Doug Thompsonddff8762009-04-27 16:14:52 +02001084 if (csrow < 0) {
1085 edac_mc_handle_ce_no_info(src_mci, EDAC_MOD_STR);
1086 } else {
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001087 error_address_to_page_and_offset(sys_addr, &page, &offset);
Doug Thompsonddff8762009-04-27 16:14:52 +02001088
1089 edac_mc_handle_ce(src_mci, page, offset, syndrome, csrow,
1090 channel, EDAC_MOD_STR);
1091 }
1092}
1093
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001094static int ddr2_cs_size(unsigned i, bool dct_width)
Doug Thompsonddff8762009-04-27 16:14:52 +02001095{
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001096 unsigned shift = 0;
Doug Thompsonddff8762009-04-27 16:14:52 +02001097
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001098 if (i <= 2)
1099 shift = i;
1100 else if (!(i & 0x1))
1101 shift = i >> 1;
Borislav Petkov1433eb92009-10-21 13:44:36 +02001102 else
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001103 shift = (i + 1) >> 1;
Doug Thompsonddff8762009-04-27 16:14:52 +02001104
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001105 return 128 << (shift + !!dct_width);
1106}
1107
1108static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1109 unsigned cs_mode)
1110{
1111 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1112
1113 if (pvt->ext_model >= K8_REV_F) {
1114 WARN_ON(cs_mode > 11);
1115 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1116 }
1117 else if (pvt->ext_model >= K8_REV_D) {
1118 WARN_ON(cs_mode > 10);
1119
1120 if (cs_mode == 3 || cs_mode == 8)
1121 return 32 << (cs_mode - 1);
1122 else
1123 return 32 << cs_mode;
1124 }
1125 else {
1126 WARN_ON(cs_mode > 6);
1127 return 32 << cs_mode;
1128 }
Doug Thompsonddff8762009-04-27 16:14:52 +02001129}
1130
Doug Thompson1afd3c92009-04-27 16:16:50 +02001131/*
1132 * Get the number of DCT channels in use.
1133 *
1134 * Return:
1135 * number of Memory Channels in operation
1136 * Pass back:
1137 * contents of the DCL0_LOW register
1138 */
Borislav Petkov7d20d142011-01-07 17:58:04 +01001139static int f1x_early_channel_count(struct amd64_pvt *pvt)
Doug Thompson1afd3c92009-04-27 16:16:50 +02001140{
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02001141 int i, j, channels = 0;
Doug Thompsonddff8762009-04-27 16:14:52 +02001142
Borislav Petkov7d20d142011-01-07 17:58:04 +01001143 /* On F10h, if we are in 128 bit mode, then we are using 2 channels */
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001144 if (boot_cpu_data.x86 == 0x10 && (pvt->dclr0 & WIDTH_128))
Borislav Petkov7d20d142011-01-07 17:58:04 +01001145 return 2;
Doug Thompson1afd3c92009-04-27 16:16:50 +02001146
1147 /*
Borislav Petkovd16149e2009-10-16 19:55:49 +02001148 * Need to check if in unganged mode: In such, there are 2 channels,
1149 * but they are not in 128 bit mode and thus the above 'dclr0' status
1150 * bit will be OFF.
Doug Thompson1afd3c92009-04-27 16:16:50 +02001151 *
1152 * Need to check DCT0[0] and DCT1[0] to see if only one of them has
1153 * their CSEnable bit on. If so, then SINGLE DIMM case.
1154 */
Borislav Petkovd16149e2009-10-16 19:55:49 +02001155 debugf0("Data width is not 128 bits - need more decoding\n");
Doug Thompson1afd3c92009-04-27 16:16:50 +02001156
1157 /*
1158 * Check DRAM Bank Address Mapping values for each DIMM to see if there
1159 * is more than just one DIMM present in unganged mode. Need to check
1160 * both controllers since DIMMs can be placed in either one.
1161 */
Borislav Petkov525a1b22010-12-21 15:53:27 +01001162 for (i = 0; i < 2; i++) {
1163 u32 dbam = (i ? pvt->dbam1 : pvt->dbam0);
Doug Thompson1afd3c92009-04-27 16:16:50 +02001164
Wan Wei57a30852009-08-07 17:04:49 +02001165 for (j = 0; j < 4; j++) {
1166 if (DBAM_DIMM(j, dbam) > 0) {
1167 channels++;
1168 break;
1169 }
1170 }
Doug Thompson1afd3c92009-04-27 16:16:50 +02001171 }
1172
Borislav Petkovd16149e2009-10-16 19:55:49 +02001173 if (channels > 2)
1174 channels = 2;
1175
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001176 amd64_info("MCT channel count: %d\n", channels);
Doug Thompson1afd3c92009-04-27 16:16:50 +02001177
1178 return channels;
Doug Thompson1afd3c92009-04-27 16:16:50 +02001179}
1180
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001181static int ddr3_cs_size(unsigned i, bool dct_width)
Doug Thompson1afd3c92009-04-27 16:16:50 +02001182{
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001183 unsigned shift = 0;
1184 int cs_size = 0;
1185
1186 if (i == 0 || i == 3 || i == 4)
1187 cs_size = -1;
1188 else if (i <= 2)
1189 shift = i;
1190 else if (i == 12)
1191 shift = 7;
1192 else if (!(i & 0x1))
1193 shift = i >> 1;
1194 else
1195 shift = (i + 1) >> 1;
1196
1197 if (cs_size != -1)
1198 cs_size = (128 * (1 << !!dct_width)) << shift;
1199
1200 return cs_size;
1201}
1202
1203static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1204 unsigned cs_mode)
1205{
1206 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1207
1208 WARN_ON(cs_mode > 11);
Borislav Petkov1433eb92009-10-21 13:44:36 +02001209
1210 if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001211 return ddr3_cs_size(cs_mode, dclr & WIDTH_128);
Borislav Petkov1433eb92009-10-21 13:44:36 +02001212 else
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001213 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1214}
Borislav Petkov1433eb92009-10-21 13:44:36 +02001215
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001216/*
1217 * F15h supports only 64bit DCT interfaces
1218 */
1219static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1220 unsigned cs_mode)
1221{
1222 WARN_ON(cs_mode > 12);
1223
1224 return ddr3_cs_size(cs_mode, false);
Doug Thompson1afd3c92009-04-27 16:16:50 +02001225}
1226
Borislav Petkov5a5d2372011-01-17 17:52:57 +01001227static void read_dram_ctl_register(struct amd64_pvt *pvt)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001228{
Doug Thompson6163b5d2009-04-27 16:20:17 +02001229
Borislav Petkov5a5d2372011-01-17 17:52:57 +01001230 if (boot_cpu_data.x86 == 0xf)
1231 return;
1232
Borislav Petkov78da1212010-12-22 19:31:45 +01001233 if (!amd64_read_dct_pci_cfg(pvt, DCT_SEL_LO, &pvt->dct_sel_lo)) {
1234 debugf0("F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
1235 pvt->dct_sel_lo, dct_sel_baseaddr(pvt));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001236
Borislav Petkov5a5d2372011-01-17 17:52:57 +01001237 debugf0(" DCTs operate in %s mode.\n",
1238 (dct_ganging_enabled(pvt) ? "ganged" : "unganged"));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001239
Borislav Petkov72381bd2009-10-09 19:14:43 +02001240 if (!dct_ganging_enabled(pvt))
1241 debugf0(" Address range split per DCT: %s\n",
1242 (dct_high_range_enabled(pvt) ? "yes" : "no"));
1243
Borislav Petkov78da1212010-12-22 19:31:45 +01001244 debugf0(" data interleave for ECC: %s, "
Borislav Petkov72381bd2009-10-09 19:14:43 +02001245 "DRAM cleared since last warm reset: %s\n",
1246 (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
1247 (dct_memory_cleared(pvt) ? "yes" : "no"));
1248
Borislav Petkov78da1212010-12-22 19:31:45 +01001249 debugf0(" channel interleave: %s, "
1250 "interleave bits selector: 0x%x\n",
Borislav Petkov72381bd2009-10-09 19:14:43 +02001251 (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
Doug Thompson6163b5d2009-04-27 16:20:17 +02001252 dct_sel_interleave_addr(pvt));
1253 }
1254
Borislav Petkov78da1212010-12-22 19:31:45 +01001255 amd64_read_dct_pci_cfg(pvt, DCT_SEL_HI, &pvt->dct_sel_hi);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001256}
1257
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001258/*
Borislav Petkov229a7a12010-12-09 18:57:54 +01001259 * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001260 * Interleaving Modes.
1261 */
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001262static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
Borislav Petkov229a7a12010-12-09 18:57:54 +01001263 bool hi_range_sel, u8 intlv_en)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001264{
Borislav Petkov151fa712011-02-21 19:33:10 +01001265 u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001266
1267 if (dct_ganging_enabled(pvt))
Borislav Petkov229a7a12010-12-09 18:57:54 +01001268 return 0;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001269
Borislav Petkov229a7a12010-12-09 18:57:54 +01001270 if (hi_range_sel)
1271 return dct_sel_high;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001272
Borislav Petkov229a7a12010-12-09 18:57:54 +01001273 /*
1274 * see F2x110[DctSelIntLvAddr] - channel interleave mode
1275 */
1276 if (dct_interleave_enabled(pvt)) {
1277 u8 intlv_addr = dct_sel_interleave_addr(pvt);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001278
Borislav Petkov229a7a12010-12-09 18:57:54 +01001279 /* return DCT select function: 0=DCT0, 1=DCT1 */
1280 if (!intlv_addr)
1281 return sys_addr >> 6 & 1;
1282
1283 if (intlv_addr & 0x2) {
1284 u8 shift = intlv_addr & 0x1 ? 9 : 6;
1285 u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2;
1286
1287 return ((sys_addr >> shift) & 1) ^ temp;
1288 }
1289
1290 return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
1291 }
1292
1293 if (dct_high_range_enabled(pvt))
1294 return ~dct_sel_high & 1;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001295
1296 return 0;
1297}
1298
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001299/* Convert the sys_addr to the normalized DCT address */
Borislav Petkove7613592011-02-21 19:49:01 +01001300static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, unsigned range,
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001301 u64 sys_addr, bool hi_rng,
1302 u32 dct_sel_base_addr)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001303{
1304 u64 chan_off;
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001305 u64 dram_base = get_dram_base(pvt, range);
1306 u64 hole_off = f10_dhar_offset(pvt);
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001307 u64 dct_sel_base_off = (pvt->dct_sel_hi & 0xFFFFFC00) << 16;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001308
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001309 if (hi_rng) {
1310 /*
1311 * if
1312 * base address of high range is below 4Gb
1313 * (bits [47:27] at [31:11])
1314 * DRAM address space on this DCT is hoisted above 4Gb &&
1315 * sys_addr > 4Gb
1316 *
1317 * remove hole offset from sys_addr
1318 * else
1319 * remove high range offset from sys_addr
1320 */
1321 if ((!(dct_sel_base_addr >> 16) ||
1322 dct_sel_base_addr < dhar_base(pvt)) &&
Borislav Petkov972ea172011-02-21 19:43:02 +01001323 dhar_valid(pvt) &&
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001324 (sys_addr >= BIT_64(32)))
Borislav Petkovbc21fa52010-11-11 17:29:13 +01001325 chan_off = hole_off;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001326 else
1327 chan_off = dct_sel_base_off;
1328 } else {
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001329 /*
1330 * if
1331 * we have a valid hole &&
1332 * sys_addr > 4Gb
1333 *
1334 * remove hole
1335 * else
1336 * remove dram base to normalize to DCT address
1337 */
Borislav Petkov972ea172011-02-21 19:43:02 +01001338 if (dhar_valid(pvt) && (sys_addr >= BIT_64(32)))
Borislav Petkovbc21fa52010-11-11 17:29:13 +01001339 chan_off = hole_off;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001340 else
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001341 chan_off = dram_base;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001342 }
1343
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001344 return (sys_addr & GENMASK(6,47)) - (chan_off & GENMASK(23,47));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001345}
1346
Doug Thompson6163b5d2009-04-27 16:20:17 +02001347/*
1348 * checks if the csrow passed in is marked as SPARED, if so returns the new
1349 * spare row
1350 */
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001351static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001352{
Borislav Petkov614ec9d2011-01-13 18:02:22 +01001353 int tmp_cs;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001354
Borislav Petkov614ec9d2011-01-13 18:02:22 +01001355 if (online_spare_swap_done(pvt, dct) &&
1356 csrow == online_spare_bad_dramcs(pvt, dct)) {
1357
1358 for_each_chip_select(tmp_cs, dct, pvt) {
1359 if (chip_select_base(tmp_cs, dct, pvt) & 0x2) {
1360 csrow = tmp_cs;
1361 break;
1362 }
1363 }
Doug Thompson6163b5d2009-04-27 16:20:17 +02001364 }
1365 return csrow;
1366}
1367
1368/*
1369 * Iterate over the DRAM DCT "base" and "mask" registers looking for a
1370 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
1371 *
1372 * Return:
1373 * -EINVAL: NOT FOUND
1374 * 0..csrow = Chip-Select Row
1375 */
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001376static int f1x_lookup_addr_in_dct(u64 in_addr, u32 nid, u8 dct)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001377{
1378 struct mem_ctl_info *mci;
1379 struct amd64_pvt *pvt;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001380 u64 cs_base, cs_mask;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001381 int cs_found = -EINVAL;
1382 int csrow;
1383
Borislav Petkovcc4d8862010-10-13 16:11:59 +02001384 mci = mcis[nid];
Doug Thompson6163b5d2009-04-27 16:20:17 +02001385 if (!mci)
1386 return cs_found;
1387
1388 pvt = mci->pvt_info;
1389
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001390 debugf1("input addr: 0x%llx, DCT: %d\n", in_addr, dct);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001391
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001392 for_each_chip_select(csrow, dct, pvt) {
1393 if (!csrow_enabled(csrow, dct, pvt))
Doug Thompson6163b5d2009-04-27 16:20:17 +02001394 continue;
1395
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001396 get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001397
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001398 debugf1(" CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
1399 csrow, cs_base, cs_mask);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001400
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001401 cs_mask = ~cs_mask;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001402
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001403 debugf1(" (InputAddr & ~CSMask)=0x%llx "
1404 "(CSBase & ~CSMask)=0x%llx\n",
1405 (in_addr & cs_mask), (cs_base & cs_mask));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001406
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001407 if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
1408 cs_found = f10_process_possible_spare(pvt, dct, csrow);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001409
1410 debugf1(" MATCH csrow=%d\n", cs_found);
1411 break;
1412 }
1413 }
1414 return cs_found;
1415}
1416
Borislav Petkov95b0ef52011-01-11 22:08:07 +01001417/*
1418 * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is
1419 * swapped with a region located at the bottom of memory so that the GPU can use
1420 * the interleaved region and thus two channels.
1421 */
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001422static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr)
Borislav Petkov95b0ef52011-01-11 22:08:07 +01001423{
1424 u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr;
1425
1426 if (boot_cpu_data.x86 == 0x10) {
1427 /* only revC3 and revE have that feature */
1428 if (boot_cpu_data.x86_model < 4 ||
1429 (boot_cpu_data.x86_model < 0xa &&
1430 boot_cpu_data.x86_mask < 3))
1431 return sys_addr;
1432 }
1433
1434 amd64_read_dct_pci_cfg(pvt, SWAP_INTLV_REG, &swap_reg);
1435
1436 if (!(swap_reg & 0x1))
1437 return sys_addr;
1438
1439 swap_base = (swap_reg >> 3) & 0x7f;
1440 swap_limit = (swap_reg >> 11) & 0x7f;
1441 rgn_size = (swap_reg >> 20) & 0x7f;
1442 tmp_addr = sys_addr >> 27;
1443
1444 if (!(sys_addr >> 34) &&
1445 (((tmp_addr >= swap_base) &&
1446 (tmp_addr <= swap_limit)) ||
1447 (tmp_addr < rgn_size)))
1448 return sys_addr ^ (u64)swap_base << 27;
1449
1450 return sys_addr;
1451}
1452
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001453/* For a given @dram_range, check if @sys_addr falls within it. */
Borislav Petkove7613592011-02-21 19:49:01 +01001454static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001455 u64 sys_addr, int *nid, int *chan_sel)
1456{
Borislav Petkov229a7a12010-12-09 18:57:54 +01001457 int cs_found = -EINVAL;
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001458 u64 chan_addr;
Borislav Petkov5d4b58e2011-01-13 16:01:13 +01001459 u32 dct_sel_base;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001460 u8 channel;
Borislav Petkov229a7a12010-12-09 18:57:54 +01001461 bool high_range = false;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001462
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001463 u8 node_id = dram_dst_node(pvt, range);
Borislav Petkov229a7a12010-12-09 18:57:54 +01001464 u8 intlv_en = dram_intlv_en(pvt, range);
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001465 u32 intlv_sel = dram_intlv_sel(pvt, range);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001466
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001467 debugf1("(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1468 range, sys_addr, get_dram_limit(pvt, range));
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001469
Borislav Petkov355fba62011-01-17 13:03:26 +01001470 if (dhar_valid(pvt) &&
1471 dhar_base(pvt) <= sys_addr &&
1472 sys_addr < BIT_64(32)) {
1473 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1474 sys_addr);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001475 return -EINVAL;
Borislav Petkov355fba62011-01-17 13:03:26 +01001476 }
1477
Borislav Petkovf030ddf2011-04-08 15:05:21 +02001478 if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en)))
Borislav Petkov355fba62011-01-17 13:03:26 +01001479 return -EINVAL;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001480
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001481 sys_addr = f1x_swap_interleaved_region(pvt, sys_addr);
Borislav Petkov95b0ef52011-01-11 22:08:07 +01001482
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001483 dct_sel_base = dct_sel_baseaddr(pvt);
1484
1485 /*
1486 * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
1487 * select between DCT0 and DCT1.
1488 */
1489 if (dct_high_range_enabled(pvt) &&
1490 !dct_ganging_enabled(pvt) &&
1491 ((sys_addr >> 27) >= (dct_sel_base >> 11)))
Borislav Petkov229a7a12010-12-09 18:57:54 +01001492 high_range = true;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001493
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001494 channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001495
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001496 chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr,
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001497 high_range, dct_sel_base);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001498
Borislav Petkove2f79db2011-01-13 14:57:34 +01001499 /* Remove node interleaving, see F1x120 */
1500 if (intlv_en)
1501 chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) |
1502 (chan_addr & 0xfff);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001503
Borislav Petkov5d4b58e2011-01-13 16:01:13 +01001504 /* remove channel interleave */
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001505 if (dct_interleave_enabled(pvt) &&
1506 !dct_high_range_enabled(pvt) &&
1507 !dct_ganging_enabled(pvt)) {
Borislav Petkov5d4b58e2011-01-13 16:01:13 +01001508
1509 if (dct_sel_interleave_addr(pvt) != 1) {
1510 if (dct_sel_interleave_addr(pvt) == 0x3)
1511 /* hash 9 */
1512 chan_addr = ((chan_addr >> 10) << 9) |
1513 (chan_addr & 0x1ff);
1514 else
1515 /* A[6] or hash 6 */
1516 chan_addr = ((chan_addr >> 7) << 6) |
1517 (chan_addr & 0x3f);
1518 } else
1519 /* A[12] */
1520 chan_addr = ((chan_addr >> 13) << 12) |
1521 (chan_addr & 0xfff);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001522 }
1523
Borislav Petkov5d4b58e2011-01-13 16:01:13 +01001524 debugf1(" Normalized DCT addr: 0x%llx\n", chan_addr);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001525
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001526 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001527
1528 if (cs_found >= 0) {
1529 *nid = node_id;
1530 *chan_sel = channel;
1531 }
1532 return cs_found;
1533}
1534
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001535static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt, u64 sys_addr,
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001536 int *node, int *chan_sel)
1537{
Borislav Petkove7613592011-02-21 19:49:01 +01001538 int cs_found = -EINVAL;
1539 unsigned range;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001540
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001541 for (range = 0; range < DRAM_RANGES; range++) {
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001542
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001543 if (!dram_rw(pvt, range))
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001544 continue;
1545
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001546 if ((get_dram_base(pvt, range) <= sys_addr) &&
1547 (get_dram_limit(pvt, range) >= sys_addr)) {
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001548
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001549 cs_found = f1x_match_to_this_node(pvt, range,
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001550 sys_addr, node,
1551 chan_sel);
1552 if (cs_found >= 0)
1553 break;
1554 }
1555 }
1556 return cs_found;
1557}
1558
1559/*
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001560 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
1561 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001562 *
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001563 * The @sys_addr is usually an error address received from the hardware
1564 * (MCX_ADDR).
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001565 */
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001566static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001567 u16 syndrome)
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001568{
1569 struct amd64_pvt *pvt = mci->pvt_info;
1570 u32 page, offset;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001571 int nid, csrow, chan = 0;
1572
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001573 csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &nid, &chan);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001574
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001575 if (csrow < 0) {
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001576 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001577 return;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001578 }
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001579
1580 error_address_to_page_and_offset(sys_addr, &page, &offset);
1581
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001582 /*
1583 * We need the syndromes for channel detection only when we're
1584 * ganged. Otherwise @chan should already contain the channel at
1585 * this point.
1586 */
Borislav Petkova97fa682010-12-23 14:07:18 +01001587 if (dct_ganging_enabled(pvt))
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001588 chan = get_channel_from_ecc_syndrome(mci, syndrome);
1589
1590 if (chan >= 0)
1591 edac_mc_handle_ce(mci, page, offset, syndrome, csrow, chan,
1592 EDAC_MOD_STR);
1593 else
1594 /*
1595 * Channel unknown, report all channels on this CSROW as failed.
1596 */
1597 for (chan = 0; chan < mci->csrows[csrow].nr_channels; chan++)
1598 edac_mc_handle_ce(mci, page, offset, syndrome,
1599 csrow, chan, EDAC_MOD_STR);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001600}
1601
1602/*
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001603 * debug routine to display the memory sizes of all logical DIMMs and its
Borislav Petkovcb328502010-12-22 14:28:24 +01001604 * CSROWs
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001605 */
Borislav Petkov8c671752011-02-23 17:25:12 +01001606static void amd64_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001607{
Borislav Petkov603adaf2009-12-21 14:52:53 +01001608 int dimm, size0, size1, factor = 0;
Borislav Petkov525a1b22010-12-21 15:53:27 +01001609 u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
1610 u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001611
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001612 if (boot_cpu_data.x86 == 0xf) {
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001613 if (pvt->dclr0 & WIDTH_128)
Borislav Petkov603adaf2009-12-21 14:52:53 +01001614 factor = 1;
1615
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001616 /* K8 families < revF not supported yet */
Borislav Petkov1433eb92009-10-21 13:44:36 +02001617 if (pvt->ext_model < K8_REV_F)
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001618 return;
1619 else
1620 WARN_ON(ctrl != 0);
1621 }
1622
Borislav Petkov4d796362011-02-03 15:59:57 +01001623 dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1 : pvt->dbam0;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001624 dcsb = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->csels[1].csbases
1625 : pvt->csels[0].csbases;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001626
Borislav Petkov4d796362011-02-03 15:59:57 +01001627 debugf1("F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n", ctrl, dbam);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001628
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001629 edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
1630
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001631 /* Dump memory sizes for DIMM and its CSROWs */
1632 for (dimm = 0; dimm < 4; dimm++) {
1633
1634 size0 = 0;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001635 if (dcsb[dimm*2] & DCSB_CS_ENABLE)
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001636 size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
1637 DBAM_DIMM(dimm, dbam));
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001638
1639 size1 = 0;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001640 if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001641 size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
1642 DBAM_DIMM(dimm, dbam));
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001643
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001644 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
1645 dimm * 2, size0 << factor,
1646 dimm * 2 + 1, size1 << factor);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001647 }
1648}
1649
Doug Thompson4d376072009-04-27 16:25:05 +02001650static struct amd64_family_type amd64_family_types[] = {
1651 [K8_CPUS] = {
Borislav Petkov0092b202010-10-01 19:20:05 +02001652 .ctl_name = "K8",
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001653 .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
1654 .f3_id = PCI_DEVICE_ID_AMD_K8_NB_MISC,
Doug Thompson4d376072009-04-27 16:25:05 +02001655 .ops = {
Borislav Petkov1433eb92009-10-21 13:44:36 +02001656 .early_channel_count = k8_early_channel_count,
Borislav Petkov1433eb92009-10-21 13:44:36 +02001657 .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow,
1658 .dbam_to_cs = k8_dbam_to_chip_select,
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001659 .read_dct_pci_cfg = k8_read_dct_pci_cfg,
Doug Thompson4d376072009-04-27 16:25:05 +02001660 }
1661 },
1662 [F10_CPUS] = {
Borislav Petkov0092b202010-10-01 19:20:05 +02001663 .ctl_name = "F10h",
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001664 .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
1665 .f3_id = PCI_DEVICE_ID_AMD_10H_NB_MISC,
Doug Thompson4d376072009-04-27 16:25:05 +02001666 .ops = {
Borislav Petkov7d20d142011-01-07 17:58:04 +01001667 .early_channel_count = f1x_early_channel_count,
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001668 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
Borislav Petkov1433eb92009-10-21 13:44:36 +02001669 .dbam_to_cs = f10_dbam_to_chip_select,
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001670 .read_dct_pci_cfg = f10_read_dct_pci_cfg,
1671 }
1672 },
1673 [F15_CPUS] = {
1674 .ctl_name = "F15h",
Borislav Petkovdf71a052011-01-19 18:15:10 +01001675 .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1,
1676 .f3_id = PCI_DEVICE_ID_AMD_15H_NB_F3,
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001677 .ops = {
Borislav Petkov7d20d142011-01-07 17:58:04 +01001678 .early_channel_count = f1x_early_channel_count,
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001679 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001680 .dbam_to_cs = f15_dbam_to_chip_select,
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001681 .read_dct_pci_cfg = f15_read_dct_pci_cfg,
Doug Thompson4d376072009-04-27 16:25:05 +02001682 }
1683 },
Doug Thompson4d376072009-04-27 16:25:05 +02001684};
1685
1686static struct pci_dev *pci_get_related_function(unsigned int vendor,
1687 unsigned int device,
1688 struct pci_dev *related)
1689{
1690 struct pci_dev *dev = NULL;
1691
1692 dev = pci_get_device(vendor, device, dev);
1693 while (dev) {
1694 if ((dev->bus->number == related->bus->number) &&
1695 (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
1696 break;
1697 dev = pci_get_device(vendor, device, dev);
1698 }
1699
1700 return dev;
1701}
1702
Doug Thompsonb1289d62009-04-27 16:37:05 +02001703/*
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001704 * These are tables of eigenvectors (one per line) which can be used for the
1705 * construction of the syndrome tables. The modified syndrome search algorithm
1706 * uses those to find the symbol in error and thus the DIMM.
Doug Thompsonb1289d62009-04-27 16:37:05 +02001707 *
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001708 * Algorithm courtesy of Ross LaFetra from AMD.
Doug Thompsonb1289d62009-04-27 16:37:05 +02001709 */
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001710static u16 x4_vectors[] = {
1711 0x2f57, 0x1afe, 0x66cc, 0xdd88,
1712 0x11eb, 0x3396, 0x7f4c, 0xeac8,
1713 0x0001, 0x0002, 0x0004, 0x0008,
1714 0x1013, 0x3032, 0x4044, 0x8088,
1715 0x106b, 0x30d6, 0x70fc, 0xe0a8,
1716 0x4857, 0xc4fe, 0x13cc, 0x3288,
1717 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
1718 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
1719 0x15c1, 0x2a42, 0x89ac, 0x4758,
1720 0x2b03, 0x1602, 0x4f0c, 0xca08,
1721 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
1722 0x8ba7, 0x465e, 0x244c, 0x1cc8,
1723 0x2b87, 0x164e, 0x642c, 0xdc18,
1724 0x40b9, 0x80de, 0x1094, 0x20e8,
1725 0x27db, 0x1eb6, 0x9dac, 0x7b58,
1726 0x11c1, 0x2242, 0x84ac, 0x4c58,
1727 0x1be5, 0x2d7a, 0x5e34, 0xa718,
1728 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
1729 0x4c97, 0xc87e, 0x11fc, 0x33a8,
1730 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
1731 0x16b3, 0x3d62, 0x4f34, 0x8518,
1732 0x1e2f, 0x391a, 0x5cac, 0xf858,
1733 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
1734 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
1735 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
1736 0x4397, 0xc27e, 0x17fc, 0x3ea8,
1737 0x1617, 0x3d3e, 0x6464, 0xb8b8,
1738 0x23ff, 0x12aa, 0xab6c, 0x56d8,
1739 0x2dfb, 0x1ba6, 0x913c, 0x7328,
1740 0x185d, 0x2ca6, 0x7914, 0x9e28,
1741 0x171b, 0x3e36, 0x7d7c, 0xebe8,
1742 0x4199, 0x82ee, 0x19f4, 0x2e58,
1743 0x4807, 0xc40e, 0x130c, 0x3208,
1744 0x1905, 0x2e0a, 0x5804, 0xac08,
1745 0x213f, 0x132a, 0xadfc, 0x5ba8,
1746 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
Doug Thompsonb1289d62009-04-27 16:37:05 +02001747};
1748
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001749static u16 x8_vectors[] = {
1750 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
1751 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
1752 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
1753 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
1754 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
1755 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
1756 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
1757 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
1758 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
1759 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
1760 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
1761 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
1762 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
1763 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
1764 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
1765 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
1766 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
1767 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
1768 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
1769};
1770
Borislav Petkovd34a6ec2011-02-23 17:41:50 +01001771static int decode_syndrome(u16 syndrome, u16 *vectors, unsigned num_vecs,
1772 unsigned v_dim)
Doug Thompsonb1289d62009-04-27 16:37:05 +02001773{
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001774 unsigned int i, err_sym;
Doug Thompsonb1289d62009-04-27 16:37:05 +02001775
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001776 for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
1777 u16 s = syndrome;
Borislav Petkovd34a6ec2011-02-23 17:41:50 +01001778 unsigned v_idx = err_sym * v_dim;
1779 unsigned v_end = (err_sym + 1) * v_dim;
Doug Thompsonb1289d62009-04-27 16:37:05 +02001780
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001781 /* walk over all 16 bits of the syndrome */
1782 for (i = 1; i < (1U << 16); i <<= 1) {
1783
1784 /* if bit is set in that eigenvector... */
1785 if (v_idx < v_end && vectors[v_idx] & i) {
1786 u16 ev_comp = vectors[v_idx++];
1787
1788 /* ... and bit set in the modified syndrome, */
1789 if (s & i) {
1790 /* remove it. */
1791 s ^= ev_comp;
1792
1793 if (!s)
1794 return err_sym;
1795 }
1796
1797 } else if (s & i)
1798 /* can't get to zero, move to next symbol */
1799 break;
1800 }
Doug Thompsonb1289d62009-04-27 16:37:05 +02001801 }
1802
1803 debugf0("syndrome(%x) not found\n", syndrome);
1804 return -1;
1805}
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001806
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001807static int map_err_sym_to_channel(int err_sym, int sym_size)
1808{
1809 if (sym_size == 4)
1810 switch (err_sym) {
1811 case 0x20:
1812 case 0x21:
1813 return 0;
1814 break;
1815 case 0x22:
1816 case 0x23:
1817 return 1;
1818 break;
1819 default:
1820 return err_sym >> 4;
1821 break;
1822 }
1823 /* x8 symbols */
1824 else
1825 switch (err_sym) {
1826 /* imaginary bits not in a DIMM */
1827 case 0x10:
1828 WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
1829 err_sym);
1830 return -1;
1831 break;
1832
1833 case 0x11:
1834 return 0;
1835 break;
1836 case 0x12:
1837 return 1;
1838 break;
1839 default:
1840 return err_sym >> 3;
1841 break;
1842 }
1843 return -1;
1844}
1845
1846static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
1847{
1848 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001849 int err_sym = -1;
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001850
Borislav Petkova3b7db02011-01-19 20:35:12 +01001851 if (pvt->ecc_sym_sz == 8)
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001852 err_sym = decode_syndrome(syndrome, x8_vectors,
1853 ARRAY_SIZE(x8_vectors),
Borislav Petkova3b7db02011-01-19 20:35:12 +01001854 pvt->ecc_sym_sz);
1855 else if (pvt->ecc_sym_sz == 4)
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001856 err_sym = decode_syndrome(syndrome, x4_vectors,
1857 ARRAY_SIZE(x4_vectors),
Borislav Petkova3b7db02011-01-19 20:35:12 +01001858 pvt->ecc_sym_sz);
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001859 else {
Borislav Petkova3b7db02011-01-19 20:35:12 +01001860 amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz);
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001861 return err_sym;
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001862 }
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001863
Borislav Petkova3b7db02011-01-19 20:35:12 +01001864 return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz);
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001865}
1866
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001867/*
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001868 * Handle any Correctable Errors (CEs) that have occurred. Check for valid ERROR
1869 * ADDRESS and process.
1870 */
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001871static void amd64_handle_ce(struct mem_ctl_info *mci, struct mce *m)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001872{
1873 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001874 u64 sys_addr;
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001875 u16 syndrome;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001876
1877 /* Ensure that the Error Address is VALID */
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001878 if (!(m->status & MCI_STATUS_ADDRV)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001879 amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n");
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001880 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1881 return;
1882 }
1883
Borislav Petkov70046622011-01-10 14:37:27 +01001884 sys_addr = get_error_address(m);
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001885 syndrome = extract_syndrome(m->status);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001886
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001887 amd64_mc_err(mci, "CE ERROR_ADDRESS= 0x%llx\n", sys_addr);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001888
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001889 pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, syndrome);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001890}
1891
1892/* Handle any Un-correctable Errors (UEs) */
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001893static void amd64_handle_ue(struct mem_ctl_info *mci, struct mce *m)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001894{
Borislav Petkov1f6bcee2009-11-13 14:02:57 +01001895 struct mem_ctl_info *log_mci, *src_mci = NULL;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001896 int csrow;
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001897 u64 sys_addr;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001898 u32 page, offset;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001899
1900 log_mci = mci;
1901
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001902 if (!(m->status & MCI_STATUS_ADDRV)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001903 amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n");
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001904 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1905 return;
1906 }
1907
Borislav Petkov70046622011-01-10 14:37:27 +01001908 sys_addr = get_error_address(m);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001909
1910 /*
1911 * Find out which node the error address belongs to. This may be
1912 * different from the node that detected the error.
1913 */
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001914 src_mci = find_mc_by_sys_addr(mci, sys_addr);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001915 if (!src_mci) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001916 amd64_mc_err(mci, "ERROR ADDRESS (0x%lx) NOT mapped to a MC\n",
1917 (unsigned long)sys_addr);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001918 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1919 return;
1920 }
1921
1922 log_mci = src_mci;
1923
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001924 csrow = sys_addr_to_csrow(log_mci, sys_addr);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001925 if (csrow < 0) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001926 amd64_mc_err(mci, "ERROR_ADDRESS (0x%lx) NOT mapped to CS\n",
1927 (unsigned long)sys_addr);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001928 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1929 } else {
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001930 error_address_to_page_and_offset(sys_addr, &page, &offset);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001931 edac_mc_handle_ue(log_mci, page, offset, csrow, EDAC_MOD_STR);
1932 }
1933}
1934
Borislav Petkov549d0422009-07-24 13:51:42 +02001935static inline void __amd64_decode_bus_error(struct mem_ctl_info *mci,
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001936 struct mce *m)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001937{
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001938 u16 ec = EC(m->status);
1939 u8 xec = XEC(m->status, 0x1f);
1940 u8 ecc_type = (m->status >> 45) & 0x3;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001941
Borislav Petkovb70ef012009-06-25 19:32:38 +02001942 /* Bail early out if this was an 'observed' error */
Borislav Petkov5980bb92011-01-07 16:26:49 +01001943 if (PP(ec) == NBSL_PP_OBS)
Borislav Petkovb70ef012009-06-25 19:32:38 +02001944 return;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001945
Borislav Petkovecaf5602009-07-23 16:32:01 +02001946 /* Do only ECC errors */
1947 if (xec && xec != F10_NBSL_EXT_ERR_ECC)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001948 return;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001949
Borislav Petkovecaf5602009-07-23 16:32:01 +02001950 if (ecc_type == 2)
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001951 amd64_handle_ce(mci, m);
Borislav Petkovecaf5602009-07-23 16:32:01 +02001952 else if (ecc_type == 1)
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001953 amd64_handle_ue(mci, m);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001954}
1955
Borislav Petkov7cfd4a82010-09-01 14:45:20 +02001956void amd64_decode_bus_error(int node_id, struct mce *m, u32 nbcfg)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001957{
Borislav Petkovcc4d8862010-10-13 16:11:59 +02001958 struct mem_ctl_info *mci = mcis[node_id];
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001959
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001960 __amd64_decode_bus_error(mci, m);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001961}
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001962
Doug Thompson0ec449e2009-04-27 19:41:25 +02001963/*
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001964 * Use pvt->F2 which contains the F2 CPU PCI device to get the related
Borislav Petkovbbd0c1f2010-10-01 19:27:58 +02001965 * F1 (AddrMap) and F3 (Misc) devices. Return negative value on error.
Doug Thompson0ec449e2009-04-27 19:41:25 +02001966 */
Borislav Petkov360b7f32010-10-15 19:25:38 +02001967static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 f1_id, u16 f3_id)
Doug Thompson0ec449e2009-04-27 19:41:25 +02001968{
Doug Thompson0ec449e2009-04-27 19:41:25 +02001969 /* Reserve the ADDRESS MAP Device */
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001970 pvt->F1 = pci_get_related_function(pvt->F2->vendor, f1_id, pvt->F2);
1971 if (!pvt->F1) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001972 amd64_err("error address map device not found: "
1973 "vendor %x device 0x%x (broken BIOS?)\n",
1974 PCI_VENDOR_ID_AMD, f1_id);
Borislav Petkovbbd0c1f2010-10-01 19:27:58 +02001975 return -ENODEV;
Doug Thompson0ec449e2009-04-27 19:41:25 +02001976 }
1977
1978 /* Reserve the MISC Device */
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001979 pvt->F3 = pci_get_related_function(pvt->F2->vendor, f3_id, pvt->F2);
1980 if (!pvt->F3) {
1981 pci_dev_put(pvt->F1);
1982 pvt->F1 = NULL;
Doug Thompson0ec449e2009-04-27 19:41:25 +02001983
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001984 amd64_err("error F3 device not found: "
1985 "vendor %x device 0x%x (broken BIOS?)\n",
1986 PCI_VENDOR_ID_AMD, f3_id);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001987
Borislav Petkovbbd0c1f2010-10-01 19:27:58 +02001988 return -ENODEV;
Doug Thompson0ec449e2009-04-27 19:41:25 +02001989 }
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001990 debugf1("F1: %s\n", pci_name(pvt->F1));
1991 debugf1("F2: %s\n", pci_name(pvt->F2));
1992 debugf1("F3: %s\n", pci_name(pvt->F3));
Doug Thompson0ec449e2009-04-27 19:41:25 +02001993
1994 return 0;
1995}
1996
Borislav Petkov360b7f32010-10-15 19:25:38 +02001997static void free_mc_sibling_devs(struct amd64_pvt *pvt)
Doug Thompson0ec449e2009-04-27 19:41:25 +02001998{
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001999 pci_dev_put(pvt->F1);
2000 pci_dev_put(pvt->F3);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002001}
2002
2003/*
2004 * Retrieve the hardware registers of the memory controller (this includes the
2005 * 'Address Map' and 'Misc' device regs)
2006 */
Borislav Petkov360b7f32010-10-15 19:25:38 +02002007static void read_mc_regs(struct amd64_pvt *pvt)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002008{
Borislav Petkova3b7db02011-01-19 20:35:12 +01002009 struct cpuinfo_x86 *c = &boot_cpu_data;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002010 u64 msr_val;
Borislav Petkovad6a32e2010-03-09 12:46:00 +01002011 u32 tmp;
Borislav Petkove7613592011-02-21 19:49:01 +01002012 unsigned range;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002013
2014 /*
2015 * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
2016 * those are Read-As-Zero
2017 */
Borislav Petkove97f8bb2009-10-12 15:27:45 +02002018 rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
2019 debugf0(" TOP_MEM: 0x%016llx\n", pvt->top_mem);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002020
2021 /* check first whether TOP_MEM2 is enabled */
2022 rdmsrl(MSR_K8_SYSCFG, msr_val);
2023 if (msr_val & (1U << 21)) {
Borislav Petkove97f8bb2009-10-12 15:27:45 +02002024 rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
2025 debugf0(" TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002026 } else
2027 debugf0(" TOP_MEM2 disabled.\n");
2028
Borislav Petkov5980bb92011-01-07 16:26:49 +01002029 amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002030
Borislav Petkov5a5d2372011-01-17 17:52:57 +01002031 read_dram_ctl_register(pvt);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002032
Borislav Petkov7f19bf72010-10-21 18:52:53 +02002033 for (range = 0; range < DRAM_RANGES; range++) {
2034 u8 rw;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002035
Borislav Petkov7f19bf72010-10-21 18:52:53 +02002036 /* read settings for this DRAM range */
2037 read_dram_base_limit_regs(pvt, range);
Borislav Petkove97f8bb2009-10-12 15:27:45 +02002038
Borislav Petkov7f19bf72010-10-21 18:52:53 +02002039 rw = dram_rw(pvt, range);
2040 if (!rw)
2041 continue;
2042
2043 debugf1(" DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
2044 range,
2045 get_dram_base(pvt, range),
2046 get_dram_limit(pvt, range));
2047
2048 debugf1(" IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
2049 dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
2050 (rw & 0x1) ? "R" : "-",
2051 (rw & 0x2) ? "W" : "-",
2052 dram_intlv_sel(pvt, range),
2053 dram_dst_node(pvt, range));
Doug Thompson0ec449e2009-04-27 19:41:25 +02002054 }
2055
Borislav Petkovb2b0c602010-10-08 18:32:29 +02002056 read_dct_base_mask(pvt);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002057
Borislav Petkovbc21fa52010-11-11 17:29:13 +01002058 amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
Borislav Petkov525a1b22010-12-21 15:53:27 +01002059 amd64_read_dct_pci_cfg(pvt, DBAM0, &pvt->dbam0);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002060
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002061 amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002062
Borislav Petkovcb328502010-12-22 14:28:24 +01002063 amd64_read_dct_pci_cfg(pvt, DCLR0, &pvt->dclr0);
2064 amd64_read_dct_pci_cfg(pvt, DCHR0, &pvt->dchr0);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002065
Borislav Petkov78da1212010-12-22 19:31:45 +01002066 if (!dct_ganging_enabled(pvt)) {
Borislav Petkovcb328502010-12-22 14:28:24 +01002067 amd64_read_dct_pci_cfg(pvt, DCLR1, &pvt->dclr1);
2068 amd64_read_dct_pci_cfg(pvt, DCHR1, &pvt->dchr1);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002069 }
Borislav Petkovad6a32e2010-03-09 12:46:00 +01002070
Borislav Petkova3b7db02011-01-19 20:35:12 +01002071 pvt->ecc_sym_sz = 4;
2072
2073 if (c->x86 >= 0x10) {
Borislav Petkovb2b0c602010-10-08 18:32:29 +02002074 amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
Borislav Petkov525a1b22010-12-21 15:53:27 +01002075 amd64_read_dct_pci_cfg(pvt, DBAM1, &pvt->dbam1);
Borislav Petkova3b7db02011-01-19 20:35:12 +01002076
2077 /* F10h, revD and later can do x8 ECC too */
2078 if ((c->x86 > 0x10 || c->x86_model > 7) && tmp & BIT(25))
2079 pvt->ecc_sym_sz = 8;
Borislav Petkov525a1b22010-12-21 15:53:27 +01002080 }
Borislav Petkovb2b0c602010-10-08 18:32:29 +02002081 dump_misc_regs(pvt);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002082}
2083
2084/*
2085 * NOTE: CPU Revision Dependent code
2086 *
2087 * Input:
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002088 * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002089 * k8 private pointer to -->
2090 * DRAM Bank Address mapping register
2091 * node_id
2092 * DCL register where dual_channel_active is
2093 *
2094 * The DBAM register consists of 4 sets of 4 bits each definitions:
2095 *
2096 * Bits: CSROWs
2097 * 0-3 CSROWs 0 and 1
2098 * 4-7 CSROWs 2 and 3
2099 * 8-11 CSROWs 4 and 5
2100 * 12-15 CSROWs 6 and 7
2101 *
2102 * Values range from: 0 to 15
2103 * The meaning of the values depends on CPU revision and dual-channel state,
2104 * see relevant BKDG more info.
2105 *
2106 * The memory controller provides for total of only 8 CSROWs in its current
2107 * architecture. Each "pair" of CSROWs normally represents just one DIMM in
2108 * single channel or two (2) DIMMs in dual channel mode.
2109 *
2110 * The following code logic collapses the various tables for CSROW based on CPU
2111 * revision.
2112 *
2113 * Returns:
2114 * The number of PAGE_SIZE pages on the specified CSROW number it
2115 * encompasses
2116 *
2117 */
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01002118static u32 amd64_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002119{
Borislav Petkov1433eb92009-10-21 13:44:36 +02002120 u32 cs_mode, nr_pages;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002121
2122 /*
2123 * The math on this doesn't look right on the surface because x/2*4 can
2124 * be simplified to x*2 but this expression makes use of the fact that
2125 * it is integral math where 1/2=0. This intermediate value becomes the
2126 * number of bits to shift the DBAM register to extract the proper CSROW
2127 * field.
2128 */
Borislav Petkov1433eb92009-10-21 13:44:36 +02002129 cs_mode = (pvt->dbam0 >> ((csrow_nr / 2) * 4)) & 0xF;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002130
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01002131 nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode) << (20 - PAGE_SHIFT);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002132
2133 /*
2134 * If dual channel then double the memory size of single channel.
2135 * Channel count is 1 or 2
2136 */
2137 nr_pages <<= (pvt->channel_count - 1);
2138
Borislav Petkov1433eb92009-10-21 13:44:36 +02002139 debugf0(" (csrow=%d) DBAM map index= %d\n", csrow_nr, cs_mode);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002140 debugf0(" nr_pages= %u channel-count = %d\n",
2141 nr_pages, pvt->channel_count);
2142
2143 return nr_pages;
2144}
2145
2146/*
2147 * Initialize the array of csrow attribute instances, based on the values
2148 * from pci config hardware registers.
2149 */
Borislav Petkov360b7f32010-10-15 19:25:38 +02002150static int init_csrows(struct mem_ctl_info *mci)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002151{
2152 struct csrow_info *csrow;
Borislav Petkov2299ef72010-10-15 17:44:04 +02002153 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002154 u64 input_addr_min, input_addr_max, sys_addr, base, mask;
Borislav Petkov2299ef72010-10-15 17:44:04 +02002155 u32 val;
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02002156 int i, empty = 1;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002157
Borislav Petkova97fa682010-12-23 14:07:18 +01002158 amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002159
Borislav Petkov2299ef72010-10-15 17:44:04 +02002160 pvt->nbcfg = val;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002161
Borislav Petkov2299ef72010-10-15 17:44:04 +02002162 debugf0("node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
2163 pvt->mc_node_id, val,
Borislav Petkova97fa682010-12-23 14:07:18 +01002164 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
Doug Thompson0ec449e2009-04-27 19:41:25 +02002165
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002166 for_each_chip_select(i, 0, pvt) {
Doug Thompson0ec449e2009-04-27 19:41:25 +02002167 csrow = &mci->csrows[i];
2168
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002169 if (!csrow_enabled(i, 0, pvt)) {
Doug Thompson0ec449e2009-04-27 19:41:25 +02002170 debugf1("----CSROW %d EMPTY for node %d\n", i,
2171 pvt->mc_node_id);
2172 continue;
2173 }
2174
2175 debugf1("----CSROW %d VALID for MC node %d\n",
2176 i, pvt->mc_node_id);
2177
2178 empty = 0;
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01002179 csrow->nr_pages = amd64_csrow_nr_pages(pvt, 0, i);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002180 find_csrow_limits(mci, i, &input_addr_min, &input_addr_max);
2181 sys_addr = input_addr_to_sys_addr(mci, input_addr_min);
2182 csrow->first_page = (u32) (sys_addr >> PAGE_SHIFT);
2183 sys_addr = input_addr_to_sys_addr(mci, input_addr_max);
2184 csrow->last_page = (u32) (sys_addr >> PAGE_SHIFT);
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002185
2186 get_cs_base_and_mask(pvt, i, 0, &base, &mask);
2187 csrow->page_mask = ~mask;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002188 /* 8 bytes of resolution */
2189
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002190 csrow->mtype = amd64_determine_memory_type(pvt, i);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002191
2192 debugf1(" for MC node %d csrow %d:\n", pvt->mc_node_id, i);
2193 debugf1(" input_addr_min: 0x%lx input_addr_max: 0x%lx\n",
2194 (unsigned long)input_addr_min,
2195 (unsigned long)input_addr_max);
2196 debugf1(" sys_addr: 0x%lx page_mask: 0x%lx\n",
2197 (unsigned long)sys_addr, csrow->page_mask);
2198 debugf1(" nr_pages: %u first_page: 0x%lx "
2199 "last_page: 0x%lx\n",
2200 (unsigned)csrow->nr_pages,
2201 csrow->first_page, csrow->last_page);
2202
2203 /*
2204 * determine whether CHIPKILL or JUST ECC or NO ECC is operating
2205 */
Borislav Petkova97fa682010-12-23 14:07:18 +01002206 if (pvt->nbcfg & NBCFG_ECC_ENABLE)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002207 csrow->edac_mode =
Borislav Petkova97fa682010-12-23 14:07:18 +01002208 (pvt->nbcfg & NBCFG_CHIPKILL) ?
Doug Thompson0ec449e2009-04-27 19:41:25 +02002209 EDAC_S4ECD4ED : EDAC_SECDED;
2210 else
2211 csrow->edac_mode = EDAC_NONE;
2212 }
2213
2214 return empty;
2215}
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002216
Borislav Petkov06724532009-09-16 13:05:46 +02002217/* get all cores on this DCT */
Borislav Petkovb487c332011-02-21 18:55:00 +01002218static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, unsigned nid)
Doug Thompsonf9431992009-04-27 19:46:08 +02002219{
Borislav Petkov06724532009-09-16 13:05:46 +02002220 int cpu;
Doug Thompsonf9431992009-04-27 19:46:08 +02002221
Borislav Petkov06724532009-09-16 13:05:46 +02002222 for_each_online_cpu(cpu)
2223 if (amd_get_nb_id(cpu) == nid)
2224 cpumask_set_cpu(cpu, mask);
Doug Thompsonf9431992009-04-27 19:46:08 +02002225}
2226
2227/* check MCG_CTL on all the cpus on this node */
Borislav Petkovb487c332011-02-21 18:55:00 +01002228static bool amd64_nb_mce_bank_enabled_on_node(unsigned nid)
Doug Thompsonf9431992009-04-27 19:46:08 +02002229{
Rusty Russellba578cb2009-11-03 14:56:35 +10302230 cpumask_var_t mask;
Borislav Petkov50542252009-12-11 18:14:40 +01002231 int cpu, nbe;
Borislav Petkov06724532009-09-16 13:05:46 +02002232 bool ret = false;
Doug Thompsonf9431992009-04-27 19:46:08 +02002233
Rusty Russellba578cb2009-11-03 14:56:35 +10302234 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002235 amd64_warn("%s: Error allocating mask\n", __func__);
Rusty Russellba578cb2009-11-03 14:56:35 +10302236 return false;
2237 }
Borislav Petkov06724532009-09-16 13:05:46 +02002238
Rusty Russellba578cb2009-11-03 14:56:35 +10302239 get_cpus_on_this_dct_cpumask(mask, nid);
Borislav Petkov06724532009-09-16 13:05:46 +02002240
Rusty Russellba578cb2009-11-03 14:56:35 +10302241 rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
Borislav Petkov06724532009-09-16 13:05:46 +02002242
Rusty Russellba578cb2009-11-03 14:56:35 +10302243 for_each_cpu(cpu, mask) {
Borislav Petkov50542252009-12-11 18:14:40 +01002244 struct msr *reg = per_cpu_ptr(msrs, cpu);
Borislav Petkov5980bb92011-01-07 16:26:49 +01002245 nbe = reg->l & MSR_MCGCTL_NBE;
Borislav Petkov06724532009-09-16 13:05:46 +02002246
2247 debugf0("core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
Borislav Petkov50542252009-12-11 18:14:40 +01002248 cpu, reg->q,
Borislav Petkov06724532009-09-16 13:05:46 +02002249 (nbe ? "enabled" : "disabled"));
2250
2251 if (!nbe)
2252 goto out;
Borislav Petkov06724532009-09-16 13:05:46 +02002253 }
2254 ret = true;
2255
2256out:
Rusty Russellba578cb2009-11-03 14:56:35 +10302257 free_cpumask_var(mask);
Doug Thompsonf9431992009-04-27 19:46:08 +02002258 return ret;
2259}
2260
Borislav Petkov2299ef72010-10-15 17:44:04 +02002261static int toggle_ecc_err_reporting(struct ecc_settings *s, u8 nid, bool on)
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002262{
2263 cpumask_var_t cmask;
Borislav Petkov50542252009-12-11 18:14:40 +01002264 int cpu;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002265
2266 if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002267 amd64_warn("%s: error allocating mask\n", __func__);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002268 return false;
2269 }
2270
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002271 get_cpus_on_this_dct_cpumask(cmask, nid);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002272
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002273 rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2274
2275 for_each_cpu(cpu, cmask) {
2276
Borislav Petkov50542252009-12-11 18:14:40 +01002277 struct msr *reg = per_cpu_ptr(msrs, cpu);
2278
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002279 if (on) {
Borislav Petkov5980bb92011-01-07 16:26:49 +01002280 if (reg->l & MSR_MCGCTL_NBE)
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002281 s->flags.nb_mce_enable = 1;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002282
Borislav Petkov5980bb92011-01-07 16:26:49 +01002283 reg->l |= MSR_MCGCTL_NBE;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002284 } else {
2285 /*
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002286 * Turn off NB MCE reporting only when it was off before
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002287 */
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002288 if (!s->flags.nb_mce_enable)
Borislav Petkov5980bb92011-01-07 16:26:49 +01002289 reg->l &= ~MSR_MCGCTL_NBE;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002290 }
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002291 }
2292 wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2293
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002294 free_cpumask_var(cmask);
2295
2296 return 0;
2297}
2298
Borislav Petkov2299ef72010-10-15 17:44:04 +02002299static bool enable_ecc_error_reporting(struct ecc_settings *s, u8 nid,
2300 struct pci_dev *F3)
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002301{
Borislav Petkov2299ef72010-10-15 17:44:04 +02002302 bool ret = true;
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002303 u32 value, mask = 0x3; /* UECC/CECC enable */
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002304
Borislav Petkov2299ef72010-10-15 17:44:04 +02002305 if (toggle_ecc_err_reporting(s, nid, ON)) {
2306 amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
2307 return false;
2308 }
2309
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002310 amd64_read_pci_cfg(F3, NBCTL, &value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002311
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002312 s->old_nbctl = value & mask;
2313 s->nbctl_valid = true;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002314
2315 value |= mask;
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002316 amd64_write_pci_cfg(F3, NBCTL, value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002317
Borislav Petkova97fa682010-12-23 14:07:18 +01002318 amd64_read_pci_cfg(F3, NBCFG, &value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002319
Borislav Petkova97fa682010-12-23 14:07:18 +01002320 debugf0("1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2321 nid, value, !!(value & NBCFG_ECC_ENABLE));
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002322
Borislav Petkova97fa682010-12-23 14:07:18 +01002323 if (!(value & NBCFG_ECC_ENABLE)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002324 amd64_warn("DRAM ECC disabled on this node, enabling...\n");
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002325
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002326 s->flags.nb_ecc_prev = 0;
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002327
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002328 /* Attempt to turn on DRAM ECC Enable */
Borislav Petkova97fa682010-12-23 14:07:18 +01002329 value |= NBCFG_ECC_ENABLE;
2330 amd64_write_pci_cfg(F3, NBCFG, value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002331
Borislav Petkova97fa682010-12-23 14:07:18 +01002332 amd64_read_pci_cfg(F3, NBCFG, &value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002333
Borislav Petkova97fa682010-12-23 14:07:18 +01002334 if (!(value & NBCFG_ECC_ENABLE)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002335 amd64_warn("Hardware rejected DRAM ECC enable,"
2336 "check memory DIMM configuration.\n");
Borislav Petkov2299ef72010-10-15 17:44:04 +02002337 ret = false;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002338 } else {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002339 amd64_info("Hardware accepted DRAM ECC Enable\n");
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002340 }
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002341 } else {
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002342 s->flags.nb_ecc_prev = 1;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002343 }
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002344
Borislav Petkova97fa682010-12-23 14:07:18 +01002345 debugf0("2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2346 nid, value, !!(value & NBCFG_ECC_ENABLE));
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002347
Borislav Petkov2299ef72010-10-15 17:44:04 +02002348 return ret;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002349}
2350
Borislav Petkov360b7f32010-10-15 19:25:38 +02002351static void restore_ecc_error_reporting(struct ecc_settings *s, u8 nid,
2352 struct pci_dev *F3)
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002353{
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002354 u32 value, mask = 0x3; /* UECC/CECC enable */
2355
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002356
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002357 if (!s->nbctl_valid)
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002358 return;
2359
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002360 amd64_read_pci_cfg(F3, NBCTL, &value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002361 value &= ~mask;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002362 value |= s->old_nbctl;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002363
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002364 amd64_write_pci_cfg(F3, NBCTL, value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002365
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002366 /* restore previous BIOS DRAM ECC "off" setting we force-enabled */
2367 if (!s->flags.nb_ecc_prev) {
Borislav Petkova97fa682010-12-23 14:07:18 +01002368 amd64_read_pci_cfg(F3, NBCFG, &value);
2369 value &= ~NBCFG_ECC_ENABLE;
2370 amd64_write_pci_cfg(F3, NBCFG, value);
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002371 }
2372
2373 /* restore the NB Enable MCGCTL bit */
Borislav Petkov2299ef72010-10-15 17:44:04 +02002374 if (toggle_ecc_err_reporting(s, nid, OFF))
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002375 amd64_warn("Error restoring NB MCGCTL settings!\n");
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002376}
2377
Doug Thompsonf9431992009-04-27 19:46:08 +02002378/*
Borislav Petkov2299ef72010-10-15 17:44:04 +02002379 * EDAC requires that the BIOS have ECC enabled before
2380 * taking over the processing of ECC errors. A command line
2381 * option allows to force-enable hardware ECC later in
2382 * enable_ecc_error_reporting().
Doug Thompsonf9431992009-04-27 19:46:08 +02002383 */
Borislav Petkovcab4d272010-02-11 17:15:57 +01002384static const char *ecc_msg =
2385 "ECC disabled in the BIOS or no ECC capability, module will not load.\n"
2386 " Either enable ECC checking or force module loading by setting "
2387 "'ecc_enable_override'.\n"
2388 " (Note that use of the override may cause unknown side effects.)\n";
Borislav Petkovbe3468e2009-08-05 15:47:22 +02002389
Borislav Petkov2299ef72010-10-15 17:44:04 +02002390static bool ecc_enabled(struct pci_dev *F3, u8 nid)
Doug Thompsonf9431992009-04-27 19:46:08 +02002391{
2392 u32 value;
Borislav Petkov2299ef72010-10-15 17:44:04 +02002393 u8 ecc_en = 0;
Borislav Petkov06724532009-09-16 13:05:46 +02002394 bool nb_mce_en = false;
Doug Thompsonf9431992009-04-27 19:46:08 +02002395
Borislav Petkova97fa682010-12-23 14:07:18 +01002396 amd64_read_pci_cfg(F3, NBCFG, &value);
Doug Thompsonf9431992009-04-27 19:46:08 +02002397
Borislav Petkova97fa682010-12-23 14:07:18 +01002398 ecc_en = !!(value & NBCFG_ECC_ENABLE);
Borislav Petkov2299ef72010-10-15 17:44:04 +02002399 amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled"));
Doug Thompsonf9431992009-04-27 19:46:08 +02002400
Borislav Petkov2299ef72010-10-15 17:44:04 +02002401 nb_mce_en = amd64_nb_mce_bank_enabled_on_node(nid);
Borislav Petkov06724532009-09-16 13:05:46 +02002402 if (!nb_mce_en)
Borislav Petkov2299ef72010-10-15 17:44:04 +02002403 amd64_notice("NB MCE bank disabled, set MSR "
2404 "0x%08x[4] on node %d to enable.\n",
2405 MSR_IA32_MCG_CTL, nid);
Doug Thompsonf9431992009-04-27 19:46:08 +02002406
Borislav Petkov2299ef72010-10-15 17:44:04 +02002407 if (!ecc_en || !nb_mce_en) {
2408 amd64_notice("%s", ecc_msg);
2409 return false;
Borislav Petkov43f5e682009-12-21 18:55:18 +01002410 }
Borislav Petkov2299ef72010-10-15 17:44:04 +02002411 return true;
Doug Thompsonf9431992009-04-27 19:46:08 +02002412}
2413
Doug Thompson7d6034d2009-04-27 20:01:01 +02002414struct mcidev_sysfs_attribute sysfs_attrs[ARRAY_SIZE(amd64_dbg_attrs) +
2415 ARRAY_SIZE(amd64_inj_attrs) +
2416 1];
2417
2418struct mcidev_sysfs_attribute terminator = { .attr = { .name = NULL } };
2419
Borislav Petkov360b7f32010-10-15 19:25:38 +02002420static void set_mc_sysfs_attrs(struct mem_ctl_info *mci)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002421{
2422 unsigned int i = 0, j = 0;
2423
2424 for (; i < ARRAY_SIZE(amd64_dbg_attrs); i++)
2425 sysfs_attrs[i] = amd64_dbg_attrs[i];
2426
Borislav Petkova135cef2010-11-26 19:24:44 +01002427 if (boot_cpu_data.x86 >= 0x10)
2428 for (j = 0; j < ARRAY_SIZE(amd64_inj_attrs); j++, i++)
2429 sysfs_attrs[i] = amd64_inj_attrs[j];
Doug Thompson7d6034d2009-04-27 20:01:01 +02002430
2431 sysfs_attrs[i] = terminator;
2432
2433 mci->mc_driver_sysfs_attributes = sysfs_attrs;
2434}
2435
Borislav Petkovdf71a052011-01-19 18:15:10 +01002436static void setup_mci_misc_attrs(struct mem_ctl_info *mci,
2437 struct amd64_family_type *fam)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002438{
2439 struct amd64_pvt *pvt = mci->pvt_info;
2440
2441 mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
2442 mci->edac_ctl_cap = EDAC_FLAG_NONE;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002443
Borislav Petkov5980bb92011-01-07 16:26:49 +01002444 if (pvt->nbcap & NBCAP_SECDED)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002445 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
2446
Borislav Petkov5980bb92011-01-07 16:26:49 +01002447 if (pvt->nbcap & NBCAP_CHIPKILL)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002448 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
2449
2450 mci->edac_cap = amd64_determine_edac_cap(pvt);
2451 mci->mod_name = EDAC_MOD_STR;
2452 mci->mod_ver = EDAC_AMD64_VERSION;
Borislav Petkovdf71a052011-01-19 18:15:10 +01002453 mci->ctl_name = fam->ctl_name;
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002454 mci->dev_name = pci_name(pvt->F2);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002455 mci->ctl_page_to_phys = NULL;
2456
Doug Thompson7d6034d2009-04-27 20:01:01 +02002457 /* memory scrubber interface */
2458 mci->set_sdram_scrub_rate = amd64_set_scrub_rate;
2459 mci->get_sdram_scrub_rate = amd64_get_scrub_rate;
2460}
2461
Borislav Petkov0092b202010-10-01 19:20:05 +02002462/*
2463 * returns a pointer to the family descriptor on success, NULL otherwise.
2464 */
2465static struct amd64_family_type *amd64_per_family_init(struct amd64_pvt *pvt)
Borislav Petkov395ae782010-10-01 18:38:19 +02002466{
Borislav Petkov0092b202010-10-01 19:20:05 +02002467 u8 fam = boot_cpu_data.x86;
2468 struct amd64_family_type *fam_type = NULL;
2469
2470 switch (fam) {
Borislav Petkov395ae782010-10-01 18:38:19 +02002471 case 0xf:
Borislav Petkov0092b202010-10-01 19:20:05 +02002472 fam_type = &amd64_family_types[K8_CPUS];
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002473 pvt->ops = &amd64_family_types[K8_CPUS].ops;
Borislav Petkov395ae782010-10-01 18:38:19 +02002474 break;
Borislav Petkovdf71a052011-01-19 18:15:10 +01002475
Borislav Petkov395ae782010-10-01 18:38:19 +02002476 case 0x10:
Borislav Petkov0092b202010-10-01 19:20:05 +02002477 fam_type = &amd64_family_types[F10_CPUS];
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002478 pvt->ops = &amd64_family_types[F10_CPUS].ops;
Borislav Petkovdf71a052011-01-19 18:15:10 +01002479 break;
2480
2481 case 0x15:
2482 fam_type = &amd64_family_types[F15_CPUS];
2483 pvt->ops = &amd64_family_types[F15_CPUS].ops;
Borislav Petkov395ae782010-10-01 18:38:19 +02002484 break;
2485
2486 default:
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002487 amd64_err("Unsupported family!\n");
Borislav Petkov0092b202010-10-01 19:20:05 +02002488 return NULL;
Borislav Petkov395ae782010-10-01 18:38:19 +02002489 }
Borislav Petkov0092b202010-10-01 19:20:05 +02002490
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002491 pvt->ext_model = boot_cpu_data.x86_model >> 4;
2492
Borislav Petkovdf71a052011-01-19 18:15:10 +01002493 amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name,
Borislav Petkov0092b202010-10-01 19:20:05 +02002494 (fam == 0xf ?
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002495 (pvt->ext_model >= K8_REV_F ? "revF or later "
2496 : "revE or earlier ")
2497 : ""), pvt->mc_node_id);
Borislav Petkov0092b202010-10-01 19:20:05 +02002498 return fam_type;
Borislav Petkov395ae782010-10-01 18:38:19 +02002499}
2500
Borislav Petkov2299ef72010-10-15 17:44:04 +02002501static int amd64_init_one_instance(struct pci_dev *F2)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002502{
2503 struct amd64_pvt *pvt = NULL;
Borislav Petkov0092b202010-10-01 19:20:05 +02002504 struct amd64_family_type *fam_type = NULL;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002505 struct mem_ctl_info *mci = NULL;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002506 int err = 0, ret;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002507 u8 nid = get_node_id(F2);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002508
2509 ret = -ENOMEM;
2510 pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
2511 if (!pvt)
Borislav Petkov360b7f32010-10-15 19:25:38 +02002512 goto err_ret;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002513
Borislav Petkov360b7f32010-10-15 19:25:38 +02002514 pvt->mc_node_id = nid;
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002515 pvt->F2 = F2;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002516
Borislav Petkov395ae782010-10-01 18:38:19 +02002517 ret = -EINVAL;
Borislav Petkov0092b202010-10-01 19:20:05 +02002518 fam_type = amd64_per_family_init(pvt);
2519 if (!fam_type)
Borislav Petkov395ae782010-10-01 18:38:19 +02002520 goto err_free;
2521
Doug Thompson7d6034d2009-04-27 20:01:01 +02002522 ret = -ENODEV;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002523 err = reserve_mc_sibling_devs(pvt, fam_type->f1_id, fam_type->f3_id);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002524 if (err)
2525 goto err_free;
2526
Borislav Petkov360b7f32010-10-15 19:25:38 +02002527 read_mc_regs(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002528
Doug Thompson7d6034d2009-04-27 20:01:01 +02002529 /*
2530 * We need to determine how many memory channels there are. Then use
2531 * that information for calculating the size of the dynamic instance
Borislav Petkov360b7f32010-10-15 19:25:38 +02002532 * tables in the 'mci' structure.
Doug Thompson7d6034d2009-04-27 20:01:01 +02002533 */
Borislav Petkov360b7f32010-10-15 19:25:38 +02002534 ret = -EINVAL;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002535 pvt->channel_count = pvt->ops->early_channel_count(pvt);
2536 if (pvt->channel_count < 0)
Borislav Petkov360b7f32010-10-15 19:25:38 +02002537 goto err_siblings;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002538
2539 ret = -ENOMEM;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002540 mci = edac_mc_alloc(0, pvt->csels[0].b_cnt, pvt->channel_count, nid);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002541 if (!mci)
Borislav Petkov360b7f32010-10-15 19:25:38 +02002542 goto err_siblings;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002543
2544 mci->pvt_info = pvt;
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002545 mci->dev = &pvt->F2->dev;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002546
Borislav Petkovdf71a052011-01-19 18:15:10 +01002547 setup_mci_misc_attrs(mci, fam_type);
Borislav Petkov360b7f32010-10-15 19:25:38 +02002548
2549 if (init_csrows(mci))
Doug Thompson7d6034d2009-04-27 20:01:01 +02002550 mci->edac_cap = EDAC_FLAG_NONE;
2551
Borislav Petkov360b7f32010-10-15 19:25:38 +02002552 set_mc_sysfs_attrs(mci);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002553
2554 ret = -ENODEV;
2555 if (edac_mc_add_mc(mci)) {
2556 debugf1("failed edac_mc_add_mc()\n");
2557 goto err_add_mc;
2558 }
2559
Borislav Petkov549d0422009-07-24 13:51:42 +02002560 /* register stuff with EDAC MCE */
2561 if (report_gart_errors)
2562 amd_report_gart_errors(true);
2563
2564 amd_register_ecc_decoder(amd64_decode_bus_error);
2565
Borislav Petkov360b7f32010-10-15 19:25:38 +02002566 mcis[nid] = mci;
2567
2568 atomic_inc(&drv_instances);
2569
Doug Thompson7d6034d2009-04-27 20:01:01 +02002570 return 0;
2571
2572err_add_mc:
2573 edac_mc_free(mci);
2574
Borislav Petkov360b7f32010-10-15 19:25:38 +02002575err_siblings:
2576 free_mc_sibling_devs(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002577
Borislav Petkov360b7f32010-10-15 19:25:38 +02002578err_free:
2579 kfree(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002580
Borislav Petkov360b7f32010-10-15 19:25:38 +02002581err_ret:
Doug Thompson7d6034d2009-04-27 20:01:01 +02002582 return ret;
2583}
2584
Borislav Petkov2299ef72010-10-15 17:44:04 +02002585static int __devinit amd64_probe_one_instance(struct pci_dev *pdev,
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002586 const struct pci_device_id *mc_type)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002587{
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002588 u8 nid = get_node_id(pdev);
Borislav Petkov2299ef72010-10-15 17:44:04 +02002589 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002590 struct ecc_settings *s;
Borislav Petkov2299ef72010-10-15 17:44:04 +02002591 int ret = 0;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002592
Doug Thompson7d6034d2009-04-27 20:01:01 +02002593 ret = pci_enable_device(pdev);
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002594 if (ret < 0) {
Doug Thompson7d6034d2009-04-27 20:01:01 +02002595 debugf0("ret=%d\n", ret);
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002596 return -EIO;
2597 }
2598
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002599 ret = -ENOMEM;
2600 s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
2601 if (!s)
Borislav Petkov2299ef72010-10-15 17:44:04 +02002602 goto err_out;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002603
2604 ecc_stngs[nid] = s;
2605
Borislav Petkov2299ef72010-10-15 17:44:04 +02002606 if (!ecc_enabled(F3, nid)) {
2607 ret = -ENODEV;
2608
2609 if (!ecc_enable_override)
2610 goto err_enable;
2611
2612 amd64_warn("Forcing ECC on!\n");
2613
2614 if (!enable_ecc_error_reporting(s, nid, F3))
2615 goto err_enable;
2616 }
2617
2618 ret = amd64_init_one_instance(pdev);
Borislav Petkov360b7f32010-10-15 19:25:38 +02002619 if (ret < 0) {
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002620 amd64_err("Error probing instance: %d\n", nid);
Borislav Petkov360b7f32010-10-15 19:25:38 +02002621 restore_ecc_error_reporting(s, nid, F3);
2622 }
Doug Thompson7d6034d2009-04-27 20:01:01 +02002623
2624 return ret;
Borislav Petkov2299ef72010-10-15 17:44:04 +02002625
2626err_enable:
2627 kfree(s);
2628 ecc_stngs[nid] = NULL;
2629
2630err_out:
2631 return ret;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002632}
2633
2634static void __devexit amd64_remove_one_instance(struct pci_dev *pdev)
2635{
2636 struct mem_ctl_info *mci;
2637 struct amd64_pvt *pvt;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002638 u8 nid = get_node_id(pdev);
2639 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2640 struct ecc_settings *s = ecc_stngs[nid];
Doug Thompson7d6034d2009-04-27 20:01:01 +02002641
2642 /* Remove from EDAC CORE tracking list */
2643 mci = edac_mc_del_mc(&pdev->dev);
2644 if (!mci)
2645 return;
2646
2647 pvt = mci->pvt_info;
2648
Borislav Petkov360b7f32010-10-15 19:25:38 +02002649 restore_ecc_error_reporting(s, nid, F3);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002650
Borislav Petkov360b7f32010-10-15 19:25:38 +02002651 free_mc_sibling_devs(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002652
Borislav Petkov549d0422009-07-24 13:51:42 +02002653 /* unregister from EDAC MCE */
2654 amd_report_gart_errors(false);
2655 amd_unregister_ecc_decoder(amd64_decode_bus_error);
2656
Borislav Petkov360b7f32010-10-15 19:25:38 +02002657 kfree(ecc_stngs[nid]);
2658 ecc_stngs[nid] = NULL;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002659
Doug Thompson7d6034d2009-04-27 20:01:01 +02002660 /* Free the EDAC CORE resources */
Borislav Petkov8f68ed92009-12-21 15:15:59 +01002661 mci->pvt_info = NULL;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002662 mcis[nid] = NULL;
Borislav Petkov8f68ed92009-12-21 15:15:59 +01002663
2664 kfree(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002665 edac_mc_free(mci);
2666}
2667
2668/*
2669 * This table is part of the interface for loading drivers for PCI devices. The
2670 * PCI core identifies what devices are on a system during boot, and then
2671 * inquiry this table to see if this driver is for a given device found.
2672 */
2673static const struct pci_device_id amd64_pci_table[] __devinitdata = {
2674 {
2675 .vendor = PCI_VENDOR_ID_AMD,
2676 .device = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
2677 .subvendor = PCI_ANY_ID,
2678 .subdevice = PCI_ANY_ID,
2679 .class = 0,
2680 .class_mask = 0,
Doug Thompson7d6034d2009-04-27 20:01:01 +02002681 },
2682 {
2683 .vendor = PCI_VENDOR_ID_AMD,
2684 .device = PCI_DEVICE_ID_AMD_10H_NB_DRAM,
2685 .subvendor = PCI_ANY_ID,
2686 .subdevice = PCI_ANY_ID,
2687 .class = 0,
2688 .class_mask = 0,
Doug Thompson7d6034d2009-04-27 20:01:01 +02002689 },
Borislav Petkovdf71a052011-01-19 18:15:10 +01002690 {
2691 .vendor = PCI_VENDOR_ID_AMD,
2692 .device = PCI_DEVICE_ID_AMD_15H_NB_F2,
2693 .subvendor = PCI_ANY_ID,
2694 .subdevice = PCI_ANY_ID,
2695 .class = 0,
2696 .class_mask = 0,
2697 },
2698
Doug Thompson7d6034d2009-04-27 20:01:01 +02002699 {0, }
2700};
2701MODULE_DEVICE_TABLE(pci, amd64_pci_table);
2702
2703static struct pci_driver amd64_pci_driver = {
2704 .name = EDAC_MOD_STR,
Borislav Petkov2299ef72010-10-15 17:44:04 +02002705 .probe = amd64_probe_one_instance,
Doug Thompson7d6034d2009-04-27 20:01:01 +02002706 .remove = __devexit_p(amd64_remove_one_instance),
2707 .id_table = amd64_pci_table,
2708};
2709
Borislav Petkov360b7f32010-10-15 19:25:38 +02002710static void setup_pci_device(void)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002711{
2712 struct mem_ctl_info *mci;
2713 struct amd64_pvt *pvt;
2714
2715 if (amd64_ctl_pci)
2716 return;
2717
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002718 mci = mcis[0];
Doug Thompson7d6034d2009-04-27 20:01:01 +02002719 if (mci) {
2720
2721 pvt = mci->pvt_info;
2722 amd64_ctl_pci =
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002723 edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002724
2725 if (!amd64_ctl_pci) {
2726 pr_warning("%s(): Unable to create PCI control\n",
2727 __func__);
2728
2729 pr_warning("%s(): PCI error report via EDAC not set\n",
2730 __func__);
2731 }
2732 }
2733}
2734
2735static int __init amd64_edac_init(void)
2736{
Borislav Petkov360b7f32010-10-15 19:25:38 +02002737 int err = -ENODEV;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002738
Borislav Petkovdf71a052011-01-19 18:15:10 +01002739 printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002740
2741 opstate_init();
2742
Hans Rosenfeld9653a5c2010-10-29 17:14:31 +02002743 if (amd_cache_northbridges() < 0)
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01002744 goto err_ret;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002745
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002746 err = -ENOMEM;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002747 mcis = kzalloc(amd_nb_num() * sizeof(mcis[0]), GFP_KERNEL);
2748 ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL);
Borislav Petkov360b7f32010-10-15 19:25:38 +02002749 if (!(mcis && ecc_stngs))
Borislav Petkova9f0fbe2011-03-29 18:10:53 +02002750 goto err_free;
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002751
Borislav Petkov50542252009-12-11 18:14:40 +01002752 msrs = msrs_alloc();
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01002753 if (!msrs)
Borislav Petkov360b7f32010-10-15 19:25:38 +02002754 goto err_free;
Borislav Petkov50542252009-12-11 18:14:40 +01002755
Doug Thompson7d6034d2009-04-27 20:01:01 +02002756 err = pci_register_driver(&amd64_pci_driver);
2757 if (err)
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01002758 goto err_pci;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002759
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01002760 err = -ENODEV;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002761 if (!atomic_read(&drv_instances))
2762 goto err_no_instances;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002763
Borislav Petkov360b7f32010-10-15 19:25:38 +02002764 setup_pci_device();
2765 return 0;
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01002766
Borislav Petkov360b7f32010-10-15 19:25:38 +02002767err_no_instances:
Doug Thompson7d6034d2009-04-27 20:01:01 +02002768 pci_unregister_driver(&amd64_pci_driver);
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002769
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01002770err_pci:
2771 msrs_free(msrs);
2772 msrs = NULL;
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002773
Borislav Petkov360b7f32010-10-15 19:25:38 +02002774err_free:
2775 kfree(mcis);
2776 mcis = NULL;
2777
2778 kfree(ecc_stngs);
2779 ecc_stngs = NULL;
2780
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01002781err_ret:
Doug Thompson7d6034d2009-04-27 20:01:01 +02002782 return err;
2783}
2784
2785static void __exit amd64_edac_exit(void)
2786{
2787 if (amd64_ctl_pci)
2788 edac_pci_release_generic_ctl(amd64_ctl_pci);
2789
2790 pci_unregister_driver(&amd64_pci_driver);
Borislav Petkov50542252009-12-11 18:14:40 +01002791
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002792 kfree(ecc_stngs);
2793 ecc_stngs = NULL;
2794
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002795 kfree(mcis);
2796 mcis = NULL;
2797
Borislav Petkov50542252009-12-11 18:14:40 +01002798 msrs_free(msrs);
2799 msrs = NULL;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002800}
2801
2802module_init(amd64_edac_init);
2803module_exit(amd64_edac_exit);
2804
2805MODULE_LICENSE("GPL");
2806MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
2807 "Dave Peterson, Thayne Harbaugh");
2808MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
2809 EDAC_AMD64_VERSION);
2810
2811module_param(edac_op_state, int, 0444);
2812MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");