blob: 31e71c4fc83171e7da846740533f63131189327b [file] [log] [blame]
Doug Thompson2bc65412009-05-04 20:11:14 +02001#include "amd64_edac.h"
Andreas Herrmann23ac4ae2010-09-17 18:03:43 +02002#include <asm/amd_nb.h>
Doug Thompson2bc65412009-05-04 20:11:14 +02003
4static struct edac_pci_ctl_info *amd64_ctl_pci;
5
6static int report_gart_errors;
7module_param(report_gart_errors, int, 0644);
8
9/*
10 * Set by command line parameter. If BIOS has enabled the ECC, this override is
11 * cleared to prevent re-enabling the hardware by this driver.
12 */
13static int ecc_enable_override;
14module_param(ecc_enable_override, int, 0644);
15
Tejun Heoa29d8b82010-02-02 14:39:15 +090016static struct msr __percpu *msrs;
Borislav Petkov50542252009-12-11 18:14:40 +010017
Borislav Petkov360b7f32010-10-15 19:25:38 +020018/*
19 * count successfully initialized driver instances for setup_pci_device()
20 */
21static atomic_t drv_instances = ATOMIC_INIT(0);
22
Borislav Petkovcc4d8862010-10-13 16:11:59 +020023/* Per-node driver instances */
24static struct mem_ctl_info **mcis;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +020025static struct ecc_settings **ecc_stngs;
Doug Thompson2bc65412009-05-04 20:11:14 +020026
27/*
Borislav Petkovb70ef012009-06-25 19:32:38 +020028 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
29 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
30 * or higher value'.
31 *
32 *FIXME: Produce a better mapping/linearisation.
33 */
Borislav Petkov39094442010-11-24 19:52:09 +010034struct scrubrate {
35 u32 scrubval; /* bit pattern for scrub rate */
36 u32 bandwidth; /* bandwidth consumed (bytes/sec) */
37} scrubrates[] = {
Borislav Petkovb70ef012009-06-25 19:32:38 +020038 { 0x01, 1600000000UL},
39 { 0x02, 800000000UL},
40 { 0x03, 400000000UL},
41 { 0x04, 200000000UL},
42 { 0x05, 100000000UL},
43 { 0x06, 50000000UL},
44 { 0x07, 25000000UL},
45 { 0x08, 12284069UL},
46 { 0x09, 6274509UL},
47 { 0x0A, 3121951UL},
48 { 0x0B, 1560975UL},
49 { 0x0C, 781440UL},
50 { 0x0D, 390720UL},
51 { 0x0E, 195300UL},
52 { 0x0F, 97650UL},
53 { 0x10, 48854UL},
54 { 0x11, 24427UL},
55 { 0x12, 12213UL},
56 { 0x13, 6101UL},
57 { 0x14, 3051UL},
58 { 0x15, 1523UL},
59 { 0x16, 761UL},
60 { 0x00, 0UL}, /* scrubbing off */
61};
62
Borislav Petkovb2b0c602010-10-08 18:32:29 +020063static int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
64 u32 *val, const char *func)
65{
66 int err = 0;
67
68 err = pci_read_config_dword(pdev, offset, val);
69 if (err)
70 amd64_warn("%s: error reading F%dx%03x.\n",
71 func, PCI_FUNC(pdev->devfn), offset);
72
73 return err;
74}
75
76int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
77 u32 val, const char *func)
78{
79 int err = 0;
80
81 err = pci_write_config_dword(pdev, offset, val);
82 if (err)
83 amd64_warn("%s: error writing to F%dx%03x.\n",
84 func, PCI_FUNC(pdev->devfn), offset);
85
86 return err;
87}
88
89/*
90 *
91 * Depending on the family, F2 DCT reads need special handling:
92 *
93 * K8: has a single DCT only
94 *
95 * F10h: each DCT has its own set of regs
96 * DCT0 -> F2x040..
97 * DCT1 -> F2x140..
98 *
99 * F15h: we select which DCT we access using F1x10C[DctCfgSel]
100 *
101 */
102static int k8_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
103 const char *func)
104{
105 if (addr >= 0x100)
106 return -EINVAL;
107
108 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
109}
110
111static int f10_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
112 const char *func)
113{
114 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
115}
116
117static int f15_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
118 const char *func)
119{
120 u32 reg = 0;
121 u8 dct = 0;
122
123 if (addr >= 0x140 && addr <= 0x1a0) {
124 dct = 1;
125 addr -= 0x100;
126 }
127
128 amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, &reg);
129 reg &= 0xfffffffe;
130 reg |= dct;
131 amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
132
133 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
134}
135
Borislav Petkovb70ef012009-06-25 19:32:38 +0200136/*
Doug Thompson2bc65412009-05-04 20:11:14 +0200137 * Memory scrubber control interface. For K8, memory scrubbing is handled by
138 * hardware and can involve L2 cache, dcache as well as the main memory. With
139 * F10, this is extended to L3 cache scrubbing on CPU models sporting that
140 * functionality.
141 *
142 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
143 * (dram) over to cache lines. This is nasty, so we will use bandwidth in
144 * bytes/sec for the setting.
145 *
146 * Currently, we only do dram scrubbing. If the scrubbing is done in software on
147 * other archs, we might not have access to the caches directly.
148 */
149
150/*
151 * scan the scrub rate mapping table for a close or matching bandwidth value to
152 * issue. If requested is too big, then use last maximum value found.
153 */
Borislav Petkov395ae782010-10-01 18:38:19 +0200154static int __amd64_set_scrub_rate(struct pci_dev *ctl, u32 new_bw, u32 min_rate)
Doug Thompson2bc65412009-05-04 20:11:14 +0200155{
156 u32 scrubval;
157 int i;
158
159 /*
160 * map the configured rate (new_bw) to a value specific to the AMD64
161 * memory controller and apply to register. Search for the first
162 * bandwidth entry that is greater or equal than the setting requested
163 * and program that. If at last entry, turn off DRAM scrubbing.
164 */
165 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
166 /*
167 * skip scrub rates which aren't recommended
168 * (see F10 BKDG, F3x58)
169 */
Borislav Petkov395ae782010-10-01 18:38:19 +0200170 if (scrubrates[i].scrubval < min_rate)
Doug Thompson2bc65412009-05-04 20:11:14 +0200171 continue;
172
173 if (scrubrates[i].bandwidth <= new_bw)
174 break;
175
176 /*
177 * if no suitable bandwidth found, turn off DRAM scrubbing
178 * entirely by falling back to the last element in the
179 * scrubrates array.
180 */
181 }
182
183 scrubval = scrubrates[i].scrubval;
Doug Thompson2bc65412009-05-04 20:11:14 +0200184
Borislav Petkov5980bb92011-01-07 16:26:49 +0100185 pci_write_bits32(ctl, SCRCTRL, scrubval, 0x001F);
Doug Thompson2bc65412009-05-04 20:11:14 +0200186
Borislav Petkov39094442010-11-24 19:52:09 +0100187 if (scrubval)
188 return scrubrates[i].bandwidth;
189
Doug Thompson2bc65412009-05-04 20:11:14 +0200190 return 0;
191}
192
Borislav Petkov395ae782010-10-01 18:38:19 +0200193static int amd64_set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
Doug Thompson2bc65412009-05-04 20:11:14 +0200194{
195 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkov87b3e0e2011-01-19 20:02:38 +0100196 u32 min_scrubrate = 0x5;
Doug Thompson2bc65412009-05-04 20:11:14 +0200197
Borislav Petkov87b3e0e2011-01-19 20:02:38 +0100198 if (boot_cpu_data.x86 == 0xf)
199 min_scrubrate = 0x0;
200
201 return __amd64_set_scrub_rate(pvt->F3, bw, min_scrubrate);
Doug Thompson2bc65412009-05-04 20:11:14 +0200202}
203
Borislav Petkov39094442010-11-24 19:52:09 +0100204static int amd64_get_scrub_rate(struct mem_ctl_info *mci)
Doug Thompson2bc65412009-05-04 20:11:14 +0200205{
206 struct amd64_pvt *pvt = mci->pvt_info;
207 u32 scrubval = 0;
Borislav Petkov39094442010-11-24 19:52:09 +0100208 int i, retval = -EINVAL;
Doug Thompson2bc65412009-05-04 20:11:14 +0200209
Borislav Petkov5980bb92011-01-07 16:26:49 +0100210 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
Doug Thompson2bc65412009-05-04 20:11:14 +0200211
212 scrubval = scrubval & 0x001F;
213
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200214 amd64_debug("pci-read, sdram scrub control value: %d\n", scrubval);
Doug Thompson2bc65412009-05-04 20:11:14 +0200215
Roel Kluin926311f2010-01-11 20:58:21 +0100216 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
Doug Thompson2bc65412009-05-04 20:11:14 +0200217 if (scrubrates[i].scrubval == scrubval) {
Borislav Petkov39094442010-11-24 19:52:09 +0100218 retval = scrubrates[i].bandwidth;
Doug Thompson2bc65412009-05-04 20:11:14 +0200219 break;
220 }
221 }
Borislav Petkov39094442010-11-24 19:52:09 +0100222 return retval;
Doug Thompson2bc65412009-05-04 20:11:14 +0200223}
224
Doug Thompson67757632009-04-27 15:53:22 +0200225/*
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200226 * returns true if the SysAddr given by sys_addr matches the
227 * DRAM base/limit associated with node_id
Doug Thompson67757632009-04-27 15:53:22 +0200228 */
Borislav Petkovb487c332011-02-21 18:55:00 +0100229static bool amd64_base_limit_match(struct amd64_pvt *pvt, u64 sys_addr,
230 unsigned nid)
Doug Thompson67757632009-04-27 15:53:22 +0200231{
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200232 u64 addr;
Doug Thompson67757632009-04-27 15:53:22 +0200233
234 /* The K8 treats this as a 40-bit value. However, bits 63-40 will be
235 * all ones if the most significant implemented address bit is 1.
236 * Here we discard bits 63-40. See section 3.4.2 of AMD publication
237 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
238 * Application Programming.
239 */
240 addr = sys_addr & 0x000000ffffffffffull;
241
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200242 return ((addr >= get_dram_base(pvt, nid)) &&
243 (addr <= get_dram_limit(pvt, nid)));
Doug Thompson67757632009-04-27 15:53:22 +0200244}
245
246/*
247 * Attempt to map a SysAddr to a node. On success, return a pointer to the
248 * mem_ctl_info structure for the node that the SysAddr maps to.
249 *
250 * On failure, return NULL.
251 */
252static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
253 u64 sys_addr)
254{
255 struct amd64_pvt *pvt;
Borislav Petkovb487c332011-02-21 18:55:00 +0100256 unsigned node_id;
Doug Thompson67757632009-04-27 15:53:22 +0200257 u32 intlv_en, bits;
258
259 /*
260 * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
261 * 3.4.4.2) registers to map the SysAddr to a node ID.
262 */
263 pvt = mci->pvt_info;
264
265 /*
266 * The value of this field should be the same for all DRAM Base
267 * registers. Therefore we arbitrarily choose to read it from the
268 * register for node 0.
269 */
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200270 intlv_en = dram_intlv_en(pvt, 0);
Doug Thompson67757632009-04-27 15:53:22 +0200271
272 if (intlv_en == 0) {
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200273 for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
Doug Thompson67757632009-04-27 15:53:22 +0200274 if (amd64_base_limit_match(pvt, sys_addr, node_id))
Borislav Petkov8edc5442009-09-18 12:39:19 +0200275 goto found;
Doug Thompson67757632009-04-27 15:53:22 +0200276 }
Borislav Petkov8edc5442009-09-18 12:39:19 +0200277 goto err_no_match;
Doug Thompson67757632009-04-27 15:53:22 +0200278 }
279
Borislav Petkov72f158f2009-09-18 12:27:27 +0200280 if (unlikely((intlv_en != 0x01) &&
281 (intlv_en != 0x03) &&
282 (intlv_en != 0x07))) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200283 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
Doug Thompson67757632009-04-27 15:53:22 +0200284 return NULL;
285 }
286
287 bits = (((u32) sys_addr) >> 12) & intlv_en;
288
289 for (node_id = 0; ; ) {
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200290 if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
Doug Thompson67757632009-04-27 15:53:22 +0200291 break; /* intlv_sel field matches */
292
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200293 if (++node_id >= DRAM_RANGES)
Doug Thompson67757632009-04-27 15:53:22 +0200294 goto err_no_match;
295 }
296
297 /* sanity test for sys_addr */
298 if (unlikely(!amd64_base_limit_match(pvt, sys_addr, node_id))) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200299 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
300 "range for node %d with node interleaving enabled.\n",
301 __func__, sys_addr, node_id);
Doug Thompson67757632009-04-27 15:53:22 +0200302 return NULL;
303 }
304
305found:
Borislav Petkovb487c332011-02-21 18:55:00 +0100306 return edac_mc_find((int)node_id);
Doug Thompson67757632009-04-27 15:53:22 +0200307
308err_no_match:
309 debugf2("sys_addr 0x%lx doesn't match any node\n",
310 (unsigned long)sys_addr);
311
312 return NULL;
313}
Doug Thompsone2ce7252009-04-27 15:57:12 +0200314
315/*
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100316 * compute the CS base address of the @csrow on the DRAM controller @dct.
317 * For details see F2x[5C:40] in the processor's BKDG
Doug Thompsone2ce7252009-04-27 15:57:12 +0200318 */
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100319static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
320 u64 *base, u64 *mask)
Doug Thompsone2ce7252009-04-27 15:57:12 +0200321{
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100322 u64 csbase, csmask, base_bits, mask_bits;
323 u8 addr_shift;
324
325 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) {
326 csbase = pvt->csels[dct].csbases[csrow];
327 csmask = pvt->csels[dct].csmasks[csrow];
328 base_bits = GENMASK(21, 31) | GENMASK(9, 15);
329 mask_bits = GENMASK(21, 29) | GENMASK(9, 15);
330 addr_shift = 4;
331 } else {
332 csbase = pvt->csels[dct].csbases[csrow];
333 csmask = pvt->csels[dct].csmasks[csrow >> 1];
334 addr_shift = 8;
335
336 if (boot_cpu_data.x86 == 0x15)
337 base_bits = mask_bits = GENMASK(19,30) | GENMASK(5,13);
338 else
339 base_bits = mask_bits = GENMASK(19,28) | GENMASK(5,13);
340 }
341
342 *base = (csbase & base_bits) << addr_shift;
343
344 *mask = ~0ULL;
345 /* poke holes for the csmask */
346 *mask &= ~(mask_bits << addr_shift);
347 /* OR them in */
348 *mask |= (csmask & mask_bits) << addr_shift;
Doug Thompsone2ce7252009-04-27 15:57:12 +0200349}
350
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100351#define for_each_chip_select(i, dct, pvt) \
352 for (i = 0; i < pvt->csels[dct].b_cnt; i++)
Doug Thompsone2ce7252009-04-27 15:57:12 +0200353
Borislav Petkov614ec9d2011-01-13 18:02:22 +0100354#define chip_select_base(i, dct, pvt) \
355 pvt->csels[dct].csbases[i]
356
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100357#define for_each_chip_select_mask(i, dct, pvt) \
358 for (i = 0; i < pvt->csels[dct].m_cnt; i++)
Doug Thompsone2ce7252009-04-27 15:57:12 +0200359
360/*
361 * @input_addr is an InputAddr associated with the node given by mci. Return the
362 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
363 */
364static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
365{
366 struct amd64_pvt *pvt;
367 int csrow;
368 u64 base, mask;
369
370 pvt = mci->pvt_info;
371
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100372 for_each_chip_select(csrow, 0, pvt) {
373 if (!csrow_enabled(csrow, 0, pvt))
Doug Thompsone2ce7252009-04-27 15:57:12 +0200374 continue;
375
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100376 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
377
378 mask = ~mask;
Doug Thompsone2ce7252009-04-27 15:57:12 +0200379
380 if ((input_addr & mask) == (base & mask)) {
381 debugf2("InputAddr 0x%lx matches csrow %d (node %d)\n",
382 (unsigned long)input_addr, csrow,
383 pvt->mc_node_id);
384
385 return csrow;
386 }
387 }
Doug Thompsone2ce7252009-04-27 15:57:12 +0200388 debugf2("no matching csrow for InputAddr 0x%lx (MC node %d)\n",
389 (unsigned long)input_addr, pvt->mc_node_id);
390
391 return -1;
392}
393
394/*
Doug Thompsone2ce7252009-04-27 15:57:12 +0200395 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
396 * for the node represented by mci. Info is passed back in *hole_base,
397 * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if
398 * info is invalid. Info may be invalid for either of the following reasons:
399 *
400 * - The revision of the node is not E or greater. In this case, the DRAM Hole
401 * Address Register does not exist.
402 *
403 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
404 * indicating that its contents are not valid.
405 *
406 * The values passed back in *hole_base, *hole_offset, and *hole_size are
407 * complete 32-bit values despite the fact that the bitfields in the DHAR
408 * only represent bits 31-24 of the base and offset values.
409 */
410int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
411 u64 *hole_offset, u64 *hole_size)
412{
413 struct amd64_pvt *pvt = mci->pvt_info;
414 u64 base;
415
416 /* only revE and later have the DRAM Hole Address Register */
Borislav Petkov1433eb92009-10-21 13:44:36 +0200417 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_E) {
Doug Thompsone2ce7252009-04-27 15:57:12 +0200418 debugf1(" revision %d for node %d does not support DHAR\n",
419 pvt->ext_model, pvt->mc_node_id);
420 return 1;
421 }
422
Borislav Petkovbc21fa52010-11-11 17:29:13 +0100423 /* valid for Fam10h and above */
Borislav Petkovc8e518d2010-12-10 19:49:19 +0100424 if (boot_cpu_data.x86 >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
Doug Thompsone2ce7252009-04-27 15:57:12 +0200425 debugf1(" Dram Memory Hoisting is DISABLED on this system\n");
426 return 1;
427 }
428
Borislav Petkovc8e518d2010-12-10 19:49:19 +0100429 if (!dhar_valid(pvt)) {
Doug Thompsone2ce7252009-04-27 15:57:12 +0200430 debugf1(" Dram Memory Hoisting is DISABLED on this node %d\n",
431 pvt->mc_node_id);
432 return 1;
433 }
434
435 /* This node has Memory Hoisting */
436
437 /* +------------------+--------------------+--------------------+-----
438 * | memory | DRAM hole | relocated |
439 * | [0, (x - 1)] | [x, 0xffffffff] | addresses from |
440 * | | | DRAM hole |
441 * | | | [0x100000000, |
442 * | | | (0x100000000+ |
443 * | | | (0xffffffff-x))] |
444 * +------------------+--------------------+--------------------+-----
445 *
446 * Above is a diagram of physical memory showing the DRAM hole and the
447 * relocated addresses from the DRAM hole. As shown, the DRAM hole
448 * starts at address x (the base address) and extends through address
449 * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the
450 * addresses in the hole so that they start at 0x100000000.
451 */
452
Borislav Petkovbc21fa52010-11-11 17:29:13 +0100453 base = dhar_base(pvt);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200454
455 *hole_base = base;
456 *hole_size = (0x1ull << 32) - base;
457
458 if (boot_cpu_data.x86 > 0xf)
Borislav Petkovbc21fa52010-11-11 17:29:13 +0100459 *hole_offset = f10_dhar_offset(pvt);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200460 else
Borislav Petkovbc21fa52010-11-11 17:29:13 +0100461 *hole_offset = k8_dhar_offset(pvt);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200462
463 debugf1(" DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
464 pvt->mc_node_id, (unsigned long)*hole_base,
465 (unsigned long)*hole_offset, (unsigned long)*hole_size);
466
467 return 0;
468}
469EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info);
470
Doug Thompson93c2df52009-05-04 20:46:50 +0200471/*
472 * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is
473 * assumed that sys_addr maps to the node given by mci.
474 *
475 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
476 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
477 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
478 * then it is also involved in translating a SysAddr to a DramAddr. Sections
479 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
480 * These parts of the documentation are unclear. I interpret them as follows:
481 *
482 * When node n receives a SysAddr, it processes the SysAddr as follows:
483 *
484 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
485 * Limit registers for node n. If the SysAddr is not within the range
486 * specified by the base and limit values, then node n ignores the Sysaddr
487 * (since it does not map to node n). Otherwise continue to step 2 below.
488 *
489 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
490 * disabled so skip to step 3 below. Otherwise see if the SysAddr is within
491 * the range of relocated addresses (starting at 0x100000000) from the DRAM
492 * hole. If not, skip to step 3 below. Else get the value of the
493 * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
494 * offset defined by this value from the SysAddr.
495 *
496 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
497 * Base register for node n. To obtain the DramAddr, subtract the base
498 * address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
499 */
500static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
501{
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200502 struct amd64_pvt *pvt = mci->pvt_info;
Doug Thompson93c2df52009-05-04 20:46:50 +0200503 u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
504 int ret = 0;
505
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200506 dram_base = get_dram_base(pvt, pvt->mc_node_id);
Doug Thompson93c2df52009-05-04 20:46:50 +0200507
508 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
509 &hole_size);
510 if (!ret) {
511 if ((sys_addr >= (1ull << 32)) &&
512 (sys_addr < ((1ull << 32) + hole_size))) {
513 /* use DHAR to translate SysAddr to DramAddr */
514 dram_addr = sys_addr - hole_offset;
515
516 debugf2("using DHAR to translate SysAddr 0x%lx to "
517 "DramAddr 0x%lx\n",
518 (unsigned long)sys_addr,
519 (unsigned long)dram_addr);
520
521 return dram_addr;
522 }
523 }
524
525 /*
526 * Translate the SysAddr to a DramAddr as shown near the start of
527 * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8
528 * only deals with 40-bit values. Therefore we discard bits 63-40 of
529 * sys_addr below. If bit 39 of sys_addr is 1 then the bits we
530 * discard are all 1s. Otherwise the bits we discard are all 0s. See
531 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
532 * Programmer's Manual Volume 1 Application Programming.
533 */
Borislav Petkovf678b8c2010-12-13 19:21:07 +0100534 dram_addr = (sys_addr & GENMASK(0, 39)) - dram_base;
Doug Thompson93c2df52009-05-04 20:46:50 +0200535
536 debugf2("using DRAM Base register to translate SysAddr 0x%lx to "
537 "DramAddr 0x%lx\n", (unsigned long)sys_addr,
538 (unsigned long)dram_addr);
539 return dram_addr;
540}
541
542/*
543 * @intlv_en is the value of the IntlvEn field from a DRAM Base register
544 * (section 3.4.4.1). Return the number of bits from a SysAddr that are used
545 * for node interleaving.
546 */
547static int num_node_interleave_bits(unsigned intlv_en)
548{
549 static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
550 int n;
551
552 BUG_ON(intlv_en > 7);
553 n = intlv_shift_table[intlv_en];
554 return n;
555}
556
557/* Translate the DramAddr given by @dram_addr to an InputAddr. */
558static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
559{
560 struct amd64_pvt *pvt;
561 int intlv_shift;
562 u64 input_addr;
563
564 pvt = mci->pvt_info;
565
566 /*
567 * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
568 * concerning translating a DramAddr to an InputAddr.
569 */
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200570 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
Borislav Petkovf678b8c2010-12-13 19:21:07 +0100571 input_addr = ((dram_addr >> intlv_shift) & GENMASK(12, 35)) +
572 (dram_addr & 0xfff);
Doug Thompson93c2df52009-05-04 20:46:50 +0200573
574 debugf2(" Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
575 intlv_shift, (unsigned long)dram_addr,
576 (unsigned long)input_addr);
577
578 return input_addr;
579}
580
581/*
582 * Translate the SysAddr represented by @sys_addr to an InputAddr. It is
583 * assumed that @sys_addr maps to the node given by mci.
584 */
585static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
586{
587 u64 input_addr;
588
589 input_addr =
590 dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
591
592 debugf2("SysAdddr 0x%lx translates to InputAddr 0x%lx\n",
593 (unsigned long)sys_addr, (unsigned long)input_addr);
594
595 return input_addr;
596}
597
598
599/*
600 * @input_addr is an InputAddr associated with the node represented by mci.
601 * Translate @input_addr to a DramAddr and return the result.
602 */
603static u64 input_addr_to_dram_addr(struct mem_ctl_info *mci, u64 input_addr)
604{
605 struct amd64_pvt *pvt;
Borislav Petkovb487c332011-02-21 18:55:00 +0100606 unsigned node_id, intlv_shift;
Doug Thompson93c2df52009-05-04 20:46:50 +0200607 u64 bits, dram_addr;
608 u32 intlv_sel;
609
610 /*
611 * Near the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
612 * shows how to translate a DramAddr to an InputAddr. Here we reverse
613 * this procedure. When translating from a DramAddr to an InputAddr, the
614 * bits used for node interleaving are discarded. Here we recover these
615 * bits from the IntlvSel field of the DRAM Limit register (section
616 * 3.4.4.2) for the node that input_addr is associated with.
617 */
618 pvt = mci->pvt_info;
619 node_id = pvt->mc_node_id;
Borislav Petkovb487c332011-02-21 18:55:00 +0100620
621 BUG_ON(node_id > 7);
Doug Thompson93c2df52009-05-04 20:46:50 +0200622
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200623 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
Doug Thompson93c2df52009-05-04 20:46:50 +0200624 if (intlv_shift == 0) {
625 debugf1(" InputAddr 0x%lx translates to DramAddr of "
626 "same value\n", (unsigned long)input_addr);
627
628 return input_addr;
629 }
630
Borislav Petkovf678b8c2010-12-13 19:21:07 +0100631 bits = ((input_addr & GENMASK(12, 35)) << intlv_shift) +
632 (input_addr & 0xfff);
Doug Thompson93c2df52009-05-04 20:46:50 +0200633
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200634 intlv_sel = dram_intlv_sel(pvt, node_id) & ((1 << intlv_shift) - 1);
Doug Thompson93c2df52009-05-04 20:46:50 +0200635 dram_addr = bits + (intlv_sel << 12);
636
637 debugf1("InputAddr 0x%lx translates to DramAddr 0x%lx "
638 "(%d node interleave bits)\n", (unsigned long)input_addr,
639 (unsigned long)dram_addr, intlv_shift);
640
641 return dram_addr;
642}
643
644/*
645 * @dram_addr is a DramAddr that maps to the node represented by mci. Convert
646 * @dram_addr to a SysAddr.
647 */
648static u64 dram_addr_to_sys_addr(struct mem_ctl_info *mci, u64 dram_addr)
649{
650 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200651 u64 hole_base, hole_offset, hole_size, base, sys_addr;
Doug Thompson93c2df52009-05-04 20:46:50 +0200652 int ret = 0;
653
654 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
655 &hole_size);
656 if (!ret) {
657 if ((dram_addr >= hole_base) &&
658 (dram_addr < (hole_base + hole_size))) {
659 sys_addr = dram_addr + hole_offset;
660
661 debugf1("using DHAR to translate DramAddr 0x%lx to "
662 "SysAddr 0x%lx\n", (unsigned long)dram_addr,
663 (unsigned long)sys_addr);
664
665 return sys_addr;
666 }
667 }
668
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200669 base = get_dram_base(pvt, pvt->mc_node_id);
Doug Thompson93c2df52009-05-04 20:46:50 +0200670 sys_addr = dram_addr + base;
671
672 /*
673 * The sys_addr we have computed up to this point is a 40-bit value
674 * because the k8 deals with 40-bit values. However, the value we are
675 * supposed to return is a full 64-bit physical address. The AMD
676 * x86-64 architecture specifies that the most significant implemented
677 * address bit through bit 63 of a physical address must be either all
678 * 0s or all 1s. Therefore we sign-extend the 40-bit sys_addr to a
679 * 64-bit value below. See section 3.4.2 of AMD publication 24592:
680 * AMD x86-64 Architecture Programmer's Manual Volume 1 Application
681 * Programming.
682 */
683 sys_addr |= ~((sys_addr & (1ull << 39)) - 1);
684
685 debugf1(" Node %d, DramAddr 0x%lx to SysAddr 0x%lx\n",
686 pvt->mc_node_id, (unsigned long)dram_addr,
687 (unsigned long)sys_addr);
688
689 return sys_addr;
690}
691
692/*
693 * @input_addr is an InputAddr associated with the node given by mci. Translate
694 * @input_addr to a SysAddr.
695 */
696static inline u64 input_addr_to_sys_addr(struct mem_ctl_info *mci,
697 u64 input_addr)
698{
699 return dram_addr_to_sys_addr(mci,
700 input_addr_to_dram_addr(mci, input_addr));
701}
702
703/*
704 * Find the minimum and maximum InputAddr values that map to the given @csrow.
705 * Pass back these values in *input_addr_min and *input_addr_max.
706 */
707static void find_csrow_limits(struct mem_ctl_info *mci, int csrow,
708 u64 *input_addr_min, u64 *input_addr_max)
709{
710 struct amd64_pvt *pvt;
711 u64 base, mask;
712
713 pvt = mci->pvt_info;
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100714 BUG_ON((csrow < 0) || (csrow >= pvt->csels[0].b_cnt));
Doug Thompson93c2df52009-05-04 20:46:50 +0200715
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100716 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
Doug Thompson93c2df52009-05-04 20:46:50 +0200717
718 *input_addr_min = base & ~mask;
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100719 *input_addr_max = base | mask;
Doug Thompson93c2df52009-05-04 20:46:50 +0200720}
721
Doug Thompson93c2df52009-05-04 20:46:50 +0200722/* Map the Error address to a PAGE and PAGE OFFSET. */
723static inline void error_address_to_page_and_offset(u64 error_address,
724 u32 *page, u32 *offset)
725{
726 *page = (u32) (error_address >> PAGE_SHIFT);
727 *offset = ((u32) error_address) & ~PAGE_MASK;
728}
729
730/*
731 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
732 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
733 * of a node that detected an ECC memory error. mci represents the node that
734 * the error address maps to (possibly different from the node that detected
735 * the error). Return the number of the csrow that sys_addr maps to, or -1 on
736 * error.
737 */
738static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
739{
740 int csrow;
741
742 csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
743
744 if (csrow == -1)
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200745 amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
746 "address 0x%lx\n", (unsigned long)sys_addr);
Doug Thompson93c2df52009-05-04 20:46:50 +0200747 return csrow;
748}
Doug Thompsone2ce7252009-04-27 15:57:12 +0200749
Borislav Petkovbfc04ae2009-11-12 19:05:07 +0100750static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
Doug Thompson2da11652009-04-27 16:09:09 +0200751
Doug Thompson2da11652009-04-27 16:09:09 +0200752/*
753 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
754 * are ECC capable.
755 */
756static enum edac_type amd64_determine_edac_cap(struct amd64_pvt *pvt)
757{
Borislav Petkovcb328502010-12-22 14:28:24 +0100758 u8 bit;
Borislav Petkov584fcff2009-06-10 18:29:54 +0200759 enum dev_type edac_cap = EDAC_FLAG_NONE;
Doug Thompson2da11652009-04-27 16:09:09 +0200760
Borislav Petkov1433eb92009-10-21 13:44:36 +0200761 bit = (boot_cpu_data.x86 > 0xf || pvt->ext_model >= K8_REV_F)
Doug Thompson2da11652009-04-27 16:09:09 +0200762 ? 19
763 : 17;
764
Borislav Petkov584fcff2009-06-10 18:29:54 +0200765 if (pvt->dclr0 & BIT(bit))
Doug Thompson2da11652009-04-27 16:09:09 +0200766 edac_cap = EDAC_FLAG_SECDED;
767
768 return edac_cap;
769}
770
Borislav Petkov8c671752011-02-23 17:25:12 +0100771static void amd64_debug_display_dimm_sizes(struct amd64_pvt *, u8);
Doug Thompson2da11652009-04-27 16:09:09 +0200772
Borislav Petkov68798e12009-11-03 16:18:33 +0100773static void amd64_dump_dramcfg_low(u32 dclr, int chan)
774{
775 debugf1("F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
776
777 debugf1(" DIMM type: %sbuffered; all DIMMs support ECC: %s\n",
778 (dclr & BIT(16)) ? "un" : "",
779 (dclr & BIT(19)) ? "yes" : "no");
780
781 debugf1(" PAR/ERR parity: %s\n",
782 (dclr & BIT(8)) ? "enabled" : "disabled");
783
Borislav Petkovcb328502010-12-22 14:28:24 +0100784 if (boot_cpu_data.x86 == 0x10)
785 debugf1(" DCT 128bit mode width: %s\n",
786 (dclr & BIT(11)) ? "128b" : "64b");
Borislav Petkov68798e12009-11-03 16:18:33 +0100787
788 debugf1(" x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
789 (dclr & BIT(12)) ? "yes" : "no",
790 (dclr & BIT(13)) ? "yes" : "no",
791 (dclr & BIT(14)) ? "yes" : "no",
792 (dclr & BIT(15)) ? "yes" : "no");
793}
794
Doug Thompson2da11652009-04-27 16:09:09 +0200795/* Display and decode various NB registers for debug purposes. */
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200796static void dump_misc_regs(struct amd64_pvt *pvt)
Doug Thompson2da11652009-04-27 16:09:09 +0200797{
Borislav Petkov68798e12009-11-03 16:18:33 +0100798 debugf1("F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
Doug Thompson2da11652009-04-27 16:09:09 +0200799
Borislav Petkov68798e12009-11-03 16:18:33 +0100800 debugf1(" NB two channel DRAM capable: %s\n",
Borislav Petkov5980bb92011-01-07 16:26:49 +0100801 (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");
Borislav Petkov68798e12009-11-03 16:18:33 +0100802
803 debugf1(" ECC capable: %s, ChipKill ECC capable: %s\n",
Borislav Petkov5980bb92011-01-07 16:26:49 +0100804 (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
805 (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");
Borislav Petkov68798e12009-11-03 16:18:33 +0100806
807 amd64_dump_dramcfg_low(pvt->dclr0, 0);
Doug Thompson2da11652009-04-27 16:09:09 +0200808
Borislav Petkov8de1d912009-10-16 13:39:30 +0200809 debugf1("F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
Doug Thompson2da11652009-04-27 16:09:09 +0200810
Borislav Petkov8de1d912009-10-16 13:39:30 +0200811 debugf1("F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, "
812 "offset: 0x%08x\n",
Borislav Petkovbc21fa52010-11-11 17:29:13 +0100813 pvt->dhar, dhar_base(pvt),
814 (boot_cpu_data.x86 == 0xf) ? k8_dhar_offset(pvt)
815 : f10_dhar_offset(pvt));
Doug Thompson2da11652009-04-27 16:09:09 +0200816
Borislav Petkovc8e518d2010-12-10 19:49:19 +0100817 debugf1(" DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
Doug Thompson2da11652009-04-27 16:09:09 +0200818
Borislav Petkov8c671752011-02-23 17:25:12 +0100819 amd64_debug_display_dimm_sizes(pvt, 0);
Borislav Petkov4d796362011-02-03 15:59:57 +0100820
Borislav Petkov8de1d912009-10-16 13:39:30 +0200821 /* everything below this point is Fam10h and above */
Borislav Petkov4d796362011-02-03 15:59:57 +0100822 if (boot_cpu_data.x86 == 0xf)
Doug Thompson2da11652009-04-27 16:09:09 +0200823 return;
Borislav Petkov4d796362011-02-03 15:59:57 +0100824
Borislav Petkov8c671752011-02-23 17:25:12 +0100825 amd64_debug_display_dimm_sizes(pvt, 1);
Doug Thompson2da11652009-04-27 16:09:09 +0200826
Borislav Petkova3b7db02011-01-19 20:35:12 +0100827 amd64_info("using %s syndromes.\n", ((pvt->ecc_sym_sz == 8) ? "x8" : "x4"));
Borislav Petkovad6a32e2010-03-09 12:46:00 +0100828
Borislav Petkov8de1d912009-10-16 13:39:30 +0200829 /* Only if NOT ganged does dclr1 have valid info */
Borislav Petkov68798e12009-11-03 16:18:33 +0100830 if (!dct_ganging_enabled(pvt))
831 amd64_dump_dramcfg_low(pvt->dclr1, 1);
Doug Thompson2da11652009-04-27 16:09:09 +0200832}
833
Doug Thompson94be4bf2009-04-27 16:12:00 +0200834/*
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100835 * see BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
Doug Thompson94be4bf2009-04-27 16:12:00 +0200836 */
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100837static void prep_chip_selects(struct amd64_pvt *pvt)
Doug Thompson94be4bf2009-04-27 16:12:00 +0200838{
Borislav Petkov1433eb92009-10-21 13:44:36 +0200839 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) {
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100840 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
841 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
Borislav Petkov9d858bb2009-09-21 14:35:51 +0200842 } else {
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100843 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
844 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
Doug Thompson94be4bf2009-04-27 16:12:00 +0200845 }
846}
847
848/*
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100849 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
Doug Thompson94be4bf2009-04-27 16:12:00 +0200850 */
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200851static void read_dct_base_mask(struct amd64_pvt *pvt)
Doug Thompson94be4bf2009-04-27 16:12:00 +0200852{
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100853 int cs;
Doug Thompson94be4bf2009-04-27 16:12:00 +0200854
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100855 prep_chip_selects(pvt);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200856
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100857 for_each_chip_select(cs, 0, pvt) {
Borislav Petkov71d2a322011-02-21 19:37:24 +0100858 int reg0 = DCSB0 + (cs * 4);
859 int reg1 = DCSB1 + (cs * 4);
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100860 u32 *base0 = &pvt->csels[0].csbases[cs];
861 u32 *base1 = &pvt->csels[1].csbases[cs];
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200862
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100863 if (!amd64_read_dct_pci_cfg(pvt, reg0, base0))
Doug Thompson94be4bf2009-04-27 16:12:00 +0200864 debugf0(" DCSB0[%d]=0x%08x reg: F2x%x\n",
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100865 cs, *base0, reg0);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200866
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100867 if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt))
868 continue;
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200869
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100870 if (!amd64_read_dct_pci_cfg(pvt, reg1, base1))
871 debugf0(" DCSB1[%d]=0x%08x reg: F2x%x\n",
872 cs, *base1, reg1);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200873 }
874
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100875 for_each_chip_select_mask(cs, 0, pvt) {
Borislav Petkov71d2a322011-02-21 19:37:24 +0100876 int reg0 = DCSM0 + (cs * 4);
877 int reg1 = DCSM1 + (cs * 4);
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100878 u32 *mask0 = &pvt->csels[0].csmasks[cs];
879 u32 *mask1 = &pvt->csels[1].csmasks[cs];
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200880
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100881 if (!amd64_read_dct_pci_cfg(pvt, reg0, mask0))
Doug Thompson94be4bf2009-04-27 16:12:00 +0200882 debugf0(" DCSM0[%d]=0x%08x reg: F2x%x\n",
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100883 cs, *mask0, reg0);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200884
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100885 if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt))
886 continue;
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200887
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100888 if (!amd64_read_dct_pci_cfg(pvt, reg1, mask1))
889 debugf0(" DCSM1[%d]=0x%08x reg: F2x%x\n",
890 cs, *mask1, reg1);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200891 }
892}
893
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200894static enum mem_type amd64_determine_memory_type(struct amd64_pvt *pvt, int cs)
Doug Thompson94be4bf2009-04-27 16:12:00 +0200895{
896 enum mem_type type;
897
Borislav Petkovcb328502010-12-22 14:28:24 +0100898 /* F15h supports only DDR3 */
899 if (boot_cpu_data.x86 >= 0x15)
900 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
901 else if (boot_cpu_data.x86 == 0x10 || pvt->ext_model >= K8_REV_F) {
Borislav Petkov6b4c0bd2009-11-12 15:37:57 +0100902 if (pvt->dchr0 & DDR3_MODE)
903 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
904 else
905 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
Doug Thompson94be4bf2009-04-27 16:12:00 +0200906 } else {
Doug Thompson94be4bf2009-04-27 16:12:00 +0200907 type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
908 }
909
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200910 amd64_info("CS%d: %s\n", cs, edac_mem_types[type]);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200911
912 return type;
913}
914
Borislav Petkovcb328502010-12-22 14:28:24 +0100915/* Get the number of DCT channels the memory controller is using. */
Doug Thompsonddff8762009-04-27 16:14:52 +0200916static int k8_early_channel_count(struct amd64_pvt *pvt)
917{
Borislav Petkovcb328502010-12-22 14:28:24 +0100918 int flag;
Doug Thompsonddff8762009-04-27 16:14:52 +0200919
Borislav Petkov9f56da02010-10-01 19:44:53 +0200920 if (pvt->ext_model >= K8_REV_F)
Doug Thompsonddff8762009-04-27 16:14:52 +0200921 /* RevF (NPT) and later */
Borislav Petkov41d8bfa2011-01-18 19:16:08 +0100922 flag = pvt->dclr0 & WIDTH_128;
Borislav Petkov9f56da02010-10-01 19:44:53 +0200923 else
Doug Thompsonddff8762009-04-27 16:14:52 +0200924 /* RevE and earlier */
925 flag = pvt->dclr0 & REVE_WIDTH_128;
Doug Thompsonddff8762009-04-27 16:14:52 +0200926
927 /* not used */
928 pvt->dclr1 = 0;
929
930 return (flag) ? 2 : 1;
931}
932
Borislav Petkov70046622011-01-10 14:37:27 +0100933/* On F10h and later ErrAddr is MC4_ADDR[47:1] */
934static u64 get_error_address(struct mce *m)
Doug Thompsonddff8762009-04-27 16:14:52 +0200935{
Borislav Petkov70046622011-01-10 14:37:27 +0100936 u8 start_bit = 1;
937 u8 end_bit = 47;
938
939 if (boot_cpu_data.x86 == 0xf) {
940 start_bit = 3;
941 end_bit = 39;
942 }
943
944 return m->addr & GENMASK(start_bit, end_bit);
Doug Thompsonddff8762009-04-27 16:14:52 +0200945}
946
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200947static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
Doug Thompsonddff8762009-04-27 16:14:52 +0200948{
Borislav Petkov71d2a322011-02-21 19:37:24 +0100949 int off = range << 3;
Doug Thompsonddff8762009-04-27 16:14:52 +0200950
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200951 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo);
952 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
Doug Thompsonddff8762009-04-27 16:14:52 +0200953
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200954 if (boot_cpu_data.x86 == 0xf)
955 return;
Doug Thompsonddff8762009-04-27 16:14:52 +0200956
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200957 if (!dram_rw(pvt, range))
958 return;
Doug Thompsonddff8762009-04-27 16:14:52 +0200959
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200960 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi);
961 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
Doug Thompsonddff8762009-04-27 16:14:52 +0200962}
963
Borislav Petkovf192c7b2011-01-10 14:24:32 +0100964static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
965 u16 syndrome)
Doug Thompsonddff8762009-04-27 16:14:52 +0200966{
967 struct mem_ctl_info *src_mci;
Borislav Petkovf192c7b2011-01-10 14:24:32 +0100968 struct amd64_pvt *pvt = mci->pvt_info;
Doug Thompsonddff8762009-04-27 16:14:52 +0200969 int channel, csrow;
970 u32 page, offset;
Doug Thompsonddff8762009-04-27 16:14:52 +0200971
972 /* CHIPKILL enabled */
Borislav Petkovf192c7b2011-01-10 14:24:32 +0100973 if (pvt->nbcfg & NBCFG_CHIPKILL) {
Borislav Petkovbfc04ae2009-11-12 19:05:07 +0100974 channel = get_channel_from_ecc_syndrome(mci, syndrome);
Doug Thompsonddff8762009-04-27 16:14:52 +0200975 if (channel < 0) {
976 /*
977 * Syndrome didn't map, so we don't know which of the
978 * 2 DIMMs is in error. So we need to ID 'both' of them
979 * as suspect.
980 */
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200981 amd64_mc_warn(mci, "unknown syndrome 0x%04x - possible "
982 "error reporting race\n", syndrome);
Doug Thompsonddff8762009-04-27 16:14:52 +0200983 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
984 return;
985 }
986 } else {
987 /*
988 * non-chipkill ecc mode
989 *
990 * The k8 documentation is unclear about how to determine the
991 * channel number when using non-chipkill memory. This method
992 * was obtained from email communication with someone at AMD.
993 * (Wish the email was placed in this comment - norsk)
994 */
Borislav Petkov44e9e2e2009-10-26 15:00:19 +0100995 channel = ((sys_addr & BIT(3)) != 0);
Doug Thompsonddff8762009-04-27 16:14:52 +0200996 }
997
998 /*
999 * Find out which node the error address belongs to. This may be
1000 * different from the node that detected the error.
1001 */
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001002 src_mci = find_mc_by_sys_addr(mci, sys_addr);
Keith Mannthey2cff18c2009-09-18 14:35:23 +02001003 if (!src_mci) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001004 amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001005 (unsigned long)sys_addr);
Doug Thompsonddff8762009-04-27 16:14:52 +02001006 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1007 return;
1008 }
1009
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001010 /* Now map the sys_addr to a CSROW */
1011 csrow = sys_addr_to_csrow(src_mci, sys_addr);
Doug Thompsonddff8762009-04-27 16:14:52 +02001012 if (csrow < 0) {
1013 edac_mc_handle_ce_no_info(src_mci, EDAC_MOD_STR);
1014 } else {
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001015 error_address_to_page_and_offset(sys_addr, &page, &offset);
Doug Thompsonddff8762009-04-27 16:14:52 +02001016
1017 edac_mc_handle_ce(src_mci, page, offset, syndrome, csrow,
1018 channel, EDAC_MOD_STR);
1019 }
1020}
1021
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001022static int ddr2_cs_size(unsigned i, bool dct_width)
Doug Thompsonddff8762009-04-27 16:14:52 +02001023{
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001024 unsigned shift = 0;
Doug Thompsonddff8762009-04-27 16:14:52 +02001025
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001026 if (i <= 2)
1027 shift = i;
1028 else if (!(i & 0x1))
1029 shift = i >> 1;
Borislav Petkov1433eb92009-10-21 13:44:36 +02001030 else
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001031 shift = (i + 1) >> 1;
Doug Thompsonddff8762009-04-27 16:14:52 +02001032
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001033 return 128 << (shift + !!dct_width);
1034}
1035
1036static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1037 unsigned cs_mode)
1038{
1039 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1040
1041 if (pvt->ext_model >= K8_REV_F) {
1042 WARN_ON(cs_mode > 11);
1043 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1044 }
1045 else if (pvt->ext_model >= K8_REV_D) {
1046 WARN_ON(cs_mode > 10);
1047
1048 if (cs_mode == 3 || cs_mode == 8)
1049 return 32 << (cs_mode - 1);
1050 else
1051 return 32 << cs_mode;
1052 }
1053 else {
1054 WARN_ON(cs_mode > 6);
1055 return 32 << cs_mode;
1056 }
Doug Thompsonddff8762009-04-27 16:14:52 +02001057}
1058
Doug Thompson1afd3c92009-04-27 16:16:50 +02001059/*
1060 * Get the number of DCT channels in use.
1061 *
1062 * Return:
1063 * number of Memory Channels in operation
1064 * Pass back:
1065 * contents of the DCL0_LOW register
1066 */
Borislav Petkov7d20d142011-01-07 17:58:04 +01001067static int f1x_early_channel_count(struct amd64_pvt *pvt)
Doug Thompson1afd3c92009-04-27 16:16:50 +02001068{
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02001069 int i, j, channels = 0;
Doug Thompsonddff8762009-04-27 16:14:52 +02001070
Borislav Petkov7d20d142011-01-07 17:58:04 +01001071 /* On F10h, if we are in 128 bit mode, then we are using 2 channels */
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001072 if (boot_cpu_data.x86 == 0x10 && (pvt->dclr0 & WIDTH_128))
Borislav Petkov7d20d142011-01-07 17:58:04 +01001073 return 2;
Doug Thompson1afd3c92009-04-27 16:16:50 +02001074
1075 /*
Borislav Petkovd16149e2009-10-16 19:55:49 +02001076 * Need to check if in unganged mode: In such, there are 2 channels,
1077 * but they are not in 128 bit mode and thus the above 'dclr0' status
1078 * bit will be OFF.
Doug Thompson1afd3c92009-04-27 16:16:50 +02001079 *
1080 * Need to check DCT0[0] and DCT1[0] to see if only one of them has
1081 * their CSEnable bit on. If so, then SINGLE DIMM case.
1082 */
Borislav Petkovd16149e2009-10-16 19:55:49 +02001083 debugf0("Data width is not 128 bits - need more decoding\n");
Doug Thompson1afd3c92009-04-27 16:16:50 +02001084
1085 /*
1086 * Check DRAM Bank Address Mapping values for each DIMM to see if there
1087 * is more than just one DIMM present in unganged mode. Need to check
1088 * both controllers since DIMMs can be placed in either one.
1089 */
Borislav Petkov525a1b22010-12-21 15:53:27 +01001090 for (i = 0; i < 2; i++) {
1091 u32 dbam = (i ? pvt->dbam1 : pvt->dbam0);
Doug Thompson1afd3c92009-04-27 16:16:50 +02001092
Wan Wei57a30852009-08-07 17:04:49 +02001093 for (j = 0; j < 4; j++) {
1094 if (DBAM_DIMM(j, dbam) > 0) {
1095 channels++;
1096 break;
1097 }
1098 }
Doug Thompson1afd3c92009-04-27 16:16:50 +02001099 }
1100
Borislav Petkovd16149e2009-10-16 19:55:49 +02001101 if (channels > 2)
1102 channels = 2;
1103
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001104 amd64_info("MCT channel count: %d\n", channels);
Doug Thompson1afd3c92009-04-27 16:16:50 +02001105
1106 return channels;
Doug Thompson1afd3c92009-04-27 16:16:50 +02001107}
1108
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001109static int ddr3_cs_size(unsigned i, bool dct_width)
Doug Thompson1afd3c92009-04-27 16:16:50 +02001110{
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001111 unsigned shift = 0;
1112 int cs_size = 0;
1113
1114 if (i == 0 || i == 3 || i == 4)
1115 cs_size = -1;
1116 else if (i <= 2)
1117 shift = i;
1118 else if (i == 12)
1119 shift = 7;
1120 else if (!(i & 0x1))
1121 shift = i >> 1;
1122 else
1123 shift = (i + 1) >> 1;
1124
1125 if (cs_size != -1)
1126 cs_size = (128 * (1 << !!dct_width)) << shift;
1127
1128 return cs_size;
1129}
1130
1131static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1132 unsigned cs_mode)
1133{
1134 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1135
1136 WARN_ON(cs_mode > 11);
Borislav Petkov1433eb92009-10-21 13:44:36 +02001137
1138 if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001139 return ddr3_cs_size(cs_mode, dclr & WIDTH_128);
Borislav Petkov1433eb92009-10-21 13:44:36 +02001140 else
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001141 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1142}
Borislav Petkov1433eb92009-10-21 13:44:36 +02001143
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001144/*
1145 * F15h supports only 64bit DCT interfaces
1146 */
1147static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1148 unsigned cs_mode)
1149{
1150 WARN_ON(cs_mode > 12);
1151
1152 return ddr3_cs_size(cs_mode, false);
Doug Thompson1afd3c92009-04-27 16:16:50 +02001153}
1154
Borislav Petkov5a5d2372011-01-17 17:52:57 +01001155static void read_dram_ctl_register(struct amd64_pvt *pvt)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001156{
Doug Thompson6163b5d2009-04-27 16:20:17 +02001157
Borislav Petkov5a5d2372011-01-17 17:52:57 +01001158 if (boot_cpu_data.x86 == 0xf)
1159 return;
1160
Borislav Petkov78da1212010-12-22 19:31:45 +01001161 if (!amd64_read_dct_pci_cfg(pvt, DCT_SEL_LO, &pvt->dct_sel_lo)) {
1162 debugf0("F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
1163 pvt->dct_sel_lo, dct_sel_baseaddr(pvt));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001164
Borislav Petkov5a5d2372011-01-17 17:52:57 +01001165 debugf0(" DCTs operate in %s mode.\n",
1166 (dct_ganging_enabled(pvt) ? "ganged" : "unganged"));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001167
Borislav Petkov72381bd2009-10-09 19:14:43 +02001168 if (!dct_ganging_enabled(pvt))
1169 debugf0(" Address range split per DCT: %s\n",
1170 (dct_high_range_enabled(pvt) ? "yes" : "no"));
1171
Borislav Petkov78da1212010-12-22 19:31:45 +01001172 debugf0(" data interleave for ECC: %s, "
Borislav Petkov72381bd2009-10-09 19:14:43 +02001173 "DRAM cleared since last warm reset: %s\n",
1174 (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
1175 (dct_memory_cleared(pvt) ? "yes" : "no"));
1176
Borislav Petkov78da1212010-12-22 19:31:45 +01001177 debugf0(" channel interleave: %s, "
1178 "interleave bits selector: 0x%x\n",
Borislav Petkov72381bd2009-10-09 19:14:43 +02001179 (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
Doug Thompson6163b5d2009-04-27 16:20:17 +02001180 dct_sel_interleave_addr(pvt));
1181 }
1182
Borislav Petkov78da1212010-12-22 19:31:45 +01001183 amd64_read_dct_pci_cfg(pvt, DCT_SEL_HI, &pvt->dct_sel_hi);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001184}
1185
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001186/*
Borislav Petkov229a7a12010-12-09 18:57:54 +01001187 * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001188 * Interleaving Modes.
1189 */
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001190static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
Borislav Petkov229a7a12010-12-09 18:57:54 +01001191 bool hi_range_sel, u8 intlv_en)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001192{
Borislav Petkov151fa712011-02-21 19:33:10 +01001193 u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001194
1195 if (dct_ganging_enabled(pvt))
Borislav Petkov229a7a12010-12-09 18:57:54 +01001196 return 0;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001197
Borislav Petkov229a7a12010-12-09 18:57:54 +01001198 if (hi_range_sel)
1199 return dct_sel_high;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001200
Borislav Petkov229a7a12010-12-09 18:57:54 +01001201 /*
1202 * see F2x110[DctSelIntLvAddr] - channel interleave mode
1203 */
1204 if (dct_interleave_enabled(pvt)) {
1205 u8 intlv_addr = dct_sel_interleave_addr(pvt);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001206
Borislav Petkov229a7a12010-12-09 18:57:54 +01001207 /* return DCT select function: 0=DCT0, 1=DCT1 */
1208 if (!intlv_addr)
1209 return sys_addr >> 6 & 1;
1210
1211 if (intlv_addr & 0x2) {
1212 u8 shift = intlv_addr & 0x1 ? 9 : 6;
1213 u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2;
1214
1215 return ((sys_addr >> shift) & 1) ^ temp;
1216 }
1217
1218 return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
1219 }
1220
1221 if (dct_high_range_enabled(pvt))
1222 return ~dct_sel_high & 1;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001223
1224 return 0;
1225}
1226
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001227/* Convert the sys_addr to the normalized DCT address */
Borislav Petkove7613592011-02-21 19:49:01 +01001228static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, unsigned range,
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001229 u64 sys_addr, bool hi_rng,
1230 u32 dct_sel_base_addr)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001231{
1232 u64 chan_off;
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001233 u64 dram_base = get_dram_base(pvt, range);
1234 u64 hole_off = f10_dhar_offset(pvt);
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001235 u64 dct_sel_base_off = (pvt->dct_sel_hi & 0xFFFFFC00) << 16;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001236
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001237 if (hi_rng) {
1238 /*
1239 * if
1240 * base address of high range is below 4Gb
1241 * (bits [47:27] at [31:11])
1242 * DRAM address space on this DCT is hoisted above 4Gb &&
1243 * sys_addr > 4Gb
1244 *
1245 * remove hole offset from sys_addr
1246 * else
1247 * remove high range offset from sys_addr
1248 */
1249 if ((!(dct_sel_base_addr >> 16) ||
1250 dct_sel_base_addr < dhar_base(pvt)) &&
Borislav Petkov972ea172011-02-21 19:43:02 +01001251 dhar_valid(pvt) &&
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001252 (sys_addr >= BIT_64(32)))
Borislav Petkovbc21fa52010-11-11 17:29:13 +01001253 chan_off = hole_off;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001254 else
1255 chan_off = dct_sel_base_off;
1256 } else {
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001257 /*
1258 * if
1259 * we have a valid hole &&
1260 * sys_addr > 4Gb
1261 *
1262 * remove hole
1263 * else
1264 * remove dram base to normalize to DCT address
1265 */
Borislav Petkov972ea172011-02-21 19:43:02 +01001266 if (dhar_valid(pvt) && (sys_addr >= BIT_64(32)))
Borislav Petkovbc21fa52010-11-11 17:29:13 +01001267 chan_off = hole_off;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001268 else
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001269 chan_off = dram_base;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001270 }
1271
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001272 return (sys_addr & GENMASK(6,47)) - (chan_off & GENMASK(23,47));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001273}
1274
Doug Thompson6163b5d2009-04-27 16:20:17 +02001275/*
1276 * checks if the csrow passed in is marked as SPARED, if so returns the new
1277 * spare row
1278 */
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001279static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001280{
Borislav Petkov614ec9d2011-01-13 18:02:22 +01001281 int tmp_cs;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001282
Borislav Petkov614ec9d2011-01-13 18:02:22 +01001283 if (online_spare_swap_done(pvt, dct) &&
1284 csrow == online_spare_bad_dramcs(pvt, dct)) {
1285
1286 for_each_chip_select(tmp_cs, dct, pvt) {
1287 if (chip_select_base(tmp_cs, dct, pvt) & 0x2) {
1288 csrow = tmp_cs;
1289 break;
1290 }
1291 }
Doug Thompson6163b5d2009-04-27 16:20:17 +02001292 }
1293 return csrow;
1294}
1295
1296/*
1297 * Iterate over the DRAM DCT "base" and "mask" registers looking for a
1298 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
1299 *
1300 * Return:
1301 * -EINVAL: NOT FOUND
1302 * 0..csrow = Chip-Select Row
1303 */
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001304static int f1x_lookup_addr_in_dct(u64 in_addr, u32 nid, u8 dct)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001305{
1306 struct mem_ctl_info *mci;
1307 struct amd64_pvt *pvt;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001308 u64 cs_base, cs_mask;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001309 int cs_found = -EINVAL;
1310 int csrow;
1311
Borislav Petkovcc4d8862010-10-13 16:11:59 +02001312 mci = mcis[nid];
Doug Thompson6163b5d2009-04-27 16:20:17 +02001313 if (!mci)
1314 return cs_found;
1315
1316 pvt = mci->pvt_info;
1317
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001318 debugf1("input addr: 0x%llx, DCT: %d\n", in_addr, dct);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001319
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001320 for_each_chip_select(csrow, dct, pvt) {
1321 if (!csrow_enabled(csrow, dct, pvt))
Doug Thompson6163b5d2009-04-27 16:20:17 +02001322 continue;
1323
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001324 get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001325
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001326 debugf1(" CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
1327 csrow, cs_base, cs_mask);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001328
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001329 cs_mask = ~cs_mask;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001330
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001331 debugf1(" (InputAddr & ~CSMask)=0x%llx "
1332 "(CSBase & ~CSMask)=0x%llx\n",
1333 (in_addr & cs_mask), (cs_base & cs_mask));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001334
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001335 if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
1336 cs_found = f10_process_possible_spare(pvt, dct, csrow);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001337
1338 debugf1(" MATCH csrow=%d\n", cs_found);
1339 break;
1340 }
1341 }
1342 return cs_found;
1343}
1344
Borislav Petkov95b0ef52011-01-11 22:08:07 +01001345/*
1346 * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is
1347 * swapped with a region located at the bottom of memory so that the GPU can use
1348 * the interleaved region and thus two channels.
1349 */
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001350static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr)
Borislav Petkov95b0ef52011-01-11 22:08:07 +01001351{
1352 u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr;
1353
1354 if (boot_cpu_data.x86 == 0x10) {
1355 /* only revC3 and revE have that feature */
1356 if (boot_cpu_data.x86_model < 4 ||
1357 (boot_cpu_data.x86_model < 0xa &&
1358 boot_cpu_data.x86_mask < 3))
1359 return sys_addr;
1360 }
1361
1362 amd64_read_dct_pci_cfg(pvt, SWAP_INTLV_REG, &swap_reg);
1363
1364 if (!(swap_reg & 0x1))
1365 return sys_addr;
1366
1367 swap_base = (swap_reg >> 3) & 0x7f;
1368 swap_limit = (swap_reg >> 11) & 0x7f;
1369 rgn_size = (swap_reg >> 20) & 0x7f;
1370 tmp_addr = sys_addr >> 27;
1371
1372 if (!(sys_addr >> 34) &&
1373 (((tmp_addr >= swap_base) &&
1374 (tmp_addr <= swap_limit)) ||
1375 (tmp_addr < rgn_size)))
1376 return sys_addr ^ (u64)swap_base << 27;
1377
1378 return sys_addr;
1379}
1380
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001381/* For a given @dram_range, check if @sys_addr falls within it. */
Borislav Petkove7613592011-02-21 19:49:01 +01001382static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001383 u64 sys_addr, int *nid, int *chan_sel)
1384{
Borislav Petkov229a7a12010-12-09 18:57:54 +01001385 int cs_found = -EINVAL;
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001386 u64 chan_addr;
Borislav Petkov5d4b58e2011-01-13 16:01:13 +01001387 u32 dct_sel_base;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001388 u8 channel;
Borislav Petkov229a7a12010-12-09 18:57:54 +01001389 bool high_range = false;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001390
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001391 u8 node_id = dram_dst_node(pvt, range);
Borislav Petkov229a7a12010-12-09 18:57:54 +01001392 u8 intlv_en = dram_intlv_en(pvt, range);
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001393 u32 intlv_sel = dram_intlv_sel(pvt, range);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001394
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001395 debugf1("(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1396 range, sys_addr, get_dram_limit(pvt, range));
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001397
Borislav Petkov355fba62011-01-17 13:03:26 +01001398 if (dhar_valid(pvt) &&
1399 dhar_base(pvt) <= sys_addr &&
1400 sys_addr < BIT_64(32)) {
1401 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1402 sys_addr);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001403 return -EINVAL;
Borislav Petkov355fba62011-01-17 13:03:26 +01001404 }
1405
1406 if (intlv_en &&
1407 (intlv_sel != ((sys_addr >> 12) & intlv_en))) {
1408 amd64_warn("Botched intlv bits, en: 0x%x, sel: 0x%x\n",
1409 intlv_en, intlv_sel);
1410 return -EINVAL;
1411 }
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001412
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001413 sys_addr = f1x_swap_interleaved_region(pvt, sys_addr);
Borislav Petkov95b0ef52011-01-11 22:08:07 +01001414
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001415 dct_sel_base = dct_sel_baseaddr(pvt);
1416
1417 /*
1418 * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
1419 * select between DCT0 and DCT1.
1420 */
1421 if (dct_high_range_enabled(pvt) &&
1422 !dct_ganging_enabled(pvt) &&
1423 ((sys_addr >> 27) >= (dct_sel_base >> 11)))
Borislav Petkov229a7a12010-12-09 18:57:54 +01001424 high_range = true;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001425
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001426 channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001427
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001428 chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr,
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001429 high_range, dct_sel_base);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001430
Borislav Petkove2f79db2011-01-13 14:57:34 +01001431 /* Remove node interleaving, see F1x120 */
1432 if (intlv_en)
1433 chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) |
1434 (chan_addr & 0xfff);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001435
Borislav Petkov5d4b58e2011-01-13 16:01:13 +01001436 /* remove channel interleave */
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001437 if (dct_interleave_enabled(pvt) &&
1438 !dct_high_range_enabled(pvt) &&
1439 !dct_ganging_enabled(pvt)) {
Borislav Petkov5d4b58e2011-01-13 16:01:13 +01001440
1441 if (dct_sel_interleave_addr(pvt) != 1) {
1442 if (dct_sel_interleave_addr(pvt) == 0x3)
1443 /* hash 9 */
1444 chan_addr = ((chan_addr >> 10) << 9) |
1445 (chan_addr & 0x1ff);
1446 else
1447 /* A[6] or hash 6 */
1448 chan_addr = ((chan_addr >> 7) << 6) |
1449 (chan_addr & 0x3f);
1450 } else
1451 /* A[12] */
1452 chan_addr = ((chan_addr >> 13) << 12) |
1453 (chan_addr & 0xfff);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001454 }
1455
Borislav Petkov5d4b58e2011-01-13 16:01:13 +01001456 debugf1(" Normalized DCT addr: 0x%llx\n", chan_addr);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001457
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001458 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001459
1460 if (cs_found >= 0) {
1461 *nid = node_id;
1462 *chan_sel = channel;
1463 }
1464 return cs_found;
1465}
1466
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001467static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt, u64 sys_addr,
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001468 int *node, int *chan_sel)
1469{
Borislav Petkove7613592011-02-21 19:49:01 +01001470 int cs_found = -EINVAL;
1471 unsigned range;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001472
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001473 for (range = 0; range < DRAM_RANGES; range++) {
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001474
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001475 if (!dram_rw(pvt, range))
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001476 continue;
1477
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001478 if ((get_dram_base(pvt, range) <= sys_addr) &&
1479 (get_dram_limit(pvt, range) >= sys_addr)) {
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001480
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001481 cs_found = f1x_match_to_this_node(pvt, range,
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001482 sys_addr, node,
1483 chan_sel);
1484 if (cs_found >= 0)
1485 break;
1486 }
1487 }
1488 return cs_found;
1489}
1490
1491/*
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001492 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
1493 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001494 *
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001495 * The @sys_addr is usually an error address received from the hardware
1496 * (MCX_ADDR).
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001497 */
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001498static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001499 u16 syndrome)
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001500{
1501 struct amd64_pvt *pvt = mci->pvt_info;
1502 u32 page, offset;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001503 int nid, csrow, chan = 0;
1504
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001505 csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &nid, &chan);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001506
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001507 if (csrow < 0) {
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001508 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001509 return;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001510 }
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001511
1512 error_address_to_page_and_offset(sys_addr, &page, &offset);
1513
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001514 /*
1515 * We need the syndromes for channel detection only when we're
1516 * ganged. Otherwise @chan should already contain the channel at
1517 * this point.
1518 */
Borislav Petkova97fa682010-12-23 14:07:18 +01001519 if (dct_ganging_enabled(pvt))
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001520 chan = get_channel_from_ecc_syndrome(mci, syndrome);
1521
1522 if (chan >= 0)
1523 edac_mc_handle_ce(mci, page, offset, syndrome, csrow, chan,
1524 EDAC_MOD_STR);
1525 else
1526 /*
1527 * Channel unknown, report all channels on this CSROW as failed.
1528 */
1529 for (chan = 0; chan < mci->csrows[csrow].nr_channels; chan++)
1530 edac_mc_handle_ce(mci, page, offset, syndrome,
1531 csrow, chan, EDAC_MOD_STR);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001532}
1533
1534/*
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001535 * debug routine to display the memory sizes of all logical DIMMs and its
Borislav Petkovcb328502010-12-22 14:28:24 +01001536 * CSROWs
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001537 */
Borislav Petkov8c671752011-02-23 17:25:12 +01001538static void amd64_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001539{
Borislav Petkov603adaf2009-12-21 14:52:53 +01001540 int dimm, size0, size1, factor = 0;
Borislav Petkov525a1b22010-12-21 15:53:27 +01001541 u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
1542 u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001543
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001544 if (boot_cpu_data.x86 == 0xf) {
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001545 if (pvt->dclr0 & WIDTH_128)
Borislav Petkov603adaf2009-12-21 14:52:53 +01001546 factor = 1;
1547
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001548 /* K8 families < revF not supported yet */
Borislav Petkov1433eb92009-10-21 13:44:36 +02001549 if (pvt->ext_model < K8_REV_F)
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001550 return;
1551 else
1552 WARN_ON(ctrl != 0);
1553 }
1554
Borislav Petkov4d796362011-02-03 15:59:57 +01001555 dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1 : pvt->dbam0;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001556 dcsb = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->csels[1].csbases
1557 : pvt->csels[0].csbases;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001558
Borislav Petkov4d796362011-02-03 15:59:57 +01001559 debugf1("F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n", ctrl, dbam);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001560
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001561 edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
1562
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001563 /* Dump memory sizes for DIMM and its CSROWs */
1564 for (dimm = 0; dimm < 4; dimm++) {
1565
1566 size0 = 0;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001567 if (dcsb[dimm*2] & DCSB_CS_ENABLE)
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001568 size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
1569 DBAM_DIMM(dimm, dbam));
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001570
1571 size1 = 0;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001572 if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001573 size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
1574 DBAM_DIMM(dimm, dbam));
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001575
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001576 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
1577 dimm * 2, size0 << factor,
1578 dimm * 2 + 1, size1 << factor);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001579 }
1580}
1581
Doug Thompson4d376072009-04-27 16:25:05 +02001582static struct amd64_family_type amd64_family_types[] = {
1583 [K8_CPUS] = {
Borislav Petkov0092b202010-10-01 19:20:05 +02001584 .ctl_name = "K8",
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001585 .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
1586 .f3_id = PCI_DEVICE_ID_AMD_K8_NB_MISC,
Doug Thompson4d376072009-04-27 16:25:05 +02001587 .ops = {
Borislav Petkov1433eb92009-10-21 13:44:36 +02001588 .early_channel_count = k8_early_channel_count,
Borislav Petkov1433eb92009-10-21 13:44:36 +02001589 .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow,
1590 .dbam_to_cs = k8_dbam_to_chip_select,
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001591 .read_dct_pci_cfg = k8_read_dct_pci_cfg,
Doug Thompson4d376072009-04-27 16:25:05 +02001592 }
1593 },
1594 [F10_CPUS] = {
Borislav Petkov0092b202010-10-01 19:20:05 +02001595 .ctl_name = "F10h",
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001596 .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
1597 .f3_id = PCI_DEVICE_ID_AMD_10H_NB_MISC,
Doug Thompson4d376072009-04-27 16:25:05 +02001598 .ops = {
Borislav Petkov7d20d142011-01-07 17:58:04 +01001599 .early_channel_count = f1x_early_channel_count,
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001600 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
Borislav Petkov1433eb92009-10-21 13:44:36 +02001601 .dbam_to_cs = f10_dbam_to_chip_select,
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001602 .read_dct_pci_cfg = f10_read_dct_pci_cfg,
1603 }
1604 },
1605 [F15_CPUS] = {
1606 .ctl_name = "F15h",
Borislav Petkovdf71a052011-01-19 18:15:10 +01001607 .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1,
1608 .f3_id = PCI_DEVICE_ID_AMD_15H_NB_F3,
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001609 .ops = {
Borislav Petkov7d20d142011-01-07 17:58:04 +01001610 .early_channel_count = f1x_early_channel_count,
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001611 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001612 .dbam_to_cs = f15_dbam_to_chip_select,
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001613 .read_dct_pci_cfg = f15_read_dct_pci_cfg,
Doug Thompson4d376072009-04-27 16:25:05 +02001614 }
1615 },
Doug Thompson4d376072009-04-27 16:25:05 +02001616};
1617
1618static struct pci_dev *pci_get_related_function(unsigned int vendor,
1619 unsigned int device,
1620 struct pci_dev *related)
1621{
1622 struct pci_dev *dev = NULL;
1623
1624 dev = pci_get_device(vendor, device, dev);
1625 while (dev) {
1626 if ((dev->bus->number == related->bus->number) &&
1627 (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
1628 break;
1629 dev = pci_get_device(vendor, device, dev);
1630 }
1631
1632 return dev;
1633}
1634
Doug Thompsonb1289d62009-04-27 16:37:05 +02001635/*
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001636 * These are tables of eigenvectors (one per line) which can be used for the
1637 * construction of the syndrome tables. The modified syndrome search algorithm
1638 * uses those to find the symbol in error and thus the DIMM.
Doug Thompsonb1289d62009-04-27 16:37:05 +02001639 *
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001640 * Algorithm courtesy of Ross LaFetra from AMD.
Doug Thompsonb1289d62009-04-27 16:37:05 +02001641 */
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001642static u16 x4_vectors[] = {
1643 0x2f57, 0x1afe, 0x66cc, 0xdd88,
1644 0x11eb, 0x3396, 0x7f4c, 0xeac8,
1645 0x0001, 0x0002, 0x0004, 0x0008,
1646 0x1013, 0x3032, 0x4044, 0x8088,
1647 0x106b, 0x30d6, 0x70fc, 0xe0a8,
1648 0x4857, 0xc4fe, 0x13cc, 0x3288,
1649 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
1650 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
1651 0x15c1, 0x2a42, 0x89ac, 0x4758,
1652 0x2b03, 0x1602, 0x4f0c, 0xca08,
1653 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
1654 0x8ba7, 0x465e, 0x244c, 0x1cc8,
1655 0x2b87, 0x164e, 0x642c, 0xdc18,
1656 0x40b9, 0x80de, 0x1094, 0x20e8,
1657 0x27db, 0x1eb6, 0x9dac, 0x7b58,
1658 0x11c1, 0x2242, 0x84ac, 0x4c58,
1659 0x1be5, 0x2d7a, 0x5e34, 0xa718,
1660 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
1661 0x4c97, 0xc87e, 0x11fc, 0x33a8,
1662 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
1663 0x16b3, 0x3d62, 0x4f34, 0x8518,
1664 0x1e2f, 0x391a, 0x5cac, 0xf858,
1665 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
1666 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
1667 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
1668 0x4397, 0xc27e, 0x17fc, 0x3ea8,
1669 0x1617, 0x3d3e, 0x6464, 0xb8b8,
1670 0x23ff, 0x12aa, 0xab6c, 0x56d8,
1671 0x2dfb, 0x1ba6, 0x913c, 0x7328,
1672 0x185d, 0x2ca6, 0x7914, 0x9e28,
1673 0x171b, 0x3e36, 0x7d7c, 0xebe8,
1674 0x4199, 0x82ee, 0x19f4, 0x2e58,
1675 0x4807, 0xc40e, 0x130c, 0x3208,
1676 0x1905, 0x2e0a, 0x5804, 0xac08,
1677 0x213f, 0x132a, 0xadfc, 0x5ba8,
1678 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
Doug Thompsonb1289d62009-04-27 16:37:05 +02001679};
1680
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001681static u16 x8_vectors[] = {
1682 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
1683 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
1684 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
1685 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
1686 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
1687 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
1688 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
1689 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
1690 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
1691 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
1692 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
1693 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
1694 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
1695 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
1696 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
1697 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
1698 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
1699 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
1700 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
1701};
1702
Borislav Petkovd34a6ec2011-02-23 17:41:50 +01001703static int decode_syndrome(u16 syndrome, u16 *vectors, unsigned num_vecs,
1704 unsigned v_dim)
Doug Thompsonb1289d62009-04-27 16:37:05 +02001705{
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001706 unsigned int i, err_sym;
Doug Thompsonb1289d62009-04-27 16:37:05 +02001707
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001708 for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
1709 u16 s = syndrome;
Borislav Petkovd34a6ec2011-02-23 17:41:50 +01001710 unsigned v_idx = err_sym * v_dim;
1711 unsigned v_end = (err_sym + 1) * v_dim;
Doug Thompsonb1289d62009-04-27 16:37:05 +02001712
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001713 /* walk over all 16 bits of the syndrome */
1714 for (i = 1; i < (1U << 16); i <<= 1) {
1715
1716 /* if bit is set in that eigenvector... */
1717 if (v_idx < v_end && vectors[v_idx] & i) {
1718 u16 ev_comp = vectors[v_idx++];
1719
1720 /* ... and bit set in the modified syndrome, */
1721 if (s & i) {
1722 /* remove it. */
1723 s ^= ev_comp;
1724
1725 if (!s)
1726 return err_sym;
1727 }
1728
1729 } else if (s & i)
1730 /* can't get to zero, move to next symbol */
1731 break;
1732 }
Doug Thompsonb1289d62009-04-27 16:37:05 +02001733 }
1734
1735 debugf0("syndrome(%x) not found\n", syndrome);
1736 return -1;
1737}
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001738
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001739static int map_err_sym_to_channel(int err_sym, int sym_size)
1740{
1741 if (sym_size == 4)
1742 switch (err_sym) {
1743 case 0x20:
1744 case 0x21:
1745 return 0;
1746 break;
1747 case 0x22:
1748 case 0x23:
1749 return 1;
1750 break;
1751 default:
1752 return err_sym >> 4;
1753 break;
1754 }
1755 /* x8 symbols */
1756 else
1757 switch (err_sym) {
1758 /* imaginary bits not in a DIMM */
1759 case 0x10:
1760 WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
1761 err_sym);
1762 return -1;
1763 break;
1764
1765 case 0x11:
1766 return 0;
1767 break;
1768 case 0x12:
1769 return 1;
1770 break;
1771 default:
1772 return err_sym >> 3;
1773 break;
1774 }
1775 return -1;
1776}
1777
1778static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
1779{
1780 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001781 int err_sym = -1;
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001782
Borislav Petkova3b7db02011-01-19 20:35:12 +01001783 if (pvt->ecc_sym_sz == 8)
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001784 err_sym = decode_syndrome(syndrome, x8_vectors,
1785 ARRAY_SIZE(x8_vectors),
Borislav Petkova3b7db02011-01-19 20:35:12 +01001786 pvt->ecc_sym_sz);
1787 else if (pvt->ecc_sym_sz == 4)
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001788 err_sym = decode_syndrome(syndrome, x4_vectors,
1789 ARRAY_SIZE(x4_vectors),
Borislav Petkova3b7db02011-01-19 20:35:12 +01001790 pvt->ecc_sym_sz);
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001791 else {
Borislav Petkova3b7db02011-01-19 20:35:12 +01001792 amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz);
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001793 return err_sym;
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001794 }
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001795
Borislav Petkova3b7db02011-01-19 20:35:12 +01001796 return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz);
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001797}
1798
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001799/*
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001800 * Handle any Correctable Errors (CEs) that have occurred. Check for valid ERROR
1801 * ADDRESS and process.
1802 */
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001803static void amd64_handle_ce(struct mem_ctl_info *mci, struct mce *m)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001804{
1805 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001806 u64 sys_addr;
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001807 u16 syndrome;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001808
1809 /* Ensure that the Error Address is VALID */
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001810 if (!(m->status & MCI_STATUS_ADDRV)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001811 amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n");
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001812 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1813 return;
1814 }
1815
Borislav Petkov70046622011-01-10 14:37:27 +01001816 sys_addr = get_error_address(m);
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001817 syndrome = extract_syndrome(m->status);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001818
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001819 amd64_mc_err(mci, "CE ERROR_ADDRESS= 0x%llx\n", sys_addr);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001820
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001821 pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, syndrome);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001822}
1823
1824/* Handle any Un-correctable Errors (UEs) */
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001825static void amd64_handle_ue(struct mem_ctl_info *mci, struct mce *m)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001826{
Borislav Petkov1f6bcee2009-11-13 14:02:57 +01001827 struct mem_ctl_info *log_mci, *src_mci = NULL;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001828 int csrow;
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001829 u64 sys_addr;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001830 u32 page, offset;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001831
1832 log_mci = mci;
1833
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001834 if (!(m->status & MCI_STATUS_ADDRV)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001835 amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n");
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001836 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1837 return;
1838 }
1839
Borislav Petkov70046622011-01-10 14:37:27 +01001840 sys_addr = get_error_address(m);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001841
1842 /*
1843 * Find out which node the error address belongs to. This may be
1844 * different from the node that detected the error.
1845 */
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001846 src_mci = find_mc_by_sys_addr(mci, sys_addr);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001847 if (!src_mci) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001848 amd64_mc_err(mci, "ERROR ADDRESS (0x%lx) NOT mapped to a MC\n",
1849 (unsigned long)sys_addr);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001850 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1851 return;
1852 }
1853
1854 log_mci = src_mci;
1855
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001856 csrow = sys_addr_to_csrow(log_mci, sys_addr);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001857 if (csrow < 0) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001858 amd64_mc_err(mci, "ERROR_ADDRESS (0x%lx) NOT mapped to CS\n",
1859 (unsigned long)sys_addr);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001860 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1861 } else {
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001862 error_address_to_page_and_offset(sys_addr, &page, &offset);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001863 edac_mc_handle_ue(log_mci, page, offset, csrow, EDAC_MOD_STR);
1864 }
1865}
1866
Borislav Petkov549d0422009-07-24 13:51:42 +02001867static inline void __amd64_decode_bus_error(struct mem_ctl_info *mci,
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001868 struct mce *m)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001869{
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001870 u16 ec = EC(m->status);
1871 u8 xec = XEC(m->status, 0x1f);
1872 u8 ecc_type = (m->status >> 45) & 0x3;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001873
Borislav Petkovb70ef012009-06-25 19:32:38 +02001874 /* Bail early out if this was an 'observed' error */
Borislav Petkov5980bb92011-01-07 16:26:49 +01001875 if (PP(ec) == NBSL_PP_OBS)
Borislav Petkovb70ef012009-06-25 19:32:38 +02001876 return;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001877
Borislav Petkovecaf5602009-07-23 16:32:01 +02001878 /* Do only ECC errors */
1879 if (xec && xec != F10_NBSL_EXT_ERR_ECC)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001880 return;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001881
Borislav Petkovecaf5602009-07-23 16:32:01 +02001882 if (ecc_type == 2)
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001883 amd64_handle_ce(mci, m);
Borislav Petkovecaf5602009-07-23 16:32:01 +02001884 else if (ecc_type == 1)
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001885 amd64_handle_ue(mci, m);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001886}
1887
Borislav Petkov7cfd4a82010-09-01 14:45:20 +02001888void amd64_decode_bus_error(int node_id, struct mce *m, u32 nbcfg)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001889{
Borislav Petkovcc4d8862010-10-13 16:11:59 +02001890 struct mem_ctl_info *mci = mcis[node_id];
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001891
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001892 __amd64_decode_bus_error(mci, m);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001893}
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001894
Doug Thompson0ec449e2009-04-27 19:41:25 +02001895/*
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001896 * Use pvt->F2 which contains the F2 CPU PCI device to get the related
Borislav Petkovbbd0c1f2010-10-01 19:27:58 +02001897 * F1 (AddrMap) and F3 (Misc) devices. Return negative value on error.
Doug Thompson0ec449e2009-04-27 19:41:25 +02001898 */
Borislav Petkov360b7f32010-10-15 19:25:38 +02001899static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 f1_id, u16 f3_id)
Doug Thompson0ec449e2009-04-27 19:41:25 +02001900{
Doug Thompson0ec449e2009-04-27 19:41:25 +02001901 /* Reserve the ADDRESS MAP Device */
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001902 pvt->F1 = pci_get_related_function(pvt->F2->vendor, f1_id, pvt->F2);
1903 if (!pvt->F1) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001904 amd64_err("error address map device not found: "
1905 "vendor %x device 0x%x (broken BIOS?)\n",
1906 PCI_VENDOR_ID_AMD, f1_id);
Borislav Petkovbbd0c1f2010-10-01 19:27:58 +02001907 return -ENODEV;
Doug Thompson0ec449e2009-04-27 19:41:25 +02001908 }
1909
1910 /* Reserve the MISC Device */
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001911 pvt->F3 = pci_get_related_function(pvt->F2->vendor, f3_id, pvt->F2);
1912 if (!pvt->F3) {
1913 pci_dev_put(pvt->F1);
1914 pvt->F1 = NULL;
Doug Thompson0ec449e2009-04-27 19:41:25 +02001915
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001916 amd64_err("error F3 device not found: "
1917 "vendor %x device 0x%x (broken BIOS?)\n",
1918 PCI_VENDOR_ID_AMD, f3_id);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001919
Borislav Petkovbbd0c1f2010-10-01 19:27:58 +02001920 return -ENODEV;
Doug Thompson0ec449e2009-04-27 19:41:25 +02001921 }
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001922 debugf1("F1: %s\n", pci_name(pvt->F1));
1923 debugf1("F2: %s\n", pci_name(pvt->F2));
1924 debugf1("F3: %s\n", pci_name(pvt->F3));
Doug Thompson0ec449e2009-04-27 19:41:25 +02001925
1926 return 0;
1927}
1928
Borislav Petkov360b7f32010-10-15 19:25:38 +02001929static void free_mc_sibling_devs(struct amd64_pvt *pvt)
Doug Thompson0ec449e2009-04-27 19:41:25 +02001930{
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001931 pci_dev_put(pvt->F1);
1932 pci_dev_put(pvt->F3);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001933}
1934
1935/*
1936 * Retrieve the hardware registers of the memory controller (this includes the
1937 * 'Address Map' and 'Misc' device regs)
1938 */
Borislav Petkov360b7f32010-10-15 19:25:38 +02001939static void read_mc_regs(struct amd64_pvt *pvt)
Doug Thompson0ec449e2009-04-27 19:41:25 +02001940{
Borislav Petkova3b7db02011-01-19 20:35:12 +01001941 struct cpuinfo_x86 *c = &boot_cpu_data;
Doug Thompson0ec449e2009-04-27 19:41:25 +02001942 u64 msr_val;
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001943 u32 tmp;
Borislav Petkove7613592011-02-21 19:49:01 +01001944 unsigned range;
Doug Thompson0ec449e2009-04-27 19:41:25 +02001945
1946 /*
1947 * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
1948 * those are Read-As-Zero
1949 */
Borislav Petkove97f8bb2009-10-12 15:27:45 +02001950 rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
1951 debugf0(" TOP_MEM: 0x%016llx\n", pvt->top_mem);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001952
1953 /* check first whether TOP_MEM2 is enabled */
1954 rdmsrl(MSR_K8_SYSCFG, msr_val);
1955 if (msr_val & (1U << 21)) {
Borislav Petkove97f8bb2009-10-12 15:27:45 +02001956 rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
1957 debugf0(" TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001958 } else
1959 debugf0(" TOP_MEM2 disabled.\n");
1960
Borislav Petkov5980bb92011-01-07 16:26:49 +01001961 amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001962
Borislav Petkov5a5d2372011-01-17 17:52:57 +01001963 read_dram_ctl_register(pvt);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001964
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001965 for (range = 0; range < DRAM_RANGES; range++) {
1966 u8 rw;
Doug Thompson0ec449e2009-04-27 19:41:25 +02001967
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001968 /* read settings for this DRAM range */
1969 read_dram_base_limit_regs(pvt, range);
Borislav Petkove97f8bb2009-10-12 15:27:45 +02001970
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001971 rw = dram_rw(pvt, range);
1972 if (!rw)
1973 continue;
1974
1975 debugf1(" DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
1976 range,
1977 get_dram_base(pvt, range),
1978 get_dram_limit(pvt, range));
1979
1980 debugf1(" IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
1981 dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
1982 (rw & 0x1) ? "R" : "-",
1983 (rw & 0x2) ? "W" : "-",
1984 dram_intlv_sel(pvt, range),
1985 dram_dst_node(pvt, range));
Doug Thompson0ec449e2009-04-27 19:41:25 +02001986 }
1987
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001988 read_dct_base_mask(pvt);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001989
Borislav Petkovbc21fa52010-11-11 17:29:13 +01001990 amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
Borislav Petkov525a1b22010-12-21 15:53:27 +01001991 amd64_read_dct_pci_cfg(pvt, DBAM0, &pvt->dbam0);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001992
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001993 amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001994
Borislav Petkovcb328502010-12-22 14:28:24 +01001995 amd64_read_dct_pci_cfg(pvt, DCLR0, &pvt->dclr0);
1996 amd64_read_dct_pci_cfg(pvt, DCHR0, &pvt->dchr0);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001997
Borislav Petkov78da1212010-12-22 19:31:45 +01001998 if (!dct_ganging_enabled(pvt)) {
Borislav Petkovcb328502010-12-22 14:28:24 +01001999 amd64_read_dct_pci_cfg(pvt, DCLR1, &pvt->dclr1);
2000 amd64_read_dct_pci_cfg(pvt, DCHR1, &pvt->dchr1);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002001 }
Borislav Petkovad6a32e2010-03-09 12:46:00 +01002002
Borislav Petkova3b7db02011-01-19 20:35:12 +01002003 pvt->ecc_sym_sz = 4;
2004
2005 if (c->x86 >= 0x10) {
Borislav Petkovb2b0c602010-10-08 18:32:29 +02002006 amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
Borislav Petkov525a1b22010-12-21 15:53:27 +01002007 amd64_read_dct_pci_cfg(pvt, DBAM1, &pvt->dbam1);
Borislav Petkova3b7db02011-01-19 20:35:12 +01002008
2009 /* F10h, revD and later can do x8 ECC too */
2010 if ((c->x86 > 0x10 || c->x86_model > 7) && tmp & BIT(25))
2011 pvt->ecc_sym_sz = 8;
Borislav Petkov525a1b22010-12-21 15:53:27 +01002012 }
Borislav Petkovb2b0c602010-10-08 18:32:29 +02002013 dump_misc_regs(pvt);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002014}
2015
2016/*
2017 * NOTE: CPU Revision Dependent code
2018 *
2019 * Input:
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002020 * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002021 * k8 private pointer to -->
2022 * DRAM Bank Address mapping register
2023 * node_id
2024 * DCL register where dual_channel_active is
2025 *
2026 * The DBAM register consists of 4 sets of 4 bits each definitions:
2027 *
2028 * Bits: CSROWs
2029 * 0-3 CSROWs 0 and 1
2030 * 4-7 CSROWs 2 and 3
2031 * 8-11 CSROWs 4 and 5
2032 * 12-15 CSROWs 6 and 7
2033 *
2034 * Values range from: 0 to 15
2035 * The meaning of the values depends on CPU revision and dual-channel state,
2036 * see relevant BKDG more info.
2037 *
2038 * The memory controller provides for total of only 8 CSROWs in its current
2039 * architecture. Each "pair" of CSROWs normally represents just one DIMM in
2040 * single channel or two (2) DIMMs in dual channel mode.
2041 *
2042 * The following code logic collapses the various tables for CSROW based on CPU
2043 * revision.
2044 *
2045 * Returns:
2046 * The number of PAGE_SIZE pages on the specified CSROW number it
2047 * encompasses
2048 *
2049 */
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01002050static u32 amd64_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002051{
Borislav Petkov1433eb92009-10-21 13:44:36 +02002052 u32 cs_mode, nr_pages;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002053
2054 /*
2055 * The math on this doesn't look right on the surface because x/2*4 can
2056 * be simplified to x*2 but this expression makes use of the fact that
2057 * it is integral math where 1/2=0. This intermediate value becomes the
2058 * number of bits to shift the DBAM register to extract the proper CSROW
2059 * field.
2060 */
Borislav Petkov1433eb92009-10-21 13:44:36 +02002061 cs_mode = (pvt->dbam0 >> ((csrow_nr / 2) * 4)) & 0xF;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002062
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01002063 nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode) << (20 - PAGE_SHIFT);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002064
2065 /*
2066 * If dual channel then double the memory size of single channel.
2067 * Channel count is 1 or 2
2068 */
2069 nr_pages <<= (pvt->channel_count - 1);
2070
Borislav Petkov1433eb92009-10-21 13:44:36 +02002071 debugf0(" (csrow=%d) DBAM map index= %d\n", csrow_nr, cs_mode);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002072 debugf0(" nr_pages= %u channel-count = %d\n",
2073 nr_pages, pvt->channel_count);
2074
2075 return nr_pages;
2076}
2077
2078/*
2079 * Initialize the array of csrow attribute instances, based on the values
2080 * from pci config hardware registers.
2081 */
Borislav Petkov360b7f32010-10-15 19:25:38 +02002082static int init_csrows(struct mem_ctl_info *mci)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002083{
2084 struct csrow_info *csrow;
Borislav Petkov2299ef72010-10-15 17:44:04 +02002085 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002086 u64 input_addr_min, input_addr_max, sys_addr, base, mask;
Borislav Petkov2299ef72010-10-15 17:44:04 +02002087 u32 val;
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02002088 int i, empty = 1;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002089
Borislav Petkova97fa682010-12-23 14:07:18 +01002090 amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002091
Borislav Petkov2299ef72010-10-15 17:44:04 +02002092 pvt->nbcfg = val;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002093
Borislav Petkov2299ef72010-10-15 17:44:04 +02002094 debugf0("node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
2095 pvt->mc_node_id, val,
Borislav Petkova97fa682010-12-23 14:07:18 +01002096 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
Doug Thompson0ec449e2009-04-27 19:41:25 +02002097
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002098 for_each_chip_select(i, 0, pvt) {
Doug Thompson0ec449e2009-04-27 19:41:25 +02002099 csrow = &mci->csrows[i];
2100
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002101 if (!csrow_enabled(i, 0, pvt)) {
Doug Thompson0ec449e2009-04-27 19:41:25 +02002102 debugf1("----CSROW %d EMPTY for node %d\n", i,
2103 pvt->mc_node_id);
2104 continue;
2105 }
2106
2107 debugf1("----CSROW %d VALID for MC node %d\n",
2108 i, pvt->mc_node_id);
2109
2110 empty = 0;
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01002111 csrow->nr_pages = amd64_csrow_nr_pages(pvt, 0, i);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002112 find_csrow_limits(mci, i, &input_addr_min, &input_addr_max);
2113 sys_addr = input_addr_to_sys_addr(mci, input_addr_min);
2114 csrow->first_page = (u32) (sys_addr >> PAGE_SHIFT);
2115 sys_addr = input_addr_to_sys_addr(mci, input_addr_max);
2116 csrow->last_page = (u32) (sys_addr >> PAGE_SHIFT);
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002117
2118 get_cs_base_and_mask(pvt, i, 0, &base, &mask);
2119 csrow->page_mask = ~mask;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002120 /* 8 bytes of resolution */
2121
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002122 csrow->mtype = amd64_determine_memory_type(pvt, i);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002123
2124 debugf1(" for MC node %d csrow %d:\n", pvt->mc_node_id, i);
2125 debugf1(" input_addr_min: 0x%lx input_addr_max: 0x%lx\n",
2126 (unsigned long)input_addr_min,
2127 (unsigned long)input_addr_max);
2128 debugf1(" sys_addr: 0x%lx page_mask: 0x%lx\n",
2129 (unsigned long)sys_addr, csrow->page_mask);
2130 debugf1(" nr_pages: %u first_page: 0x%lx "
2131 "last_page: 0x%lx\n",
2132 (unsigned)csrow->nr_pages,
2133 csrow->first_page, csrow->last_page);
2134
2135 /*
2136 * determine whether CHIPKILL or JUST ECC or NO ECC is operating
2137 */
Borislav Petkova97fa682010-12-23 14:07:18 +01002138 if (pvt->nbcfg & NBCFG_ECC_ENABLE)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002139 csrow->edac_mode =
Borislav Petkova97fa682010-12-23 14:07:18 +01002140 (pvt->nbcfg & NBCFG_CHIPKILL) ?
Doug Thompson0ec449e2009-04-27 19:41:25 +02002141 EDAC_S4ECD4ED : EDAC_SECDED;
2142 else
2143 csrow->edac_mode = EDAC_NONE;
2144 }
2145
2146 return empty;
2147}
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002148
Borislav Petkov06724532009-09-16 13:05:46 +02002149/* get all cores on this DCT */
Borislav Petkovb487c332011-02-21 18:55:00 +01002150static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, unsigned nid)
Doug Thompsonf9431992009-04-27 19:46:08 +02002151{
Borislav Petkov06724532009-09-16 13:05:46 +02002152 int cpu;
Doug Thompsonf9431992009-04-27 19:46:08 +02002153
Borislav Petkov06724532009-09-16 13:05:46 +02002154 for_each_online_cpu(cpu)
2155 if (amd_get_nb_id(cpu) == nid)
2156 cpumask_set_cpu(cpu, mask);
Doug Thompsonf9431992009-04-27 19:46:08 +02002157}
2158
2159/* check MCG_CTL on all the cpus on this node */
Borislav Petkovb487c332011-02-21 18:55:00 +01002160static bool amd64_nb_mce_bank_enabled_on_node(unsigned nid)
Doug Thompsonf9431992009-04-27 19:46:08 +02002161{
Rusty Russellba578cb2009-11-03 14:56:35 +10302162 cpumask_var_t mask;
Borislav Petkov50542252009-12-11 18:14:40 +01002163 int cpu, nbe;
Borislav Petkov06724532009-09-16 13:05:46 +02002164 bool ret = false;
Doug Thompsonf9431992009-04-27 19:46:08 +02002165
Rusty Russellba578cb2009-11-03 14:56:35 +10302166 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002167 amd64_warn("%s: Error allocating mask\n", __func__);
Rusty Russellba578cb2009-11-03 14:56:35 +10302168 return false;
2169 }
Borislav Petkov06724532009-09-16 13:05:46 +02002170
Rusty Russellba578cb2009-11-03 14:56:35 +10302171 get_cpus_on_this_dct_cpumask(mask, nid);
Borislav Petkov06724532009-09-16 13:05:46 +02002172
Rusty Russellba578cb2009-11-03 14:56:35 +10302173 rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
Borislav Petkov06724532009-09-16 13:05:46 +02002174
Rusty Russellba578cb2009-11-03 14:56:35 +10302175 for_each_cpu(cpu, mask) {
Borislav Petkov50542252009-12-11 18:14:40 +01002176 struct msr *reg = per_cpu_ptr(msrs, cpu);
Borislav Petkov5980bb92011-01-07 16:26:49 +01002177 nbe = reg->l & MSR_MCGCTL_NBE;
Borislav Petkov06724532009-09-16 13:05:46 +02002178
2179 debugf0("core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
Borislav Petkov50542252009-12-11 18:14:40 +01002180 cpu, reg->q,
Borislav Petkov06724532009-09-16 13:05:46 +02002181 (nbe ? "enabled" : "disabled"));
2182
2183 if (!nbe)
2184 goto out;
Borislav Petkov06724532009-09-16 13:05:46 +02002185 }
2186 ret = true;
2187
2188out:
Rusty Russellba578cb2009-11-03 14:56:35 +10302189 free_cpumask_var(mask);
Doug Thompsonf9431992009-04-27 19:46:08 +02002190 return ret;
2191}
2192
Borislav Petkov2299ef72010-10-15 17:44:04 +02002193static int toggle_ecc_err_reporting(struct ecc_settings *s, u8 nid, bool on)
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002194{
2195 cpumask_var_t cmask;
Borislav Petkov50542252009-12-11 18:14:40 +01002196 int cpu;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002197
2198 if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002199 amd64_warn("%s: error allocating mask\n", __func__);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002200 return false;
2201 }
2202
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002203 get_cpus_on_this_dct_cpumask(cmask, nid);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002204
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002205 rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2206
2207 for_each_cpu(cpu, cmask) {
2208
Borislav Petkov50542252009-12-11 18:14:40 +01002209 struct msr *reg = per_cpu_ptr(msrs, cpu);
2210
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002211 if (on) {
Borislav Petkov5980bb92011-01-07 16:26:49 +01002212 if (reg->l & MSR_MCGCTL_NBE)
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002213 s->flags.nb_mce_enable = 1;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002214
Borislav Petkov5980bb92011-01-07 16:26:49 +01002215 reg->l |= MSR_MCGCTL_NBE;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002216 } else {
2217 /*
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002218 * Turn off NB MCE reporting only when it was off before
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002219 */
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002220 if (!s->flags.nb_mce_enable)
Borislav Petkov5980bb92011-01-07 16:26:49 +01002221 reg->l &= ~MSR_MCGCTL_NBE;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002222 }
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002223 }
2224 wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2225
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002226 free_cpumask_var(cmask);
2227
2228 return 0;
2229}
2230
Borislav Petkov2299ef72010-10-15 17:44:04 +02002231static bool enable_ecc_error_reporting(struct ecc_settings *s, u8 nid,
2232 struct pci_dev *F3)
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002233{
Borislav Petkov2299ef72010-10-15 17:44:04 +02002234 bool ret = true;
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002235 u32 value, mask = 0x3; /* UECC/CECC enable */
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002236
Borislav Petkov2299ef72010-10-15 17:44:04 +02002237 if (toggle_ecc_err_reporting(s, nid, ON)) {
2238 amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
2239 return false;
2240 }
2241
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002242 amd64_read_pci_cfg(F3, NBCTL, &value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002243
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002244 s->old_nbctl = value & mask;
2245 s->nbctl_valid = true;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002246
2247 value |= mask;
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002248 amd64_write_pci_cfg(F3, NBCTL, value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002249
Borislav Petkova97fa682010-12-23 14:07:18 +01002250 amd64_read_pci_cfg(F3, NBCFG, &value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002251
Borislav Petkova97fa682010-12-23 14:07:18 +01002252 debugf0("1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2253 nid, value, !!(value & NBCFG_ECC_ENABLE));
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002254
Borislav Petkova97fa682010-12-23 14:07:18 +01002255 if (!(value & NBCFG_ECC_ENABLE)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002256 amd64_warn("DRAM ECC disabled on this node, enabling...\n");
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002257
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002258 s->flags.nb_ecc_prev = 0;
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002259
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002260 /* Attempt to turn on DRAM ECC Enable */
Borislav Petkova97fa682010-12-23 14:07:18 +01002261 value |= NBCFG_ECC_ENABLE;
2262 amd64_write_pci_cfg(F3, NBCFG, value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002263
Borislav Petkova97fa682010-12-23 14:07:18 +01002264 amd64_read_pci_cfg(F3, NBCFG, &value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002265
Borislav Petkova97fa682010-12-23 14:07:18 +01002266 if (!(value & NBCFG_ECC_ENABLE)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002267 amd64_warn("Hardware rejected DRAM ECC enable,"
2268 "check memory DIMM configuration.\n");
Borislav Petkov2299ef72010-10-15 17:44:04 +02002269 ret = false;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002270 } else {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002271 amd64_info("Hardware accepted DRAM ECC Enable\n");
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002272 }
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002273 } else {
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002274 s->flags.nb_ecc_prev = 1;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002275 }
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002276
Borislav Petkova97fa682010-12-23 14:07:18 +01002277 debugf0("2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2278 nid, value, !!(value & NBCFG_ECC_ENABLE));
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002279
Borislav Petkov2299ef72010-10-15 17:44:04 +02002280 return ret;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002281}
2282
Borislav Petkov360b7f32010-10-15 19:25:38 +02002283static void restore_ecc_error_reporting(struct ecc_settings *s, u8 nid,
2284 struct pci_dev *F3)
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002285{
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002286 u32 value, mask = 0x3; /* UECC/CECC enable */
2287
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002288
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002289 if (!s->nbctl_valid)
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002290 return;
2291
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002292 amd64_read_pci_cfg(F3, NBCTL, &value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002293 value &= ~mask;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002294 value |= s->old_nbctl;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002295
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002296 amd64_write_pci_cfg(F3, NBCTL, value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002297
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002298 /* restore previous BIOS DRAM ECC "off" setting we force-enabled */
2299 if (!s->flags.nb_ecc_prev) {
Borislav Petkova97fa682010-12-23 14:07:18 +01002300 amd64_read_pci_cfg(F3, NBCFG, &value);
2301 value &= ~NBCFG_ECC_ENABLE;
2302 amd64_write_pci_cfg(F3, NBCFG, value);
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002303 }
2304
2305 /* restore the NB Enable MCGCTL bit */
Borislav Petkov2299ef72010-10-15 17:44:04 +02002306 if (toggle_ecc_err_reporting(s, nid, OFF))
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002307 amd64_warn("Error restoring NB MCGCTL settings!\n");
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002308}
2309
Doug Thompsonf9431992009-04-27 19:46:08 +02002310/*
Borislav Petkov2299ef72010-10-15 17:44:04 +02002311 * EDAC requires that the BIOS have ECC enabled before
2312 * taking over the processing of ECC errors. A command line
2313 * option allows to force-enable hardware ECC later in
2314 * enable_ecc_error_reporting().
Doug Thompsonf9431992009-04-27 19:46:08 +02002315 */
Borislav Petkovcab4d272010-02-11 17:15:57 +01002316static const char *ecc_msg =
2317 "ECC disabled in the BIOS or no ECC capability, module will not load.\n"
2318 " Either enable ECC checking or force module loading by setting "
2319 "'ecc_enable_override'.\n"
2320 " (Note that use of the override may cause unknown side effects.)\n";
Borislav Petkovbe3468e2009-08-05 15:47:22 +02002321
Borislav Petkov2299ef72010-10-15 17:44:04 +02002322static bool ecc_enabled(struct pci_dev *F3, u8 nid)
Doug Thompsonf9431992009-04-27 19:46:08 +02002323{
2324 u32 value;
Borislav Petkov2299ef72010-10-15 17:44:04 +02002325 u8 ecc_en = 0;
Borislav Petkov06724532009-09-16 13:05:46 +02002326 bool nb_mce_en = false;
Doug Thompsonf9431992009-04-27 19:46:08 +02002327
Borislav Petkova97fa682010-12-23 14:07:18 +01002328 amd64_read_pci_cfg(F3, NBCFG, &value);
Doug Thompsonf9431992009-04-27 19:46:08 +02002329
Borislav Petkova97fa682010-12-23 14:07:18 +01002330 ecc_en = !!(value & NBCFG_ECC_ENABLE);
Borislav Petkov2299ef72010-10-15 17:44:04 +02002331 amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled"));
Doug Thompsonf9431992009-04-27 19:46:08 +02002332
Borislav Petkov2299ef72010-10-15 17:44:04 +02002333 nb_mce_en = amd64_nb_mce_bank_enabled_on_node(nid);
Borislav Petkov06724532009-09-16 13:05:46 +02002334 if (!nb_mce_en)
Borislav Petkov2299ef72010-10-15 17:44:04 +02002335 amd64_notice("NB MCE bank disabled, set MSR "
2336 "0x%08x[4] on node %d to enable.\n",
2337 MSR_IA32_MCG_CTL, nid);
Doug Thompsonf9431992009-04-27 19:46:08 +02002338
Borislav Petkov2299ef72010-10-15 17:44:04 +02002339 if (!ecc_en || !nb_mce_en) {
2340 amd64_notice("%s", ecc_msg);
2341 return false;
Borislav Petkov43f5e682009-12-21 18:55:18 +01002342 }
Borislav Petkov2299ef72010-10-15 17:44:04 +02002343 return true;
Doug Thompsonf9431992009-04-27 19:46:08 +02002344}
2345
Doug Thompson7d6034d2009-04-27 20:01:01 +02002346struct mcidev_sysfs_attribute sysfs_attrs[ARRAY_SIZE(amd64_dbg_attrs) +
2347 ARRAY_SIZE(amd64_inj_attrs) +
2348 1];
2349
2350struct mcidev_sysfs_attribute terminator = { .attr = { .name = NULL } };
2351
Borislav Petkov360b7f32010-10-15 19:25:38 +02002352static void set_mc_sysfs_attrs(struct mem_ctl_info *mci)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002353{
2354 unsigned int i = 0, j = 0;
2355
2356 for (; i < ARRAY_SIZE(amd64_dbg_attrs); i++)
2357 sysfs_attrs[i] = amd64_dbg_attrs[i];
2358
Borislav Petkova135cef2010-11-26 19:24:44 +01002359 if (boot_cpu_data.x86 >= 0x10)
2360 for (j = 0; j < ARRAY_SIZE(amd64_inj_attrs); j++, i++)
2361 sysfs_attrs[i] = amd64_inj_attrs[j];
Doug Thompson7d6034d2009-04-27 20:01:01 +02002362
2363 sysfs_attrs[i] = terminator;
2364
2365 mci->mc_driver_sysfs_attributes = sysfs_attrs;
2366}
2367
Borislav Petkovdf71a052011-01-19 18:15:10 +01002368static void setup_mci_misc_attrs(struct mem_ctl_info *mci,
2369 struct amd64_family_type *fam)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002370{
2371 struct amd64_pvt *pvt = mci->pvt_info;
2372
2373 mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
2374 mci->edac_ctl_cap = EDAC_FLAG_NONE;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002375
Borislav Petkov5980bb92011-01-07 16:26:49 +01002376 if (pvt->nbcap & NBCAP_SECDED)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002377 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
2378
Borislav Petkov5980bb92011-01-07 16:26:49 +01002379 if (pvt->nbcap & NBCAP_CHIPKILL)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002380 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
2381
2382 mci->edac_cap = amd64_determine_edac_cap(pvt);
2383 mci->mod_name = EDAC_MOD_STR;
2384 mci->mod_ver = EDAC_AMD64_VERSION;
Borislav Petkovdf71a052011-01-19 18:15:10 +01002385 mci->ctl_name = fam->ctl_name;
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002386 mci->dev_name = pci_name(pvt->F2);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002387 mci->ctl_page_to_phys = NULL;
2388
Doug Thompson7d6034d2009-04-27 20:01:01 +02002389 /* memory scrubber interface */
2390 mci->set_sdram_scrub_rate = amd64_set_scrub_rate;
2391 mci->get_sdram_scrub_rate = amd64_get_scrub_rate;
2392}
2393
Borislav Petkov0092b202010-10-01 19:20:05 +02002394/*
2395 * returns a pointer to the family descriptor on success, NULL otherwise.
2396 */
2397static struct amd64_family_type *amd64_per_family_init(struct amd64_pvt *pvt)
Borislav Petkov395ae782010-10-01 18:38:19 +02002398{
Borislav Petkov0092b202010-10-01 19:20:05 +02002399 u8 fam = boot_cpu_data.x86;
2400 struct amd64_family_type *fam_type = NULL;
2401
2402 switch (fam) {
Borislav Petkov395ae782010-10-01 18:38:19 +02002403 case 0xf:
Borislav Petkov0092b202010-10-01 19:20:05 +02002404 fam_type = &amd64_family_types[K8_CPUS];
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002405 pvt->ops = &amd64_family_types[K8_CPUS].ops;
Borislav Petkov395ae782010-10-01 18:38:19 +02002406 break;
Borislav Petkovdf71a052011-01-19 18:15:10 +01002407
Borislav Petkov395ae782010-10-01 18:38:19 +02002408 case 0x10:
Borislav Petkov0092b202010-10-01 19:20:05 +02002409 fam_type = &amd64_family_types[F10_CPUS];
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002410 pvt->ops = &amd64_family_types[F10_CPUS].ops;
Borislav Petkovdf71a052011-01-19 18:15:10 +01002411 break;
2412
2413 case 0x15:
2414 fam_type = &amd64_family_types[F15_CPUS];
2415 pvt->ops = &amd64_family_types[F15_CPUS].ops;
Borislav Petkov395ae782010-10-01 18:38:19 +02002416 break;
2417
2418 default:
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002419 amd64_err("Unsupported family!\n");
Borislav Petkov0092b202010-10-01 19:20:05 +02002420 return NULL;
Borislav Petkov395ae782010-10-01 18:38:19 +02002421 }
Borislav Petkov0092b202010-10-01 19:20:05 +02002422
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002423 pvt->ext_model = boot_cpu_data.x86_model >> 4;
2424
Borislav Petkovdf71a052011-01-19 18:15:10 +01002425 amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name,
Borislav Petkov0092b202010-10-01 19:20:05 +02002426 (fam == 0xf ?
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002427 (pvt->ext_model >= K8_REV_F ? "revF or later "
2428 : "revE or earlier ")
2429 : ""), pvt->mc_node_id);
Borislav Petkov0092b202010-10-01 19:20:05 +02002430 return fam_type;
Borislav Petkov395ae782010-10-01 18:38:19 +02002431}
2432
Borislav Petkov2299ef72010-10-15 17:44:04 +02002433static int amd64_init_one_instance(struct pci_dev *F2)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002434{
2435 struct amd64_pvt *pvt = NULL;
Borislav Petkov0092b202010-10-01 19:20:05 +02002436 struct amd64_family_type *fam_type = NULL;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002437 struct mem_ctl_info *mci = NULL;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002438 int err = 0, ret;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002439 u8 nid = get_node_id(F2);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002440
2441 ret = -ENOMEM;
2442 pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
2443 if (!pvt)
Borislav Petkov360b7f32010-10-15 19:25:38 +02002444 goto err_ret;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002445
Borislav Petkov360b7f32010-10-15 19:25:38 +02002446 pvt->mc_node_id = nid;
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002447 pvt->F2 = F2;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002448
Borislav Petkov395ae782010-10-01 18:38:19 +02002449 ret = -EINVAL;
Borislav Petkov0092b202010-10-01 19:20:05 +02002450 fam_type = amd64_per_family_init(pvt);
2451 if (!fam_type)
Borislav Petkov395ae782010-10-01 18:38:19 +02002452 goto err_free;
2453
Doug Thompson7d6034d2009-04-27 20:01:01 +02002454 ret = -ENODEV;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002455 err = reserve_mc_sibling_devs(pvt, fam_type->f1_id, fam_type->f3_id);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002456 if (err)
2457 goto err_free;
2458
Borislav Petkov360b7f32010-10-15 19:25:38 +02002459 read_mc_regs(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002460
Doug Thompson7d6034d2009-04-27 20:01:01 +02002461 /*
2462 * We need to determine how many memory channels there are. Then use
2463 * that information for calculating the size of the dynamic instance
Borislav Petkov360b7f32010-10-15 19:25:38 +02002464 * tables in the 'mci' structure.
Doug Thompson7d6034d2009-04-27 20:01:01 +02002465 */
Borislav Petkov360b7f32010-10-15 19:25:38 +02002466 ret = -EINVAL;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002467 pvt->channel_count = pvt->ops->early_channel_count(pvt);
2468 if (pvt->channel_count < 0)
Borislav Petkov360b7f32010-10-15 19:25:38 +02002469 goto err_siblings;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002470
2471 ret = -ENOMEM;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002472 mci = edac_mc_alloc(0, pvt->csels[0].b_cnt, pvt->channel_count, nid);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002473 if (!mci)
Borislav Petkov360b7f32010-10-15 19:25:38 +02002474 goto err_siblings;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002475
2476 mci->pvt_info = pvt;
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002477 mci->dev = &pvt->F2->dev;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002478
Borislav Petkovdf71a052011-01-19 18:15:10 +01002479 setup_mci_misc_attrs(mci, fam_type);
Borislav Petkov360b7f32010-10-15 19:25:38 +02002480
2481 if (init_csrows(mci))
Doug Thompson7d6034d2009-04-27 20:01:01 +02002482 mci->edac_cap = EDAC_FLAG_NONE;
2483
Borislav Petkov360b7f32010-10-15 19:25:38 +02002484 set_mc_sysfs_attrs(mci);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002485
2486 ret = -ENODEV;
2487 if (edac_mc_add_mc(mci)) {
2488 debugf1("failed edac_mc_add_mc()\n");
2489 goto err_add_mc;
2490 }
2491
Borislav Petkov549d0422009-07-24 13:51:42 +02002492 /* register stuff with EDAC MCE */
2493 if (report_gart_errors)
2494 amd_report_gart_errors(true);
2495
2496 amd_register_ecc_decoder(amd64_decode_bus_error);
2497
Borislav Petkov360b7f32010-10-15 19:25:38 +02002498 mcis[nid] = mci;
2499
2500 atomic_inc(&drv_instances);
2501
Doug Thompson7d6034d2009-04-27 20:01:01 +02002502 return 0;
2503
2504err_add_mc:
2505 edac_mc_free(mci);
2506
Borislav Petkov360b7f32010-10-15 19:25:38 +02002507err_siblings:
2508 free_mc_sibling_devs(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002509
Borislav Petkov360b7f32010-10-15 19:25:38 +02002510err_free:
2511 kfree(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002512
Borislav Petkov360b7f32010-10-15 19:25:38 +02002513err_ret:
Doug Thompson7d6034d2009-04-27 20:01:01 +02002514 return ret;
2515}
2516
Borislav Petkov2299ef72010-10-15 17:44:04 +02002517static int __devinit amd64_probe_one_instance(struct pci_dev *pdev,
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002518 const struct pci_device_id *mc_type)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002519{
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002520 u8 nid = get_node_id(pdev);
Borislav Petkov2299ef72010-10-15 17:44:04 +02002521 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002522 struct ecc_settings *s;
Borislav Petkov2299ef72010-10-15 17:44:04 +02002523 int ret = 0;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002524
Doug Thompson7d6034d2009-04-27 20:01:01 +02002525 ret = pci_enable_device(pdev);
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002526 if (ret < 0) {
Doug Thompson7d6034d2009-04-27 20:01:01 +02002527 debugf0("ret=%d\n", ret);
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002528 return -EIO;
2529 }
2530
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002531 ret = -ENOMEM;
2532 s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
2533 if (!s)
Borislav Petkov2299ef72010-10-15 17:44:04 +02002534 goto err_out;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002535
2536 ecc_stngs[nid] = s;
2537
Borislav Petkov2299ef72010-10-15 17:44:04 +02002538 if (!ecc_enabled(F3, nid)) {
2539 ret = -ENODEV;
2540
2541 if (!ecc_enable_override)
2542 goto err_enable;
2543
2544 amd64_warn("Forcing ECC on!\n");
2545
2546 if (!enable_ecc_error_reporting(s, nid, F3))
2547 goto err_enable;
2548 }
2549
2550 ret = amd64_init_one_instance(pdev);
Borislav Petkov360b7f32010-10-15 19:25:38 +02002551 if (ret < 0) {
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002552 amd64_err("Error probing instance: %d\n", nid);
Borislav Petkov360b7f32010-10-15 19:25:38 +02002553 restore_ecc_error_reporting(s, nid, F3);
2554 }
Doug Thompson7d6034d2009-04-27 20:01:01 +02002555
2556 return ret;
Borislav Petkov2299ef72010-10-15 17:44:04 +02002557
2558err_enable:
2559 kfree(s);
2560 ecc_stngs[nid] = NULL;
2561
2562err_out:
2563 return ret;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002564}
2565
2566static void __devexit amd64_remove_one_instance(struct pci_dev *pdev)
2567{
2568 struct mem_ctl_info *mci;
2569 struct amd64_pvt *pvt;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002570 u8 nid = get_node_id(pdev);
2571 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2572 struct ecc_settings *s = ecc_stngs[nid];
Doug Thompson7d6034d2009-04-27 20:01:01 +02002573
2574 /* Remove from EDAC CORE tracking list */
2575 mci = edac_mc_del_mc(&pdev->dev);
2576 if (!mci)
2577 return;
2578
2579 pvt = mci->pvt_info;
2580
Borislav Petkov360b7f32010-10-15 19:25:38 +02002581 restore_ecc_error_reporting(s, nid, F3);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002582
Borislav Petkov360b7f32010-10-15 19:25:38 +02002583 free_mc_sibling_devs(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002584
Borislav Petkov549d0422009-07-24 13:51:42 +02002585 /* unregister from EDAC MCE */
2586 amd_report_gart_errors(false);
2587 amd_unregister_ecc_decoder(amd64_decode_bus_error);
2588
Borislav Petkov360b7f32010-10-15 19:25:38 +02002589 kfree(ecc_stngs[nid]);
2590 ecc_stngs[nid] = NULL;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002591
Doug Thompson7d6034d2009-04-27 20:01:01 +02002592 /* Free the EDAC CORE resources */
Borislav Petkov8f68ed92009-12-21 15:15:59 +01002593 mci->pvt_info = NULL;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002594 mcis[nid] = NULL;
Borislav Petkov8f68ed92009-12-21 15:15:59 +01002595
2596 kfree(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002597 edac_mc_free(mci);
2598}
2599
2600/*
2601 * This table is part of the interface for loading drivers for PCI devices. The
2602 * PCI core identifies what devices are on a system during boot, and then
2603 * inquiry this table to see if this driver is for a given device found.
2604 */
2605static const struct pci_device_id amd64_pci_table[] __devinitdata = {
2606 {
2607 .vendor = PCI_VENDOR_ID_AMD,
2608 .device = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
2609 .subvendor = PCI_ANY_ID,
2610 .subdevice = PCI_ANY_ID,
2611 .class = 0,
2612 .class_mask = 0,
Doug Thompson7d6034d2009-04-27 20:01:01 +02002613 },
2614 {
2615 .vendor = PCI_VENDOR_ID_AMD,
2616 .device = PCI_DEVICE_ID_AMD_10H_NB_DRAM,
2617 .subvendor = PCI_ANY_ID,
2618 .subdevice = PCI_ANY_ID,
2619 .class = 0,
2620 .class_mask = 0,
Doug Thompson7d6034d2009-04-27 20:01:01 +02002621 },
Borislav Petkovdf71a052011-01-19 18:15:10 +01002622 {
2623 .vendor = PCI_VENDOR_ID_AMD,
2624 .device = PCI_DEVICE_ID_AMD_15H_NB_F2,
2625 .subvendor = PCI_ANY_ID,
2626 .subdevice = PCI_ANY_ID,
2627 .class = 0,
2628 .class_mask = 0,
2629 },
2630
Doug Thompson7d6034d2009-04-27 20:01:01 +02002631 {0, }
2632};
2633MODULE_DEVICE_TABLE(pci, amd64_pci_table);
2634
2635static struct pci_driver amd64_pci_driver = {
2636 .name = EDAC_MOD_STR,
Borislav Petkov2299ef72010-10-15 17:44:04 +02002637 .probe = amd64_probe_one_instance,
Doug Thompson7d6034d2009-04-27 20:01:01 +02002638 .remove = __devexit_p(amd64_remove_one_instance),
2639 .id_table = amd64_pci_table,
2640};
2641
Borislav Petkov360b7f32010-10-15 19:25:38 +02002642static void setup_pci_device(void)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002643{
2644 struct mem_ctl_info *mci;
2645 struct amd64_pvt *pvt;
2646
2647 if (amd64_ctl_pci)
2648 return;
2649
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002650 mci = mcis[0];
Doug Thompson7d6034d2009-04-27 20:01:01 +02002651 if (mci) {
2652
2653 pvt = mci->pvt_info;
2654 amd64_ctl_pci =
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002655 edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002656
2657 if (!amd64_ctl_pci) {
2658 pr_warning("%s(): Unable to create PCI control\n",
2659 __func__);
2660
2661 pr_warning("%s(): PCI error report via EDAC not set\n",
2662 __func__);
2663 }
2664 }
2665}
2666
2667static int __init amd64_edac_init(void)
2668{
Borislav Petkov360b7f32010-10-15 19:25:38 +02002669 int err = -ENODEV;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002670
Borislav Petkovdf71a052011-01-19 18:15:10 +01002671 printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002672
2673 opstate_init();
2674
Hans Rosenfeld9653a5c2010-10-29 17:14:31 +02002675 if (amd_cache_northbridges() < 0)
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01002676 goto err_ret;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002677
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002678 err = -ENOMEM;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002679 mcis = kzalloc(amd_nb_num() * sizeof(mcis[0]), GFP_KERNEL);
2680 ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL);
Borislav Petkov360b7f32010-10-15 19:25:38 +02002681 if (!(mcis && ecc_stngs))
Borislav Petkova9f0fbe2011-03-29 18:10:53 +02002682 goto err_free;
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002683
Borislav Petkov50542252009-12-11 18:14:40 +01002684 msrs = msrs_alloc();
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01002685 if (!msrs)
Borislav Petkov360b7f32010-10-15 19:25:38 +02002686 goto err_free;
Borislav Petkov50542252009-12-11 18:14:40 +01002687
Doug Thompson7d6034d2009-04-27 20:01:01 +02002688 err = pci_register_driver(&amd64_pci_driver);
2689 if (err)
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01002690 goto err_pci;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002691
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01002692 err = -ENODEV;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002693 if (!atomic_read(&drv_instances))
2694 goto err_no_instances;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002695
Borislav Petkov360b7f32010-10-15 19:25:38 +02002696 setup_pci_device();
2697 return 0;
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01002698
Borislav Petkov360b7f32010-10-15 19:25:38 +02002699err_no_instances:
Doug Thompson7d6034d2009-04-27 20:01:01 +02002700 pci_unregister_driver(&amd64_pci_driver);
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002701
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01002702err_pci:
2703 msrs_free(msrs);
2704 msrs = NULL;
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002705
Borislav Petkov360b7f32010-10-15 19:25:38 +02002706err_free:
2707 kfree(mcis);
2708 mcis = NULL;
2709
2710 kfree(ecc_stngs);
2711 ecc_stngs = NULL;
2712
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01002713err_ret:
Doug Thompson7d6034d2009-04-27 20:01:01 +02002714 return err;
2715}
2716
2717static void __exit amd64_edac_exit(void)
2718{
2719 if (amd64_ctl_pci)
2720 edac_pci_release_generic_ctl(amd64_ctl_pci);
2721
2722 pci_unregister_driver(&amd64_pci_driver);
Borislav Petkov50542252009-12-11 18:14:40 +01002723
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002724 kfree(ecc_stngs);
2725 ecc_stngs = NULL;
2726
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002727 kfree(mcis);
2728 mcis = NULL;
2729
Borislav Petkov50542252009-12-11 18:14:40 +01002730 msrs_free(msrs);
2731 msrs = NULL;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002732}
2733
2734module_init(amd64_edac_init);
2735module_exit(amd64_edac_exit);
2736
2737MODULE_LICENSE("GPL");
2738MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
2739 "Dave Peterson, Thayne Harbaugh");
2740MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
2741 EDAC_AMD64_VERSION);
2742
2743module_param(edac_op_state, int, 0444);
2744MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");