blob: 7ef73c919c5da24a0e260a600f97c775e7fb8117 [file] [log] [blame]
Doug Thompson2bc65412009-05-04 20:11:14 +02001#include "amd64_edac.h"
Andreas Herrmann23ac4ae2010-09-17 18:03:43 +02002#include <asm/amd_nb.h>
Doug Thompson2bc65412009-05-04 20:11:14 +02003
4static struct edac_pci_ctl_info *amd64_ctl_pci;
5
6static int report_gart_errors;
7module_param(report_gart_errors, int, 0644);
8
9/*
10 * Set by command line parameter. If BIOS has enabled the ECC, this override is
11 * cleared to prevent re-enabling the hardware by this driver.
12 */
13static int ecc_enable_override;
14module_param(ecc_enable_override, int, 0644);
15
Tejun Heoa29d8b82010-02-02 14:39:15 +090016static struct msr __percpu *msrs;
Borislav Petkov50542252009-12-11 18:14:40 +010017
Borislav Petkov360b7f32010-10-15 19:25:38 +020018/*
19 * count successfully initialized driver instances for setup_pci_device()
20 */
21static atomic_t drv_instances = ATOMIC_INIT(0);
22
Borislav Petkovcc4d8862010-10-13 16:11:59 +020023/* Per-node driver instances */
24static struct mem_ctl_info **mcis;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +020025static struct ecc_settings **ecc_stngs;
Doug Thompson2bc65412009-05-04 20:11:14 +020026
27/*
Borislav Petkovb70ef012009-06-25 19:32:38 +020028 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
29 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
30 * or higher value'.
31 *
32 *FIXME: Produce a better mapping/linearisation.
33 */
Borislav Petkov39094442010-11-24 19:52:09 +010034struct scrubrate {
35 u32 scrubval; /* bit pattern for scrub rate */
36 u32 bandwidth; /* bandwidth consumed (bytes/sec) */
37} scrubrates[] = {
Borislav Petkovb70ef012009-06-25 19:32:38 +020038 { 0x01, 1600000000UL},
39 { 0x02, 800000000UL},
40 { 0x03, 400000000UL},
41 { 0x04, 200000000UL},
42 { 0x05, 100000000UL},
43 { 0x06, 50000000UL},
44 { 0x07, 25000000UL},
45 { 0x08, 12284069UL},
46 { 0x09, 6274509UL},
47 { 0x0A, 3121951UL},
48 { 0x0B, 1560975UL},
49 { 0x0C, 781440UL},
50 { 0x0D, 390720UL},
51 { 0x0E, 195300UL},
52 { 0x0F, 97650UL},
53 { 0x10, 48854UL},
54 { 0x11, 24427UL},
55 { 0x12, 12213UL},
56 { 0x13, 6101UL},
57 { 0x14, 3051UL},
58 { 0x15, 1523UL},
59 { 0x16, 761UL},
60 { 0x00, 0UL}, /* scrubbing off */
61};
62
Borislav Petkovb2b0c602010-10-08 18:32:29 +020063static int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
64 u32 *val, const char *func)
65{
66 int err = 0;
67
68 err = pci_read_config_dword(pdev, offset, val);
69 if (err)
70 amd64_warn("%s: error reading F%dx%03x.\n",
71 func, PCI_FUNC(pdev->devfn), offset);
72
73 return err;
74}
75
76int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
77 u32 val, const char *func)
78{
79 int err = 0;
80
81 err = pci_write_config_dword(pdev, offset, val);
82 if (err)
83 amd64_warn("%s: error writing to F%dx%03x.\n",
84 func, PCI_FUNC(pdev->devfn), offset);
85
86 return err;
87}
88
89/*
90 *
91 * Depending on the family, F2 DCT reads need special handling:
92 *
93 * K8: has a single DCT only
94 *
95 * F10h: each DCT has its own set of regs
96 * DCT0 -> F2x040..
97 * DCT1 -> F2x140..
98 *
99 * F15h: we select which DCT we access using F1x10C[DctCfgSel]
100 *
101 */
102static int k8_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
103 const char *func)
104{
105 if (addr >= 0x100)
106 return -EINVAL;
107
108 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
109}
110
111static int f10_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
112 const char *func)
113{
114 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
115}
116
Borislav Petkov73ba8592011-09-19 17:34:45 +0200117/*
118 * Select DCT to which PCI cfg accesses are routed
119 */
120static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct)
121{
122 u32 reg = 0;
123
124 amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, &reg);
125 reg &= 0xfffffffe;
126 reg |= dct;
127 amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
128}
129
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200130static int f15_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
131 const char *func)
132{
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200133 u8 dct = 0;
134
135 if (addr >= 0x140 && addr <= 0x1a0) {
136 dct = 1;
137 addr -= 0x100;
138 }
139
Borislav Petkov73ba8592011-09-19 17:34:45 +0200140 f15h_select_dct(pvt, dct);
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200141
142 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
143}
144
Borislav Petkovb70ef012009-06-25 19:32:38 +0200145/*
Doug Thompson2bc65412009-05-04 20:11:14 +0200146 * Memory scrubber control interface. For K8, memory scrubbing is handled by
147 * hardware and can involve L2 cache, dcache as well as the main memory. With
148 * F10, this is extended to L3 cache scrubbing on CPU models sporting that
149 * functionality.
150 *
151 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
152 * (dram) over to cache lines. This is nasty, so we will use bandwidth in
153 * bytes/sec for the setting.
154 *
155 * Currently, we only do dram scrubbing. If the scrubbing is done in software on
156 * other archs, we might not have access to the caches directly.
157 */
158
159/*
160 * scan the scrub rate mapping table for a close or matching bandwidth value to
161 * issue. If requested is too big, then use last maximum value found.
162 */
Borislav Petkov395ae782010-10-01 18:38:19 +0200163static int __amd64_set_scrub_rate(struct pci_dev *ctl, u32 new_bw, u32 min_rate)
Doug Thompson2bc65412009-05-04 20:11:14 +0200164{
165 u32 scrubval;
166 int i;
167
168 /*
169 * map the configured rate (new_bw) to a value specific to the AMD64
170 * memory controller and apply to register. Search for the first
171 * bandwidth entry that is greater or equal than the setting requested
172 * and program that. If at last entry, turn off DRAM scrubbing.
173 */
174 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
175 /*
176 * skip scrub rates which aren't recommended
177 * (see F10 BKDG, F3x58)
178 */
Borislav Petkov395ae782010-10-01 18:38:19 +0200179 if (scrubrates[i].scrubval < min_rate)
Doug Thompson2bc65412009-05-04 20:11:14 +0200180 continue;
181
182 if (scrubrates[i].bandwidth <= new_bw)
183 break;
184
185 /*
186 * if no suitable bandwidth found, turn off DRAM scrubbing
187 * entirely by falling back to the last element in the
188 * scrubrates array.
189 */
190 }
191
192 scrubval = scrubrates[i].scrubval;
Doug Thompson2bc65412009-05-04 20:11:14 +0200193
Borislav Petkov5980bb92011-01-07 16:26:49 +0100194 pci_write_bits32(ctl, SCRCTRL, scrubval, 0x001F);
Doug Thompson2bc65412009-05-04 20:11:14 +0200195
Borislav Petkov39094442010-11-24 19:52:09 +0100196 if (scrubval)
197 return scrubrates[i].bandwidth;
198
Doug Thompson2bc65412009-05-04 20:11:14 +0200199 return 0;
200}
201
Borislav Petkov395ae782010-10-01 18:38:19 +0200202static int amd64_set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
Doug Thompson2bc65412009-05-04 20:11:14 +0200203{
204 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkov87b3e0e2011-01-19 20:02:38 +0100205 u32 min_scrubrate = 0x5;
Doug Thompson2bc65412009-05-04 20:11:14 +0200206
Borislav Petkov87b3e0e2011-01-19 20:02:38 +0100207 if (boot_cpu_data.x86 == 0xf)
208 min_scrubrate = 0x0;
209
Borislav Petkov73ba8592011-09-19 17:34:45 +0200210 /* F15h Erratum #505 */
211 if (boot_cpu_data.x86 == 0x15)
212 f15h_select_dct(pvt, 0);
213
Borislav Petkov87b3e0e2011-01-19 20:02:38 +0100214 return __amd64_set_scrub_rate(pvt->F3, bw, min_scrubrate);
Doug Thompson2bc65412009-05-04 20:11:14 +0200215}
216
Borislav Petkov39094442010-11-24 19:52:09 +0100217static int amd64_get_scrub_rate(struct mem_ctl_info *mci)
Doug Thompson2bc65412009-05-04 20:11:14 +0200218{
219 struct amd64_pvt *pvt = mci->pvt_info;
220 u32 scrubval = 0;
Borislav Petkov39094442010-11-24 19:52:09 +0100221 int i, retval = -EINVAL;
Doug Thompson2bc65412009-05-04 20:11:14 +0200222
Borislav Petkov73ba8592011-09-19 17:34:45 +0200223 /* F15h Erratum #505 */
224 if (boot_cpu_data.x86 == 0x15)
225 f15h_select_dct(pvt, 0);
226
Borislav Petkov5980bb92011-01-07 16:26:49 +0100227 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
Doug Thompson2bc65412009-05-04 20:11:14 +0200228
229 scrubval = scrubval & 0x001F;
230
Roel Kluin926311f2010-01-11 20:58:21 +0100231 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
Doug Thompson2bc65412009-05-04 20:11:14 +0200232 if (scrubrates[i].scrubval == scrubval) {
Borislav Petkov39094442010-11-24 19:52:09 +0100233 retval = scrubrates[i].bandwidth;
Doug Thompson2bc65412009-05-04 20:11:14 +0200234 break;
235 }
236 }
Borislav Petkov39094442010-11-24 19:52:09 +0100237 return retval;
Doug Thompson2bc65412009-05-04 20:11:14 +0200238}
239
Doug Thompson67757632009-04-27 15:53:22 +0200240/*
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200241 * returns true if the SysAddr given by sys_addr matches the
242 * DRAM base/limit associated with node_id
Doug Thompson67757632009-04-27 15:53:22 +0200243 */
Borislav Petkovb487c332011-02-21 18:55:00 +0100244static bool amd64_base_limit_match(struct amd64_pvt *pvt, u64 sys_addr,
245 unsigned nid)
Doug Thompson67757632009-04-27 15:53:22 +0200246{
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200247 u64 addr;
Doug Thompson67757632009-04-27 15:53:22 +0200248
249 /* The K8 treats this as a 40-bit value. However, bits 63-40 will be
250 * all ones if the most significant implemented address bit is 1.
251 * Here we discard bits 63-40. See section 3.4.2 of AMD publication
252 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
253 * Application Programming.
254 */
255 addr = sys_addr & 0x000000ffffffffffull;
256
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200257 return ((addr >= get_dram_base(pvt, nid)) &&
258 (addr <= get_dram_limit(pvt, nid)));
Doug Thompson67757632009-04-27 15:53:22 +0200259}
260
261/*
262 * Attempt to map a SysAddr to a node. On success, return a pointer to the
263 * mem_ctl_info structure for the node that the SysAddr maps to.
264 *
265 * On failure, return NULL.
266 */
267static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
268 u64 sys_addr)
269{
270 struct amd64_pvt *pvt;
Borislav Petkovb487c332011-02-21 18:55:00 +0100271 unsigned node_id;
Doug Thompson67757632009-04-27 15:53:22 +0200272 u32 intlv_en, bits;
273
274 /*
275 * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
276 * 3.4.4.2) registers to map the SysAddr to a node ID.
277 */
278 pvt = mci->pvt_info;
279
280 /*
281 * The value of this field should be the same for all DRAM Base
282 * registers. Therefore we arbitrarily choose to read it from the
283 * register for node 0.
284 */
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200285 intlv_en = dram_intlv_en(pvt, 0);
Doug Thompson67757632009-04-27 15:53:22 +0200286
287 if (intlv_en == 0) {
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200288 for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
Doug Thompson67757632009-04-27 15:53:22 +0200289 if (amd64_base_limit_match(pvt, sys_addr, node_id))
Borislav Petkov8edc5442009-09-18 12:39:19 +0200290 goto found;
Doug Thompson67757632009-04-27 15:53:22 +0200291 }
Borislav Petkov8edc5442009-09-18 12:39:19 +0200292 goto err_no_match;
Doug Thompson67757632009-04-27 15:53:22 +0200293 }
294
Borislav Petkov72f158f2009-09-18 12:27:27 +0200295 if (unlikely((intlv_en != 0x01) &&
296 (intlv_en != 0x03) &&
297 (intlv_en != 0x07))) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200298 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
Doug Thompson67757632009-04-27 15:53:22 +0200299 return NULL;
300 }
301
302 bits = (((u32) sys_addr) >> 12) & intlv_en;
303
304 for (node_id = 0; ; ) {
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200305 if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
Doug Thompson67757632009-04-27 15:53:22 +0200306 break; /* intlv_sel field matches */
307
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200308 if (++node_id >= DRAM_RANGES)
Doug Thompson67757632009-04-27 15:53:22 +0200309 goto err_no_match;
310 }
311
312 /* sanity test for sys_addr */
313 if (unlikely(!amd64_base_limit_match(pvt, sys_addr, node_id))) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200314 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
315 "range for node %d with node interleaving enabled.\n",
316 __func__, sys_addr, node_id);
Doug Thompson67757632009-04-27 15:53:22 +0200317 return NULL;
318 }
319
320found:
Borislav Petkovb487c332011-02-21 18:55:00 +0100321 return edac_mc_find((int)node_id);
Doug Thompson67757632009-04-27 15:53:22 +0200322
323err_no_match:
324 debugf2("sys_addr 0x%lx doesn't match any node\n",
325 (unsigned long)sys_addr);
326
327 return NULL;
328}
Doug Thompsone2ce7252009-04-27 15:57:12 +0200329
330/*
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100331 * compute the CS base address of the @csrow on the DRAM controller @dct.
332 * For details see F2x[5C:40] in the processor's BKDG
Doug Thompsone2ce7252009-04-27 15:57:12 +0200333 */
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100334static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
335 u64 *base, u64 *mask)
Doug Thompsone2ce7252009-04-27 15:57:12 +0200336{
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100337 u64 csbase, csmask, base_bits, mask_bits;
338 u8 addr_shift;
339
340 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) {
341 csbase = pvt->csels[dct].csbases[csrow];
342 csmask = pvt->csels[dct].csmasks[csrow];
343 base_bits = GENMASK(21, 31) | GENMASK(9, 15);
344 mask_bits = GENMASK(21, 29) | GENMASK(9, 15);
345 addr_shift = 4;
346 } else {
347 csbase = pvt->csels[dct].csbases[csrow];
348 csmask = pvt->csels[dct].csmasks[csrow >> 1];
349 addr_shift = 8;
350
351 if (boot_cpu_data.x86 == 0x15)
352 base_bits = mask_bits = GENMASK(19,30) | GENMASK(5,13);
353 else
354 base_bits = mask_bits = GENMASK(19,28) | GENMASK(5,13);
355 }
356
357 *base = (csbase & base_bits) << addr_shift;
358
359 *mask = ~0ULL;
360 /* poke holes for the csmask */
361 *mask &= ~(mask_bits << addr_shift);
362 /* OR them in */
363 *mask |= (csmask & mask_bits) << addr_shift;
Doug Thompsone2ce7252009-04-27 15:57:12 +0200364}
365
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100366#define for_each_chip_select(i, dct, pvt) \
367 for (i = 0; i < pvt->csels[dct].b_cnt; i++)
Doug Thompsone2ce7252009-04-27 15:57:12 +0200368
Borislav Petkov614ec9d2011-01-13 18:02:22 +0100369#define chip_select_base(i, dct, pvt) \
370 pvt->csels[dct].csbases[i]
371
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100372#define for_each_chip_select_mask(i, dct, pvt) \
373 for (i = 0; i < pvt->csels[dct].m_cnt; i++)
Doug Thompsone2ce7252009-04-27 15:57:12 +0200374
375/*
376 * @input_addr is an InputAddr associated with the node given by mci. Return the
377 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
378 */
379static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
380{
381 struct amd64_pvt *pvt;
382 int csrow;
383 u64 base, mask;
384
385 pvt = mci->pvt_info;
386
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100387 for_each_chip_select(csrow, 0, pvt) {
388 if (!csrow_enabled(csrow, 0, pvt))
Doug Thompsone2ce7252009-04-27 15:57:12 +0200389 continue;
390
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100391 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
392
393 mask = ~mask;
Doug Thompsone2ce7252009-04-27 15:57:12 +0200394
395 if ((input_addr & mask) == (base & mask)) {
396 debugf2("InputAddr 0x%lx matches csrow %d (node %d)\n",
397 (unsigned long)input_addr, csrow,
398 pvt->mc_node_id);
399
400 return csrow;
401 }
402 }
Doug Thompsone2ce7252009-04-27 15:57:12 +0200403 debugf2("no matching csrow for InputAddr 0x%lx (MC node %d)\n",
404 (unsigned long)input_addr, pvt->mc_node_id);
405
406 return -1;
407}
408
409/*
Doug Thompsone2ce7252009-04-27 15:57:12 +0200410 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
411 * for the node represented by mci. Info is passed back in *hole_base,
412 * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if
413 * info is invalid. Info may be invalid for either of the following reasons:
414 *
415 * - The revision of the node is not E or greater. In this case, the DRAM Hole
416 * Address Register does not exist.
417 *
418 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
419 * indicating that its contents are not valid.
420 *
421 * The values passed back in *hole_base, *hole_offset, and *hole_size are
422 * complete 32-bit values despite the fact that the bitfields in the DHAR
423 * only represent bits 31-24 of the base and offset values.
424 */
425int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
426 u64 *hole_offset, u64 *hole_size)
427{
428 struct amd64_pvt *pvt = mci->pvt_info;
429 u64 base;
430
431 /* only revE and later have the DRAM Hole Address Register */
Borislav Petkov1433eb92009-10-21 13:44:36 +0200432 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_E) {
Doug Thompsone2ce7252009-04-27 15:57:12 +0200433 debugf1(" revision %d for node %d does not support DHAR\n",
434 pvt->ext_model, pvt->mc_node_id);
435 return 1;
436 }
437
Borislav Petkovbc21fa52010-11-11 17:29:13 +0100438 /* valid for Fam10h and above */
Borislav Petkovc8e518d2010-12-10 19:49:19 +0100439 if (boot_cpu_data.x86 >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
Doug Thompsone2ce7252009-04-27 15:57:12 +0200440 debugf1(" Dram Memory Hoisting is DISABLED on this system\n");
441 return 1;
442 }
443
Borislav Petkovc8e518d2010-12-10 19:49:19 +0100444 if (!dhar_valid(pvt)) {
Doug Thompsone2ce7252009-04-27 15:57:12 +0200445 debugf1(" Dram Memory Hoisting is DISABLED on this node %d\n",
446 pvt->mc_node_id);
447 return 1;
448 }
449
450 /* This node has Memory Hoisting */
451
452 /* +------------------+--------------------+--------------------+-----
453 * | memory | DRAM hole | relocated |
454 * | [0, (x - 1)] | [x, 0xffffffff] | addresses from |
455 * | | | DRAM hole |
456 * | | | [0x100000000, |
457 * | | | (0x100000000+ |
458 * | | | (0xffffffff-x))] |
459 * +------------------+--------------------+--------------------+-----
460 *
461 * Above is a diagram of physical memory showing the DRAM hole and the
462 * relocated addresses from the DRAM hole. As shown, the DRAM hole
463 * starts at address x (the base address) and extends through address
464 * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the
465 * addresses in the hole so that they start at 0x100000000.
466 */
467
Borislav Petkovbc21fa52010-11-11 17:29:13 +0100468 base = dhar_base(pvt);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200469
470 *hole_base = base;
471 *hole_size = (0x1ull << 32) - base;
472
473 if (boot_cpu_data.x86 > 0xf)
Borislav Petkovbc21fa52010-11-11 17:29:13 +0100474 *hole_offset = f10_dhar_offset(pvt);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200475 else
Borislav Petkovbc21fa52010-11-11 17:29:13 +0100476 *hole_offset = k8_dhar_offset(pvt);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200477
478 debugf1(" DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
479 pvt->mc_node_id, (unsigned long)*hole_base,
480 (unsigned long)*hole_offset, (unsigned long)*hole_size);
481
482 return 0;
483}
484EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info);
485
Doug Thompson93c2df52009-05-04 20:46:50 +0200486/*
487 * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is
488 * assumed that sys_addr maps to the node given by mci.
489 *
490 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
491 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
492 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
493 * then it is also involved in translating a SysAddr to a DramAddr. Sections
494 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
495 * These parts of the documentation are unclear. I interpret them as follows:
496 *
497 * When node n receives a SysAddr, it processes the SysAddr as follows:
498 *
499 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
500 * Limit registers for node n. If the SysAddr is not within the range
501 * specified by the base and limit values, then node n ignores the Sysaddr
502 * (since it does not map to node n). Otherwise continue to step 2 below.
503 *
504 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
505 * disabled so skip to step 3 below. Otherwise see if the SysAddr is within
506 * the range of relocated addresses (starting at 0x100000000) from the DRAM
507 * hole. If not, skip to step 3 below. Else get the value of the
508 * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
509 * offset defined by this value from the SysAddr.
510 *
511 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
512 * Base register for node n. To obtain the DramAddr, subtract the base
513 * address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
514 */
515static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
516{
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200517 struct amd64_pvt *pvt = mci->pvt_info;
Doug Thompson93c2df52009-05-04 20:46:50 +0200518 u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
519 int ret = 0;
520
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200521 dram_base = get_dram_base(pvt, pvt->mc_node_id);
Doug Thompson93c2df52009-05-04 20:46:50 +0200522
523 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
524 &hole_size);
525 if (!ret) {
526 if ((sys_addr >= (1ull << 32)) &&
527 (sys_addr < ((1ull << 32) + hole_size))) {
528 /* use DHAR to translate SysAddr to DramAddr */
529 dram_addr = sys_addr - hole_offset;
530
531 debugf2("using DHAR to translate SysAddr 0x%lx to "
532 "DramAddr 0x%lx\n",
533 (unsigned long)sys_addr,
534 (unsigned long)dram_addr);
535
536 return dram_addr;
537 }
538 }
539
540 /*
541 * Translate the SysAddr to a DramAddr as shown near the start of
542 * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8
543 * only deals with 40-bit values. Therefore we discard bits 63-40 of
544 * sys_addr below. If bit 39 of sys_addr is 1 then the bits we
545 * discard are all 1s. Otherwise the bits we discard are all 0s. See
546 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
547 * Programmer's Manual Volume 1 Application Programming.
548 */
Borislav Petkovf678b8c2010-12-13 19:21:07 +0100549 dram_addr = (sys_addr & GENMASK(0, 39)) - dram_base;
Doug Thompson93c2df52009-05-04 20:46:50 +0200550
551 debugf2("using DRAM Base register to translate SysAddr 0x%lx to "
552 "DramAddr 0x%lx\n", (unsigned long)sys_addr,
553 (unsigned long)dram_addr);
554 return dram_addr;
555}
556
557/*
558 * @intlv_en is the value of the IntlvEn field from a DRAM Base register
559 * (section 3.4.4.1). Return the number of bits from a SysAddr that are used
560 * for node interleaving.
561 */
562static int num_node_interleave_bits(unsigned intlv_en)
563{
564 static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
565 int n;
566
567 BUG_ON(intlv_en > 7);
568 n = intlv_shift_table[intlv_en];
569 return n;
570}
571
572/* Translate the DramAddr given by @dram_addr to an InputAddr. */
573static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
574{
575 struct amd64_pvt *pvt;
576 int intlv_shift;
577 u64 input_addr;
578
579 pvt = mci->pvt_info;
580
581 /*
582 * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
583 * concerning translating a DramAddr to an InputAddr.
584 */
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200585 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
Borislav Petkovf678b8c2010-12-13 19:21:07 +0100586 input_addr = ((dram_addr >> intlv_shift) & GENMASK(12, 35)) +
587 (dram_addr & 0xfff);
Doug Thompson93c2df52009-05-04 20:46:50 +0200588
589 debugf2(" Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
590 intlv_shift, (unsigned long)dram_addr,
591 (unsigned long)input_addr);
592
593 return input_addr;
594}
595
596/*
597 * Translate the SysAddr represented by @sys_addr to an InputAddr. It is
598 * assumed that @sys_addr maps to the node given by mci.
599 */
600static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
601{
602 u64 input_addr;
603
604 input_addr =
605 dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
606
607 debugf2("SysAdddr 0x%lx translates to InputAddr 0x%lx\n",
608 (unsigned long)sys_addr, (unsigned long)input_addr);
609
610 return input_addr;
611}
612
613
614/*
615 * @input_addr is an InputAddr associated with the node represented by mci.
616 * Translate @input_addr to a DramAddr and return the result.
617 */
618static u64 input_addr_to_dram_addr(struct mem_ctl_info *mci, u64 input_addr)
619{
620 struct amd64_pvt *pvt;
Borislav Petkovb487c332011-02-21 18:55:00 +0100621 unsigned node_id, intlv_shift;
Doug Thompson93c2df52009-05-04 20:46:50 +0200622 u64 bits, dram_addr;
623 u32 intlv_sel;
624
625 /*
626 * Near the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
627 * shows how to translate a DramAddr to an InputAddr. Here we reverse
628 * this procedure. When translating from a DramAddr to an InputAddr, the
629 * bits used for node interleaving are discarded. Here we recover these
630 * bits from the IntlvSel field of the DRAM Limit register (section
631 * 3.4.4.2) for the node that input_addr is associated with.
632 */
633 pvt = mci->pvt_info;
634 node_id = pvt->mc_node_id;
Borislav Petkovb487c332011-02-21 18:55:00 +0100635
636 BUG_ON(node_id > 7);
Doug Thompson93c2df52009-05-04 20:46:50 +0200637
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200638 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
Doug Thompson93c2df52009-05-04 20:46:50 +0200639 if (intlv_shift == 0) {
640 debugf1(" InputAddr 0x%lx translates to DramAddr of "
641 "same value\n", (unsigned long)input_addr);
642
643 return input_addr;
644 }
645
Borislav Petkovf678b8c2010-12-13 19:21:07 +0100646 bits = ((input_addr & GENMASK(12, 35)) << intlv_shift) +
647 (input_addr & 0xfff);
Doug Thompson93c2df52009-05-04 20:46:50 +0200648
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200649 intlv_sel = dram_intlv_sel(pvt, node_id) & ((1 << intlv_shift) - 1);
Doug Thompson93c2df52009-05-04 20:46:50 +0200650 dram_addr = bits + (intlv_sel << 12);
651
652 debugf1("InputAddr 0x%lx translates to DramAddr 0x%lx "
653 "(%d node interleave bits)\n", (unsigned long)input_addr,
654 (unsigned long)dram_addr, intlv_shift);
655
656 return dram_addr;
657}
658
659/*
660 * @dram_addr is a DramAddr that maps to the node represented by mci. Convert
661 * @dram_addr to a SysAddr.
662 */
663static u64 dram_addr_to_sys_addr(struct mem_ctl_info *mci, u64 dram_addr)
664{
665 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200666 u64 hole_base, hole_offset, hole_size, base, sys_addr;
Doug Thompson93c2df52009-05-04 20:46:50 +0200667 int ret = 0;
668
669 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
670 &hole_size);
671 if (!ret) {
672 if ((dram_addr >= hole_base) &&
673 (dram_addr < (hole_base + hole_size))) {
674 sys_addr = dram_addr + hole_offset;
675
676 debugf1("using DHAR to translate DramAddr 0x%lx to "
677 "SysAddr 0x%lx\n", (unsigned long)dram_addr,
678 (unsigned long)sys_addr);
679
680 return sys_addr;
681 }
682 }
683
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200684 base = get_dram_base(pvt, pvt->mc_node_id);
Doug Thompson93c2df52009-05-04 20:46:50 +0200685 sys_addr = dram_addr + base;
686
687 /*
688 * The sys_addr we have computed up to this point is a 40-bit value
689 * because the k8 deals with 40-bit values. However, the value we are
690 * supposed to return is a full 64-bit physical address. The AMD
691 * x86-64 architecture specifies that the most significant implemented
692 * address bit through bit 63 of a physical address must be either all
693 * 0s or all 1s. Therefore we sign-extend the 40-bit sys_addr to a
694 * 64-bit value below. See section 3.4.2 of AMD publication 24592:
695 * AMD x86-64 Architecture Programmer's Manual Volume 1 Application
696 * Programming.
697 */
698 sys_addr |= ~((sys_addr & (1ull << 39)) - 1);
699
700 debugf1(" Node %d, DramAddr 0x%lx to SysAddr 0x%lx\n",
701 pvt->mc_node_id, (unsigned long)dram_addr,
702 (unsigned long)sys_addr);
703
704 return sys_addr;
705}
706
707/*
708 * @input_addr is an InputAddr associated with the node given by mci. Translate
709 * @input_addr to a SysAddr.
710 */
711static inline u64 input_addr_to_sys_addr(struct mem_ctl_info *mci,
712 u64 input_addr)
713{
714 return dram_addr_to_sys_addr(mci,
715 input_addr_to_dram_addr(mci, input_addr));
716}
717
718/*
719 * Find the minimum and maximum InputAddr values that map to the given @csrow.
720 * Pass back these values in *input_addr_min and *input_addr_max.
721 */
722static void find_csrow_limits(struct mem_ctl_info *mci, int csrow,
723 u64 *input_addr_min, u64 *input_addr_max)
724{
725 struct amd64_pvt *pvt;
726 u64 base, mask;
727
728 pvt = mci->pvt_info;
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100729 BUG_ON((csrow < 0) || (csrow >= pvt->csels[0].b_cnt));
Doug Thompson93c2df52009-05-04 20:46:50 +0200730
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100731 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
Doug Thompson93c2df52009-05-04 20:46:50 +0200732
733 *input_addr_min = base & ~mask;
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100734 *input_addr_max = base | mask;
Doug Thompson93c2df52009-05-04 20:46:50 +0200735}
736
Doug Thompson93c2df52009-05-04 20:46:50 +0200737/* Map the Error address to a PAGE and PAGE OFFSET. */
738static inline void error_address_to_page_and_offset(u64 error_address,
739 u32 *page, u32 *offset)
740{
741 *page = (u32) (error_address >> PAGE_SHIFT);
742 *offset = ((u32) error_address) & ~PAGE_MASK;
743}
744
745/*
746 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
747 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
748 * of a node that detected an ECC memory error. mci represents the node that
749 * the error address maps to (possibly different from the node that detected
750 * the error). Return the number of the csrow that sys_addr maps to, or -1 on
751 * error.
752 */
753static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
754{
755 int csrow;
756
757 csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
758
759 if (csrow == -1)
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200760 amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
761 "address 0x%lx\n", (unsigned long)sys_addr);
Doug Thompson93c2df52009-05-04 20:46:50 +0200762 return csrow;
763}
Doug Thompsone2ce7252009-04-27 15:57:12 +0200764
Borislav Petkovbfc04ae2009-11-12 19:05:07 +0100765static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
Doug Thompson2da11652009-04-27 16:09:09 +0200766
Doug Thompson2da11652009-04-27 16:09:09 +0200767/*
768 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
769 * are ECC capable.
770 */
Dan Carpenter1f6189e2011-10-06 02:30:25 -0400771static unsigned long amd64_determine_edac_cap(struct amd64_pvt *pvt)
Doug Thompson2da11652009-04-27 16:09:09 +0200772{
Borislav Petkovcb328502010-12-22 14:28:24 +0100773 u8 bit;
Dan Carpenter1f6189e2011-10-06 02:30:25 -0400774 unsigned long edac_cap = EDAC_FLAG_NONE;
Doug Thompson2da11652009-04-27 16:09:09 +0200775
Borislav Petkov1433eb92009-10-21 13:44:36 +0200776 bit = (boot_cpu_data.x86 > 0xf || pvt->ext_model >= K8_REV_F)
Doug Thompson2da11652009-04-27 16:09:09 +0200777 ? 19
778 : 17;
779
Borislav Petkov584fcff2009-06-10 18:29:54 +0200780 if (pvt->dclr0 & BIT(bit))
Doug Thompson2da11652009-04-27 16:09:09 +0200781 edac_cap = EDAC_FLAG_SECDED;
782
783 return edac_cap;
784}
785
Borislav Petkov8c671752011-02-23 17:25:12 +0100786static void amd64_debug_display_dimm_sizes(struct amd64_pvt *, u8);
Doug Thompson2da11652009-04-27 16:09:09 +0200787
Borislav Petkov68798e12009-11-03 16:18:33 +0100788static void amd64_dump_dramcfg_low(u32 dclr, int chan)
789{
790 debugf1("F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
791
792 debugf1(" DIMM type: %sbuffered; all DIMMs support ECC: %s\n",
793 (dclr & BIT(16)) ? "un" : "",
794 (dclr & BIT(19)) ? "yes" : "no");
795
796 debugf1(" PAR/ERR parity: %s\n",
797 (dclr & BIT(8)) ? "enabled" : "disabled");
798
Borislav Petkovcb328502010-12-22 14:28:24 +0100799 if (boot_cpu_data.x86 == 0x10)
800 debugf1(" DCT 128bit mode width: %s\n",
801 (dclr & BIT(11)) ? "128b" : "64b");
Borislav Petkov68798e12009-11-03 16:18:33 +0100802
803 debugf1(" x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
804 (dclr & BIT(12)) ? "yes" : "no",
805 (dclr & BIT(13)) ? "yes" : "no",
806 (dclr & BIT(14)) ? "yes" : "no",
807 (dclr & BIT(15)) ? "yes" : "no");
808}
809
Doug Thompson2da11652009-04-27 16:09:09 +0200810/* Display and decode various NB registers for debug purposes. */
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200811static void dump_misc_regs(struct amd64_pvt *pvt)
Doug Thompson2da11652009-04-27 16:09:09 +0200812{
Borislav Petkov68798e12009-11-03 16:18:33 +0100813 debugf1("F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
Doug Thompson2da11652009-04-27 16:09:09 +0200814
Borislav Petkov68798e12009-11-03 16:18:33 +0100815 debugf1(" NB two channel DRAM capable: %s\n",
Borislav Petkov5980bb92011-01-07 16:26:49 +0100816 (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");
Borislav Petkov68798e12009-11-03 16:18:33 +0100817
818 debugf1(" ECC capable: %s, ChipKill ECC capable: %s\n",
Borislav Petkov5980bb92011-01-07 16:26:49 +0100819 (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
820 (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");
Borislav Petkov68798e12009-11-03 16:18:33 +0100821
822 amd64_dump_dramcfg_low(pvt->dclr0, 0);
Doug Thompson2da11652009-04-27 16:09:09 +0200823
Borislav Petkov8de1d912009-10-16 13:39:30 +0200824 debugf1("F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
Doug Thompson2da11652009-04-27 16:09:09 +0200825
Borislav Petkov8de1d912009-10-16 13:39:30 +0200826 debugf1("F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, "
827 "offset: 0x%08x\n",
Borislav Petkovbc21fa52010-11-11 17:29:13 +0100828 pvt->dhar, dhar_base(pvt),
829 (boot_cpu_data.x86 == 0xf) ? k8_dhar_offset(pvt)
830 : f10_dhar_offset(pvt));
Doug Thompson2da11652009-04-27 16:09:09 +0200831
Borislav Petkovc8e518d2010-12-10 19:49:19 +0100832 debugf1(" DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
Doug Thompson2da11652009-04-27 16:09:09 +0200833
Borislav Petkov8c671752011-02-23 17:25:12 +0100834 amd64_debug_display_dimm_sizes(pvt, 0);
Borislav Petkov4d796362011-02-03 15:59:57 +0100835
Borislav Petkov8de1d912009-10-16 13:39:30 +0200836 /* everything below this point is Fam10h and above */
Borislav Petkov4d796362011-02-03 15:59:57 +0100837 if (boot_cpu_data.x86 == 0xf)
Doug Thompson2da11652009-04-27 16:09:09 +0200838 return;
Borislav Petkov4d796362011-02-03 15:59:57 +0100839
Borislav Petkov8c671752011-02-23 17:25:12 +0100840 amd64_debug_display_dimm_sizes(pvt, 1);
Doug Thompson2da11652009-04-27 16:09:09 +0200841
Borislav Petkova3b7db02011-01-19 20:35:12 +0100842 amd64_info("using %s syndromes.\n", ((pvt->ecc_sym_sz == 8) ? "x8" : "x4"));
Borislav Petkovad6a32e2010-03-09 12:46:00 +0100843
Borislav Petkov8de1d912009-10-16 13:39:30 +0200844 /* Only if NOT ganged does dclr1 have valid info */
Borislav Petkov68798e12009-11-03 16:18:33 +0100845 if (!dct_ganging_enabled(pvt))
846 amd64_dump_dramcfg_low(pvt->dclr1, 1);
Doug Thompson2da11652009-04-27 16:09:09 +0200847}
848
Doug Thompson94be4bf2009-04-27 16:12:00 +0200849/*
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100850 * see BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
Doug Thompson94be4bf2009-04-27 16:12:00 +0200851 */
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100852static void prep_chip_selects(struct amd64_pvt *pvt)
Doug Thompson94be4bf2009-04-27 16:12:00 +0200853{
Borislav Petkov1433eb92009-10-21 13:44:36 +0200854 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) {
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100855 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
856 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
Borislav Petkov9d858bb2009-09-21 14:35:51 +0200857 } else {
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100858 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
859 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
Doug Thompson94be4bf2009-04-27 16:12:00 +0200860 }
861}
862
863/*
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100864 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
Doug Thompson94be4bf2009-04-27 16:12:00 +0200865 */
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200866static void read_dct_base_mask(struct amd64_pvt *pvt)
Doug Thompson94be4bf2009-04-27 16:12:00 +0200867{
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100868 int cs;
Doug Thompson94be4bf2009-04-27 16:12:00 +0200869
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100870 prep_chip_selects(pvt);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200871
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100872 for_each_chip_select(cs, 0, pvt) {
Borislav Petkov71d2a322011-02-21 19:37:24 +0100873 int reg0 = DCSB0 + (cs * 4);
874 int reg1 = DCSB1 + (cs * 4);
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100875 u32 *base0 = &pvt->csels[0].csbases[cs];
876 u32 *base1 = &pvt->csels[1].csbases[cs];
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200877
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100878 if (!amd64_read_dct_pci_cfg(pvt, reg0, base0))
Doug Thompson94be4bf2009-04-27 16:12:00 +0200879 debugf0(" DCSB0[%d]=0x%08x reg: F2x%x\n",
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100880 cs, *base0, reg0);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200881
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100882 if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt))
883 continue;
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200884
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100885 if (!amd64_read_dct_pci_cfg(pvt, reg1, base1))
886 debugf0(" DCSB1[%d]=0x%08x reg: F2x%x\n",
887 cs, *base1, reg1);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200888 }
889
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100890 for_each_chip_select_mask(cs, 0, pvt) {
Borislav Petkov71d2a322011-02-21 19:37:24 +0100891 int reg0 = DCSM0 + (cs * 4);
892 int reg1 = DCSM1 + (cs * 4);
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100893 u32 *mask0 = &pvt->csels[0].csmasks[cs];
894 u32 *mask1 = &pvt->csels[1].csmasks[cs];
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200895
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100896 if (!amd64_read_dct_pci_cfg(pvt, reg0, mask0))
Doug Thompson94be4bf2009-04-27 16:12:00 +0200897 debugf0(" DCSM0[%d]=0x%08x reg: F2x%x\n",
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100898 cs, *mask0, reg0);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200899
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100900 if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt))
901 continue;
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200902
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100903 if (!amd64_read_dct_pci_cfg(pvt, reg1, mask1))
904 debugf0(" DCSM1[%d]=0x%08x reg: F2x%x\n",
905 cs, *mask1, reg1);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200906 }
907}
908
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200909static enum mem_type amd64_determine_memory_type(struct amd64_pvt *pvt, int cs)
Doug Thompson94be4bf2009-04-27 16:12:00 +0200910{
911 enum mem_type type;
912
Borislav Petkovcb328502010-12-22 14:28:24 +0100913 /* F15h supports only DDR3 */
914 if (boot_cpu_data.x86 >= 0x15)
915 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
916 else if (boot_cpu_data.x86 == 0x10 || pvt->ext_model >= K8_REV_F) {
Borislav Petkov6b4c0bd2009-11-12 15:37:57 +0100917 if (pvt->dchr0 & DDR3_MODE)
918 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
919 else
920 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
Doug Thompson94be4bf2009-04-27 16:12:00 +0200921 } else {
Doug Thompson94be4bf2009-04-27 16:12:00 +0200922 type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
923 }
924
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200925 amd64_info("CS%d: %s\n", cs, edac_mem_types[type]);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200926
927 return type;
928}
929
Borislav Petkovcb328502010-12-22 14:28:24 +0100930/* Get the number of DCT channels the memory controller is using. */
Doug Thompsonddff8762009-04-27 16:14:52 +0200931static int k8_early_channel_count(struct amd64_pvt *pvt)
932{
Borislav Petkovcb328502010-12-22 14:28:24 +0100933 int flag;
Doug Thompsonddff8762009-04-27 16:14:52 +0200934
Borislav Petkov9f56da02010-10-01 19:44:53 +0200935 if (pvt->ext_model >= K8_REV_F)
Doug Thompsonddff8762009-04-27 16:14:52 +0200936 /* RevF (NPT) and later */
Borislav Petkov41d8bfa2011-01-18 19:16:08 +0100937 flag = pvt->dclr0 & WIDTH_128;
Borislav Petkov9f56da02010-10-01 19:44:53 +0200938 else
Doug Thompsonddff8762009-04-27 16:14:52 +0200939 /* RevE and earlier */
940 flag = pvt->dclr0 & REVE_WIDTH_128;
Doug Thompsonddff8762009-04-27 16:14:52 +0200941
942 /* not used */
943 pvt->dclr1 = 0;
944
945 return (flag) ? 2 : 1;
946}
947
Borislav Petkov70046622011-01-10 14:37:27 +0100948/* On F10h and later ErrAddr is MC4_ADDR[47:1] */
949static u64 get_error_address(struct mce *m)
Doug Thompsonddff8762009-04-27 16:14:52 +0200950{
Borislav Petkovc1ae6832011-03-30 15:42:10 +0200951 struct cpuinfo_x86 *c = &boot_cpu_data;
952 u64 addr;
Borislav Petkov70046622011-01-10 14:37:27 +0100953 u8 start_bit = 1;
954 u8 end_bit = 47;
955
Borislav Petkovc1ae6832011-03-30 15:42:10 +0200956 if (c->x86 == 0xf) {
Borislav Petkov70046622011-01-10 14:37:27 +0100957 start_bit = 3;
958 end_bit = 39;
959 }
960
Borislav Petkovc1ae6832011-03-30 15:42:10 +0200961 addr = m->addr & GENMASK(start_bit, end_bit);
962
963 /*
964 * Erratum 637 workaround
965 */
966 if (c->x86 == 0x15) {
967 struct amd64_pvt *pvt;
968 u64 cc6_base, tmp_addr;
969 u32 tmp;
970 u8 mce_nid, intlv_en;
971
972 if ((addr & GENMASK(24, 47)) >> 24 != 0x00fdf7)
973 return addr;
974
975 mce_nid = amd_get_nb_id(m->extcpu);
976 pvt = mcis[mce_nid]->pvt_info;
977
978 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp);
979 intlv_en = tmp >> 21 & 0x7;
980
981 /* add [47:27] + 3 trailing bits */
982 cc6_base = (tmp & GENMASK(0, 20)) << 3;
983
984 /* reverse and add DramIntlvEn */
985 cc6_base |= intlv_en ^ 0x7;
986
987 /* pin at [47:24] */
988 cc6_base <<= 24;
989
990 if (!intlv_en)
991 return cc6_base | (addr & GENMASK(0, 23));
992
993 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp);
994
995 /* faster log2 */
996 tmp_addr = (addr & GENMASK(12, 23)) << __fls(intlv_en + 1);
997
998 /* OR DramIntlvSel into bits [14:12] */
999 tmp_addr |= (tmp & GENMASK(21, 23)) >> 9;
1000
1001 /* add remaining [11:0] bits from original MC4_ADDR */
1002 tmp_addr |= addr & GENMASK(0, 11);
1003
1004 return cc6_base | tmp_addr;
1005 }
1006
1007 return addr;
Doug Thompsonddff8762009-04-27 16:14:52 +02001008}
1009
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001010static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
Doug Thompsonddff8762009-04-27 16:14:52 +02001011{
Borislav Petkovf08e4572011-03-21 20:45:06 +01001012 struct cpuinfo_x86 *c = &boot_cpu_data;
Borislav Petkov71d2a322011-02-21 19:37:24 +01001013 int off = range << 3;
Doug Thompsonddff8762009-04-27 16:14:52 +02001014
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001015 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo);
1016 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
Doug Thompsonddff8762009-04-27 16:14:52 +02001017
Borislav Petkovf08e4572011-03-21 20:45:06 +01001018 if (c->x86 == 0xf)
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001019 return;
Doug Thompsonddff8762009-04-27 16:14:52 +02001020
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001021 if (!dram_rw(pvt, range))
1022 return;
Doug Thompsonddff8762009-04-27 16:14:52 +02001023
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001024 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi);
1025 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
Borislav Petkovf08e4572011-03-21 20:45:06 +01001026
1027 /* Factor in CC6 save area by reading dst node's limit reg */
1028 if (c->x86 == 0x15) {
1029 struct pci_dev *f1 = NULL;
1030 u8 nid = dram_dst_node(pvt, range);
1031 u32 llim;
1032
1033 f1 = pci_get_domain_bus_and_slot(0, 0, PCI_DEVFN(0x18 + nid, 1));
1034 if (WARN_ON(!f1))
1035 return;
1036
1037 amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim);
1038
1039 pvt->ranges[range].lim.lo &= GENMASK(0, 15);
1040
1041 /* {[39:27],111b} */
1042 pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16;
1043
1044 pvt->ranges[range].lim.hi &= GENMASK(0, 7);
1045
1046 /* [47:40] */
1047 pvt->ranges[range].lim.hi |= llim >> 13;
1048
1049 pci_dev_put(f1);
1050 }
Doug Thompsonddff8762009-04-27 16:14:52 +02001051}
1052
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001053static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
1054 u16 syndrome)
Doug Thompsonddff8762009-04-27 16:14:52 +02001055{
1056 struct mem_ctl_info *src_mci;
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001057 struct amd64_pvt *pvt = mci->pvt_info;
Doug Thompsonddff8762009-04-27 16:14:52 +02001058 int channel, csrow;
1059 u32 page, offset;
Doug Thompsonddff8762009-04-27 16:14:52 +02001060
1061 /* CHIPKILL enabled */
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001062 if (pvt->nbcfg & NBCFG_CHIPKILL) {
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001063 channel = get_channel_from_ecc_syndrome(mci, syndrome);
Doug Thompsonddff8762009-04-27 16:14:52 +02001064 if (channel < 0) {
1065 /*
1066 * Syndrome didn't map, so we don't know which of the
1067 * 2 DIMMs is in error. So we need to ID 'both' of them
1068 * as suspect.
1069 */
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001070 amd64_mc_warn(mci, "unknown syndrome 0x%04x - possible "
1071 "error reporting race\n", syndrome);
Doug Thompsonddff8762009-04-27 16:14:52 +02001072 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1073 return;
1074 }
1075 } else {
1076 /*
1077 * non-chipkill ecc mode
1078 *
1079 * The k8 documentation is unclear about how to determine the
1080 * channel number when using non-chipkill memory. This method
1081 * was obtained from email communication with someone at AMD.
1082 * (Wish the email was placed in this comment - norsk)
1083 */
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001084 channel = ((sys_addr & BIT(3)) != 0);
Doug Thompsonddff8762009-04-27 16:14:52 +02001085 }
1086
1087 /*
1088 * Find out which node the error address belongs to. This may be
1089 * different from the node that detected the error.
1090 */
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001091 src_mci = find_mc_by_sys_addr(mci, sys_addr);
Keith Mannthey2cff18c2009-09-18 14:35:23 +02001092 if (!src_mci) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001093 amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001094 (unsigned long)sys_addr);
Doug Thompsonddff8762009-04-27 16:14:52 +02001095 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1096 return;
1097 }
1098
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001099 /* Now map the sys_addr to a CSROW */
1100 csrow = sys_addr_to_csrow(src_mci, sys_addr);
Doug Thompsonddff8762009-04-27 16:14:52 +02001101 if (csrow < 0) {
1102 edac_mc_handle_ce_no_info(src_mci, EDAC_MOD_STR);
1103 } else {
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001104 error_address_to_page_and_offset(sys_addr, &page, &offset);
Doug Thompsonddff8762009-04-27 16:14:52 +02001105
1106 edac_mc_handle_ce(src_mci, page, offset, syndrome, csrow,
1107 channel, EDAC_MOD_STR);
1108 }
1109}
1110
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001111static int ddr2_cs_size(unsigned i, bool dct_width)
Doug Thompsonddff8762009-04-27 16:14:52 +02001112{
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001113 unsigned shift = 0;
Doug Thompsonddff8762009-04-27 16:14:52 +02001114
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001115 if (i <= 2)
1116 shift = i;
1117 else if (!(i & 0x1))
1118 shift = i >> 1;
Borislav Petkov1433eb92009-10-21 13:44:36 +02001119 else
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001120 shift = (i + 1) >> 1;
Doug Thompsonddff8762009-04-27 16:14:52 +02001121
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001122 return 128 << (shift + !!dct_width);
1123}
1124
1125static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1126 unsigned cs_mode)
1127{
1128 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1129
1130 if (pvt->ext_model >= K8_REV_F) {
1131 WARN_ON(cs_mode > 11);
1132 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1133 }
1134 else if (pvt->ext_model >= K8_REV_D) {
Borislav Petkov11b0a312011-11-09 21:28:43 +01001135 unsigned diff;
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001136 WARN_ON(cs_mode > 10);
1137
Borislav Petkov11b0a312011-11-09 21:28:43 +01001138 /*
1139 * the below calculation, besides trying to win an obfuscated C
1140 * contest, maps cs_mode values to DIMM chip select sizes. The
1141 * mappings are:
1142 *
1143 * cs_mode CS size (mb)
1144 * ======= ============
1145 * 0 32
1146 * 1 64
1147 * 2 128
1148 * 3 128
1149 * 4 256
1150 * 5 512
1151 * 6 256
1152 * 7 512
1153 * 8 1024
1154 * 9 1024
1155 * 10 2048
1156 *
1157 * Basically, it calculates a value with which to shift the
1158 * smallest CS size of 32MB.
1159 *
1160 * ddr[23]_cs_size have a similar purpose.
1161 */
1162 diff = cs_mode/3 + (unsigned)(cs_mode > 5);
1163
1164 return 32 << (cs_mode - diff);
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001165 }
1166 else {
1167 WARN_ON(cs_mode > 6);
1168 return 32 << cs_mode;
1169 }
Doug Thompsonddff8762009-04-27 16:14:52 +02001170}
1171
Doug Thompson1afd3c92009-04-27 16:16:50 +02001172/*
1173 * Get the number of DCT channels in use.
1174 *
1175 * Return:
1176 * number of Memory Channels in operation
1177 * Pass back:
1178 * contents of the DCL0_LOW register
1179 */
Borislav Petkov7d20d142011-01-07 17:58:04 +01001180static int f1x_early_channel_count(struct amd64_pvt *pvt)
Doug Thompson1afd3c92009-04-27 16:16:50 +02001181{
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02001182 int i, j, channels = 0;
Doug Thompsonddff8762009-04-27 16:14:52 +02001183
Borislav Petkov7d20d142011-01-07 17:58:04 +01001184 /* On F10h, if we are in 128 bit mode, then we are using 2 channels */
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001185 if (boot_cpu_data.x86 == 0x10 && (pvt->dclr0 & WIDTH_128))
Borislav Petkov7d20d142011-01-07 17:58:04 +01001186 return 2;
Doug Thompson1afd3c92009-04-27 16:16:50 +02001187
1188 /*
Borislav Petkovd16149e2009-10-16 19:55:49 +02001189 * Need to check if in unganged mode: In such, there are 2 channels,
1190 * but they are not in 128 bit mode and thus the above 'dclr0' status
1191 * bit will be OFF.
Doug Thompson1afd3c92009-04-27 16:16:50 +02001192 *
1193 * Need to check DCT0[0] and DCT1[0] to see if only one of them has
1194 * their CSEnable bit on. If so, then SINGLE DIMM case.
1195 */
Borislav Petkovd16149e2009-10-16 19:55:49 +02001196 debugf0("Data width is not 128 bits - need more decoding\n");
Doug Thompson1afd3c92009-04-27 16:16:50 +02001197
1198 /*
1199 * Check DRAM Bank Address Mapping values for each DIMM to see if there
1200 * is more than just one DIMM present in unganged mode. Need to check
1201 * both controllers since DIMMs can be placed in either one.
1202 */
Borislav Petkov525a1b22010-12-21 15:53:27 +01001203 for (i = 0; i < 2; i++) {
1204 u32 dbam = (i ? pvt->dbam1 : pvt->dbam0);
Doug Thompson1afd3c92009-04-27 16:16:50 +02001205
Wan Wei57a30852009-08-07 17:04:49 +02001206 for (j = 0; j < 4; j++) {
1207 if (DBAM_DIMM(j, dbam) > 0) {
1208 channels++;
1209 break;
1210 }
1211 }
Doug Thompson1afd3c92009-04-27 16:16:50 +02001212 }
1213
Borislav Petkovd16149e2009-10-16 19:55:49 +02001214 if (channels > 2)
1215 channels = 2;
1216
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001217 amd64_info("MCT channel count: %d\n", channels);
Doug Thompson1afd3c92009-04-27 16:16:50 +02001218
1219 return channels;
Doug Thompson1afd3c92009-04-27 16:16:50 +02001220}
1221
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001222static int ddr3_cs_size(unsigned i, bool dct_width)
Doug Thompson1afd3c92009-04-27 16:16:50 +02001223{
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001224 unsigned shift = 0;
1225 int cs_size = 0;
1226
1227 if (i == 0 || i == 3 || i == 4)
1228 cs_size = -1;
1229 else if (i <= 2)
1230 shift = i;
1231 else if (i == 12)
1232 shift = 7;
1233 else if (!(i & 0x1))
1234 shift = i >> 1;
1235 else
1236 shift = (i + 1) >> 1;
1237
1238 if (cs_size != -1)
1239 cs_size = (128 * (1 << !!dct_width)) << shift;
1240
1241 return cs_size;
1242}
1243
1244static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1245 unsigned cs_mode)
1246{
1247 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1248
1249 WARN_ON(cs_mode > 11);
Borislav Petkov1433eb92009-10-21 13:44:36 +02001250
1251 if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001252 return ddr3_cs_size(cs_mode, dclr & WIDTH_128);
Borislav Petkov1433eb92009-10-21 13:44:36 +02001253 else
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001254 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1255}
Borislav Petkov1433eb92009-10-21 13:44:36 +02001256
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001257/*
1258 * F15h supports only 64bit DCT interfaces
1259 */
1260static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1261 unsigned cs_mode)
1262{
1263 WARN_ON(cs_mode > 12);
1264
1265 return ddr3_cs_size(cs_mode, false);
Doug Thompson1afd3c92009-04-27 16:16:50 +02001266}
1267
Borislav Petkov5a5d2372011-01-17 17:52:57 +01001268static void read_dram_ctl_register(struct amd64_pvt *pvt)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001269{
Doug Thompson6163b5d2009-04-27 16:20:17 +02001270
Borislav Petkov5a5d2372011-01-17 17:52:57 +01001271 if (boot_cpu_data.x86 == 0xf)
1272 return;
1273
Borislav Petkov78da1212010-12-22 19:31:45 +01001274 if (!amd64_read_dct_pci_cfg(pvt, DCT_SEL_LO, &pvt->dct_sel_lo)) {
1275 debugf0("F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
1276 pvt->dct_sel_lo, dct_sel_baseaddr(pvt));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001277
Borislav Petkov5a5d2372011-01-17 17:52:57 +01001278 debugf0(" DCTs operate in %s mode.\n",
1279 (dct_ganging_enabled(pvt) ? "ganged" : "unganged"));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001280
Borislav Petkov72381bd2009-10-09 19:14:43 +02001281 if (!dct_ganging_enabled(pvt))
1282 debugf0(" Address range split per DCT: %s\n",
1283 (dct_high_range_enabled(pvt) ? "yes" : "no"));
1284
Borislav Petkov78da1212010-12-22 19:31:45 +01001285 debugf0(" data interleave for ECC: %s, "
Borislav Petkov72381bd2009-10-09 19:14:43 +02001286 "DRAM cleared since last warm reset: %s\n",
1287 (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
1288 (dct_memory_cleared(pvt) ? "yes" : "no"));
1289
Borislav Petkov78da1212010-12-22 19:31:45 +01001290 debugf0(" channel interleave: %s, "
1291 "interleave bits selector: 0x%x\n",
Borislav Petkov72381bd2009-10-09 19:14:43 +02001292 (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
Doug Thompson6163b5d2009-04-27 16:20:17 +02001293 dct_sel_interleave_addr(pvt));
1294 }
1295
Borislav Petkov78da1212010-12-22 19:31:45 +01001296 amd64_read_dct_pci_cfg(pvt, DCT_SEL_HI, &pvt->dct_sel_hi);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001297}
1298
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001299/*
Borislav Petkov229a7a12010-12-09 18:57:54 +01001300 * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001301 * Interleaving Modes.
1302 */
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001303static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
Borislav Petkov229a7a12010-12-09 18:57:54 +01001304 bool hi_range_sel, u8 intlv_en)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001305{
Borislav Petkov151fa712011-02-21 19:33:10 +01001306 u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001307
1308 if (dct_ganging_enabled(pvt))
Borislav Petkov229a7a12010-12-09 18:57:54 +01001309 return 0;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001310
Borislav Petkov229a7a12010-12-09 18:57:54 +01001311 if (hi_range_sel)
1312 return dct_sel_high;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001313
Borislav Petkov229a7a12010-12-09 18:57:54 +01001314 /*
1315 * see F2x110[DctSelIntLvAddr] - channel interleave mode
1316 */
1317 if (dct_interleave_enabled(pvt)) {
1318 u8 intlv_addr = dct_sel_interleave_addr(pvt);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001319
Borislav Petkov229a7a12010-12-09 18:57:54 +01001320 /* return DCT select function: 0=DCT0, 1=DCT1 */
1321 if (!intlv_addr)
1322 return sys_addr >> 6 & 1;
1323
1324 if (intlv_addr & 0x2) {
1325 u8 shift = intlv_addr & 0x1 ? 9 : 6;
1326 u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2;
1327
1328 return ((sys_addr >> shift) & 1) ^ temp;
1329 }
1330
1331 return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
1332 }
1333
1334 if (dct_high_range_enabled(pvt))
1335 return ~dct_sel_high & 1;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001336
1337 return 0;
1338}
1339
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001340/* Convert the sys_addr to the normalized DCT address */
Borislav Petkove7613592011-02-21 19:49:01 +01001341static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, unsigned range,
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001342 u64 sys_addr, bool hi_rng,
1343 u32 dct_sel_base_addr)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001344{
1345 u64 chan_off;
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001346 u64 dram_base = get_dram_base(pvt, range);
1347 u64 hole_off = f10_dhar_offset(pvt);
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001348 u64 dct_sel_base_off = (pvt->dct_sel_hi & 0xFFFFFC00) << 16;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001349
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001350 if (hi_rng) {
1351 /*
1352 * if
1353 * base address of high range is below 4Gb
1354 * (bits [47:27] at [31:11])
1355 * DRAM address space on this DCT is hoisted above 4Gb &&
1356 * sys_addr > 4Gb
1357 *
1358 * remove hole offset from sys_addr
1359 * else
1360 * remove high range offset from sys_addr
1361 */
1362 if ((!(dct_sel_base_addr >> 16) ||
1363 dct_sel_base_addr < dhar_base(pvt)) &&
Borislav Petkov972ea172011-02-21 19:43:02 +01001364 dhar_valid(pvt) &&
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001365 (sys_addr >= BIT_64(32)))
Borislav Petkovbc21fa52010-11-11 17:29:13 +01001366 chan_off = hole_off;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001367 else
1368 chan_off = dct_sel_base_off;
1369 } else {
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001370 /*
1371 * if
1372 * we have a valid hole &&
1373 * sys_addr > 4Gb
1374 *
1375 * remove hole
1376 * else
1377 * remove dram base to normalize to DCT address
1378 */
Borislav Petkov972ea172011-02-21 19:43:02 +01001379 if (dhar_valid(pvt) && (sys_addr >= BIT_64(32)))
Borislav Petkovbc21fa52010-11-11 17:29:13 +01001380 chan_off = hole_off;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001381 else
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001382 chan_off = dram_base;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001383 }
1384
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001385 return (sys_addr & GENMASK(6,47)) - (chan_off & GENMASK(23,47));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001386}
1387
Doug Thompson6163b5d2009-04-27 16:20:17 +02001388/*
1389 * checks if the csrow passed in is marked as SPARED, if so returns the new
1390 * spare row
1391 */
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001392static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001393{
Borislav Petkov614ec9d2011-01-13 18:02:22 +01001394 int tmp_cs;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001395
Borislav Petkov614ec9d2011-01-13 18:02:22 +01001396 if (online_spare_swap_done(pvt, dct) &&
1397 csrow == online_spare_bad_dramcs(pvt, dct)) {
1398
1399 for_each_chip_select(tmp_cs, dct, pvt) {
1400 if (chip_select_base(tmp_cs, dct, pvt) & 0x2) {
1401 csrow = tmp_cs;
1402 break;
1403 }
1404 }
Doug Thompson6163b5d2009-04-27 16:20:17 +02001405 }
1406 return csrow;
1407}
1408
1409/*
1410 * Iterate over the DRAM DCT "base" and "mask" registers looking for a
1411 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
1412 *
1413 * Return:
1414 * -EINVAL: NOT FOUND
1415 * 0..csrow = Chip-Select Row
1416 */
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001417static int f1x_lookup_addr_in_dct(u64 in_addr, u32 nid, u8 dct)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001418{
1419 struct mem_ctl_info *mci;
1420 struct amd64_pvt *pvt;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001421 u64 cs_base, cs_mask;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001422 int cs_found = -EINVAL;
1423 int csrow;
1424
Borislav Petkovcc4d8862010-10-13 16:11:59 +02001425 mci = mcis[nid];
Doug Thompson6163b5d2009-04-27 16:20:17 +02001426 if (!mci)
1427 return cs_found;
1428
1429 pvt = mci->pvt_info;
1430
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001431 debugf1("input addr: 0x%llx, DCT: %d\n", in_addr, dct);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001432
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001433 for_each_chip_select(csrow, dct, pvt) {
1434 if (!csrow_enabled(csrow, dct, pvt))
Doug Thompson6163b5d2009-04-27 16:20:17 +02001435 continue;
1436
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001437 get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001438
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001439 debugf1(" CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
1440 csrow, cs_base, cs_mask);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001441
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001442 cs_mask = ~cs_mask;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001443
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001444 debugf1(" (InputAddr & ~CSMask)=0x%llx "
1445 "(CSBase & ~CSMask)=0x%llx\n",
1446 (in_addr & cs_mask), (cs_base & cs_mask));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001447
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001448 if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
1449 cs_found = f10_process_possible_spare(pvt, dct, csrow);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001450
1451 debugf1(" MATCH csrow=%d\n", cs_found);
1452 break;
1453 }
1454 }
1455 return cs_found;
1456}
1457
Borislav Petkov95b0ef52011-01-11 22:08:07 +01001458/*
1459 * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is
1460 * swapped with a region located at the bottom of memory so that the GPU can use
1461 * the interleaved region and thus two channels.
1462 */
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001463static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr)
Borislav Petkov95b0ef52011-01-11 22:08:07 +01001464{
1465 u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr;
1466
1467 if (boot_cpu_data.x86 == 0x10) {
1468 /* only revC3 and revE have that feature */
1469 if (boot_cpu_data.x86_model < 4 ||
1470 (boot_cpu_data.x86_model < 0xa &&
1471 boot_cpu_data.x86_mask < 3))
1472 return sys_addr;
1473 }
1474
1475 amd64_read_dct_pci_cfg(pvt, SWAP_INTLV_REG, &swap_reg);
1476
1477 if (!(swap_reg & 0x1))
1478 return sys_addr;
1479
1480 swap_base = (swap_reg >> 3) & 0x7f;
1481 swap_limit = (swap_reg >> 11) & 0x7f;
1482 rgn_size = (swap_reg >> 20) & 0x7f;
1483 tmp_addr = sys_addr >> 27;
1484
1485 if (!(sys_addr >> 34) &&
1486 (((tmp_addr >= swap_base) &&
1487 (tmp_addr <= swap_limit)) ||
1488 (tmp_addr < rgn_size)))
1489 return sys_addr ^ (u64)swap_base << 27;
1490
1491 return sys_addr;
1492}
1493
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001494/* For a given @dram_range, check if @sys_addr falls within it. */
Borislav Petkove7613592011-02-21 19:49:01 +01001495static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001496 u64 sys_addr, int *nid, int *chan_sel)
1497{
Borislav Petkov229a7a12010-12-09 18:57:54 +01001498 int cs_found = -EINVAL;
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001499 u64 chan_addr;
Borislav Petkov5d4b58e2011-01-13 16:01:13 +01001500 u32 dct_sel_base;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001501 u8 channel;
Borislav Petkov229a7a12010-12-09 18:57:54 +01001502 bool high_range = false;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001503
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001504 u8 node_id = dram_dst_node(pvt, range);
Borislav Petkov229a7a12010-12-09 18:57:54 +01001505 u8 intlv_en = dram_intlv_en(pvt, range);
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001506 u32 intlv_sel = dram_intlv_sel(pvt, range);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001507
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001508 debugf1("(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1509 range, sys_addr, get_dram_limit(pvt, range));
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001510
Borislav Petkov355fba62011-01-17 13:03:26 +01001511 if (dhar_valid(pvt) &&
1512 dhar_base(pvt) <= sys_addr &&
1513 sys_addr < BIT_64(32)) {
1514 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1515 sys_addr);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001516 return -EINVAL;
Borislav Petkov355fba62011-01-17 13:03:26 +01001517 }
1518
Borislav Petkovf030ddf2011-04-08 15:05:21 +02001519 if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en)))
Borislav Petkov355fba62011-01-17 13:03:26 +01001520 return -EINVAL;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001521
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001522 sys_addr = f1x_swap_interleaved_region(pvt, sys_addr);
Borislav Petkov95b0ef52011-01-11 22:08:07 +01001523
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001524 dct_sel_base = dct_sel_baseaddr(pvt);
1525
1526 /*
1527 * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
1528 * select between DCT0 and DCT1.
1529 */
1530 if (dct_high_range_enabled(pvt) &&
1531 !dct_ganging_enabled(pvt) &&
1532 ((sys_addr >> 27) >= (dct_sel_base >> 11)))
Borislav Petkov229a7a12010-12-09 18:57:54 +01001533 high_range = true;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001534
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001535 channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001536
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001537 chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr,
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001538 high_range, dct_sel_base);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001539
Borislav Petkove2f79db2011-01-13 14:57:34 +01001540 /* Remove node interleaving, see F1x120 */
1541 if (intlv_en)
1542 chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) |
1543 (chan_addr & 0xfff);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001544
Borislav Petkov5d4b58e2011-01-13 16:01:13 +01001545 /* remove channel interleave */
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001546 if (dct_interleave_enabled(pvt) &&
1547 !dct_high_range_enabled(pvt) &&
1548 !dct_ganging_enabled(pvt)) {
Borislav Petkov5d4b58e2011-01-13 16:01:13 +01001549
1550 if (dct_sel_interleave_addr(pvt) != 1) {
1551 if (dct_sel_interleave_addr(pvt) == 0x3)
1552 /* hash 9 */
1553 chan_addr = ((chan_addr >> 10) << 9) |
1554 (chan_addr & 0x1ff);
1555 else
1556 /* A[6] or hash 6 */
1557 chan_addr = ((chan_addr >> 7) << 6) |
1558 (chan_addr & 0x3f);
1559 } else
1560 /* A[12] */
1561 chan_addr = ((chan_addr >> 13) << 12) |
1562 (chan_addr & 0xfff);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001563 }
1564
Borislav Petkov5d4b58e2011-01-13 16:01:13 +01001565 debugf1(" Normalized DCT addr: 0x%llx\n", chan_addr);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001566
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001567 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001568
1569 if (cs_found >= 0) {
1570 *nid = node_id;
1571 *chan_sel = channel;
1572 }
1573 return cs_found;
1574}
1575
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001576static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt, u64 sys_addr,
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001577 int *node, int *chan_sel)
1578{
Borislav Petkove7613592011-02-21 19:49:01 +01001579 int cs_found = -EINVAL;
1580 unsigned range;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001581
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001582 for (range = 0; range < DRAM_RANGES; range++) {
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001583
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001584 if (!dram_rw(pvt, range))
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001585 continue;
1586
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001587 if ((get_dram_base(pvt, range) <= sys_addr) &&
1588 (get_dram_limit(pvt, range) >= sys_addr)) {
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001589
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001590 cs_found = f1x_match_to_this_node(pvt, range,
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001591 sys_addr, node,
1592 chan_sel);
1593 if (cs_found >= 0)
1594 break;
1595 }
1596 }
1597 return cs_found;
1598}
1599
1600/*
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001601 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
1602 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001603 *
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001604 * The @sys_addr is usually an error address received from the hardware
1605 * (MCX_ADDR).
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001606 */
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001607static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001608 u16 syndrome)
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001609{
1610 struct amd64_pvt *pvt = mci->pvt_info;
1611 u32 page, offset;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001612 int nid, csrow, chan = 0;
1613
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001614 csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &nid, &chan);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001615
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001616 if (csrow < 0) {
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001617 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001618 return;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001619 }
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001620
1621 error_address_to_page_and_offset(sys_addr, &page, &offset);
1622
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001623 /*
1624 * We need the syndromes for channel detection only when we're
1625 * ganged. Otherwise @chan should already contain the channel at
1626 * this point.
1627 */
Borislav Petkova97fa682010-12-23 14:07:18 +01001628 if (dct_ganging_enabled(pvt))
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001629 chan = get_channel_from_ecc_syndrome(mci, syndrome);
1630
1631 if (chan >= 0)
1632 edac_mc_handle_ce(mci, page, offset, syndrome, csrow, chan,
1633 EDAC_MOD_STR);
1634 else
1635 /*
1636 * Channel unknown, report all channels on this CSROW as failed.
1637 */
1638 for (chan = 0; chan < mci->csrows[csrow].nr_channels; chan++)
1639 edac_mc_handle_ce(mci, page, offset, syndrome,
1640 csrow, chan, EDAC_MOD_STR);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001641}
1642
1643/*
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001644 * debug routine to display the memory sizes of all logical DIMMs and its
Borislav Petkovcb328502010-12-22 14:28:24 +01001645 * CSROWs
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001646 */
Borislav Petkov8c671752011-02-23 17:25:12 +01001647static void amd64_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001648{
Borislav Petkov603adaf2009-12-21 14:52:53 +01001649 int dimm, size0, size1, factor = 0;
Borislav Petkov525a1b22010-12-21 15:53:27 +01001650 u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
1651 u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001652
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001653 if (boot_cpu_data.x86 == 0xf) {
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001654 if (pvt->dclr0 & WIDTH_128)
Borislav Petkov603adaf2009-12-21 14:52:53 +01001655 factor = 1;
1656
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001657 /* K8 families < revF not supported yet */
Borislav Petkov1433eb92009-10-21 13:44:36 +02001658 if (pvt->ext_model < K8_REV_F)
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001659 return;
1660 else
1661 WARN_ON(ctrl != 0);
1662 }
1663
Borislav Petkov4d796362011-02-03 15:59:57 +01001664 dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1 : pvt->dbam0;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001665 dcsb = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->csels[1].csbases
1666 : pvt->csels[0].csbases;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001667
Borislav Petkov4d796362011-02-03 15:59:57 +01001668 debugf1("F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n", ctrl, dbam);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001669
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001670 edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
1671
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001672 /* Dump memory sizes for DIMM and its CSROWs */
1673 for (dimm = 0; dimm < 4; dimm++) {
1674
1675 size0 = 0;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001676 if (dcsb[dimm*2] & DCSB_CS_ENABLE)
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001677 size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
1678 DBAM_DIMM(dimm, dbam));
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001679
1680 size1 = 0;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001681 if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001682 size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
1683 DBAM_DIMM(dimm, dbam));
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001684
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001685 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
1686 dimm * 2, size0 << factor,
1687 dimm * 2 + 1, size1 << factor);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001688 }
1689}
1690
Doug Thompson4d376072009-04-27 16:25:05 +02001691static struct amd64_family_type amd64_family_types[] = {
1692 [K8_CPUS] = {
Borislav Petkov0092b202010-10-01 19:20:05 +02001693 .ctl_name = "K8",
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001694 .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
1695 .f3_id = PCI_DEVICE_ID_AMD_K8_NB_MISC,
Doug Thompson4d376072009-04-27 16:25:05 +02001696 .ops = {
Borislav Petkov1433eb92009-10-21 13:44:36 +02001697 .early_channel_count = k8_early_channel_count,
Borislav Petkov1433eb92009-10-21 13:44:36 +02001698 .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow,
1699 .dbam_to_cs = k8_dbam_to_chip_select,
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001700 .read_dct_pci_cfg = k8_read_dct_pci_cfg,
Doug Thompson4d376072009-04-27 16:25:05 +02001701 }
1702 },
1703 [F10_CPUS] = {
Borislav Petkov0092b202010-10-01 19:20:05 +02001704 .ctl_name = "F10h",
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001705 .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
1706 .f3_id = PCI_DEVICE_ID_AMD_10H_NB_MISC,
Doug Thompson4d376072009-04-27 16:25:05 +02001707 .ops = {
Borislav Petkov7d20d142011-01-07 17:58:04 +01001708 .early_channel_count = f1x_early_channel_count,
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001709 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
Borislav Petkov1433eb92009-10-21 13:44:36 +02001710 .dbam_to_cs = f10_dbam_to_chip_select,
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001711 .read_dct_pci_cfg = f10_read_dct_pci_cfg,
1712 }
1713 },
1714 [F15_CPUS] = {
1715 .ctl_name = "F15h",
Borislav Petkovdf71a052011-01-19 18:15:10 +01001716 .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1,
1717 .f3_id = PCI_DEVICE_ID_AMD_15H_NB_F3,
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001718 .ops = {
Borislav Petkov7d20d142011-01-07 17:58:04 +01001719 .early_channel_count = f1x_early_channel_count,
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001720 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001721 .dbam_to_cs = f15_dbam_to_chip_select,
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001722 .read_dct_pci_cfg = f15_read_dct_pci_cfg,
Doug Thompson4d376072009-04-27 16:25:05 +02001723 }
1724 },
Doug Thompson4d376072009-04-27 16:25:05 +02001725};
1726
1727static struct pci_dev *pci_get_related_function(unsigned int vendor,
1728 unsigned int device,
1729 struct pci_dev *related)
1730{
1731 struct pci_dev *dev = NULL;
1732
1733 dev = pci_get_device(vendor, device, dev);
1734 while (dev) {
1735 if ((dev->bus->number == related->bus->number) &&
1736 (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
1737 break;
1738 dev = pci_get_device(vendor, device, dev);
1739 }
1740
1741 return dev;
1742}
1743
Doug Thompsonb1289d62009-04-27 16:37:05 +02001744/*
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001745 * These are tables of eigenvectors (one per line) which can be used for the
1746 * construction of the syndrome tables. The modified syndrome search algorithm
1747 * uses those to find the symbol in error and thus the DIMM.
Doug Thompsonb1289d62009-04-27 16:37:05 +02001748 *
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001749 * Algorithm courtesy of Ross LaFetra from AMD.
Doug Thompsonb1289d62009-04-27 16:37:05 +02001750 */
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001751static u16 x4_vectors[] = {
1752 0x2f57, 0x1afe, 0x66cc, 0xdd88,
1753 0x11eb, 0x3396, 0x7f4c, 0xeac8,
1754 0x0001, 0x0002, 0x0004, 0x0008,
1755 0x1013, 0x3032, 0x4044, 0x8088,
1756 0x106b, 0x30d6, 0x70fc, 0xe0a8,
1757 0x4857, 0xc4fe, 0x13cc, 0x3288,
1758 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
1759 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
1760 0x15c1, 0x2a42, 0x89ac, 0x4758,
1761 0x2b03, 0x1602, 0x4f0c, 0xca08,
1762 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
1763 0x8ba7, 0x465e, 0x244c, 0x1cc8,
1764 0x2b87, 0x164e, 0x642c, 0xdc18,
1765 0x40b9, 0x80de, 0x1094, 0x20e8,
1766 0x27db, 0x1eb6, 0x9dac, 0x7b58,
1767 0x11c1, 0x2242, 0x84ac, 0x4c58,
1768 0x1be5, 0x2d7a, 0x5e34, 0xa718,
1769 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
1770 0x4c97, 0xc87e, 0x11fc, 0x33a8,
1771 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
1772 0x16b3, 0x3d62, 0x4f34, 0x8518,
1773 0x1e2f, 0x391a, 0x5cac, 0xf858,
1774 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
1775 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
1776 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
1777 0x4397, 0xc27e, 0x17fc, 0x3ea8,
1778 0x1617, 0x3d3e, 0x6464, 0xb8b8,
1779 0x23ff, 0x12aa, 0xab6c, 0x56d8,
1780 0x2dfb, 0x1ba6, 0x913c, 0x7328,
1781 0x185d, 0x2ca6, 0x7914, 0x9e28,
1782 0x171b, 0x3e36, 0x7d7c, 0xebe8,
1783 0x4199, 0x82ee, 0x19f4, 0x2e58,
1784 0x4807, 0xc40e, 0x130c, 0x3208,
1785 0x1905, 0x2e0a, 0x5804, 0xac08,
1786 0x213f, 0x132a, 0xadfc, 0x5ba8,
1787 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
Doug Thompsonb1289d62009-04-27 16:37:05 +02001788};
1789
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001790static u16 x8_vectors[] = {
1791 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
1792 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
1793 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
1794 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
1795 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
1796 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
1797 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
1798 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
1799 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
1800 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
1801 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
1802 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
1803 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
1804 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
1805 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
1806 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
1807 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
1808 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
1809 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
1810};
1811
Borislav Petkovd34a6ec2011-02-23 17:41:50 +01001812static int decode_syndrome(u16 syndrome, u16 *vectors, unsigned num_vecs,
1813 unsigned v_dim)
Doug Thompsonb1289d62009-04-27 16:37:05 +02001814{
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001815 unsigned int i, err_sym;
Doug Thompsonb1289d62009-04-27 16:37:05 +02001816
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001817 for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
1818 u16 s = syndrome;
Borislav Petkovd34a6ec2011-02-23 17:41:50 +01001819 unsigned v_idx = err_sym * v_dim;
1820 unsigned v_end = (err_sym + 1) * v_dim;
Doug Thompsonb1289d62009-04-27 16:37:05 +02001821
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001822 /* walk over all 16 bits of the syndrome */
1823 for (i = 1; i < (1U << 16); i <<= 1) {
1824
1825 /* if bit is set in that eigenvector... */
1826 if (v_idx < v_end && vectors[v_idx] & i) {
1827 u16 ev_comp = vectors[v_idx++];
1828
1829 /* ... and bit set in the modified syndrome, */
1830 if (s & i) {
1831 /* remove it. */
1832 s ^= ev_comp;
1833
1834 if (!s)
1835 return err_sym;
1836 }
1837
1838 } else if (s & i)
1839 /* can't get to zero, move to next symbol */
1840 break;
1841 }
Doug Thompsonb1289d62009-04-27 16:37:05 +02001842 }
1843
1844 debugf0("syndrome(%x) not found\n", syndrome);
1845 return -1;
1846}
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001847
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001848static int map_err_sym_to_channel(int err_sym, int sym_size)
1849{
1850 if (sym_size == 4)
1851 switch (err_sym) {
1852 case 0x20:
1853 case 0x21:
1854 return 0;
1855 break;
1856 case 0x22:
1857 case 0x23:
1858 return 1;
1859 break;
1860 default:
1861 return err_sym >> 4;
1862 break;
1863 }
1864 /* x8 symbols */
1865 else
1866 switch (err_sym) {
1867 /* imaginary bits not in a DIMM */
1868 case 0x10:
1869 WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
1870 err_sym);
1871 return -1;
1872 break;
1873
1874 case 0x11:
1875 return 0;
1876 break;
1877 case 0x12:
1878 return 1;
1879 break;
1880 default:
1881 return err_sym >> 3;
1882 break;
1883 }
1884 return -1;
1885}
1886
1887static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
1888{
1889 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001890 int err_sym = -1;
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001891
Borislav Petkova3b7db02011-01-19 20:35:12 +01001892 if (pvt->ecc_sym_sz == 8)
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001893 err_sym = decode_syndrome(syndrome, x8_vectors,
1894 ARRAY_SIZE(x8_vectors),
Borislav Petkova3b7db02011-01-19 20:35:12 +01001895 pvt->ecc_sym_sz);
1896 else if (pvt->ecc_sym_sz == 4)
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001897 err_sym = decode_syndrome(syndrome, x4_vectors,
1898 ARRAY_SIZE(x4_vectors),
Borislav Petkova3b7db02011-01-19 20:35:12 +01001899 pvt->ecc_sym_sz);
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001900 else {
Borislav Petkova3b7db02011-01-19 20:35:12 +01001901 amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz);
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001902 return err_sym;
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001903 }
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001904
Borislav Petkova3b7db02011-01-19 20:35:12 +01001905 return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz);
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001906}
1907
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001908/*
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001909 * Handle any Correctable Errors (CEs) that have occurred. Check for valid ERROR
1910 * ADDRESS and process.
1911 */
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001912static void amd64_handle_ce(struct mem_ctl_info *mci, struct mce *m)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001913{
1914 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001915 u64 sys_addr;
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001916 u16 syndrome;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001917
1918 /* Ensure that the Error Address is VALID */
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001919 if (!(m->status & MCI_STATUS_ADDRV)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001920 amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n");
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001921 edac_mc_handle_ce_no_info(mci, EDAC_MOD_STR);
1922 return;
1923 }
1924
Borislav Petkov70046622011-01-10 14:37:27 +01001925 sys_addr = get_error_address(m);
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001926 syndrome = extract_syndrome(m->status);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001927
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001928 amd64_mc_err(mci, "CE ERROR_ADDRESS= 0x%llx\n", sys_addr);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001929
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001930 pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, syndrome);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001931}
1932
1933/* Handle any Un-correctable Errors (UEs) */
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001934static void amd64_handle_ue(struct mem_ctl_info *mci, struct mce *m)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001935{
Borislav Petkov1f6bcee2009-11-13 14:02:57 +01001936 struct mem_ctl_info *log_mci, *src_mci = NULL;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001937 int csrow;
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001938 u64 sys_addr;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001939 u32 page, offset;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001940
1941 log_mci = mci;
1942
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001943 if (!(m->status & MCI_STATUS_ADDRV)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001944 amd64_mc_err(mci, "HW has no ERROR_ADDRESS available\n");
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001945 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1946 return;
1947 }
1948
Borislav Petkov70046622011-01-10 14:37:27 +01001949 sys_addr = get_error_address(m);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001950
1951 /*
1952 * Find out which node the error address belongs to. This may be
1953 * different from the node that detected the error.
1954 */
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001955 src_mci = find_mc_by_sys_addr(mci, sys_addr);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001956 if (!src_mci) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001957 amd64_mc_err(mci, "ERROR ADDRESS (0x%lx) NOT mapped to a MC\n",
1958 (unsigned long)sys_addr);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001959 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1960 return;
1961 }
1962
1963 log_mci = src_mci;
1964
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001965 csrow = sys_addr_to_csrow(log_mci, sys_addr);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001966 if (csrow < 0) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001967 amd64_mc_err(mci, "ERROR_ADDRESS (0x%lx) NOT mapped to CS\n",
1968 (unsigned long)sys_addr);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001969 edac_mc_handle_ue_no_info(log_mci, EDAC_MOD_STR);
1970 } else {
Borislav Petkov44e9e2e2009-10-26 15:00:19 +01001971 error_address_to_page_and_offset(sys_addr, &page, &offset);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001972 edac_mc_handle_ue(log_mci, page, offset, csrow, EDAC_MOD_STR);
1973 }
1974}
1975
Borislav Petkov549d0422009-07-24 13:51:42 +02001976static inline void __amd64_decode_bus_error(struct mem_ctl_info *mci,
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001977 struct mce *m)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001978{
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001979 u16 ec = EC(m->status);
1980 u8 xec = XEC(m->status, 0x1f);
1981 u8 ecc_type = (m->status >> 45) & 0x3;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001982
Borislav Petkovb70ef012009-06-25 19:32:38 +02001983 /* Bail early out if this was an 'observed' error */
Borislav Petkov5980bb92011-01-07 16:26:49 +01001984 if (PP(ec) == NBSL_PP_OBS)
Borislav Petkovb70ef012009-06-25 19:32:38 +02001985 return;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001986
Borislav Petkovecaf5602009-07-23 16:32:01 +02001987 /* Do only ECC errors */
1988 if (xec && xec != F10_NBSL_EXT_ERR_ECC)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001989 return;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001990
Borislav Petkovecaf5602009-07-23 16:32:01 +02001991 if (ecc_type == 2)
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001992 amd64_handle_ce(mci, m);
Borislav Petkovecaf5602009-07-23 16:32:01 +02001993 else if (ecc_type == 1)
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001994 amd64_handle_ue(mci, m);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001995}
1996
Borislav Petkovb0b07a22011-08-24 18:44:22 +02001997void amd64_decode_bus_error(int node_id, struct mce *m)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001998{
Borislav Petkovb0b07a22011-08-24 18:44:22 +02001999 __amd64_decode_bus_error(mcis[node_id], m);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002000}
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002001
Doug Thompson0ec449e2009-04-27 19:41:25 +02002002/*
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002003 * Use pvt->F2 which contains the F2 CPU PCI device to get the related
Borislav Petkovbbd0c1f2010-10-01 19:27:58 +02002004 * F1 (AddrMap) and F3 (Misc) devices. Return negative value on error.
Doug Thompson0ec449e2009-04-27 19:41:25 +02002005 */
Borislav Petkov360b7f32010-10-15 19:25:38 +02002006static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 f1_id, u16 f3_id)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002007{
Doug Thompson0ec449e2009-04-27 19:41:25 +02002008 /* Reserve the ADDRESS MAP Device */
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002009 pvt->F1 = pci_get_related_function(pvt->F2->vendor, f1_id, pvt->F2);
2010 if (!pvt->F1) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002011 amd64_err("error address map device not found: "
2012 "vendor %x device 0x%x (broken BIOS?)\n",
2013 PCI_VENDOR_ID_AMD, f1_id);
Borislav Petkovbbd0c1f2010-10-01 19:27:58 +02002014 return -ENODEV;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002015 }
2016
2017 /* Reserve the MISC Device */
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002018 pvt->F3 = pci_get_related_function(pvt->F2->vendor, f3_id, pvt->F2);
2019 if (!pvt->F3) {
2020 pci_dev_put(pvt->F1);
2021 pvt->F1 = NULL;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002022
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002023 amd64_err("error F3 device not found: "
2024 "vendor %x device 0x%x (broken BIOS?)\n",
2025 PCI_VENDOR_ID_AMD, f3_id);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002026
Borislav Petkovbbd0c1f2010-10-01 19:27:58 +02002027 return -ENODEV;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002028 }
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002029 debugf1("F1: %s\n", pci_name(pvt->F1));
2030 debugf1("F2: %s\n", pci_name(pvt->F2));
2031 debugf1("F3: %s\n", pci_name(pvt->F3));
Doug Thompson0ec449e2009-04-27 19:41:25 +02002032
2033 return 0;
2034}
2035
Borislav Petkov360b7f32010-10-15 19:25:38 +02002036static void free_mc_sibling_devs(struct amd64_pvt *pvt)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002037{
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002038 pci_dev_put(pvt->F1);
2039 pci_dev_put(pvt->F3);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002040}
2041
2042/*
2043 * Retrieve the hardware registers of the memory controller (this includes the
2044 * 'Address Map' and 'Misc' device regs)
2045 */
Borislav Petkov360b7f32010-10-15 19:25:38 +02002046static void read_mc_regs(struct amd64_pvt *pvt)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002047{
Borislav Petkova3b7db02011-01-19 20:35:12 +01002048 struct cpuinfo_x86 *c = &boot_cpu_data;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002049 u64 msr_val;
Borislav Petkovad6a32e2010-03-09 12:46:00 +01002050 u32 tmp;
Borislav Petkove7613592011-02-21 19:49:01 +01002051 unsigned range;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002052
2053 /*
2054 * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
2055 * those are Read-As-Zero
2056 */
Borislav Petkove97f8bb2009-10-12 15:27:45 +02002057 rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
2058 debugf0(" TOP_MEM: 0x%016llx\n", pvt->top_mem);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002059
2060 /* check first whether TOP_MEM2 is enabled */
2061 rdmsrl(MSR_K8_SYSCFG, msr_val);
2062 if (msr_val & (1U << 21)) {
Borislav Petkove97f8bb2009-10-12 15:27:45 +02002063 rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
2064 debugf0(" TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002065 } else
2066 debugf0(" TOP_MEM2 disabled.\n");
2067
Borislav Petkov5980bb92011-01-07 16:26:49 +01002068 amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002069
Borislav Petkov5a5d2372011-01-17 17:52:57 +01002070 read_dram_ctl_register(pvt);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002071
Borislav Petkov7f19bf72010-10-21 18:52:53 +02002072 for (range = 0; range < DRAM_RANGES; range++) {
2073 u8 rw;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002074
Borislav Petkov7f19bf72010-10-21 18:52:53 +02002075 /* read settings for this DRAM range */
2076 read_dram_base_limit_regs(pvt, range);
Borislav Petkove97f8bb2009-10-12 15:27:45 +02002077
Borislav Petkov7f19bf72010-10-21 18:52:53 +02002078 rw = dram_rw(pvt, range);
2079 if (!rw)
2080 continue;
2081
2082 debugf1(" DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
2083 range,
2084 get_dram_base(pvt, range),
2085 get_dram_limit(pvt, range));
2086
2087 debugf1(" IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
2088 dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
2089 (rw & 0x1) ? "R" : "-",
2090 (rw & 0x2) ? "W" : "-",
2091 dram_intlv_sel(pvt, range),
2092 dram_dst_node(pvt, range));
Doug Thompson0ec449e2009-04-27 19:41:25 +02002093 }
2094
Borislav Petkovb2b0c602010-10-08 18:32:29 +02002095 read_dct_base_mask(pvt);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002096
Borislav Petkovbc21fa52010-11-11 17:29:13 +01002097 amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
Borislav Petkov525a1b22010-12-21 15:53:27 +01002098 amd64_read_dct_pci_cfg(pvt, DBAM0, &pvt->dbam0);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002099
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002100 amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002101
Borislav Petkovcb328502010-12-22 14:28:24 +01002102 amd64_read_dct_pci_cfg(pvt, DCLR0, &pvt->dclr0);
2103 amd64_read_dct_pci_cfg(pvt, DCHR0, &pvt->dchr0);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002104
Borislav Petkov78da1212010-12-22 19:31:45 +01002105 if (!dct_ganging_enabled(pvt)) {
Borislav Petkovcb328502010-12-22 14:28:24 +01002106 amd64_read_dct_pci_cfg(pvt, DCLR1, &pvt->dclr1);
2107 amd64_read_dct_pci_cfg(pvt, DCHR1, &pvt->dchr1);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002108 }
Borislav Petkovad6a32e2010-03-09 12:46:00 +01002109
Borislav Petkova3b7db02011-01-19 20:35:12 +01002110 pvt->ecc_sym_sz = 4;
2111
2112 if (c->x86 >= 0x10) {
Borislav Petkovb2b0c602010-10-08 18:32:29 +02002113 amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
Borislav Petkov525a1b22010-12-21 15:53:27 +01002114 amd64_read_dct_pci_cfg(pvt, DBAM1, &pvt->dbam1);
Borislav Petkova3b7db02011-01-19 20:35:12 +01002115
2116 /* F10h, revD and later can do x8 ECC too */
2117 if ((c->x86 > 0x10 || c->x86_model > 7) && tmp & BIT(25))
2118 pvt->ecc_sym_sz = 8;
Borislav Petkov525a1b22010-12-21 15:53:27 +01002119 }
Borislav Petkovb2b0c602010-10-08 18:32:29 +02002120 dump_misc_regs(pvt);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002121}
2122
2123/*
2124 * NOTE: CPU Revision Dependent code
2125 *
2126 * Input:
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002127 * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002128 * k8 private pointer to -->
2129 * DRAM Bank Address mapping register
2130 * node_id
2131 * DCL register where dual_channel_active is
2132 *
2133 * The DBAM register consists of 4 sets of 4 bits each definitions:
2134 *
2135 * Bits: CSROWs
2136 * 0-3 CSROWs 0 and 1
2137 * 4-7 CSROWs 2 and 3
2138 * 8-11 CSROWs 4 and 5
2139 * 12-15 CSROWs 6 and 7
2140 *
2141 * Values range from: 0 to 15
2142 * The meaning of the values depends on CPU revision and dual-channel state,
2143 * see relevant BKDG more info.
2144 *
2145 * The memory controller provides for total of only 8 CSROWs in its current
2146 * architecture. Each "pair" of CSROWs normally represents just one DIMM in
2147 * single channel or two (2) DIMMs in dual channel mode.
2148 *
2149 * The following code logic collapses the various tables for CSROW based on CPU
2150 * revision.
2151 *
2152 * Returns:
2153 * The number of PAGE_SIZE pages on the specified CSROW number it
2154 * encompasses
2155 *
2156 */
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01002157static u32 amd64_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002158{
Borislav Petkov1433eb92009-10-21 13:44:36 +02002159 u32 cs_mode, nr_pages;
Ashish Shenoyf92cae42012-02-22 17:20:38 -08002160 u32 dbam = dct ? pvt->dbam1 : pvt->dbam0;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002161
2162 /*
2163 * The math on this doesn't look right on the surface because x/2*4 can
2164 * be simplified to x*2 but this expression makes use of the fact that
2165 * it is integral math where 1/2=0. This intermediate value becomes the
2166 * number of bits to shift the DBAM register to extract the proper CSROW
2167 * field.
2168 */
Ashish Shenoyf92cae42012-02-22 17:20:38 -08002169 cs_mode = (dbam >> ((csrow_nr / 2) * 4)) & 0xF;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002170
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01002171 nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode) << (20 - PAGE_SHIFT);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002172
Borislav Petkov1433eb92009-10-21 13:44:36 +02002173 debugf0(" (csrow=%d) DBAM map index= %d\n", csrow_nr, cs_mode);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002174 debugf0(" nr_pages= %u channel-count = %d\n",
2175 nr_pages, pvt->channel_count);
2176
2177 return nr_pages;
2178}
2179
2180/*
2181 * Initialize the array of csrow attribute instances, based on the values
2182 * from pci config hardware registers.
2183 */
Borislav Petkov360b7f32010-10-15 19:25:38 +02002184static int init_csrows(struct mem_ctl_info *mci)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002185{
2186 struct csrow_info *csrow;
Borislav Petkov2299ef72010-10-15 17:44:04 +02002187 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002188 u64 input_addr_min, input_addr_max, sys_addr, base, mask;
Borislav Petkov2299ef72010-10-15 17:44:04 +02002189 u32 val;
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02002190 int i, empty = 1;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002191
Borislav Petkova97fa682010-12-23 14:07:18 +01002192 amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002193
Borislav Petkov2299ef72010-10-15 17:44:04 +02002194 pvt->nbcfg = val;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002195
Borislav Petkov2299ef72010-10-15 17:44:04 +02002196 debugf0("node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
2197 pvt->mc_node_id, val,
Borislav Petkova97fa682010-12-23 14:07:18 +01002198 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
Doug Thompson0ec449e2009-04-27 19:41:25 +02002199
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002200 for_each_chip_select(i, 0, pvt) {
Doug Thompson0ec449e2009-04-27 19:41:25 +02002201 csrow = &mci->csrows[i];
2202
Ashish Shenoyf92cae42012-02-22 17:20:38 -08002203 if (!csrow_enabled(i, 0, pvt) && !csrow_enabled(i, 1, pvt)) {
Doug Thompson0ec449e2009-04-27 19:41:25 +02002204 debugf1("----CSROW %d EMPTY for node %d\n", i,
2205 pvt->mc_node_id);
2206 continue;
2207 }
2208
2209 debugf1("----CSROW %d VALID for MC node %d\n",
2210 i, pvt->mc_node_id);
2211
2212 empty = 0;
Ashish Shenoyf92cae42012-02-22 17:20:38 -08002213 if (csrow_enabled(i, 0, pvt))
2214 csrow->nr_pages = amd64_csrow_nr_pages(pvt, 0, i);
2215 if (csrow_enabled(i, 1, pvt))
2216 csrow->nr_pages += amd64_csrow_nr_pages(pvt, 1, i);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002217 find_csrow_limits(mci, i, &input_addr_min, &input_addr_max);
2218 sys_addr = input_addr_to_sys_addr(mci, input_addr_min);
2219 csrow->first_page = (u32) (sys_addr >> PAGE_SHIFT);
2220 sys_addr = input_addr_to_sys_addr(mci, input_addr_max);
2221 csrow->last_page = (u32) (sys_addr >> PAGE_SHIFT);
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002222
2223 get_cs_base_and_mask(pvt, i, 0, &base, &mask);
2224 csrow->page_mask = ~mask;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002225 /* 8 bytes of resolution */
2226
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002227 csrow->mtype = amd64_determine_memory_type(pvt, i);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002228
2229 debugf1(" for MC node %d csrow %d:\n", pvt->mc_node_id, i);
2230 debugf1(" input_addr_min: 0x%lx input_addr_max: 0x%lx\n",
2231 (unsigned long)input_addr_min,
2232 (unsigned long)input_addr_max);
2233 debugf1(" sys_addr: 0x%lx page_mask: 0x%lx\n",
2234 (unsigned long)sys_addr, csrow->page_mask);
2235 debugf1(" nr_pages: %u first_page: 0x%lx "
2236 "last_page: 0x%lx\n",
2237 (unsigned)csrow->nr_pages,
2238 csrow->first_page, csrow->last_page);
2239
2240 /*
2241 * determine whether CHIPKILL or JUST ECC or NO ECC is operating
2242 */
Borislav Petkova97fa682010-12-23 14:07:18 +01002243 if (pvt->nbcfg & NBCFG_ECC_ENABLE)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002244 csrow->edac_mode =
Borislav Petkova97fa682010-12-23 14:07:18 +01002245 (pvt->nbcfg & NBCFG_CHIPKILL) ?
Doug Thompson0ec449e2009-04-27 19:41:25 +02002246 EDAC_S4ECD4ED : EDAC_SECDED;
2247 else
2248 csrow->edac_mode = EDAC_NONE;
2249 }
2250
2251 return empty;
2252}
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002253
Borislav Petkov06724532009-09-16 13:05:46 +02002254/* get all cores on this DCT */
Borislav Petkovb487c332011-02-21 18:55:00 +01002255static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, unsigned nid)
Doug Thompsonf9431992009-04-27 19:46:08 +02002256{
Borislav Petkov06724532009-09-16 13:05:46 +02002257 int cpu;
Doug Thompsonf9431992009-04-27 19:46:08 +02002258
Borislav Petkov06724532009-09-16 13:05:46 +02002259 for_each_online_cpu(cpu)
2260 if (amd_get_nb_id(cpu) == nid)
2261 cpumask_set_cpu(cpu, mask);
Doug Thompsonf9431992009-04-27 19:46:08 +02002262}
2263
2264/* check MCG_CTL on all the cpus on this node */
Borislav Petkovb487c332011-02-21 18:55:00 +01002265static bool amd64_nb_mce_bank_enabled_on_node(unsigned nid)
Doug Thompsonf9431992009-04-27 19:46:08 +02002266{
Rusty Russellba578cb2009-11-03 14:56:35 +10302267 cpumask_var_t mask;
Borislav Petkov50542252009-12-11 18:14:40 +01002268 int cpu, nbe;
Borislav Petkov06724532009-09-16 13:05:46 +02002269 bool ret = false;
Doug Thompsonf9431992009-04-27 19:46:08 +02002270
Rusty Russellba578cb2009-11-03 14:56:35 +10302271 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002272 amd64_warn("%s: Error allocating mask\n", __func__);
Rusty Russellba578cb2009-11-03 14:56:35 +10302273 return false;
2274 }
Borislav Petkov06724532009-09-16 13:05:46 +02002275
Rusty Russellba578cb2009-11-03 14:56:35 +10302276 get_cpus_on_this_dct_cpumask(mask, nid);
Borislav Petkov06724532009-09-16 13:05:46 +02002277
Rusty Russellba578cb2009-11-03 14:56:35 +10302278 rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
Borislav Petkov06724532009-09-16 13:05:46 +02002279
Rusty Russellba578cb2009-11-03 14:56:35 +10302280 for_each_cpu(cpu, mask) {
Borislav Petkov50542252009-12-11 18:14:40 +01002281 struct msr *reg = per_cpu_ptr(msrs, cpu);
Borislav Petkov5980bb92011-01-07 16:26:49 +01002282 nbe = reg->l & MSR_MCGCTL_NBE;
Borislav Petkov06724532009-09-16 13:05:46 +02002283
2284 debugf0("core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
Borislav Petkov50542252009-12-11 18:14:40 +01002285 cpu, reg->q,
Borislav Petkov06724532009-09-16 13:05:46 +02002286 (nbe ? "enabled" : "disabled"));
2287
2288 if (!nbe)
2289 goto out;
Borislav Petkov06724532009-09-16 13:05:46 +02002290 }
2291 ret = true;
2292
2293out:
Rusty Russellba578cb2009-11-03 14:56:35 +10302294 free_cpumask_var(mask);
Doug Thompsonf9431992009-04-27 19:46:08 +02002295 return ret;
2296}
2297
Borislav Petkov2299ef72010-10-15 17:44:04 +02002298static int toggle_ecc_err_reporting(struct ecc_settings *s, u8 nid, bool on)
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002299{
2300 cpumask_var_t cmask;
Borislav Petkov50542252009-12-11 18:14:40 +01002301 int cpu;
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002302
2303 if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002304 amd64_warn("%s: error allocating mask\n", __func__);
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002305 return false;
2306 }
2307
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002308 get_cpus_on_this_dct_cpumask(cmask, nid);
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002309
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002310 rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2311
2312 for_each_cpu(cpu, cmask) {
2313
Borislav Petkov50542252009-12-11 18:14:40 +01002314 struct msr *reg = per_cpu_ptr(msrs, cpu);
2315
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002316 if (on) {
Borislav Petkov5980bb92011-01-07 16:26:49 +01002317 if (reg->l & MSR_MCGCTL_NBE)
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002318 s->flags.nb_mce_enable = 1;
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002319
Borislav Petkov5980bb92011-01-07 16:26:49 +01002320 reg->l |= MSR_MCGCTL_NBE;
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002321 } else {
2322 /*
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002323 * Turn off NB MCE reporting only when it was off before
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002324 */
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002325 if (!s->flags.nb_mce_enable)
Borislav Petkov5980bb92011-01-07 16:26:49 +01002326 reg->l &= ~MSR_MCGCTL_NBE;
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002327 }
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002328 }
2329 wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2330
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002331 free_cpumask_var(cmask);
2332
2333 return 0;
2334}
2335
Borislav Petkov2299ef72010-10-15 17:44:04 +02002336static bool enable_ecc_error_reporting(struct ecc_settings *s, u8 nid,
2337 struct pci_dev *F3)
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002338{
Borislav Petkov2299ef72010-10-15 17:44:04 +02002339 bool ret = true;
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002340 u32 value, mask = 0x3; /* UECC/CECC enable */
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002341
Borislav Petkov2299ef72010-10-15 17:44:04 +02002342 if (toggle_ecc_err_reporting(s, nid, ON)) {
2343 amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
2344 return false;
2345 }
2346
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002347 amd64_read_pci_cfg(F3, NBCTL, &value);
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002348
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002349 s->old_nbctl = value & mask;
2350 s->nbctl_valid = true;
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002351
2352 value |= mask;
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002353 amd64_write_pci_cfg(F3, NBCTL, value);
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002354
Borislav Petkova97fa682010-12-23 14:07:18 +01002355 amd64_read_pci_cfg(F3, NBCFG, &value);
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002356
Borislav Petkova97fa682010-12-23 14:07:18 +01002357 debugf0("1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2358 nid, value, !!(value & NBCFG_ECC_ENABLE));
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002359
Borislav Petkova97fa682010-12-23 14:07:18 +01002360 if (!(value & NBCFG_ECC_ENABLE)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002361 amd64_warn("DRAM ECC disabled on this node, enabling...\n");
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002362
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002363 s->flags.nb_ecc_prev = 0;
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002364
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002365 /* Attempt to turn on DRAM ECC Enable */
Borislav Petkova97fa682010-12-23 14:07:18 +01002366 value |= NBCFG_ECC_ENABLE;
2367 amd64_write_pci_cfg(F3, NBCFG, value);
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002368
Borislav Petkova97fa682010-12-23 14:07:18 +01002369 amd64_read_pci_cfg(F3, NBCFG, &value);
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002370
Borislav Petkova97fa682010-12-23 14:07:18 +01002371 if (!(value & NBCFG_ECC_ENABLE)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002372 amd64_warn("Hardware rejected DRAM ECC enable,"
2373 "check memory DIMM configuration.\n");
Borislav Petkov2299ef72010-10-15 17:44:04 +02002374 ret = false;
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002375 } else {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002376 amd64_info("Hardware accepted DRAM ECC Enable\n");
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002377 }
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002378 } else {
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002379 s->flags.nb_ecc_prev = 1;
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002380 }
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002381
Borislav Petkova97fa682010-12-23 14:07:18 +01002382 debugf0("2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2383 nid, value, !!(value & NBCFG_ECC_ENABLE));
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002384
Borislav Petkov2299ef72010-10-15 17:44:04 +02002385 return ret;
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002386}
2387
Borislav Petkov360b7f32010-10-15 19:25:38 +02002388static void restore_ecc_error_reporting(struct ecc_settings *s, u8 nid,
2389 struct pci_dev *F3)
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002390{
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002391 u32 value, mask = 0x3; /* UECC/CECC enable */
2392
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002393
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002394 if (!s->nbctl_valid)
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002395 return;
2396
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002397 amd64_read_pci_cfg(F3, NBCTL, &value);
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002398 value &= ~mask;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002399 value |= s->old_nbctl;
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002400
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002401 amd64_write_pci_cfg(F3, NBCTL, value);
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002402
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002403 /* restore previous BIOS DRAM ECC "off" setting we force-enabled */
2404 if (!s->flags.nb_ecc_prev) {
Borislav Petkova97fa682010-12-23 14:07:18 +01002405 amd64_read_pci_cfg(F3, NBCFG, &value);
2406 value &= ~NBCFG_ECC_ENABLE;
2407 amd64_write_pci_cfg(F3, NBCFG, value);
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002408 }
2409
2410 /* restore the NB Enable MCGCTL bit */
Borislav Petkov2299ef72010-10-15 17:44:04 +02002411 if (toggle_ecc_err_reporting(s, nid, OFF))
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002412 amd64_warn("Error restoring NB MCGCTL settings!\n");
Borislav Petkovf6d6ae962009-11-03 15:29:26 +01002413}
2414
Doug Thompsonf9431992009-04-27 19:46:08 +02002415/*
Borislav Petkov2299ef72010-10-15 17:44:04 +02002416 * EDAC requires that the BIOS have ECC enabled before
2417 * taking over the processing of ECC errors. A command line
2418 * option allows to force-enable hardware ECC later in
2419 * enable_ecc_error_reporting().
Doug Thompsonf9431992009-04-27 19:46:08 +02002420 */
Borislav Petkovcab4d272010-02-11 17:15:57 +01002421static const char *ecc_msg =
2422 "ECC disabled in the BIOS or no ECC capability, module will not load.\n"
2423 " Either enable ECC checking or force module loading by setting "
2424 "'ecc_enable_override'.\n"
2425 " (Note that use of the override may cause unknown side effects.)\n";
Borislav Petkovbe3468e2009-08-05 15:47:22 +02002426
Borislav Petkov2299ef72010-10-15 17:44:04 +02002427static bool ecc_enabled(struct pci_dev *F3, u8 nid)
Doug Thompsonf9431992009-04-27 19:46:08 +02002428{
2429 u32 value;
Borislav Petkov2299ef72010-10-15 17:44:04 +02002430 u8 ecc_en = 0;
Borislav Petkov06724532009-09-16 13:05:46 +02002431 bool nb_mce_en = false;
Doug Thompsonf9431992009-04-27 19:46:08 +02002432
Borislav Petkova97fa682010-12-23 14:07:18 +01002433 amd64_read_pci_cfg(F3, NBCFG, &value);
Doug Thompsonf9431992009-04-27 19:46:08 +02002434
Borislav Petkova97fa682010-12-23 14:07:18 +01002435 ecc_en = !!(value & NBCFG_ECC_ENABLE);
Borislav Petkov2299ef72010-10-15 17:44:04 +02002436 amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled"));
Doug Thompsonf9431992009-04-27 19:46:08 +02002437
Borislav Petkov2299ef72010-10-15 17:44:04 +02002438 nb_mce_en = amd64_nb_mce_bank_enabled_on_node(nid);
Borislav Petkov06724532009-09-16 13:05:46 +02002439 if (!nb_mce_en)
Borislav Petkov2299ef72010-10-15 17:44:04 +02002440 amd64_notice("NB MCE bank disabled, set MSR "
2441 "0x%08x[4] on node %d to enable.\n",
2442 MSR_IA32_MCG_CTL, nid);
Doug Thompsonf9431992009-04-27 19:46:08 +02002443
Borislav Petkov2299ef72010-10-15 17:44:04 +02002444 if (!ecc_en || !nb_mce_en) {
2445 amd64_notice("%s", ecc_msg);
2446 return false;
Borislav Petkov43f5e682009-12-21 18:55:18 +01002447 }
Borislav Petkov2299ef72010-10-15 17:44:04 +02002448 return true;
Doug Thompsonf9431992009-04-27 19:46:08 +02002449}
2450
Doug Thompson7d6034d2009-04-27 20:01:01 +02002451struct mcidev_sysfs_attribute sysfs_attrs[ARRAY_SIZE(amd64_dbg_attrs) +
2452 ARRAY_SIZE(amd64_inj_attrs) +
2453 1];
2454
2455struct mcidev_sysfs_attribute terminator = { .attr = { .name = NULL } };
2456
Borislav Petkov360b7f32010-10-15 19:25:38 +02002457static void set_mc_sysfs_attrs(struct mem_ctl_info *mci)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002458{
2459 unsigned int i = 0, j = 0;
2460
2461 for (; i < ARRAY_SIZE(amd64_dbg_attrs); i++)
2462 sysfs_attrs[i] = amd64_dbg_attrs[i];
2463
Borislav Petkova135cef2010-11-26 19:24:44 +01002464 if (boot_cpu_data.x86 >= 0x10)
2465 for (j = 0; j < ARRAY_SIZE(amd64_inj_attrs); j++, i++)
2466 sysfs_attrs[i] = amd64_inj_attrs[j];
Doug Thompson7d6034d2009-04-27 20:01:01 +02002467
2468 sysfs_attrs[i] = terminator;
2469
2470 mci->mc_driver_sysfs_attributes = sysfs_attrs;
2471}
2472
Borislav Petkovdf71a052011-01-19 18:15:10 +01002473static void setup_mci_misc_attrs(struct mem_ctl_info *mci,
2474 struct amd64_family_type *fam)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002475{
2476 struct amd64_pvt *pvt = mci->pvt_info;
2477
2478 mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
2479 mci->edac_ctl_cap = EDAC_FLAG_NONE;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002480
Borislav Petkov5980bb92011-01-07 16:26:49 +01002481 if (pvt->nbcap & NBCAP_SECDED)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002482 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
2483
Borislav Petkov5980bb92011-01-07 16:26:49 +01002484 if (pvt->nbcap & NBCAP_CHIPKILL)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002485 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
2486
2487 mci->edac_cap = amd64_determine_edac_cap(pvt);
2488 mci->mod_name = EDAC_MOD_STR;
2489 mci->mod_ver = EDAC_AMD64_VERSION;
Borislav Petkovdf71a052011-01-19 18:15:10 +01002490 mci->ctl_name = fam->ctl_name;
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002491 mci->dev_name = pci_name(pvt->F2);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002492 mci->ctl_page_to_phys = NULL;
2493
Doug Thompson7d6034d2009-04-27 20:01:01 +02002494 /* memory scrubber interface */
2495 mci->set_sdram_scrub_rate = amd64_set_scrub_rate;
2496 mci->get_sdram_scrub_rate = amd64_get_scrub_rate;
2497}
2498
Borislav Petkov0092b202010-10-01 19:20:05 +02002499/*
2500 * returns a pointer to the family descriptor on success, NULL otherwise.
2501 */
2502static struct amd64_family_type *amd64_per_family_init(struct amd64_pvt *pvt)
Borislav Petkov395ae782010-10-01 18:38:19 +02002503{
Borislav Petkov0092b202010-10-01 19:20:05 +02002504 u8 fam = boot_cpu_data.x86;
2505 struct amd64_family_type *fam_type = NULL;
2506
2507 switch (fam) {
Borislav Petkov395ae782010-10-01 18:38:19 +02002508 case 0xf:
Borislav Petkov0092b202010-10-01 19:20:05 +02002509 fam_type = &amd64_family_types[K8_CPUS];
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002510 pvt->ops = &amd64_family_types[K8_CPUS].ops;
Borislav Petkov395ae782010-10-01 18:38:19 +02002511 break;
Borislav Petkovdf71a052011-01-19 18:15:10 +01002512
Borislav Petkov395ae782010-10-01 18:38:19 +02002513 case 0x10:
Borislav Petkov0092b202010-10-01 19:20:05 +02002514 fam_type = &amd64_family_types[F10_CPUS];
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002515 pvt->ops = &amd64_family_types[F10_CPUS].ops;
Borislav Petkovdf71a052011-01-19 18:15:10 +01002516 break;
2517
2518 case 0x15:
2519 fam_type = &amd64_family_types[F15_CPUS];
2520 pvt->ops = &amd64_family_types[F15_CPUS].ops;
Borislav Petkov395ae782010-10-01 18:38:19 +02002521 break;
2522
2523 default:
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002524 amd64_err("Unsupported family!\n");
Borislav Petkov0092b202010-10-01 19:20:05 +02002525 return NULL;
Borislav Petkov395ae782010-10-01 18:38:19 +02002526 }
Borislav Petkov0092b202010-10-01 19:20:05 +02002527
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002528 pvt->ext_model = boot_cpu_data.x86_model >> 4;
2529
Borislav Petkovdf71a052011-01-19 18:15:10 +01002530 amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name,
Borislav Petkov0092b202010-10-01 19:20:05 +02002531 (fam == 0xf ?
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002532 (pvt->ext_model >= K8_REV_F ? "revF or later "
2533 : "revE or earlier ")
2534 : ""), pvt->mc_node_id);
Borislav Petkov0092b202010-10-01 19:20:05 +02002535 return fam_type;
Borislav Petkov395ae782010-10-01 18:38:19 +02002536}
2537
Borislav Petkov2299ef72010-10-15 17:44:04 +02002538static int amd64_init_one_instance(struct pci_dev *F2)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002539{
2540 struct amd64_pvt *pvt = NULL;
Borislav Petkov0092b202010-10-01 19:20:05 +02002541 struct amd64_family_type *fam_type = NULL;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002542 struct mem_ctl_info *mci = NULL;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002543 int err = 0, ret;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002544 u8 nid = get_node_id(F2);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002545
2546 ret = -ENOMEM;
2547 pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
2548 if (!pvt)
Borislav Petkov360b7f32010-10-15 19:25:38 +02002549 goto err_ret;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002550
Borislav Petkov360b7f32010-10-15 19:25:38 +02002551 pvt->mc_node_id = nid;
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002552 pvt->F2 = F2;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002553
Borislav Petkov395ae782010-10-01 18:38:19 +02002554 ret = -EINVAL;
Borislav Petkov0092b202010-10-01 19:20:05 +02002555 fam_type = amd64_per_family_init(pvt);
2556 if (!fam_type)
Borislav Petkov395ae782010-10-01 18:38:19 +02002557 goto err_free;
2558
Doug Thompson7d6034d2009-04-27 20:01:01 +02002559 ret = -ENODEV;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002560 err = reserve_mc_sibling_devs(pvt, fam_type->f1_id, fam_type->f3_id);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002561 if (err)
2562 goto err_free;
2563
Borislav Petkov360b7f32010-10-15 19:25:38 +02002564 read_mc_regs(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002565
Doug Thompson7d6034d2009-04-27 20:01:01 +02002566 /*
2567 * We need to determine how many memory channels there are. Then use
2568 * that information for calculating the size of the dynamic instance
Borislav Petkov360b7f32010-10-15 19:25:38 +02002569 * tables in the 'mci' structure.
Doug Thompson7d6034d2009-04-27 20:01:01 +02002570 */
Borislav Petkov360b7f32010-10-15 19:25:38 +02002571 ret = -EINVAL;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002572 pvt->channel_count = pvt->ops->early_channel_count(pvt);
2573 if (pvt->channel_count < 0)
Borislav Petkov360b7f32010-10-15 19:25:38 +02002574 goto err_siblings;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002575
2576 ret = -ENOMEM;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002577 mci = edac_mc_alloc(0, pvt->csels[0].b_cnt, pvt->channel_count, nid);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002578 if (!mci)
Borislav Petkov360b7f32010-10-15 19:25:38 +02002579 goto err_siblings;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002580
2581 mci->pvt_info = pvt;
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002582 mci->dev = &pvt->F2->dev;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002583
Borislav Petkovdf71a052011-01-19 18:15:10 +01002584 setup_mci_misc_attrs(mci, fam_type);
Borislav Petkov360b7f32010-10-15 19:25:38 +02002585
2586 if (init_csrows(mci))
Doug Thompson7d6034d2009-04-27 20:01:01 +02002587 mci->edac_cap = EDAC_FLAG_NONE;
2588
Borislav Petkov360b7f32010-10-15 19:25:38 +02002589 set_mc_sysfs_attrs(mci);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002590
2591 ret = -ENODEV;
2592 if (edac_mc_add_mc(mci)) {
2593 debugf1("failed edac_mc_add_mc()\n");
2594 goto err_add_mc;
2595 }
2596
Borislav Petkov549d0422009-07-24 13:51:42 +02002597 /* register stuff with EDAC MCE */
2598 if (report_gart_errors)
2599 amd_report_gart_errors(true);
2600
2601 amd_register_ecc_decoder(amd64_decode_bus_error);
2602
Borislav Petkov360b7f32010-10-15 19:25:38 +02002603 mcis[nid] = mci;
2604
2605 atomic_inc(&drv_instances);
2606
Doug Thompson7d6034d2009-04-27 20:01:01 +02002607 return 0;
2608
2609err_add_mc:
2610 edac_mc_free(mci);
2611
Borislav Petkov360b7f32010-10-15 19:25:38 +02002612err_siblings:
2613 free_mc_sibling_devs(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002614
Borislav Petkov360b7f32010-10-15 19:25:38 +02002615err_free:
2616 kfree(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002617
Borislav Petkov360b7f32010-10-15 19:25:38 +02002618err_ret:
Doug Thompson7d6034d2009-04-27 20:01:01 +02002619 return ret;
2620}
2621
Borislav Petkov2299ef72010-10-15 17:44:04 +02002622static int __devinit amd64_probe_one_instance(struct pci_dev *pdev,
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002623 const struct pci_device_id *mc_type)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002624{
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002625 u8 nid = get_node_id(pdev);
Borislav Petkov2299ef72010-10-15 17:44:04 +02002626 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002627 struct ecc_settings *s;
Borislav Petkov2299ef72010-10-15 17:44:04 +02002628 int ret = 0;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002629
Doug Thompson7d6034d2009-04-27 20:01:01 +02002630 ret = pci_enable_device(pdev);
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002631 if (ret < 0) {
Doug Thompson7d6034d2009-04-27 20:01:01 +02002632 debugf0("ret=%d\n", ret);
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002633 return -EIO;
2634 }
2635
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002636 ret = -ENOMEM;
2637 s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
2638 if (!s)
Borislav Petkov2299ef72010-10-15 17:44:04 +02002639 goto err_out;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002640
2641 ecc_stngs[nid] = s;
2642
Borislav Petkov2299ef72010-10-15 17:44:04 +02002643 if (!ecc_enabled(F3, nid)) {
2644 ret = -ENODEV;
2645
2646 if (!ecc_enable_override)
2647 goto err_enable;
2648
2649 amd64_warn("Forcing ECC on!\n");
2650
2651 if (!enable_ecc_error_reporting(s, nid, F3))
2652 goto err_enable;
2653 }
2654
2655 ret = amd64_init_one_instance(pdev);
Borislav Petkov360b7f32010-10-15 19:25:38 +02002656 if (ret < 0) {
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002657 amd64_err("Error probing instance: %d\n", nid);
Borislav Petkov360b7f32010-10-15 19:25:38 +02002658 restore_ecc_error_reporting(s, nid, F3);
2659 }
Doug Thompson7d6034d2009-04-27 20:01:01 +02002660
2661 return ret;
Borislav Petkov2299ef72010-10-15 17:44:04 +02002662
2663err_enable:
2664 kfree(s);
2665 ecc_stngs[nid] = NULL;
2666
2667err_out:
2668 return ret;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002669}
2670
2671static void __devexit amd64_remove_one_instance(struct pci_dev *pdev)
2672{
2673 struct mem_ctl_info *mci;
2674 struct amd64_pvt *pvt;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002675 u8 nid = get_node_id(pdev);
2676 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2677 struct ecc_settings *s = ecc_stngs[nid];
Doug Thompson7d6034d2009-04-27 20:01:01 +02002678
2679 /* Remove from EDAC CORE tracking list */
2680 mci = edac_mc_del_mc(&pdev->dev);
2681 if (!mci)
2682 return;
2683
2684 pvt = mci->pvt_info;
2685
Borislav Petkov360b7f32010-10-15 19:25:38 +02002686 restore_ecc_error_reporting(s, nid, F3);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002687
Borislav Petkov360b7f32010-10-15 19:25:38 +02002688 free_mc_sibling_devs(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002689
Borislav Petkov549d0422009-07-24 13:51:42 +02002690 /* unregister from EDAC MCE */
2691 amd_report_gart_errors(false);
2692 amd_unregister_ecc_decoder(amd64_decode_bus_error);
2693
Borislav Petkov360b7f32010-10-15 19:25:38 +02002694 kfree(ecc_stngs[nid]);
2695 ecc_stngs[nid] = NULL;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002696
Doug Thompson7d6034d2009-04-27 20:01:01 +02002697 /* Free the EDAC CORE resources */
Borislav Petkov8f68ed92009-12-21 15:15:59 +01002698 mci->pvt_info = NULL;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002699 mcis[nid] = NULL;
Borislav Petkov8f68ed92009-12-21 15:15:59 +01002700
2701 kfree(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002702 edac_mc_free(mci);
2703}
2704
2705/*
2706 * This table is part of the interface for loading drivers for PCI devices. The
2707 * PCI core identifies what devices are on a system during boot, and then
2708 * inquiry this table to see if this driver is for a given device found.
2709 */
Lionel Debroux36c46f32012-02-27 07:41:47 +01002710static DEFINE_PCI_DEVICE_TABLE(amd64_pci_table) = {
Doug Thompson7d6034d2009-04-27 20:01:01 +02002711 {
2712 .vendor = PCI_VENDOR_ID_AMD,
2713 .device = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
2714 .subvendor = PCI_ANY_ID,
2715 .subdevice = PCI_ANY_ID,
2716 .class = 0,
2717 .class_mask = 0,
Doug Thompson7d6034d2009-04-27 20:01:01 +02002718 },
2719 {
2720 .vendor = PCI_VENDOR_ID_AMD,
2721 .device = PCI_DEVICE_ID_AMD_10H_NB_DRAM,
2722 .subvendor = PCI_ANY_ID,
2723 .subdevice = PCI_ANY_ID,
2724 .class = 0,
2725 .class_mask = 0,
Doug Thompson7d6034d2009-04-27 20:01:01 +02002726 },
Borislav Petkovdf71a052011-01-19 18:15:10 +01002727 {
2728 .vendor = PCI_VENDOR_ID_AMD,
2729 .device = PCI_DEVICE_ID_AMD_15H_NB_F2,
2730 .subvendor = PCI_ANY_ID,
2731 .subdevice = PCI_ANY_ID,
2732 .class = 0,
2733 .class_mask = 0,
2734 },
2735
Doug Thompson7d6034d2009-04-27 20:01:01 +02002736 {0, }
2737};
2738MODULE_DEVICE_TABLE(pci, amd64_pci_table);
2739
2740static struct pci_driver amd64_pci_driver = {
2741 .name = EDAC_MOD_STR,
Borislav Petkov2299ef72010-10-15 17:44:04 +02002742 .probe = amd64_probe_one_instance,
Doug Thompson7d6034d2009-04-27 20:01:01 +02002743 .remove = __devexit_p(amd64_remove_one_instance),
2744 .id_table = amd64_pci_table,
2745};
2746
Borislav Petkov360b7f32010-10-15 19:25:38 +02002747static void setup_pci_device(void)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002748{
2749 struct mem_ctl_info *mci;
2750 struct amd64_pvt *pvt;
2751
2752 if (amd64_ctl_pci)
2753 return;
2754
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002755 mci = mcis[0];
Doug Thompson7d6034d2009-04-27 20:01:01 +02002756 if (mci) {
2757
2758 pvt = mci->pvt_info;
2759 amd64_ctl_pci =
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002760 edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002761
2762 if (!amd64_ctl_pci) {
2763 pr_warning("%s(): Unable to create PCI control\n",
2764 __func__);
2765
2766 pr_warning("%s(): PCI error report via EDAC not set\n",
2767 __func__);
2768 }
2769 }
2770}
2771
2772static int __init amd64_edac_init(void)
2773{
Borislav Petkov360b7f32010-10-15 19:25:38 +02002774 int err = -ENODEV;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002775
Borislav Petkovdf71a052011-01-19 18:15:10 +01002776 printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002777
2778 opstate_init();
2779
Hans Rosenfeld9653a5c2010-10-29 17:14:31 +02002780 if (amd_cache_northbridges() < 0)
Borislav Petkov56b34b92009-12-21 18:13:01 +01002781 goto err_ret;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002782
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002783 err = -ENOMEM;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002784 mcis = kzalloc(amd_nb_num() * sizeof(mcis[0]), GFP_KERNEL);
2785 ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL);
Borislav Petkov360b7f32010-10-15 19:25:38 +02002786 if (!(mcis && ecc_stngs))
Borislav Petkova9f0fbe2011-03-29 18:10:53 +02002787 goto err_free;
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002788
Borislav Petkov50542252009-12-11 18:14:40 +01002789 msrs = msrs_alloc();
Borislav Petkov56b34b92009-12-21 18:13:01 +01002790 if (!msrs)
Borislav Petkov360b7f32010-10-15 19:25:38 +02002791 goto err_free;
Borislav Petkov50542252009-12-11 18:14:40 +01002792
Doug Thompson7d6034d2009-04-27 20:01:01 +02002793 err = pci_register_driver(&amd64_pci_driver);
2794 if (err)
Borislav Petkov56b34b92009-12-21 18:13:01 +01002795 goto err_pci;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002796
Borislav Petkov56b34b92009-12-21 18:13:01 +01002797 err = -ENODEV;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002798 if (!atomic_read(&drv_instances))
2799 goto err_no_instances;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002800
Borislav Petkov360b7f32010-10-15 19:25:38 +02002801 setup_pci_device();
2802 return 0;
Borislav Petkov56b34b92009-12-21 18:13:01 +01002803
Borislav Petkov360b7f32010-10-15 19:25:38 +02002804err_no_instances:
Doug Thompson7d6034d2009-04-27 20:01:01 +02002805 pci_unregister_driver(&amd64_pci_driver);
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002806
Borislav Petkov56b34b92009-12-21 18:13:01 +01002807err_pci:
2808 msrs_free(msrs);
2809 msrs = NULL;
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002810
Borislav Petkov360b7f32010-10-15 19:25:38 +02002811err_free:
2812 kfree(mcis);
2813 mcis = NULL;
2814
2815 kfree(ecc_stngs);
2816 ecc_stngs = NULL;
2817
Borislav Petkov56b34b92009-12-21 18:13:01 +01002818err_ret:
Doug Thompson7d6034d2009-04-27 20:01:01 +02002819 return err;
2820}
2821
2822static void __exit amd64_edac_exit(void)
2823{
2824 if (amd64_ctl_pci)
2825 edac_pci_release_generic_ctl(amd64_ctl_pci);
2826
2827 pci_unregister_driver(&amd64_pci_driver);
Borislav Petkov50542252009-12-11 18:14:40 +01002828
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002829 kfree(ecc_stngs);
2830 ecc_stngs = NULL;
2831
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002832 kfree(mcis);
2833 mcis = NULL;
2834
Borislav Petkov50542252009-12-11 18:14:40 +01002835 msrs_free(msrs);
2836 msrs = NULL;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002837}
2838
2839module_init(amd64_edac_init);
2840module_exit(amd64_edac_exit);
2841
2842MODULE_LICENSE("GPL");
2843MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
2844 "Dave Peterson, Thayne Harbaugh");
2845MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
2846 EDAC_AMD64_VERSION);
2847
2848module_param(edac_op_state, int, 0444);
2849MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");