blob: 910b0116c12872aaebb5e72707ddc920d0722ce1 [file] [log] [blame]
Doug Thompson2bc65412009-05-04 20:11:14 +02001#include "amd64_edac.h"
Andreas Herrmann23ac4ae2010-09-17 18:03:43 +02002#include <asm/amd_nb.h>
Doug Thompson2bc65412009-05-04 20:11:14 +02003
4static struct edac_pci_ctl_info *amd64_ctl_pci;
5
6static int report_gart_errors;
7module_param(report_gart_errors, int, 0644);
8
9/*
10 * Set by command line parameter. If BIOS has enabled the ECC, this override is
11 * cleared to prevent re-enabling the hardware by this driver.
12 */
13static int ecc_enable_override;
14module_param(ecc_enable_override, int, 0644);
15
Tejun Heoa29d8b82010-02-02 14:39:15 +090016static struct msr __percpu *msrs;
Borislav Petkov50542252009-12-11 18:14:40 +010017
Borislav Petkov360b7f32010-10-15 19:25:38 +020018/*
19 * count successfully initialized driver instances for setup_pci_device()
20 */
21static atomic_t drv_instances = ATOMIC_INIT(0);
22
Borislav Petkovcc4d8862010-10-13 16:11:59 +020023/* Per-node driver instances */
24static struct mem_ctl_info **mcis;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +020025static struct ecc_settings **ecc_stngs;
Doug Thompson2bc65412009-05-04 20:11:14 +020026
27/*
Borislav Petkovb70ef012009-06-25 19:32:38 +020028 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
29 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
30 * or higher value'.
31 *
32 *FIXME: Produce a better mapping/linearisation.
33 */
Daniel J Bluemanc7e53012012-11-30 16:44:20 +080034static const struct scrubrate {
Borislav Petkov39094442010-11-24 19:52:09 +010035 u32 scrubval; /* bit pattern for scrub rate */
36 u32 bandwidth; /* bandwidth consumed (bytes/sec) */
37} scrubrates[] = {
Borislav Petkovb70ef012009-06-25 19:32:38 +020038 { 0x01, 1600000000UL},
39 { 0x02, 800000000UL},
40 { 0x03, 400000000UL},
41 { 0x04, 200000000UL},
42 { 0x05, 100000000UL},
43 { 0x06, 50000000UL},
44 { 0x07, 25000000UL},
45 { 0x08, 12284069UL},
46 { 0x09, 6274509UL},
47 { 0x0A, 3121951UL},
48 { 0x0B, 1560975UL},
49 { 0x0C, 781440UL},
50 { 0x0D, 390720UL},
51 { 0x0E, 195300UL},
52 { 0x0F, 97650UL},
53 { 0x10, 48854UL},
54 { 0x11, 24427UL},
55 { 0x12, 12213UL},
56 { 0x13, 6101UL},
57 { 0x14, 3051UL},
58 { 0x15, 1523UL},
59 { 0x16, 761UL},
60 { 0x00, 0UL}, /* scrubbing off */
61};
62
Borislav Petkov66fed2d2012-08-09 18:41:07 +020063int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
64 u32 *val, const char *func)
Borislav Petkovb2b0c602010-10-08 18:32:29 +020065{
66 int err = 0;
67
68 err = pci_read_config_dword(pdev, offset, val);
69 if (err)
70 amd64_warn("%s: error reading F%dx%03x.\n",
71 func, PCI_FUNC(pdev->devfn), offset);
72
73 return err;
74}
75
76int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
77 u32 val, const char *func)
78{
79 int err = 0;
80
81 err = pci_write_config_dword(pdev, offset, val);
82 if (err)
83 amd64_warn("%s: error writing to F%dx%03x.\n",
84 func, PCI_FUNC(pdev->devfn), offset);
85
86 return err;
87}
88
89/*
90 *
91 * Depending on the family, F2 DCT reads need special handling:
92 *
93 * K8: has a single DCT only
94 *
95 * F10h: each DCT has its own set of regs
96 * DCT0 -> F2x040..
97 * DCT1 -> F2x140..
98 *
99 * F15h: we select which DCT we access using F1x10C[DctCfgSel]
100 *
101 */
102static int k8_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
103 const char *func)
104{
105 if (addr >= 0x100)
106 return -EINVAL;
107
108 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
109}
110
111static int f10_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
112 const char *func)
113{
114 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
115}
116
Borislav Petkov73ba8592011-09-19 17:34:45 +0200117/*
118 * Select DCT to which PCI cfg accesses are routed
119 */
120static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct)
121{
122 u32 reg = 0;
123
124 amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, &reg);
125 reg &= 0xfffffffe;
126 reg |= dct;
127 amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
128}
129
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200130static int f15_read_dct_pci_cfg(struct amd64_pvt *pvt, int addr, u32 *val,
131 const char *func)
132{
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200133 u8 dct = 0;
134
135 if (addr >= 0x140 && addr <= 0x1a0) {
136 dct = 1;
137 addr -= 0x100;
138 }
139
Borislav Petkov73ba8592011-09-19 17:34:45 +0200140 f15h_select_dct(pvt, dct);
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200141
142 return __amd64_read_pci_cfg_dword(pvt->F2, addr, val, func);
143}
144
Borislav Petkovb70ef012009-06-25 19:32:38 +0200145/*
Doug Thompson2bc65412009-05-04 20:11:14 +0200146 * Memory scrubber control interface. For K8, memory scrubbing is handled by
147 * hardware and can involve L2 cache, dcache as well as the main memory. With
148 * F10, this is extended to L3 cache scrubbing on CPU models sporting that
149 * functionality.
150 *
151 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
152 * (dram) over to cache lines. This is nasty, so we will use bandwidth in
153 * bytes/sec for the setting.
154 *
155 * Currently, we only do dram scrubbing. If the scrubbing is done in software on
156 * other archs, we might not have access to the caches directly.
157 */
158
159/*
160 * scan the scrub rate mapping table for a close or matching bandwidth value to
161 * issue. If requested is too big, then use last maximum value found.
162 */
Borislav Petkov395ae782010-10-01 18:38:19 +0200163static int __amd64_set_scrub_rate(struct pci_dev *ctl, u32 new_bw, u32 min_rate)
Doug Thompson2bc65412009-05-04 20:11:14 +0200164{
165 u32 scrubval;
166 int i;
167
168 /*
169 * map the configured rate (new_bw) to a value specific to the AMD64
170 * memory controller and apply to register. Search for the first
171 * bandwidth entry that is greater or equal than the setting requested
172 * and program that. If at last entry, turn off DRAM scrubbing.
Andrew Morton168bfee2012-10-23 14:09:39 -0700173 *
174 * If no suitable bandwidth is found, turn off DRAM scrubbing entirely
175 * by falling back to the last element in scrubrates[].
Doug Thompson2bc65412009-05-04 20:11:14 +0200176 */
Andrew Morton168bfee2012-10-23 14:09:39 -0700177 for (i = 0; i < ARRAY_SIZE(scrubrates) - 1; i++) {
Doug Thompson2bc65412009-05-04 20:11:14 +0200178 /*
179 * skip scrub rates which aren't recommended
180 * (see F10 BKDG, F3x58)
181 */
Borislav Petkov395ae782010-10-01 18:38:19 +0200182 if (scrubrates[i].scrubval < min_rate)
Doug Thompson2bc65412009-05-04 20:11:14 +0200183 continue;
184
185 if (scrubrates[i].bandwidth <= new_bw)
186 break;
Doug Thompson2bc65412009-05-04 20:11:14 +0200187 }
188
189 scrubval = scrubrates[i].scrubval;
Doug Thompson2bc65412009-05-04 20:11:14 +0200190
Borislav Petkov5980bb92011-01-07 16:26:49 +0100191 pci_write_bits32(ctl, SCRCTRL, scrubval, 0x001F);
Doug Thompson2bc65412009-05-04 20:11:14 +0200192
Borislav Petkov39094442010-11-24 19:52:09 +0100193 if (scrubval)
194 return scrubrates[i].bandwidth;
195
Doug Thompson2bc65412009-05-04 20:11:14 +0200196 return 0;
197}
198
Borislav Petkov395ae782010-10-01 18:38:19 +0200199static int amd64_set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
Doug Thompson2bc65412009-05-04 20:11:14 +0200200{
201 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkov87b3e0e2011-01-19 20:02:38 +0100202 u32 min_scrubrate = 0x5;
Doug Thompson2bc65412009-05-04 20:11:14 +0200203
Borislav Petkov87b3e0e2011-01-19 20:02:38 +0100204 if (boot_cpu_data.x86 == 0xf)
205 min_scrubrate = 0x0;
206
Borislav Petkov73ba8592011-09-19 17:34:45 +0200207 /* F15h Erratum #505 */
208 if (boot_cpu_data.x86 == 0x15)
209 f15h_select_dct(pvt, 0);
210
Borislav Petkov87b3e0e2011-01-19 20:02:38 +0100211 return __amd64_set_scrub_rate(pvt->F3, bw, min_scrubrate);
Doug Thompson2bc65412009-05-04 20:11:14 +0200212}
213
Borislav Petkov39094442010-11-24 19:52:09 +0100214static int amd64_get_scrub_rate(struct mem_ctl_info *mci)
Doug Thompson2bc65412009-05-04 20:11:14 +0200215{
216 struct amd64_pvt *pvt = mci->pvt_info;
217 u32 scrubval = 0;
Borislav Petkov39094442010-11-24 19:52:09 +0100218 int i, retval = -EINVAL;
Doug Thompson2bc65412009-05-04 20:11:14 +0200219
Borislav Petkov73ba8592011-09-19 17:34:45 +0200220 /* F15h Erratum #505 */
221 if (boot_cpu_data.x86 == 0x15)
222 f15h_select_dct(pvt, 0);
223
Borislav Petkov5980bb92011-01-07 16:26:49 +0100224 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
Doug Thompson2bc65412009-05-04 20:11:14 +0200225
226 scrubval = scrubval & 0x001F;
227
Roel Kluin926311f2010-01-11 20:58:21 +0100228 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
Doug Thompson2bc65412009-05-04 20:11:14 +0200229 if (scrubrates[i].scrubval == scrubval) {
Borislav Petkov39094442010-11-24 19:52:09 +0100230 retval = scrubrates[i].bandwidth;
Doug Thompson2bc65412009-05-04 20:11:14 +0200231 break;
232 }
233 }
Borislav Petkov39094442010-11-24 19:52:09 +0100234 return retval;
Doug Thompson2bc65412009-05-04 20:11:14 +0200235}
236
Doug Thompson67757632009-04-27 15:53:22 +0200237/*
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200238 * returns true if the SysAddr given by sys_addr matches the
239 * DRAM base/limit associated with node_id
Doug Thompson67757632009-04-27 15:53:22 +0200240 */
Borislav Petkovb487c332011-02-21 18:55:00 +0100241static bool amd64_base_limit_match(struct amd64_pvt *pvt, u64 sys_addr,
Daniel J Bluemanc7e53012012-11-30 16:44:20 +0800242 u8 nid)
Doug Thompson67757632009-04-27 15:53:22 +0200243{
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200244 u64 addr;
Doug Thompson67757632009-04-27 15:53:22 +0200245
246 /* The K8 treats this as a 40-bit value. However, bits 63-40 will be
247 * all ones if the most significant implemented address bit is 1.
248 * Here we discard bits 63-40. See section 3.4.2 of AMD publication
249 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
250 * Application Programming.
251 */
252 addr = sys_addr & 0x000000ffffffffffull;
253
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200254 return ((addr >= get_dram_base(pvt, nid)) &&
255 (addr <= get_dram_limit(pvt, nid)));
Doug Thompson67757632009-04-27 15:53:22 +0200256}
257
258/*
259 * Attempt to map a SysAddr to a node. On success, return a pointer to the
260 * mem_ctl_info structure for the node that the SysAddr maps to.
261 *
262 * On failure, return NULL.
263 */
264static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
265 u64 sys_addr)
266{
267 struct amd64_pvt *pvt;
Daniel J Bluemanc7e53012012-11-30 16:44:20 +0800268 u8 node_id;
Doug Thompson67757632009-04-27 15:53:22 +0200269 u32 intlv_en, bits;
270
271 /*
272 * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
273 * 3.4.4.2) registers to map the SysAddr to a node ID.
274 */
275 pvt = mci->pvt_info;
276
277 /*
278 * The value of this field should be the same for all DRAM Base
279 * registers. Therefore we arbitrarily choose to read it from the
280 * register for node 0.
281 */
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200282 intlv_en = dram_intlv_en(pvt, 0);
Doug Thompson67757632009-04-27 15:53:22 +0200283
284 if (intlv_en == 0) {
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200285 for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
Doug Thompson67757632009-04-27 15:53:22 +0200286 if (amd64_base_limit_match(pvt, sys_addr, node_id))
Borislav Petkov8edc5442009-09-18 12:39:19 +0200287 goto found;
Doug Thompson67757632009-04-27 15:53:22 +0200288 }
Borislav Petkov8edc5442009-09-18 12:39:19 +0200289 goto err_no_match;
Doug Thompson67757632009-04-27 15:53:22 +0200290 }
291
Borislav Petkov72f158f2009-09-18 12:27:27 +0200292 if (unlikely((intlv_en != 0x01) &&
293 (intlv_en != 0x03) &&
294 (intlv_en != 0x07))) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200295 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
Doug Thompson67757632009-04-27 15:53:22 +0200296 return NULL;
297 }
298
299 bits = (((u32) sys_addr) >> 12) & intlv_en;
300
301 for (node_id = 0; ; ) {
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200302 if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
Doug Thompson67757632009-04-27 15:53:22 +0200303 break; /* intlv_sel field matches */
304
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200305 if (++node_id >= DRAM_RANGES)
Doug Thompson67757632009-04-27 15:53:22 +0200306 goto err_no_match;
307 }
308
309 /* sanity test for sys_addr */
310 if (unlikely(!amd64_base_limit_match(pvt, sys_addr, node_id))) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200311 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
312 "range for node %d with node interleaving enabled.\n",
313 __func__, sys_addr, node_id);
Doug Thompson67757632009-04-27 15:53:22 +0200314 return NULL;
315 }
316
317found:
Borislav Petkovb487c332011-02-21 18:55:00 +0100318 return edac_mc_find((int)node_id);
Doug Thompson67757632009-04-27 15:53:22 +0200319
320err_no_match:
Joe Perches956b9ba2012-04-29 17:08:39 -0300321 edac_dbg(2, "sys_addr 0x%lx doesn't match any node\n",
322 (unsigned long)sys_addr);
Doug Thompson67757632009-04-27 15:53:22 +0200323
324 return NULL;
325}
Doug Thompsone2ce7252009-04-27 15:57:12 +0200326
327/*
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100328 * compute the CS base address of the @csrow on the DRAM controller @dct.
329 * For details see F2x[5C:40] in the processor's BKDG
Doug Thompsone2ce7252009-04-27 15:57:12 +0200330 */
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100331static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
332 u64 *base, u64 *mask)
Doug Thompsone2ce7252009-04-27 15:57:12 +0200333{
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100334 u64 csbase, csmask, base_bits, mask_bits;
335 u8 addr_shift;
336
337 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) {
338 csbase = pvt->csels[dct].csbases[csrow];
339 csmask = pvt->csels[dct].csmasks[csrow];
340 base_bits = GENMASK(21, 31) | GENMASK(9, 15);
341 mask_bits = GENMASK(21, 29) | GENMASK(9, 15);
342 addr_shift = 4;
343 } else {
344 csbase = pvt->csels[dct].csbases[csrow];
345 csmask = pvt->csels[dct].csmasks[csrow >> 1];
346 addr_shift = 8;
347
348 if (boot_cpu_data.x86 == 0x15)
349 base_bits = mask_bits = GENMASK(19,30) | GENMASK(5,13);
350 else
351 base_bits = mask_bits = GENMASK(19,28) | GENMASK(5,13);
352 }
353
354 *base = (csbase & base_bits) << addr_shift;
355
356 *mask = ~0ULL;
357 /* poke holes for the csmask */
358 *mask &= ~(mask_bits << addr_shift);
359 /* OR them in */
360 *mask |= (csmask & mask_bits) << addr_shift;
Doug Thompsone2ce7252009-04-27 15:57:12 +0200361}
362
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100363#define for_each_chip_select(i, dct, pvt) \
364 for (i = 0; i < pvt->csels[dct].b_cnt; i++)
Doug Thompsone2ce7252009-04-27 15:57:12 +0200365
Borislav Petkov614ec9d2011-01-13 18:02:22 +0100366#define chip_select_base(i, dct, pvt) \
367 pvt->csels[dct].csbases[i]
368
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100369#define for_each_chip_select_mask(i, dct, pvt) \
370 for (i = 0; i < pvt->csels[dct].m_cnt; i++)
Doug Thompsone2ce7252009-04-27 15:57:12 +0200371
372/*
373 * @input_addr is an InputAddr associated with the node given by mci. Return the
374 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
375 */
376static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
377{
378 struct amd64_pvt *pvt;
379 int csrow;
380 u64 base, mask;
381
382 pvt = mci->pvt_info;
383
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100384 for_each_chip_select(csrow, 0, pvt) {
385 if (!csrow_enabled(csrow, 0, pvt))
Doug Thompsone2ce7252009-04-27 15:57:12 +0200386 continue;
387
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100388 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
389
390 mask = ~mask;
Doug Thompsone2ce7252009-04-27 15:57:12 +0200391
392 if ((input_addr & mask) == (base & mask)) {
Joe Perches956b9ba2012-04-29 17:08:39 -0300393 edac_dbg(2, "InputAddr 0x%lx matches csrow %d (node %d)\n",
394 (unsigned long)input_addr, csrow,
395 pvt->mc_node_id);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200396
397 return csrow;
398 }
399 }
Joe Perches956b9ba2012-04-29 17:08:39 -0300400 edac_dbg(2, "no matching csrow for InputAddr 0x%lx (MC node %d)\n",
401 (unsigned long)input_addr, pvt->mc_node_id);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200402
403 return -1;
404}
405
406/*
Doug Thompsone2ce7252009-04-27 15:57:12 +0200407 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
408 * for the node represented by mci. Info is passed back in *hole_base,
409 * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if
410 * info is invalid. Info may be invalid for either of the following reasons:
411 *
412 * - The revision of the node is not E or greater. In this case, the DRAM Hole
413 * Address Register does not exist.
414 *
415 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
416 * indicating that its contents are not valid.
417 *
418 * The values passed back in *hole_base, *hole_offset, and *hole_size are
419 * complete 32-bit values despite the fact that the bitfields in the DHAR
420 * only represent bits 31-24 of the base and offset values.
421 */
422int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
423 u64 *hole_offset, u64 *hole_size)
424{
425 struct amd64_pvt *pvt = mci->pvt_info;
Doug Thompsone2ce7252009-04-27 15:57:12 +0200426
427 /* only revE and later have the DRAM Hole Address Register */
Borislav Petkov1433eb92009-10-21 13:44:36 +0200428 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_E) {
Joe Perches956b9ba2012-04-29 17:08:39 -0300429 edac_dbg(1, " revision %d for node %d does not support DHAR\n",
430 pvt->ext_model, pvt->mc_node_id);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200431 return 1;
432 }
433
Borislav Petkovbc21fa52010-11-11 17:29:13 +0100434 /* valid for Fam10h and above */
Borislav Petkovc8e518d2010-12-10 19:49:19 +0100435 if (boot_cpu_data.x86 >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
Joe Perches956b9ba2012-04-29 17:08:39 -0300436 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this system\n");
Doug Thompsone2ce7252009-04-27 15:57:12 +0200437 return 1;
438 }
439
Borislav Petkovc8e518d2010-12-10 19:49:19 +0100440 if (!dhar_valid(pvt)) {
Joe Perches956b9ba2012-04-29 17:08:39 -0300441 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this node %d\n",
442 pvt->mc_node_id);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200443 return 1;
444 }
445
446 /* This node has Memory Hoisting */
447
448 /* +------------------+--------------------+--------------------+-----
449 * | memory | DRAM hole | relocated |
450 * | [0, (x - 1)] | [x, 0xffffffff] | addresses from |
451 * | | | DRAM hole |
452 * | | | [0x100000000, |
453 * | | | (0x100000000+ |
454 * | | | (0xffffffff-x))] |
455 * +------------------+--------------------+--------------------+-----
456 *
457 * Above is a diagram of physical memory showing the DRAM hole and the
458 * relocated addresses from the DRAM hole. As shown, the DRAM hole
459 * starts at address x (the base address) and extends through address
460 * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the
461 * addresses in the hole so that they start at 0x100000000.
462 */
463
Borislav Petkov1f316772012-08-10 12:50:50 +0200464 *hole_base = dhar_base(pvt);
465 *hole_size = (1ULL << 32) - *hole_base;
Doug Thompsone2ce7252009-04-27 15:57:12 +0200466
467 if (boot_cpu_data.x86 > 0xf)
Borislav Petkovbc21fa52010-11-11 17:29:13 +0100468 *hole_offset = f10_dhar_offset(pvt);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200469 else
Borislav Petkovbc21fa52010-11-11 17:29:13 +0100470 *hole_offset = k8_dhar_offset(pvt);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200471
Joe Perches956b9ba2012-04-29 17:08:39 -0300472 edac_dbg(1, " DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
473 pvt->mc_node_id, (unsigned long)*hole_base,
474 (unsigned long)*hole_offset, (unsigned long)*hole_size);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200475
476 return 0;
477}
478EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info);
479
Doug Thompson93c2df52009-05-04 20:46:50 +0200480/*
481 * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is
482 * assumed that sys_addr maps to the node given by mci.
483 *
484 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
485 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
486 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
487 * then it is also involved in translating a SysAddr to a DramAddr. Sections
488 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
489 * These parts of the documentation are unclear. I interpret them as follows:
490 *
491 * When node n receives a SysAddr, it processes the SysAddr as follows:
492 *
493 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
494 * Limit registers for node n. If the SysAddr is not within the range
495 * specified by the base and limit values, then node n ignores the Sysaddr
496 * (since it does not map to node n). Otherwise continue to step 2 below.
497 *
498 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
499 * disabled so skip to step 3 below. Otherwise see if the SysAddr is within
500 * the range of relocated addresses (starting at 0x100000000) from the DRAM
501 * hole. If not, skip to step 3 below. Else get the value of the
502 * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
503 * offset defined by this value from the SysAddr.
504 *
505 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
506 * Base register for node n. To obtain the DramAddr, subtract the base
507 * address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
508 */
509static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
510{
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200511 struct amd64_pvt *pvt = mci->pvt_info;
Doug Thompson93c2df52009-05-04 20:46:50 +0200512 u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
Borislav Petkov1f316772012-08-10 12:50:50 +0200513 int ret;
Doug Thompson93c2df52009-05-04 20:46:50 +0200514
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200515 dram_base = get_dram_base(pvt, pvt->mc_node_id);
Doug Thompson93c2df52009-05-04 20:46:50 +0200516
517 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
518 &hole_size);
519 if (!ret) {
Borislav Petkov1f316772012-08-10 12:50:50 +0200520 if ((sys_addr >= (1ULL << 32)) &&
521 (sys_addr < ((1ULL << 32) + hole_size))) {
Doug Thompson93c2df52009-05-04 20:46:50 +0200522 /* use DHAR to translate SysAddr to DramAddr */
523 dram_addr = sys_addr - hole_offset;
524
Joe Perches956b9ba2012-04-29 17:08:39 -0300525 edac_dbg(2, "using DHAR to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
526 (unsigned long)sys_addr,
527 (unsigned long)dram_addr);
Doug Thompson93c2df52009-05-04 20:46:50 +0200528
529 return dram_addr;
530 }
531 }
532
533 /*
534 * Translate the SysAddr to a DramAddr as shown near the start of
535 * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8
536 * only deals with 40-bit values. Therefore we discard bits 63-40 of
537 * sys_addr below. If bit 39 of sys_addr is 1 then the bits we
538 * discard are all 1s. Otherwise the bits we discard are all 0s. See
539 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
540 * Programmer's Manual Volume 1 Application Programming.
541 */
Borislav Petkovf678b8c2010-12-13 19:21:07 +0100542 dram_addr = (sys_addr & GENMASK(0, 39)) - dram_base;
Doug Thompson93c2df52009-05-04 20:46:50 +0200543
Joe Perches956b9ba2012-04-29 17:08:39 -0300544 edac_dbg(2, "using DRAM Base register to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
545 (unsigned long)sys_addr, (unsigned long)dram_addr);
Doug Thompson93c2df52009-05-04 20:46:50 +0200546 return dram_addr;
547}
548
549/*
550 * @intlv_en is the value of the IntlvEn field from a DRAM Base register
551 * (section 3.4.4.1). Return the number of bits from a SysAddr that are used
552 * for node interleaving.
553 */
554static int num_node_interleave_bits(unsigned intlv_en)
555{
556 static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
557 int n;
558
559 BUG_ON(intlv_en > 7);
560 n = intlv_shift_table[intlv_en];
561 return n;
562}
563
564/* Translate the DramAddr given by @dram_addr to an InputAddr. */
565static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
566{
567 struct amd64_pvt *pvt;
568 int intlv_shift;
569 u64 input_addr;
570
571 pvt = mci->pvt_info;
572
573 /*
574 * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
575 * concerning translating a DramAddr to an InputAddr.
576 */
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200577 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
Borislav Petkovf678b8c2010-12-13 19:21:07 +0100578 input_addr = ((dram_addr >> intlv_shift) & GENMASK(12, 35)) +
579 (dram_addr & 0xfff);
Doug Thompson93c2df52009-05-04 20:46:50 +0200580
Joe Perches956b9ba2012-04-29 17:08:39 -0300581 edac_dbg(2, " Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
582 intlv_shift, (unsigned long)dram_addr,
583 (unsigned long)input_addr);
Doug Thompson93c2df52009-05-04 20:46:50 +0200584
585 return input_addr;
586}
587
588/*
589 * Translate the SysAddr represented by @sys_addr to an InputAddr. It is
590 * assumed that @sys_addr maps to the node given by mci.
591 */
592static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
593{
594 u64 input_addr;
595
596 input_addr =
597 dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
598
Joe Perches956b9ba2012-04-29 17:08:39 -0300599 edac_dbg(2, "SysAdddr 0x%lx translates to InputAddr 0x%lx\n",
600 (unsigned long)sys_addr, (unsigned long)input_addr);
Doug Thompson93c2df52009-05-04 20:46:50 +0200601
602 return input_addr;
603}
604
Doug Thompson93c2df52009-05-04 20:46:50 +0200605/* Map the Error address to a PAGE and PAGE OFFSET. */
606static inline void error_address_to_page_and_offset(u64 error_address,
Borislav Petkov33ca0642012-08-30 18:01:36 +0200607 struct err_info *err)
Doug Thompson93c2df52009-05-04 20:46:50 +0200608{
Borislav Petkov33ca0642012-08-30 18:01:36 +0200609 err->page = (u32) (error_address >> PAGE_SHIFT);
610 err->offset = ((u32) error_address) & ~PAGE_MASK;
Doug Thompson93c2df52009-05-04 20:46:50 +0200611}
612
613/*
614 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
615 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
616 * of a node that detected an ECC memory error. mci represents the node that
617 * the error address maps to (possibly different from the node that detected
618 * the error). Return the number of the csrow that sys_addr maps to, or -1 on
619 * error.
620 */
621static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
622{
623 int csrow;
624
625 csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
626
627 if (csrow == -1)
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200628 amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
629 "address 0x%lx\n", (unsigned long)sys_addr);
Doug Thompson93c2df52009-05-04 20:46:50 +0200630 return csrow;
631}
Doug Thompsone2ce7252009-04-27 15:57:12 +0200632
Borislav Petkovbfc04ae2009-11-12 19:05:07 +0100633static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
Doug Thompson2da11652009-04-27 16:09:09 +0200634
Doug Thompson2da11652009-04-27 16:09:09 +0200635/*
636 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
637 * are ECC capable.
638 */
Dan Carpenter1f6189e2011-10-06 02:30:25 -0400639static unsigned long amd64_determine_edac_cap(struct amd64_pvt *pvt)
Doug Thompson2da11652009-04-27 16:09:09 +0200640{
Borislav Petkovcb328502010-12-22 14:28:24 +0100641 u8 bit;
Dan Carpenter1f6189e2011-10-06 02:30:25 -0400642 unsigned long edac_cap = EDAC_FLAG_NONE;
Doug Thompson2da11652009-04-27 16:09:09 +0200643
Borislav Petkov1433eb92009-10-21 13:44:36 +0200644 bit = (boot_cpu_data.x86 > 0xf || pvt->ext_model >= K8_REV_F)
Doug Thompson2da11652009-04-27 16:09:09 +0200645 ? 19
646 : 17;
647
Borislav Petkov584fcff2009-06-10 18:29:54 +0200648 if (pvt->dclr0 & BIT(bit))
Doug Thompson2da11652009-04-27 16:09:09 +0200649 edac_cap = EDAC_FLAG_SECDED;
650
651 return edac_cap;
652}
653
Borislav Petkov8c671752011-02-23 17:25:12 +0100654static void amd64_debug_display_dimm_sizes(struct amd64_pvt *, u8);
Doug Thompson2da11652009-04-27 16:09:09 +0200655
Borislav Petkov68798e12009-11-03 16:18:33 +0100656static void amd64_dump_dramcfg_low(u32 dclr, int chan)
657{
Joe Perches956b9ba2012-04-29 17:08:39 -0300658 edac_dbg(1, "F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
Borislav Petkov68798e12009-11-03 16:18:33 +0100659
Joe Perches956b9ba2012-04-29 17:08:39 -0300660 edac_dbg(1, " DIMM type: %sbuffered; all DIMMs support ECC: %s\n",
661 (dclr & BIT(16)) ? "un" : "",
662 (dclr & BIT(19)) ? "yes" : "no");
Borislav Petkov68798e12009-11-03 16:18:33 +0100663
Joe Perches956b9ba2012-04-29 17:08:39 -0300664 edac_dbg(1, " PAR/ERR parity: %s\n",
665 (dclr & BIT(8)) ? "enabled" : "disabled");
Borislav Petkov68798e12009-11-03 16:18:33 +0100666
Borislav Petkovcb328502010-12-22 14:28:24 +0100667 if (boot_cpu_data.x86 == 0x10)
Joe Perches956b9ba2012-04-29 17:08:39 -0300668 edac_dbg(1, " DCT 128bit mode width: %s\n",
669 (dclr & BIT(11)) ? "128b" : "64b");
Borislav Petkov68798e12009-11-03 16:18:33 +0100670
Joe Perches956b9ba2012-04-29 17:08:39 -0300671 edac_dbg(1, " x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
672 (dclr & BIT(12)) ? "yes" : "no",
673 (dclr & BIT(13)) ? "yes" : "no",
674 (dclr & BIT(14)) ? "yes" : "no",
675 (dclr & BIT(15)) ? "yes" : "no");
Borislav Petkov68798e12009-11-03 16:18:33 +0100676}
677
Doug Thompson2da11652009-04-27 16:09:09 +0200678/* Display and decode various NB registers for debug purposes. */
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200679static void dump_misc_regs(struct amd64_pvt *pvt)
Doug Thompson2da11652009-04-27 16:09:09 +0200680{
Joe Perches956b9ba2012-04-29 17:08:39 -0300681 edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
Doug Thompson2da11652009-04-27 16:09:09 +0200682
Joe Perches956b9ba2012-04-29 17:08:39 -0300683 edac_dbg(1, " NB two channel DRAM capable: %s\n",
684 (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");
Borislav Petkov68798e12009-11-03 16:18:33 +0100685
Joe Perches956b9ba2012-04-29 17:08:39 -0300686 edac_dbg(1, " ECC capable: %s, ChipKill ECC capable: %s\n",
687 (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
688 (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");
Borislav Petkov68798e12009-11-03 16:18:33 +0100689
690 amd64_dump_dramcfg_low(pvt->dclr0, 0);
Doug Thompson2da11652009-04-27 16:09:09 +0200691
Joe Perches956b9ba2012-04-29 17:08:39 -0300692 edac_dbg(1, "F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
Doug Thompson2da11652009-04-27 16:09:09 +0200693
Joe Perches956b9ba2012-04-29 17:08:39 -0300694 edac_dbg(1, "F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, offset: 0x%08x\n",
695 pvt->dhar, dhar_base(pvt),
696 (boot_cpu_data.x86 == 0xf) ? k8_dhar_offset(pvt)
697 : f10_dhar_offset(pvt));
Doug Thompson2da11652009-04-27 16:09:09 +0200698
Joe Perches956b9ba2012-04-29 17:08:39 -0300699 edac_dbg(1, " DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
Doug Thompson2da11652009-04-27 16:09:09 +0200700
Borislav Petkov8c671752011-02-23 17:25:12 +0100701 amd64_debug_display_dimm_sizes(pvt, 0);
Borislav Petkov4d796362011-02-03 15:59:57 +0100702
Borislav Petkov8de1d912009-10-16 13:39:30 +0200703 /* everything below this point is Fam10h and above */
Borislav Petkov4d796362011-02-03 15:59:57 +0100704 if (boot_cpu_data.x86 == 0xf)
Doug Thompson2da11652009-04-27 16:09:09 +0200705 return;
Borislav Petkov4d796362011-02-03 15:59:57 +0100706
Borislav Petkov8c671752011-02-23 17:25:12 +0100707 amd64_debug_display_dimm_sizes(pvt, 1);
Doug Thompson2da11652009-04-27 16:09:09 +0200708
Borislav Petkova3b7db02011-01-19 20:35:12 +0100709 amd64_info("using %s syndromes.\n", ((pvt->ecc_sym_sz == 8) ? "x8" : "x4"));
Borislav Petkovad6a32e2010-03-09 12:46:00 +0100710
Borislav Petkov8de1d912009-10-16 13:39:30 +0200711 /* Only if NOT ganged does dclr1 have valid info */
Borislav Petkov68798e12009-11-03 16:18:33 +0100712 if (!dct_ganging_enabled(pvt))
713 amd64_dump_dramcfg_low(pvt->dclr1, 1);
Doug Thompson2da11652009-04-27 16:09:09 +0200714}
715
Doug Thompson94be4bf2009-04-27 16:12:00 +0200716/*
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100717 * see BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
Doug Thompson94be4bf2009-04-27 16:12:00 +0200718 */
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100719static void prep_chip_selects(struct amd64_pvt *pvt)
Doug Thompson94be4bf2009-04-27 16:12:00 +0200720{
Borislav Petkov1433eb92009-10-21 13:44:36 +0200721 if (boot_cpu_data.x86 == 0xf && pvt->ext_model < K8_REV_F) {
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100722 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
723 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
Borislav Petkov9d858bb2009-09-21 14:35:51 +0200724 } else {
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100725 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
726 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
Doug Thompson94be4bf2009-04-27 16:12:00 +0200727 }
728}
729
730/*
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100731 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
Doug Thompson94be4bf2009-04-27 16:12:00 +0200732 */
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200733static void read_dct_base_mask(struct amd64_pvt *pvt)
Doug Thompson94be4bf2009-04-27 16:12:00 +0200734{
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100735 int cs;
Doug Thompson94be4bf2009-04-27 16:12:00 +0200736
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100737 prep_chip_selects(pvt);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200738
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100739 for_each_chip_select(cs, 0, pvt) {
Borislav Petkov71d2a322011-02-21 19:37:24 +0100740 int reg0 = DCSB0 + (cs * 4);
741 int reg1 = DCSB1 + (cs * 4);
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100742 u32 *base0 = &pvt->csels[0].csbases[cs];
743 u32 *base1 = &pvt->csels[1].csbases[cs];
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200744
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100745 if (!amd64_read_dct_pci_cfg(pvt, reg0, base0))
Joe Perches956b9ba2012-04-29 17:08:39 -0300746 edac_dbg(0, " DCSB0[%d]=0x%08x reg: F2x%x\n",
747 cs, *base0, reg0);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200748
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100749 if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt))
750 continue;
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200751
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100752 if (!amd64_read_dct_pci_cfg(pvt, reg1, base1))
Joe Perches956b9ba2012-04-29 17:08:39 -0300753 edac_dbg(0, " DCSB1[%d]=0x%08x reg: F2x%x\n",
754 cs, *base1, reg1);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200755 }
756
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100757 for_each_chip_select_mask(cs, 0, pvt) {
Borislav Petkov71d2a322011-02-21 19:37:24 +0100758 int reg0 = DCSM0 + (cs * 4);
759 int reg1 = DCSM1 + (cs * 4);
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100760 u32 *mask0 = &pvt->csels[0].csmasks[cs];
761 u32 *mask1 = &pvt->csels[1].csmasks[cs];
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200762
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100763 if (!amd64_read_dct_pci_cfg(pvt, reg0, mask0))
Joe Perches956b9ba2012-04-29 17:08:39 -0300764 edac_dbg(0, " DCSM0[%d]=0x%08x reg: F2x%x\n",
765 cs, *mask0, reg0);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200766
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100767 if (boot_cpu_data.x86 == 0xf || dct_ganging_enabled(pvt))
768 continue;
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200769
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100770 if (!amd64_read_dct_pci_cfg(pvt, reg1, mask1))
Joe Perches956b9ba2012-04-29 17:08:39 -0300771 edac_dbg(0, " DCSM1[%d]=0x%08x reg: F2x%x\n",
772 cs, *mask1, reg1);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200773 }
774}
775
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200776static enum mem_type amd64_determine_memory_type(struct amd64_pvt *pvt, int cs)
Doug Thompson94be4bf2009-04-27 16:12:00 +0200777{
778 enum mem_type type;
779
Borislav Petkovcb328502010-12-22 14:28:24 +0100780 /* F15h supports only DDR3 */
781 if (boot_cpu_data.x86 >= 0x15)
782 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
783 else if (boot_cpu_data.x86 == 0x10 || pvt->ext_model >= K8_REV_F) {
Borislav Petkov6b4c0bd2009-11-12 15:37:57 +0100784 if (pvt->dchr0 & DDR3_MODE)
785 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
786 else
787 type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
Doug Thompson94be4bf2009-04-27 16:12:00 +0200788 } else {
Doug Thompson94be4bf2009-04-27 16:12:00 +0200789 type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
790 }
791
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200792 amd64_info("CS%d: %s\n", cs, edac_mem_types[type]);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200793
794 return type;
795}
796
Borislav Petkovcb328502010-12-22 14:28:24 +0100797/* Get the number of DCT channels the memory controller is using. */
Doug Thompsonddff8762009-04-27 16:14:52 +0200798static int k8_early_channel_count(struct amd64_pvt *pvt)
799{
Borislav Petkovcb328502010-12-22 14:28:24 +0100800 int flag;
Doug Thompsonddff8762009-04-27 16:14:52 +0200801
Borislav Petkov9f56da02010-10-01 19:44:53 +0200802 if (pvt->ext_model >= K8_REV_F)
Doug Thompsonddff8762009-04-27 16:14:52 +0200803 /* RevF (NPT) and later */
Borislav Petkov41d8bfa2011-01-18 19:16:08 +0100804 flag = pvt->dclr0 & WIDTH_128;
Borislav Petkov9f56da02010-10-01 19:44:53 +0200805 else
Doug Thompsonddff8762009-04-27 16:14:52 +0200806 /* RevE and earlier */
807 flag = pvt->dclr0 & REVE_WIDTH_128;
Doug Thompsonddff8762009-04-27 16:14:52 +0200808
809 /* not used */
810 pvt->dclr1 = 0;
811
812 return (flag) ? 2 : 1;
813}
814
Borislav Petkov70046622011-01-10 14:37:27 +0100815/* On F10h and later ErrAddr is MC4_ADDR[47:1] */
816static u64 get_error_address(struct mce *m)
Doug Thompsonddff8762009-04-27 16:14:52 +0200817{
Borislav Petkovc1ae6832011-03-30 15:42:10 +0200818 struct cpuinfo_x86 *c = &boot_cpu_data;
819 u64 addr;
Borislav Petkov70046622011-01-10 14:37:27 +0100820 u8 start_bit = 1;
821 u8 end_bit = 47;
822
Borislav Petkovc1ae6832011-03-30 15:42:10 +0200823 if (c->x86 == 0xf) {
Borislav Petkov70046622011-01-10 14:37:27 +0100824 start_bit = 3;
825 end_bit = 39;
826 }
827
Borislav Petkovc1ae6832011-03-30 15:42:10 +0200828 addr = m->addr & GENMASK(start_bit, end_bit);
829
830 /*
831 * Erratum 637 workaround
832 */
833 if (c->x86 == 0x15) {
834 struct amd64_pvt *pvt;
835 u64 cc6_base, tmp_addr;
836 u32 tmp;
Daniel J Blueman8b84c8d2012-11-27 14:32:10 +0800837 u16 mce_nid;
838 u8 intlv_en;
Borislav Petkovc1ae6832011-03-30 15:42:10 +0200839
840 if ((addr & GENMASK(24, 47)) >> 24 != 0x00fdf7)
841 return addr;
842
843 mce_nid = amd_get_nb_id(m->extcpu);
844 pvt = mcis[mce_nid]->pvt_info;
845
846 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp);
847 intlv_en = tmp >> 21 & 0x7;
848
849 /* add [47:27] + 3 trailing bits */
850 cc6_base = (tmp & GENMASK(0, 20)) << 3;
851
852 /* reverse and add DramIntlvEn */
853 cc6_base |= intlv_en ^ 0x7;
854
855 /* pin at [47:24] */
856 cc6_base <<= 24;
857
858 if (!intlv_en)
859 return cc6_base | (addr & GENMASK(0, 23));
860
861 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp);
862
863 /* faster log2 */
864 tmp_addr = (addr & GENMASK(12, 23)) << __fls(intlv_en + 1);
865
866 /* OR DramIntlvSel into bits [14:12] */
867 tmp_addr |= (tmp & GENMASK(21, 23)) >> 9;
868
869 /* add remaining [11:0] bits from original MC4_ADDR */
870 tmp_addr |= addr & GENMASK(0, 11);
871
872 return cc6_base | tmp_addr;
873 }
874
875 return addr;
Doug Thompsonddff8762009-04-27 16:14:52 +0200876}
877
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +0800878static struct pci_dev *pci_get_related_function(unsigned int vendor,
879 unsigned int device,
880 struct pci_dev *related)
881{
882 struct pci_dev *dev = NULL;
883
884 while ((dev = pci_get_device(vendor, device, dev))) {
885 if (pci_domain_nr(dev->bus) == pci_domain_nr(related->bus) &&
886 (dev->bus->number == related->bus->number) &&
887 (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
888 break;
889 }
890
891 return dev;
892}
893
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200894static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
Doug Thompsonddff8762009-04-27 16:14:52 +0200895{
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +0800896 struct amd_northbridge *nb;
897 struct pci_dev *misc, *f1 = NULL;
Borislav Petkovf08e4572011-03-21 20:45:06 +0100898 struct cpuinfo_x86 *c = &boot_cpu_data;
Borislav Petkov71d2a322011-02-21 19:37:24 +0100899 int off = range << 3;
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +0800900 u32 llim;
Doug Thompsonddff8762009-04-27 16:14:52 +0200901
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200902 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo);
903 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
Doug Thompsonddff8762009-04-27 16:14:52 +0200904
Borislav Petkovf08e4572011-03-21 20:45:06 +0100905 if (c->x86 == 0xf)
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200906 return;
Doug Thompsonddff8762009-04-27 16:14:52 +0200907
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200908 if (!dram_rw(pvt, range))
909 return;
Doug Thompsonddff8762009-04-27 16:14:52 +0200910
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200911 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi);
912 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
Borislav Petkovf08e4572011-03-21 20:45:06 +0100913
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +0800914 /* F15h: factor in CC6 save area by reading dst node's limit reg */
915 if (c->x86 != 0x15)
916 return;
Borislav Petkovf08e4572011-03-21 20:45:06 +0100917
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +0800918 nb = node_to_amd_nb(dram_dst_node(pvt, range));
919 if (WARN_ON(!nb))
920 return;
Borislav Petkovf08e4572011-03-21 20:45:06 +0100921
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +0800922 misc = nb->misc;
923 f1 = pci_get_related_function(misc->vendor, PCI_DEVICE_ID_AMD_15H_NB_F1, misc);
924 if (WARN_ON(!f1))
925 return;
Borislav Petkovf08e4572011-03-21 20:45:06 +0100926
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +0800927 amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim);
Borislav Petkovf08e4572011-03-21 20:45:06 +0100928
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +0800929 pvt->ranges[range].lim.lo &= GENMASK(0, 15);
Borislav Petkovf08e4572011-03-21 20:45:06 +0100930
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +0800931 /* {[39:27],111b} */
932 pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16;
Borislav Petkovf08e4572011-03-21 20:45:06 +0100933
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +0800934 pvt->ranges[range].lim.hi &= GENMASK(0, 7);
Borislav Petkovf08e4572011-03-21 20:45:06 +0100935
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +0800936 /* [47:40] */
937 pvt->ranges[range].lim.hi |= llim >> 13;
938
939 pci_dev_put(f1);
Doug Thompsonddff8762009-04-27 16:14:52 +0200940}
941
Borislav Petkovf192c7b2011-01-10 14:24:32 +0100942static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
Borislav Petkov33ca0642012-08-30 18:01:36 +0200943 struct err_info *err)
Doug Thompsonddff8762009-04-27 16:14:52 +0200944{
Borislav Petkovf192c7b2011-01-10 14:24:32 +0100945 struct amd64_pvt *pvt = mci->pvt_info;
Doug Thompsonddff8762009-04-27 16:14:52 +0200946
Borislav Petkov33ca0642012-08-30 18:01:36 +0200947 error_address_to_page_and_offset(sys_addr, err);
Mauro Carvalho Chehabab5a5032012-04-16 15:03:50 -0300948
949 /*
950 * Find out which node the error address belongs to. This may be
951 * different from the node that detected the error.
952 */
Borislav Petkov33ca0642012-08-30 18:01:36 +0200953 err->src_mci = find_mc_by_sys_addr(mci, sys_addr);
954 if (!err->src_mci) {
Mauro Carvalho Chehabab5a5032012-04-16 15:03:50 -0300955 amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
956 (unsigned long)sys_addr);
Borislav Petkov33ca0642012-08-30 18:01:36 +0200957 err->err_code = ERR_NODE;
Mauro Carvalho Chehabab5a5032012-04-16 15:03:50 -0300958 return;
959 }
960
961 /* Now map the sys_addr to a CSROW */
Borislav Petkov33ca0642012-08-30 18:01:36 +0200962 err->csrow = sys_addr_to_csrow(err->src_mci, sys_addr);
963 if (err->csrow < 0) {
964 err->err_code = ERR_CSROW;
Mauro Carvalho Chehabab5a5032012-04-16 15:03:50 -0300965 return;
966 }
967
Doug Thompsonddff8762009-04-27 16:14:52 +0200968 /* CHIPKILL enabled */
Borislav Petkovf192c7b2011-01-10 14:24:32 +0100969 if (pvt->nbcfg & NBCFG_CHIPKILL) {
Borislav Petkov33ca0642012-08-30 18:01:36 +0200970 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
971 if (err->channel < 0) {
Doug Thompsonddff8762009-04-27 16:14:52 +0200972 /*
973 * Syndrome didn't map, so we don't know which of the
974 * 2 DIMMs is in error. So we need to ID 'both' of them
975 * as suspect.
976 */
Borislav Petkov33ca0642012-08-30 18:01:36 +0200977 amd64_mc_warn(err->src_mci, "unknown syndrome 0x%04x - "
Mauro Carvalho Chehabab5a5032012-04-16 15:03:50 -0300978 "possible error reporting race\n",
Borislav Petkov33ca0642012-08-30 18:01:36 +0200979 err->syndrome);
980 err->err_code = ERR_CHANNEL;
Doug Thompsonddff8762009-04-27 16:14:52 +0200981 return;
982 }
983 } else {
984 /*
985 * non-chipkill ecc mode
986 *
987 * The k8 documentation is unclear about how to determine the
988 * channel number when using non-chipkill memory. This method
989 * was obtained from email communication with someone at AMD.
990 * (Wish the email was placed in this comment - norsk)
991 */
Borislav Petkov33ca0642012-08-30 18:01:36 +0200992 err->channel = ((sys_addr & BIT(3)) != 0);
Doug Thompsonddff8762009-04-27 16:14:52 +0200993 }
Doug Thompsonddff8762009-04-27 16:14:52 +0200994}
995
Borislav Petkov41d8bfa2011-01-18 19:16:08 +0100996static int ddr2_cs_size(unsigned i, bool dct_width)
Doug Thompsonddff8762009-04-27 16:14:52 +0200997{
Borislav Petkov41d8bfa2011-01-18 19:16:08 +0100998 unsigned shift = 0;
Doug Thompsonddff8762009-04-27 16:14:52 +0200999
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001000 if (i <= 2)
1001 shift = i;
1002 else if (!(i & 0x1))
1003 shift = i >> 1;
Borislav Petkov1433eb92009-10-21 13:44:36 +02001004 else
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001005 shift = (i + 1) >> 1;
Doug Thompsonddff8762009-04-27 16:14:52 +02001006
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001007 return 128 << (shift + !!dct_width);
1008}
1009
1010static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1011 unsigned cs_mode)
1012{
1013 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1014
1015 if (pvt->ext_model >= K8_REV_F) {
1016 WARN_ON(cs_mode > 11);
1017 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1018 }
1019 else if (pvt->ext_model >= K8_REV_D) {
Borislav Petkov11b0a312011-11-09 21:28:43 +01001020 unsigned diff;
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001021 WARN_ON(cs_mode > 10);
1022
Borislav Petkov11b0a312011-11-09 21:28:43 +01001023 /*
1024 * the below calculation, besides trying to win an obfuscated C
1025 * contest, maps cs_mode values to DIMM chip select sizes. The
1026 * mappings are:
1027 *
1028 * cs_mode CS size (mb)
1029 * ======= ============
1030 * 0 32
1031 * 1 64
1032 * 2 128
1033 * 3 128
1034 * 4 256
1035 * 5 512
1036 * 6 256
1037 * 7 512
1038 * 8 1024
1039 * 9 1024
1040 * 10 2048
1041 *
1042 * Basically, it calculates a value with which to shift the
1043 * smallest CS size of 32MB.
1044 *
1045 * ddr[23]_cs_size have a similar purpose.
1046 */
1047 diff = cs_mode/3 + (unsigned)(cs_mode > 5);
1048
1049 return 32 << (cs_mode - diff);
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001050 }
1051 else {
1052 WARN_ON(cs_mode > 6);
1053 return 32 << cs_mode;
1054 }
Doug Thompsonddff8762009-04-27 16:14:52 +02001055}
1056
Doug Thompson1afd3c92009-04-27 16:16:50 +02001057/*
1058 * Get the number of DCT channels in use.
1059 *
1060 * Return:
1061 * number of Memory Channels in operation
1062 * Pass back:
1063 * contents of the DCL0_LOW register
1064 */
Borislav Petkov7d20d142011-01-07 17:58:04 +01001065static int f1x_early_channel_count(struct amd64_pvt *pvt)
Doug Thompson1afd3c92009-04-27 16:16:50 +02001066{
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02001067 int i, j, channels = 0;
Doug Thompsonddff8762009-04-27 16:14:52 +02001068
Borislav Petkov7d20d142011-01-07 17:58:04 +01001069 /* On F10h, if we are in 128 bit mode, then we are using 2 channels */
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001070 if (boot_cpu_data.x86 == 0x10 && (pvt->dclr0 & WIDTH_128))
Borislav Petkov7d20d142011-01-07 17:58:04 +01001071 return 2;
Doug Thompson1afd3c92009-04-27 16:16:50 +02001072
1073 /*
Borislav Petkovd16149e2009-10-16 19:55:49 +02001074 * Need to check if in unganged mode: In such, there are 2 channels,
1075 * but they are not in 128 bit mode and thus the above 'dclr0' status
1076 * bit will be OFF.
Doug Thompson1afd3c92009-04-27 16:16:50 +02001077 *
1078 * Need to check DCT0[0] and DCT1[0] to see if only one of them has
1079 * their CSEnable bit on. If so, then SINGLE DIMM case.
1080 */
Joe Perches956b9ba2012-04-29 17:08:39 -03001081 edac_dbg(0, "Data width is not 128 bits - need more decoding\n");
Doug Thompson1afd3c92009-04-27 16:16:50 +02001082
1083 /*
1084 * Check DRAM Bank Address Mapping values for each DIMM to see if there
1085 * is more than just one DIMM present in unganged mode. Need to check
1086 * both controllers since DIMMs can be placed in either one.
1087 */
Borislav Petkov525a1b22010-12-21 15:53:27 +01001088 for (i = 0; i < 2; i++) {
1089 u32 dbam = (i ? pvt->dbam1 : pvt->dbam0);
Doug Thompson1afd3c92009-04-27 16:16:50 +02001090
Wan Wei57a30852009-08-07 17:04:49 +02001091 for (j = 0; j < 4; j++) {
1092 if (DBAM_DIMM(j, dbam) > 0) {
1093 channels++;
1094 break;
1095 }
1096 }
Doug Thompson1afd3c92009-04-27 16:16:50 +02001097 }
1098
Borislav Petkovd16149e2009-10-16 19:55:49 +02001099 if (channels > 2)
1100 channels = 2;
1101
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001102 amd64_info("MCT channel count: %d\n", channels);
Doug Thompson1afd3c92009-04-27 16:16:50 +02001103
1104 return channels;
Doug Thompson1afd3c92009-04-27 16:16:50 +02001105}
1106
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001107static int ddr3_cs_size(unsigned i, bool dct_width)
Doug Thompson1afd3c92009-04-27 16:16:50 +02001108{
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001109 unsigned shift = 0;
1110 int cs_size = 0;
1111
1112 if (i == 0 || i == 3 || i == 4)
1113 cs_size = -1;
1114 else if (i <= 2)
1115 shift = i;
1116 else if (i == 12)
1117 shift = 7;
1118 else if (!(i & 0x1))
1119 shift = i >> 1;
1120 else
1121 shift = (i + 1) >> 1;
1122
1123 if (cs_size != -1)
1124 cs_size = (128 * (1 << !!dct_width)) << shift;
1125
1126 return cs_size;
1127}
1128
1129static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1130 unsigned cs_mode)
1131{
1132 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1133
1134 WARN_ON(cs_mode > 11);
Borislav Petkov1433eb92009-10-21 13:44:36 +02001135
1136 if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001137 return ddr3_cs_size(cs_mode, dclr & WIDTH_128);
Borislav Petkov1433eb92009-10-21 13:44:36 +02001138 else
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001139 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1140}
Borislav Petkov1433eb92009-10-21 13:44:36 +02001141
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001142/*
1143 * F15h supports only 64bit DCT interfaces
1144 */
1145static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1146 unsigned cs_mode)
1147{
1148 WARN_ON(cs_mode > 12);
1149
1150 return ddr3_cs_size(cs_mode, false);
Doug Thompson1afd3c92009-04-27 16:16:50 +02001151}
1152
Borislav Petkov5a5d2372011-01-17 17:52:57 +01001153static void read_dram_ctl_register(struct amd64_pvt *pvt)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001154{
Doug Thompson6163b5d2009-04-27 16:20:17 +02001155
Borislav Petkov5a5d2372011-01-17 17:52:57 +01001156 if (boot_cpu_data.x86 == 0xf)
1157 return;
1158
Borislav Petkov78da1212010-12-22 19:31:45 +01001159 if (!amd64_read_dct_pci_cfg(pvt, DCT_SEL_LO, &pvt->dct_sel_lo)) {
Joe Perches956b9ba2012-04-29 17:08:39 -03001160 edac_dbg(0, "F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
1161 pvt->dct_sel_lo, dct_sel_baseaddr(pvt));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001162
Joe Perches956b9ba2012-04-29 17:08:39 -03001163 edac_dbg(0, " DCTs operate in %s mode\n",
1164 (dct_ganging_enabled(pvt) ? "ganged" : "unganged"));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001165
Borislav Petkov72381bd2009-10-09 19:14:43 +02001166 if (!dct_ganging_enabled(pvt))
Joe Perches956b9ba2012-04-29 17:08:39 -03001167 edac_dbg(0, " Address range split per DCT: %s\n",
1168 (dct_high_range_enabled(pvt) ? "yes" : "no"));
Borislav Petkov72381bd2009-10-09 19:14:43 +02001169
Joe Perches956b9ba2012-04-29 17:08:39 -03001170 edac_dbg(0, " data interleave for ECC: %s, DRAM cleared since last warm reset: %s\n",
1171 (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
1172 (dct_memory_cleared(pvt) ? "yes" : "no"));
Borislav Petkov72381bd2009-10-09 19:14:43 +02001173
Joe Perches956b9ba2012-04-29 17:08:39 -03001174 edac_dbg(0, " channel interleave: %s, "
1175 "interleave bits selector: 0x%x\n",
1176 (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
1177 dct_sel_interleave_addr(pvt));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001178 }
1179
Borislav Petkov78da1212010-12-22 19:31:45 +01001180 amd64_read_dct_pci_cfg(pvt, DCT_SEL_HI, &pvt->dct_sel_hi);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001181}
1182
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001183/*
Borislav Petkov229a7a12010-12-09 18:57:54 +01001184 * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001185 * Interleaving Modes.
1186 */
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001187static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
Borislav Petkov229a7a12010-12-09 18:57:54 +01001188 bool hi_range_sel, u8 intlv_en)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001189{
Borislav Petkov151fa712011-02-21 19:33:10 +01001190 u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001191
1192 if (dct_ganging_enabled(pvt))
Borislav Petkov229a7a12010-12-09 18:57:54 +01001193 return 0;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001194
Borislav Petkov229a7a12010-12-09 18:57:54 +01001195 if (hi_range_sel)
1196 return dct_sel_high;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001197
Borislav Petkov229a7a12010-12-09 18:57:54 +01001198 /*
1199 * see F2x110[DctSelIntLvAddr] - channel interleave mode
1200 */
1201 if (dct_interleave_enabled(pvt)) {
1202 u8 intlv_addr = dct_sel_interleave_addr(pvt);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001203
Borislav Petkov229a7a12010-12-09 18:57:54 +01001204 /* return DCT select function: 0=DCT0, 1=DCT1 */
1205 if (!intlv_addr)
1206 return sys_addr >> 6 & 1;
1207
1208 if (intlv_addr & 0x2) {
1209 u8 shift = intlv_addr & 0x1 ? 9 : 6;
1210 u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2;
1211
1212 return ((sys_addr >> shift) & 1) ^ temp;
1213 }
1214
1215 return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
1216 }
1217
1218 if (dct_high_range_enabled(pvt))
1219 return ~dct_sel_high & 1;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001220
1221 return 0;
1222}
1223
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001224/* Convert the sys_addr to the normalized DCT address */
Daniel J Bluemanc7e53012012-11-30 16:44:20 +08001225static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, u8 range,
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001226 u64 sys_addr, bool hi_rng,
1227 u32 dct_sel_base_addr)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001228{
1229 u64 chan_off;
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001230 u64 dram_base = get_dram_base(pvt, range);
1231 u64 hole_off = f10_dhar_offset(pvt);
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001232 u64 dct_sel_base_off = (pvt->dct_sel_hi & 0xFFFFFC00) << 16;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001233
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001234 if (hi_rng) {
1235 /*
1236 * if
1237 * base address of high range is below 4Gb
1238 * (bits [47:27] at [31:11])
1239 * DRAM address space on this DCT is hoisted above 4Gb &&
1240 * sys_addr > 4Gb
1241 *
1242 * remove hole offset from sys_addr
1243 * else
1244 * remove high range offset from sys_addr
1245 */
1246 if ((!(dct_sel_base_addr >> 16) ||
1247 dct_sel_base_addr < dhar_base(pvt)) &&
Borislav Petkov972ea172011-02-21 19:43:02 +01001248 dhar_valid(pvt) &&
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001249 (sys_addr >= BIT_64(32)))
Borislav Petkovbc21fa52010-11-11 17:29:13 +01001250 chan_off = hole_off;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001251 else
1252 chan_off = dct_sel_base_off;
1253 } else {
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001254 /*
1255 * if
1256 * we have a valid hole &&
1257 * sys_addr > 4Gb
1258 *
1259 * remove hole
1260 * else
1261 * remove dram base to normalize to DCT address
1262 */
Borislav Petkov972ea172011-02-21 19:43:02 +01001263 if (dhar_valid(pvt) && (sys_addr >= BIT_64(32)))
Borislav Petkovbc21fa52010-11-11 17:29:13 +01001264 chan_off = hole_off;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001265 else
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001266 chan_off = dram_base;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001267 }
1268
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001269 return (sys_addr & GENMASK(6,47)) - (chan_off & GENMASK(23,47));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001270}
1271
Doug Thompson6163b5d2009-04-27 16:20:17 +02001272/*
1273 * checks if the csrow passed in is marked as SPARED, if so returns the new
1274 * spare row
1275 */
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001276static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001277{
Borislav Petkov614ec9d2011-01-13 18:02:22 +01001278 int tmp_cs;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001279
Borislav Petkov614ec9d2011-01-13 18:02:22 +01001280 if (online_spare_swap_done(pvt, dct) &&
1281 csrow == online_spare_bad_dramcs(pvt, dct)) {
1282
1283 for_each_chip_select(tmp_cs, dct, pvt) {
1284 if (chip_select_base(tmp_cs, dct, pvt) & 0x2) {
1285 csrow = tmp_cs;
1286 break;
1287 }
1288 }
Doug Thompson6163b5d2009-04-27 16:20:17 +02001289 }
1290 return csrow;
1291}
1292
1293/*
1294 * Iterate over the DRAM DCT "base" and "mask" registers looking for a
1295 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
1296 *
1297 * Return:
1298 * -EINVAL: NOT FOUND
1299 * 0..csrow = Chip-Select Row
1300 */
Daniel J Bluemanc7e53012012-11-30 16:44:20 +08001301static int f1x_lookup_addr_in_dct(u64 in_addr, u8 nid, u8 dct)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001302{
1303 struct mem_ctl_info *mci;
1304 struct amd64_pvt *pvt;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001305 u64 cs_base, cs_mask;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001306 int cs_found = -EINVAL;
1307 int csrow;
1308
Borislav Petkovcc4d8862010-10-13 16:11:59 +02001309 mci = mcis[nid];
Doug Thompson6163b5d2009-04-27 16:20:17 +02001310 if (!mci)
1311 return cs_found;
1312
1313 pvt = mci->pvt_info;
1314
Joe Perches956b9ba2012-04-29 17:08:39 -03001315 edac_dbg(1, "input addr: 0x%llx, DCT: %d\n", in_addr, dct);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001316
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001317 for_each_chip_select(csrow, dct, pvt) {
1318 if (!csrow_enabled(csrow, dct, pvt))
Doug Thompson6163b5d2009-04-27 16:20:17 +02001319 continue;
1320
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001321 get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001322
Joe Perches956b9ba2012-04-29 17:08:39 -03001323 edac_dbg(1, " CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
1324 csrow, cs_base, cs_mask);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001325
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001326 cs_mask = ~cs_mask;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001327
Joe Perches956b9ba2012-04-29 17:08:39 -03001328 edac_dbg(1, " (InputAddr & ~CSMask)=0x%llx (CSBase & ~CSMask)=0x%llx\n",
1329 (in_addr & cs_mask), (cs_base & cs_mask));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001330
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001331 if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
1332 cs_found = f10_process_possible_spare(pvt, dct, csrow);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001333
Joe Perches956b9ba2012-04-29 17:08:39 -03001334 edac_dbg(1, " MATCH csrow=%d\n", cs_found);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001335 break;
1336 }
1337 }
1338 return cs_found;
1339}
1340
Borislav Petkov95b0ef52011-01-11 22:08:07 +01001341/*
1342 * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is
1343 * swapped with a region located at the bottom of memory so that the GPU can use
1344 * the interleaved region and thus two channels.
1345 */
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001346static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr)
Borislav Petkov95b0ef52011-01-11 22:08:07 +01001347{
1348 u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr;
1349
1350 if (boot_cpu_data.x86 == 0x10) {
1351 /* only revC3 and revE have that feature */
1352 if (boot_cpu_data.x86_model < 4 ||
1353 (boot_cpu_data.x86_model < 0xa &&
1354 boot_cpu_data.x86_mask < 3))
1355 return sys_addr;
1356 }
1357
1358 amd64_read_dct_pci_cfg(pvt, SWAP_INTLV_REG, &swap_reg);
1359
1360 if (!(swap_reg & 0x1))
1361 return sys_addr;
1362
1363 swap_base = (swap_reg >> 3) & 0x7f;
1364 swap_limit = (swap_reg >> 11) & 0x7f;
1365 rgn_size = (swap_reg >> 20) & 0x7f;
1366 tmp_addr = sys_addr >> 27;
1367
1368 if (!(sys_addr >> 34) &&
1369 (((tmp_addr >= swap_base) &&
1370 (tmp_addr <= swap_limit)) ||
1371 (tmp_addr < rgn_size)))
1372 return sys_addr ^ (u64)swap_base << 27;
1373
1374 return sys_addr;
1375}
1376
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001377/* For a given @dram_range, check if @sys_addr falls within it. */
Borislav Petkove7613592011-02-21 19:49:01 +01001378static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
Borislav Petkov33ca0642012-08-30 18:01:36 +02001379 u64 sys_addr, int *chan_sel)
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001380{
Borislav Petkov229a7a12010-12-09 18:57:54 +01001381 int cs_found = -EINVAL;
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001382 u64 chan_addr;
Borislav Petkov5d4b58e2011-01-13 16:01:13 +01001383 u32 dct_sel_base;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001384 u8 channel;
Borislav Petkov229a7a12010-12-09 18:57:54 +01001385 bool high_range = false;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001386
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001387 u8 node_id = dram_dst_node(pvt, range);
Borislav Petkov229a7a12010-12-09 18:57:54 +01001388 u8 intlv_en = dram_intlv_en(pvt, range);
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001389 u32 intlv_sel = dram_intlv_sel(pvt, range);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001390
Joe Perches956b9ba2012-04-29 17:08:39 -03001391 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1392 range, sys_addr, get_dram_limit(pvt, range));
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001393
Borislav Petkov355fba62011-01-17 13:03:26 +01001394 if (dhar_valid(pvt) &&
1395 dhar_base(pvt) <= sys_addr &&
1396 sys_addr < BIT_64(32)) {
1397 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1398 sys_addr);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001399 return -EINVAL;
Borislav Petkov355fba62011-01-17 13:03:26 +01001400 }
1401
Borislav Petkovf030ddf2011-04-08 15:05:21 +02001402 if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en)))
Borislav Petkov355fba62011-01-17 13:03:26 +01001403 return -EINVAL;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001404
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001405 sys_addr = f1x_swap_interleaved_region(pvt, sys_addr);
Borislav Petkov95b0ef52011-01-11 22:08:07 +01001406
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001407 dct_sel_base = dct_sel_baseaddr(pvt);
1408
1409 /*
1410 * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
1411 * select between DCT0 and DCT1.
1412 */
1413 if (dct_high_range_enabled(pvt) &&
1414 !dct_ganging_enabled(pvt) &&
1415 ((sys_addr >> 27) >= (dct_sel_base >> 11)))
Borislav Petkov229a7a12010-12-09 18:57:54 +01001416 high_range = true;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001417
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001418 channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001419
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001420 chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr,
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001421 high_range, dct_sel_base);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001422
Borislav Petkove2f79db2011-01-13 14:57:34 +01001423 /* Remove node interleaving, see F1x120 */
1424 if (intlv_en)
1425 chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) |
1426 (chan_addr & 0xfff);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001427
Borislav Petkov5d4b58e2011-01-13 16:01:13 +01001428 /* remove channel interleave */
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001429 if (dct_interleave_enabled(pvt) &&
1430 !dct_high_range_enabled(pvt) &&
1431 !dct_ganging_enabled(pvt)) {
Borislav Petkov5d4b58e2011-01-13 16:01:13 +01001432
1433 if (dct_sel_interleave_addr(pvt) != 1) {
1434 if (dct_sel_interleave_addr(pvt) == 0x3)
1435 /* hash 9 */
1436 chan_addr = ((chan_addr >> 10) << 9) |
1437 (chan_addr & 0x1ff);
1438 else
1439 /* A[6] or hash 6 */
1440 chan_addr = ((chan_addr >> 7) << 6) |
1441 (chan_addr & 0x3f);
1442 } else
1443 /* A[12] */
1444 chan_addr = ((chan_addr >> 13) << 12) |
1445 (chan_addr & 0xfff);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001446 }
1447
Joe Perches956b9ba2012-04-29 17:08:39 -03001448 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001449
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001450 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001451
Borislav Petkov33ca0642012-08-30 18:01:36 +02001452 if (cs_found >= 0)
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001453 *chan_sel = channel;
Borislav Petkov33ca0642012-08-30 18:01:36 +02001454
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001455 return cs_found;
1456}
1457
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001458static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt, u64 sys_addr,
Borislav Petkov33ca0642012-08-30 18:01:36 +02001459 int *chan_sel)
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001460{
Borislav Petkove7613592011-02-21 19:49:01 +01001461 int cs_found = -EINVAL;
1462 unsigned range;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001463
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001464 for (range = 0; range < DRAM_RANGES; range++) {
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001465
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001466 if (!dram_rw(pvt, range))
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001467 continue;
1468
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001469 if ((get_dram_base(pvt, range) <= sys_addr) &&
1470 (get_dram_limit(pvt, range) >= sys_addr)) {
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001471
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001472 cs_found = f1x_match_to_this_node(pvt, range,
Borislav Petkov33ca0642012-08-30 18:01:36 +02001473 sys_addr, chan_sel);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001474 if (cs_found >= 0)
1475 break;
1476 }
1477 }
1478 return cs_found;
1479}
1480
1481/*
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001482 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
1483 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001484 *
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001485 * The @sys_addr is usually an error address received from the hardware
1486 * (MCX_ADDR).
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001487 */
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001488static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
Borislav Petkov33ca0642012-08-30 18:01:36 +02001489 struct err_info *err)
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001490{
1491 struct amd64_pvt *pvt = mci->pvt_info;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001492
Borislav Petkov33ca0642012-08-30 18:01:36 +02001493 error_address_to_page_and_offset(sys_addr, err);
Mauro Carvalho Chehabab5a5032012-04-16 15:03:50 -03001494
Borislav Petkov33ca0642012-08-30 18:01:36 +02001495 err->csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &err->channel);
1496 if (err->csrow < 0) {
1497 err->err_code = ERR_CSROW;
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001498 return;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001499 }
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001500
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001501 /*
1502 * We need the syndromes for channel detection only when we're
1503 * ganged. Otherwise @chan should already contain the channel at
1504 * this point.
1505 */
Borislav Petkova97fa682010-12-23 14:07:18 +01001506 if (dct_ganging_enabled(pvt))
Borislav Petkov33ca0642012-08-30 18:01:36 +02001507 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001508}
1509
1510/*
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001511 * debug routine to display the memory sizes of all logical DIMMs and its
Borislav Petkovcb328502010-12-22 14:28:24 +01001512 * CSROWs
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001513 */
Borislav Petkov8c671752011-02-23 17:25:12 +01001514static void amd64_debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001515{
Borislav Petkovbb89f5a2012-09-12 18:06:00 +02001516 int dimm, size0, size1;
Borislav Petkov525a1b22010-12-21 15:53:27 +01001517 u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
1518 u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001519
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001520 if (boot_cpu_data.x86 == 0xf) {
1521 /* K8 families < revF not supported yet */
Borislav Petkov1433eb92009-10-21 13:44:36 +02001522 if (pvt->ext_model < K8_REV_F)
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001523 return;
1524 else
1525 WARN_ON(ctrl != 0);
1526 }
1527
Borislav Petkov4d796362011-02-03 15:59:57 +01001528 dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1 : pvt->dbam0;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001529 dcsb = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->csels[1].csbases
1530 : pvt->csels[0].csbases;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001531
Joe Perches956b9ba2012-04-29 17:08:39 -03001532 edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n",
1533 ctrl, dbam);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001534
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001535 edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
1536
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001537 /* Dump memory sizes for DIMM and its CSROWs */
1538 for (dimm = 0; dimm < 4; dimm++) {
1539
1540 size0 = 0;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001541 if (dcsb[dimm*2] & DCSB_CS_ENABLE)
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001542 size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
1543 DBAM_DIMM(dimm, dbam));
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001544
1545 size1 = 0;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001546 if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001547 size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
1548 DBAM_DIMM(dimm, dbam));
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001549
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001550 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
Borislav Petkovbb89f5a2012-09-12 18:06:00 +02001551 dimm * 2, size0,
1552 dimm * 2 + 1, size1);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001553 }
1554}
1555
Doug Thompson4d376072009-04-27 16:25:05 +02001556static struct amd64_family_type amd64_family_types[] = {
1557 [K8_CPUS] = {
Borislav Petkov0092b202010-10-01 19:20:05 +02001558 .ctl_name = "K8",
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001559 .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
1560 .f3_id = PCI_DEVICE_ID_AMD_K8_NB_MISC,
Doug Thompson4d376072009-04-27 16:25:05 +02001561 .ops = {
Borislav Petkov1433eb92009-10-21 13:44:36 +02001562 .early_channel_count = k8_early_channel_count,
Borislav Petkov1433eb92009-10-21 13:44:36 +02001563 .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow,
1564 .dbam_to_cs = k8_dbam_to_chip_select,
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001565 .read_dct_pci_cfg = k8_read_dct_pci_cfg,
Doug Thompson4d376072009-04-27 16:25:05 +02001566 }
1567 },
1568 [F10_CPUS] = {
Borislav Petkov0092b202010-10-01 19:20:05 +02001569 .ctl_name = "F10h",
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001570 .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
1571 .f3_id = PCI_DEVICE_ID_AMD_10H_NB_MISC,
Doug Thompson4d376072009-04-27 16:25:05 +02001572 .ops = {
Borislav Petkov7d20d142011-01-07 17:58:04 +01001573 .early_channel_count = f1x_early_channel_count,
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001574 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
Borislav Petkov1433eb92009-10-21 13:44:36 +02001575 .dbam_to_cs = f10_dbam_to_chip_select,
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001576 .read_dct_pci_cfg = f10_read_dct_pci_cfg,
1577 }
1578 },
1579 [F15_CPUS] = {
1580 .ctl_name = "F15h",
Borislav Petkovdf71a052011-01-19 18:15:10 +01001581 .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1,
1582 .f3_id = PCI_DEVICE_ID_AMD_15H_NB_F3,
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001583 .ops = {
Borislav Petkov7d20d142011-01-07 17:58:04 +01001584 .early_channel_count = f1x_early_channel_count,
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001585 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001586 .dbam_to_cs = f15_dbam_to_chip_select,
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001587 .read_dct_pci_cfg = f15_read_dct_pci_cfg,
Doug Thompson4d376072009-04-27 16:25:05 +02001588 }
1589 },
Doug Thompson4d376072009-04-27 16:25:05 +02001590};
1591
Doug Thompsonb1289d62009-04-27 16:37:05 +02001592/*
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001593 * These are tables of eigenvectors (one per line) which can be used for the
1594 * construction of the syndrome tables. The modified syndrome search algorithm
1595 * uses those to find the symbol in error and thus the DIMM.
Doug Thompsonb1289d62009-04-27 16:37:05 +02001596 *
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001597 * Algorithm courtesy of Ross LaFetra from AMD.
Doug Thompsonb1289d62009-04-27 16:37:05 +02001598 */
Daniel J Bluemanc7e53012012-11-30 16:44:20 +08001599static const u16 x4_vectors[] = {
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001600 0x2f57, 0x1afe, 0x66cc, 0xdd88,
1601 0x11eb, 0x3396, 0x7f4c, 0xeac8,
1602 0x0001, 0x0002, 0x0004, 0x0008,
1603 0x1013, 0x3032, 0x4044, 0x8088,
1604 0x106b, 0x30d6, 0x70fc, 0xe0a8,
1605 0x4857, 0xc4fe, 0x13cc, 0x3288,
1606 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
1607 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
1608 0x15c1, 0x2a42, 0x89ac, 0x4758,
1609 0x2b03, 0x1602, 0x4f0c, 0xca08,
1610 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
1611 0x8ba7, 0x465e, 0x244c, 0x1cc8,
1612 0x2b87, 0x164e, 0x642c, 0xdc18,
1613 0x40b9, 0x80de, 0x1094, 0x20e8,
1614 0x27db, 0x1eb6, 0x9dac, 0x7b58,
1615 0x11c1, 0x2242, 0x84ac, 0x4c58,
1616 0x1be5, 0x2d7a, 0x5e34, 0xa718,
1617 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
1618 0x4c97, 0xc87e, 0x11fc, 0x33a8,
1619 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
1620 0x16b3, 0x3d62, 0x4f34, 0x8518,
1621 0x1e2f, 0x391a, 0x5cac, 0xf858,
1622 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
1623 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
1624 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
1625 0x4397, 0xc27e, 0x17fc, 0x3ea8,
1626 0x1617, 0x3d3e, 0x6464, 0xb8b8,
1627 0x23ff, 0x12aa, 0xab6c, 0x56d8,
1628 0x2dfb, 0x1ba6, 0x913c, 0x7328,
1629 0x185d, 0x2ca6, 0x7914, 0x9e28,
1630 0x171b, 0x3e36, 0x7d7c, 0xebe8,
1631 0x4199, 0x82ee, 0x19f4, 0x2e58,
1632 0x4807, 0xc40e, 0x130c, 0x3208,
1633 0x1905, 0x2e0a, 0x5804, 0xac08,
1634 0x213f, 0x132a, 0xadfc, 0x5ba8,
1635 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
Doug Thompsonb1289d62009-04-27 16:37:05 +02001636};
1637
Daniel J Bluemanc7e53012012-11-30 16:44:20 +08001638static const u16 x8_vectors[] = {
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001639 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
1640 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
1641 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
1642 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
1643 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
1644 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
1645 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
1646 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
1647 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
1648 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
1649 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
1650 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
1651 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
1652 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
1653 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
1654 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
1655 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
1656 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
1657 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
1658};
1659
Daniel J Bluemanc7e53012012-11-30 16:44:20 +08001660static int decode_syndrome(u16 syndrome, const u16 *vectors, unsigned num_vecs,
Borislav Petkovd34a6ec2011-02-23 17:41:50 +01001661 unsigned v_dim)
Doug Thompsonb1289d62009-04-27 16:37:05 +02001662{
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001663 unsigned int i, err_sym;
Doug Thompsonb1289d62009-04-27 16:37:05 +02001664
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001665 for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
1666 u16 s = syndrome;
Borislav Petkovd34a6ec2011-02-23 17:41:50 +01001667 unsigned v_idx = err_sym * v_dim;
1668 unsigned v_end = (err_sym + 1) * v_dim;
Doug Thompsonb1289d62009-04-27 16:37:05 +02001669
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001670 /* walk over all 16 bits of the syndrome */
1671 for (i = 1; i < (1U << 16); i <<= 1) {
1672
1673 /* if bit is set in that eigenvector... */
1674 if (v_idx < v_end && vectors[v_idx] & i) {
1675 u16 ev_comp = vectors[v_idx++];
1676
1677 /* ... and bit set in the modified syndrome, */
1678 if (s & i) {
1679 /* remove it. */
1680 s ^= ev_comp;
1681
1682 if (!s)
1683 return err_sym;
1684 }
1685
1686 } else if (s & i)
1687 /* can't get to zero, move to next symbol */
1688 break;
1689 }
Doug Thompsonb1289d62009-04-27 16:37:05 +02001690 }
1691
Joe Perches956b9ba2012-04-29 17:08:39 -03001692 edac_dbg(0, "syndrome(%x) not found\n", syndrome);
Doug Thompsonb1289d62009-04-27 16:37:05 +02001693 return -1;
1694}
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001695
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001696static int map_err_sym_to_channel(int err_sym, int sym_size)
1697{
1698 if (sym_size == 4)
1699 switch (err_sym) {
1700 case 0x20:
1701 case 0x21:
1702 return 0;
1703 break;
1704 case 0x22:
1705 case 0x23:
1706 return 1;
1707 break;
1708 default:
1709 return err_sym >> 4;
1710 break;
1711 }
1712 /* x8 symbols */
1713 else
1714 switch (err_sym) {
1715 /* imaginary bits not in a DIMM */
1716 case 0x10:
1717 WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
1718 err_sym);
1719 return -1;
1720 break;
1721
1722 case 0x11:
1723 return 0;
1724 break;
1725 case 0x12:
1726 return 1;
1727 break;
1728 default:
1729 return err_sym >> 3;
1730 break;
1731 }
1732 return -1;
1733}
1734
1735static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
1736{
1737 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001738 int err_sym = -1;
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001739
Borislav Petkova3b7db02011-01-19 20:35:12 +01001740 if (pvt->ecc_sym_sz == 8)
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001741 err_sym = decode_syndrome(syndrome, x8_vectors,
1742 ARRAY_SIZE(x8_vectors),
Borislav Petkova3b7db02011-01-19 20:35:12 +01001743 pvt->ecc_sym_sz);
1744 else if (pvt->ecc_sym_sz == 4)
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001745 err_sym = decode_syndrome(syndrome, x4_vectors,
1746 ARRAY_SIZE(x4_vectors),
Borislav Petkova3b7db02011-01-19 20:35:12 +01001747 pvt->ecc_sym_sz);
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001748 else {
Borislav Petkova3b7db02011-01-19 20:35:12 +01001749 amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz);
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001750 return err_sym;
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001751 }
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001752
Borislav Petkova3b7db02011-01-19 20:35:12 +01001753 return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz);
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001754}
1755
Borislav Petkov33ca0642012-08-30 18:01:36 +02001756static void __log_bus_error(struct mem_ctl_info *mci, struct err_info *err,
1757 u8 ecc_type)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001758{
Borislav Petkov33ca0642012-08-30 18:01:36 +02001759 enum hw_event_mc_err_type err_type;
1760 const char *string;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001761
Borislav Petkov33ca0642012-08-30 18:01:36 +02001762 if (ecc_type == 2)
1763 err_type = HW_EVENT_ERR_CORRECTED;
1764 else if (ecc_type == 1)
1765 err_type = HW_EVENT_ERR_UNCORRECTED;
1766 else {
1767 WARN(1, "Something is rotten in the state of Denmark.\n");
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001768 return;
1769 }
1770
Borislav Petkov33ca0642012-08-30 18:01:36 +02001771 switch (err->err_code) {
1772 case DECODE_OK:
1773 string = "";
1774 break;
1775 case ERR_NODE:
1776 string = "Failed to map error addr to a node";
1777 break;
1778 case ERR_CSROW:
1779 string = "Failed to map error addr to a csrow";
1780 break;
1781 case ERR_CHANNEL:
1782 string = "unknown syndrome - possible error reporting race";
1783 break;
1784 default:
1785 string = "WTF error";
1786 break;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001787 }
Borislav Petkov33ca0642012-08-30 18:01:36 +02001788
1789 edac_mc_handle_error(err_type, mci, 1,
1790 err->page, err->offset, err->syndrome,
1791 err->csrow, err->channel, -1,
1792 string, "");
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001793}
1794
Borislav Petkov549d0422009-07-24 13:51:42 +02001795static inline void __amd64_decode_bus_error(struct mem_ctl_info *mci,
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001796 struct mce *m)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001797{
Borislav Petkov33ca0642012-08-30 18:01:36 +02001798 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001799 u8 ecc_type = (m->status >> 45) & 0x3;
Borislav Petkov66fed2d2012-08-09 18:41:07 +02001800 u8 xec = XEC(m->status, 0x1f);
1801 u16 ec = EC(m->status);
Borislav Petkov33ca0642012-08-30 18:01:36 +02001802 u64 sys_addr;
1803 struct err_info err;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001804
Borislav Petkov66fed2d2012-08-09 18:41:07 +02001805 /* Bail out early if this was an 'observed' error */
Borislav Petkov5980bb92011-01-07 16:26:49 +01001806 if (PP(ec) == NBSL_PP_OBS)
Borislav Petkovb70ef012009-06-25 19:32:38 +02001807 return;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001808
Borislav Petkovecaf5602009-07-23 16:32:01 +02001809 /* Do only ECC errors */
1810 if (xec && xec != F10_NBSL_EXT_ERR_ECC)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001811 return;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001812
Borislav Petkov33ca0642012-08-30 18:01:36 +02001813 memset(&err, 0, sizeof(err));
1814
1815 sys_addr = get_error_address(m);
1816
Borislav Petkovecaf5602009-07-23 16:32:01 +02001817 if (ecc_type == 2)
Borislav Petkov33ca0642012-08-30 18:01:36 +02001818 err.syndrome = extract_syndrome(m->status);
1819
1820 pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, &err);
1821
1822 __log_bus_error(mci, &err, ecc_type);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001823}
1824
Borislav Petkovb0b07a22011-08-24 18:44:22 +02001825void amd64_decode_bus_error(int node_id, struct mce *m)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001826{
Borislav Petkovb0b07a22011-08-24 18:44:22 +02001827 __amd64_decode_bus_error(mcis[node_id], m);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001828}
Doug Thompsond27bf6f2009-05-06 17:55:27 +02001829
Doug Thompson0ec449e2009-04-27 19:41:25 +02001830/*
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001831 * Use pvt->F2 which contains the F2 CPU PCI device to get the related
Borislav Petkovbbd0c1f2010-10-01 19:27:58 +02001832 * F1 (AddrMap) and F3 (Misc) devices. Return negative value on error.
Doug Thompson0ec449e2009-04-27 19:41:25 +02001833 */
Borislav Petkov360b7f32010-10-15 19:25:38 +02001834static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 f1_id, u16 f3_id)
Doug Thompson0ec449e2009-04-27 19:41:25 +02001835{
Doug Thompson0ec449e2009-04-27 19:41:25 +02001836 /* Reserve the ADDRESS MAP Device */
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001837 pvt->F1 = pci_get_related_function(pvt->F2->vendor, f1_id, pvt->F2);
1838 if (!pvt->F1) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001839 amd64_err("error address map device not found: "
1840 "vendor %x device 0x%x (broken BIOS?)\n",
1841 PCI_VENDOR_ID_AMD, f1_id);
Borislav Petkovbbd0c1f2010-10-01 19:27:58 +02001842 return -ENODEV;
Doug Thompson0ec449e2009-04-27 19:41:25 +02001843 }
1844
1845 /* Reserve the MISC Device */
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001846 pvt->F3 = pci_get_related_function(pvt->F2->vendor, f3_id, pvt->F2);
1847 if (!pvt->F3) {
1848 pci_dev_put(pvt->F1);
1849 pvt->F1 = NULL;
Doug Thompson0ec449e2009-04-27 19:41:25 +02001850
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001851 amd64_err("error F3 device not found: "
1852 "vendor %x device 0x%x (broken BIOS?)\n",
1853 PCI_VENDOR_ID_AMD, f3_id);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001854
Borislav Petkovbbd0c1f2010-10-01 19:27:58 +02001855 return -ENODEV;
Doug Thompson0ec449e2009-04-27 19:41:25 +02001856 }
Joe Perches956b9ba2012-04-29 17:08:39 -03001857 edac_dbg(1, "F1: %s\n", pci_name(pvt->F1));
1858 edac_dbg(1, "F2: %s\n", pci_name(pvt->F2));
1859 edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
Doug Thompson0ec449e2009-04-27 19:41:25 +02001860
1861 return 0;
1862}
1863
Borislav Petkov360b7f32010-10-15 19:25:38 +02001864static void free_mc_sibling_devs(struct amd64_pvt *pvt)
Doug Thompson0ec449e2009-04-27 19:41:25 +02001865{
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001866 pci_dev_put(pvt->F1);
1867 pci_dev_put(pvt->F3);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001868}
1869
1870/*
1871 * Retrieve the hardware registers of the memory controller (this includes the
1872 * 'Address Map' and 'Misc' device regs)
1873 */
Borislav Petkov360b7f32010-10-15 19:25:38 +02001874static void read_mc_regs(struct amd64_pvt *pvt)
Doug Thompson0ec449e2009-04-27 19:41:25 +02001875{
Borislav Petkova3b7db02011-01-19 20:35:12 +01001876 struct cpuinfo_x86 *c = &boot_cpu_data;
Doug Thompson0ec449e2009-04-27 19:41:25 +02001877 u64 msr_val;
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001878 u32 tmp;
Borislav Petkove7613592011-02-21 19:49:01 +01001879 unsigned range;
Doug Thompson0ec449e2009-04-27 19:41:25 +02001880
1881 /*
1882 * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
1883 * those are Read-As-Zero
1884 */
Borislav Petkove97f8bb2009-10-12 15:27:45 +02001885 rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
Joe Perches956b9ba2012-04-29 17:08:39 -03001886 edac_dbg(0, " TOP_MEM: 0x%016llx\n", pvt->top_mem);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001887
1888 /* check first whether TOP_MEM2 is enabled */
1889 rdmsrl(MSR_K8_SYSCFG, msr_val);
1890 if (msr_val & (1U << 21)) {
Borislav Petkove97f8bb2009-10-12 15:27:45 +02001891 rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
Joe Perches956b9ba2012-04-29 17:08:39 -03001892 edac_dbg(0, " TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001893 } else
Joe Perches956b9ba2012-04-29 17:08:39 -03001894 edac_dbg(0, " TOP_MEM2 disabled\n");
Doug Thompson0ec449e2009-04-27 19:41:25 +02001895
Borislav Petkov5980bb92011-01-07 16:26:49 +01001896 amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001897
Borislav Petkov5a5d2372011-01-17 17:52:57 +01001898 read_dram_ctl_register(pvt);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001899
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001900 for (range = 0; range < DRAM_RANGES; range++) {
1901 u8 rw;
Doug Thompson0ec449e2009-04-27 19:41:25 +02001902
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001903 /* read settings for this DRAM range */
1904 read_dram_base_limit_regs(pvt, range);
Borislav Petkove97f8bb2009-10-12 15:27:45 +02001905
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001906 rw = dram_rw(pvt, range);
1907 if (!rw)
1908 continue;
1909
Joe Perches956b9ba2012-04-29 17:08:39 -03001910 edac_dbg(1, " DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
1911 range,
1912 get_dram_base(pvt, range),
1913 get_dram_limit(pvt, range));
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001914
Joe Perches956b9ba2012-04-29 17:08:39 -03001915 edac_dbg(1, " IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
1916 dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
1917 (rw & 0x1) ? "R" : "-",
1918 (rw & 0x2) ? "W" : "-",
1919 dram_intlv_sel(pvt, range),
1920 dram_dst_node(pvt, range));
Doug Thompson0ec449e2009-04-27 19:41:25 +02001921 }
1922
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001923 read_dct_base_mask(pvt);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001924
Borislav Petkovbc21fa52010-11-11 17:29:13 +01001925 amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
Borislav Petkov525a1b22010-12-21 15:53:27 +01001926 amd64_read_dct_pci_cfg(pvt, DBAM0, &pvt->dbam0);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001927
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001928 amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001929
Borislav Petkovcb328502010-12-22 14:28:24 +01001930 amd64_read_dct_pci_cfg(pvt, DCLR0, &pvt->dclr0);
1931 amd64_read_dct_pci_cfg(pvt, DCHR0, &pvt->dchr0);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001932
Borislav Petkov78da1212010-12-22 19:31:45 +01001933 if (!dct_ganging_enabled(pvt)) {
Borislav Petkovcb328502010-12-22 14:28:24 +01001934 amd64_read_dct_pci_cfg(pvt, DCLR1, &pvt->dclr1);
1935 amd64_read_dct_pci_cfg(pvt, DCHR1, &pvt->dchr1);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001936 }
Borislav Petkovad6a32e2010-03-09 12:46:00 +01001937
Borislav Petkova3b7db02011-01-19 20:35:12 +01001938 pvt->ecc_sym_sz = 4;
1939
1940 if (c->x86 >= 0x10) {
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001941 amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
Borislav Petkov525a1b22010-12-21 15:53:27 +01001942 amd64_read_dct_pci_cfg(pvt, DBAM1, &pvt->dbam1);
Borislav Petkova3b7db02011-01-19 20:35:12 +01001943
1944 /* F10h, revD and later can do x8 ECC too */
1945 if ((c->x86 > 0x10 || c->x86_model > 7) && tmp & BIT(25))
1946 pvt->ecc_sym_sz = 8;
Borislav Petkov525a1b22010-12-21 15:53:27 +01001947 }
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001948 dump_misc_regs(pvt);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001949}
1950
1951/*
1952 * NOTE: CPU Revision Dependent code
1953 *
1954 * Input:
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001955 * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1)
Doug Thompson0ec449e2009-04-27 19:41:25 +02001956 * k8 private pointer to -->
1957 * DRAM Bank Address mapping register
1958 * node_id
1959 * DCL register where dual_channel_active is
1960 *
1961 * The DBAM register consists of 4 sets of 4 bits each definitions:
1962 *
1963 * Bits: CSROWs
1964 * 0-3 CSROWs 0 and 1
1965 * 4-7 CSROWs 2 and 3
1966 * 8-11 CSROWs 4 and 5
1967 * 12-15 CSROWs 6 and 7
1968 *
1969 * Values range from: 0 to 15
1970 * The meaning of the values depends on CPU revision and dual-channel state,
1971 * see relevant BKDG more info.
1972 *
1973 * The memory controller provides for total of only 8 CSROWs in its current
1974 * architecture. Each "pair" of CSROWs normally represents just one DIMM in
1975 * single channel or two (2) DIMMs in dual channel mode.
1976 *
1977 * The following code logic collapses the various tables for CSROW based on CPU
1978 * revision.
1979 *
1980 * Returns:
1981 * The number of PAGE_SIZE pages on the specified CSROW number it
1982 * encompasses
1983 *
1984 */
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001985static u32 amd64_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr)
Doug Thompson0ec449e2009-04-27 19:41:25 +02001986{
Borislav Petkov1433eb92009-10-21 13:44:36 +02001987 u32 cs_mode, nr_pages;
Ashish Shenoyf92cae42012-02-22 17:20:38 -08001988 u32 dbam = dct ? pvt->dbam1 : pvt->dbam0;
Doug Thompson0ec449e2009-04-27 19:41:25 +02001989
Borislav Petkov10de6492012-09-12 19:00:38 +02001990
Doug Thompson0ec449e2009-04-27 19:41:25 +02001991 /*
1992 * The math on this doesn't look right on the surface because x/2*4 can
1993 * be simplified to x*2 but this expression makes use of the fact that
1994 * it is integral math where 1/2=0. This intermediate value becomes the
1995 * number of bits to shift the DBAM register to extract the proper CSROW
1996 * field.
1997 */
Borislav Petkov0a5dfc32012-09-12 18:16:01 +02001998 cs_mode = DBAM_DIMM(csrow_nr / 2, dbam);
Doug Thompson0ec449e2009-04-27 19:41:25 +02001999
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01002000 nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode) << (20 - PAGE_SHIFT);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002001
Borislav Petkov10de6492012-09-12 19:00:38 +02002002 edac_dbg(0, "csrow: %d, channel: %d, DBAM idx: %d\n",
2003 csrow_nr, dct, cs_mode);
2004 edac_dbg(0, "nr_pages/channel: %u\n", nr_pages);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002005
2006 return nr_pages;
2007}
2008
2009/*
2010 * Initialize the array of csrow attribute instances, based on the values
2011 * from pci config hardware registers.
2012 */
Borislav Petkov360b7f32010-10-15 19:25:38 +02002013static int init_csrows(struct mem_ctl_info *mci)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002014{
Borislav Petkov10de6492012-09-12 19:00:38 +02002015 struct amd64_pvt *pvt = mci->pvt_info;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002016 struct csrow_info *csrow;
Mauro Carvalho Chehabde3910eb2012-04-24 15:05:43 -03002017 struct dimm_info *dimm;
Mauro Carvalho Chehab084a4fc2012-01-27 18:38:08 -03002018 enum edac_type edac_mode;
Borislav Petkov10de6492012-09-12 19:00:38 +02002019 enum mem_type mtype;
2020 int i, j, empty = 1;
Mauro Carvalho Chehaba895bf82012-01-28 09:09:38 -03002021 int nr_pages = 0;
Borislav Petkov10de6492012-09-12 19:00:38 +02002022 u32 val;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002023
Borislav Petkova97fa682010-12-23 14:07:18 +01002024 amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002025
Borislav Petkov2299ef72010-10-15 17:44:04 +02002026 pvt->nbcfg = val;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002027
Joe Perches956b9ba2012-04-29 17:08:39 -03002028 edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
2029 pvt->mc_node_id, val,
2030 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
Doug Thompson0ec449e2009-04-27 19:41:25 +02002031
Borislav Petkov10de6492012-09-12 19:00:38 +02002032 /*
2033 * We iterate over DCT0 here but we look at DCT1 in parallel, if needed.
2034 */
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002035 for_each_chip_select(i, 0, pvt) {
Borislav Petkov10de6492012-09-12 19:00:38 +02002036 bool row_dct0 = !!csrow_enabled(i, 0, pvt);
2037 bool row_dct1 = false;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002038
Borislav Petkov10de6492012-09-12 19:00:38 +02002039 if (boot_cpu_data.x86 != 0xf)
2040 row_dct1 = !!csrow_enabled(i, 1, pvt);
2041
2042 if (!row_dct0 && !row_dct1)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002043 continue;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002044
Borislav Petkov10de6492012-09-12 19:00:38 +02002045 csrow = mci->csrows[i];
Doug Thompson0ec449e2009-04-27 19:41:25 +02002046 empty = 0;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002047
Borislav Petkov10de6492012-09-12 19:00:38 +02002048 edac_dbg(1, "MC node: %d, csrow: %d\n",
2049 pvt->mc_node_id, i);
2050
2051 if (row_dct0)
2052 nr_pages = amd64_csrow_nr_pages(pvt, 0, i);
2053
2054 /* K8 has only one DCT */
2055 if (boot_cpu_data.x86 != 0xf && row_dct1)
2056 nr_pages += amd64_csrow_nr_pages(pvt, 1, i);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002057
Mauro Carvalho Chehab084a4fc2012-01-27 18:38:08 -03002058 mtype = amd64_determine_memory_type(pvt, i);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002059
Borislav Petkov10de6492012-09-12 19:00:38 +02002060 edac_dbg(1, "Total csrow%d pages: %u\n", i, nr_pages);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002061
2062 /*
2063 * determine whether CHIPKILL or JUST ECC or NO ECC is operating
2064 */
Borislav Petkova97fa682010-12-23 14:07:18 +01002065 if (pvt->nbcfg & NBCFG_ECC_ENABLE)
Mauro Carvalho Chehab084a4fc2012-01-27 18:38:08 -03002066 edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL) ?
2067 EDAC_S4ECD4ED : EDAC_SECDED;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002068 else
Mauro Carvalho Chehab084a4fc2012-01-27 18:38:08 -03002069 edac_mode = EDAC_NONE;
2070
2071 for (j = 0; j < pvt->channel_count; j++) {
Mauro Carvalho Chehabde3910eb2012-04-24 15:05:43 -03002072 dimm = csrow->channels[j]->dimm;
2073 dimm->mtype = mtype;
2074 dimm->edac_mode = edac_mode;
2075 dimm->nr_pages = nr_pages;
Mauro Carvalho Chehab084a4fc2012-01-27 18:38:08 -03002076 }
Borislav Petkov16a528ee2012-09-13 18:53:58 +02002077 csrow->nr_pages = nr_pages;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002078 }
2079
2080 return empty;
2081}
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002082
Borislav Petkov06724532009-09-16 13:05:46 +02002083/* get all cores on this DCT */
Daniel J Blueman8b84c8d2012-11-27 14:32:10 +08002084static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, u16 nid)
Doug Thompsonf9431992009-04-27 19:46:08 +02002085{
Borislav Petkov06724532009-09-16 13:05:46 +02002086 int cpu;
Doug Thompsonf9431992009-04-27 19:46:08 +02002087
Borislav Petkov06724532009-09-16 13:05:46 +02002088 for_each_online_cpu(cpu)
2089 if (amd_get_nb_id(cpu) == nid)
2090 cpumask_set_cpu(cpu, mask);
Doug Thompsonf9431992009-04-27 19:46:08 +02002091}
2092
2093/* check MCG_CTL on all the cpus on this node */
Daniel J Bluemanc7e53012012-11-30 16:44:20 +08002094static bool amd64_nb_mce_bank_enabled_on_node(u16 nid)
Doug Thompsonf9431992009-04-27 19:46:08 +02002095{
Rusty Russellba578cb2009-11-03 14:56:35 +10302096 cpumask_var_t mask;
Borislav Petkov50542252009-12-11 18:14:40 +01002097 int cpu, nbe;
Borislav Petkov06724532009-09-16 13:05:46 +02002098 bool ret = false;
Doug Thompsonf9431992009-04-27 19:46:08 +02002099
Rusty Russellba578cb2009-11-03 14:56:35 +10302100 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002101 amd64_warn("%s: Error allocating mask\n", __func__);
Rusty Russellba578cb2009-11-03 14:56:35 +10302102 return false;
2103 }
Borislav Petkov06724532009-09-16 13:05:46 +02002104
Rusty Russellba578cb2009-11-03 14:56:35 +10302105 get_cpus_on_this_dct_cpumask(mask, nid);
Borislav Petkov06724532009-09-16 13:05:46 +02002106
Rusty Russellba578cb2009-11-03 14:56:35 +10302107 rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
Borislav Petkov06724532009-09-16 13:05:46 +02002108
Rusty Russellba578cb2009-11-03 14:56:35 +10302109 for_each_cpu(cpu, mask) {
Borislav Petkov50542252009-12-11 18:14:40 +01002110 struct msr *reg = per_cpu_ptr(msrs, cpu);
Borislav Petkov5980bb92011-01-07 16:26:49 +01002111 nbe = reg->l & MSR_MCGCTL_NBE;
Borislav Petkov06724532009-09-16 13:05:46 +02002112
Joe Perches956b9ba2012-04-29 17:08:39 -03002113 edac_dbg(0, "core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
2114 cpu, reg->q,
2115 (nbe ? "enabled" : "disabled"));
Borislav Petkov06724532009-09-16 13:05:46 +02002116
2117 if (!nbe)
2118 goto out;
Borislav Petkov06724532009-09-16 13:05:46 +02002119 }
2120 ret = true;
2121
2122out:
Rusty Russellba578cb2009-11-03 14:56:35 +10302123 free_cpumask_var(mask);
Doug Thompsonf9431992009-04-27 19:46:08 +02002124 return ret;
2125}
2126
Daniel J Bluemanc7e53012012-11-30 16:44:20 +08002127static int toggle_ecc_err_reporting(struct ecc_settings *s, u16 nid, bool on)
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002128{
2129 cpumask_var_t cmask;
Borislav Petkov50542252009-12-11 18:14:40 +01002130 int cpu;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002131
2132 if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002133 amd64_warn("%s: error allocating mask\n", __func__);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002134 return false;
2135 }
2136
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002137 get_cpus_on_this_dct_cpumask(cmask, nid);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002138
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002139 rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2140
2141 for_each_cpu(cpu, cmask) {
2142
Borislav Petkov50542252009-12-11 18:14:40 +01002143 struct msr *reg = per_cpu_ptr(msrs, cpu);
2144
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002145 if (on) {
Borislav Petkov5980bb92011-01-07 16:26:49 +01002146 if (reg->l & MSR_MCGCTL_NBE)
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002147 s->flags.nb_mce_enable = 1;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002148
Borislav Petkov5980bb92011-01-07 16:26:49 +01002149 reg->l |= MSR_MCGCTL_NBE;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002150 } else {
2151 /*
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002152 * Turn off NB MCE reporting only when it was off before
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002153 */
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002154 if (!s->flags.nb_mce_enable)
Borislav Petkov5980bb92011-01-07 16:26:49 +01002155 reg->l &= ~MSR_MCGCTL_NBE;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002156 }
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002157 }
2158 wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2159
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002160 free_cpumask_var(cmask);
2161
2162 return 0;
2163}
2164
Daniel J Bluemanc7e53012012-11-30 16:44:20 +08002165static bool enable_ecc_error_reporting(struct ecc_settings *s, u16 nid,
Borislav Petkov2299ef72010-10-15 17:44:04 +02002166 struct pci_dev *F3)
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002167{
Borislav Petkov2299ef72010-10-15 17:44:04 +02002168 bool ret = true;
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002169 u32 value, mask = 0x3; /* UECC/CECC enable */
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002170
Borislav Petkov2299ef72010-10-15 17:44:04 +02002171 if (toggle_ecc_err_reporting(s, nid, ON)) {
2172 amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
2173 return false;
2174 }
2175
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002176 amd64_read_pci_cfg(F3, NBCTL, &value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002177
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002178 s->old_nbctl = value & mask;
2179 s->nbctl_valid = true;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002180
2181 value |= mask;
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002182 amd64_write_pci_cfg(F3, NBCTL, value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002183
Borislav Petkova97fa682010-12-23 14:07:18 +01002184 amd64_read_pci_cfg(F3, NBCFG, &value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002185
Joe Perches956b9ba2012-04-29 17:08:39 -03002186 edac_dbg(0, "1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2187 nid, value, !!(value & NBCFG_ECC_ENABLE));
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002188
Borislav Petkova97fa682010-12-23 14:07:18 +01002189 if (!(value & NBCFG_ECC_ENABLE)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002190 amd64_warn("DRAM ECC disabled on this node, enabling...\n");
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002191
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002192 s->flags.nb_ecc_prev = 0;
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002193
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002194 /* Attempt to turn on DRAM ECC Enable */
Borislav Petkova97fa682010-12-23 14:07:18 +01002195 value |= NBCFG_ECC_ENABLE;
2196 amd64_write_pci_cfg(F3, NBCFG, value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002197
Borislav Petkova97fa682010-12-23 14:07:18 +01002198 amd64_read_pci_cfg(F3, NBCFG, &value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002199
Borislav Petkova97fa682010-12-23 14:07:18 +01002200 if (!(value & NBCFG_ECC_ENABLE)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002201 amd64_warn("Hardware rejected DRAM ECC enable,"
2202 "check memory DIMM configuration.\n");
Borislav Petkov2299ef72010-10-15 17:44:04 +02002203 ret = false;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002204 } else {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002205 amd64_info("Hardware accepted DRAM ECC Enable\n");
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002206 }
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002207 } else {
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002208 s->flags.nb_ecc_prev = 1;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002209 }
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002210
Joe Perches956b9ba2012-04-29 17:08:39 -03002211 edac_dbg(0, "2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2212 nid, value, !!(value & NBCFG_ECC_ENABLE));
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002213
Borislav Petkov2299ef72010-10-15 17:44:04 +02002214 return ret;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002215}
2216
Daniel J Bluemanc7e53012012-11-30 16:44:20 +08002217static void restore_ecc_error_reporting(struct ecc_settings *s, u16 nid,
Borislav Petkov360b7f32010-10-15 19:25:38 +02002218 struct pci_dev *F3)
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002219{
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002220 u32 value, mask = 0x3; /* UECC/CECC enable */
2221
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002222
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002223 if (!s->nbctl_valid)
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002224 return;
2225
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002226 amd64_read_pci_cfg(F3, NBCTL, &value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002227 value &= ~mask;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002228 value |= s->old_nbctl;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002229
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002230 amd64_write_pci_cfg(F3, NBCTL, value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002231
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002232 /* restore previous BIOS DRAM ECC "off" setting we force-enabled */
2233 if (!s->flags.nb_ecc_prev) {
Borislav Petkova97fa682010-12-23 14:07:18 +01002234 amd64_read_pci_cfg(F3, NBCFG, &value);
2235 value &= ~NBCFG_ECC_ENABLE;
2236 amd64_write_pci_cfg(F3, NBCFG, value);
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002237 }
2238
2239 /* restore the NB Enable MCGCTL bit */
Borislav Petkov2299ef72010-10-15 17:44:04 +02002240 if (toggle_ecc_err_reporting(s, nid, OFF))
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002241 amd64_warn("Error restoring NB MCGCTL settings!\n");
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002242}
2243
Doug Thompsonf9431992009-04-27 19:46:08 +02002244/*
Borislav Petkov2299ef72010-10-15 17:44:04 +02002245 * EDAC requires that the BIOS have ECC enabled before
2246 * taking over the processing of ECC errors. A command line
2247 * option allows to force-enable hardware ECC later in
2248 * enable_ecc_error_reporting().
Doug Thompsonf9431992009-04-27 19:46:08 +02002249 */
Borislav Petkovcab4d272010-02-11 17:15:57 +01002250static const char *ecc_msg =
2251 "ECC disabled in the BIOS or no ECC capability, module will not load.\n"
2252 " Either enable ECC checking or force module loading by setting "
2253 "'ecc_enable_override'.\n"
2254 " (Note that use of the override may cause unknown side effects.)\n";
Borislav Petkovbe3468e2009-08-05 15:47:22 +02002255
Daniel J Bluemanc7e53012012-11-30 16:44:20 +08002256static bool ecc_enabled(struct pci_dev *F3, u16 nid)
Doug Thompsonf9431992009-04-27 19:46:08 +02002257{
2258 u32 value;
Borislav Petkov2299ef72010-10-15 17:44:04 +02002259 u8 ecc_en = 0;
Borislav Petkov06724532009-09-16 13:05:46 +02002260 bool nb_mce_en = false;
Doug Thompsonf9431992009-04-27 19:46:08 +02002261
Borislav Petkova97fa682010-12-23 14:07:18 +01002262 amd64_read_pci_cfg(F3, NBCFG, &value);
Doug Thompsonf9431992009-04-27 19:46:08 +02002263
Borislav Petkova97fa682010-12-23 14:07:18 +01002264 ecc_en = !!(value & NBCFG_ECC_ENABLE);
Borislav Petkov2299ef72010-10-15 17:44:04 +02002265 amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled"));
Doug Thompsonf9431992009-04-27 19:46:08 +02002266
Borislav Petkov2299ef72010-10-15 17:44:04 +02002267 nb_mce_en = amd64_nb_mce_bank_enabled_on_node(nid);
Borislav Petkov06724532009-09-16 13:05:46 +02002268 if (!nb_mce_en)
Borislav Petkov2299ef72010-10-15 17:44:04 +02002269 amd64_notice("NB MCE bank disabled, set MSR "
2270 "0x%08x[4] on node %d to enable.\n",
2271 MSR_IA32_MCG_CTL, nid);
Doug Thompsonf9431992009-04-27 19:46:08 +02002272
Borislav Petkov2299ef72010-10-15 17:44:04 +02002273 if (!ecc_en || !nb_mce_en) {
2274 amd64_notice("%s", ecc_msg);
2275 return false;
Borislav Petkov43f5e682009-12-21 18:55:18 +01002276 }
Borislav Petkov2299ef72010-10-15 17:44:04 +02002277 return true;
Doug Thompsonf9431992009-04-27 19:46:08 +02002278}
2279
Mauro Carvalho Chehabc5608752012-03-21 14:00:44 -03002280static int set_mc_sysfs_attrs(struct mem_ctl_info *mci)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002281{
Mauro Carvalho Chehabc5608752012-03-21 14:00:44 -03002282 int rc;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002283
Mauro Carvalho Chehabc5608752012-03-21 14:00:44 -03002284 rc = amd64_create_sysfs_dbg_files(mci);
2285 if (rc < 0)
2286 return rc;
2287
2288 if (boot_cpu_data.x86 >= 0x10) {
2289 rc = amd64_create_sysfs_inject_files(mci);
2290 if (rc < 0)
2291 return rc;
2292 }
2293
2294 return 0;
2295}
2296
2297static void del_mc_sysfs_attrs(struct mem_ctl_info *mci)
2298{
2299 amd64_remove_sysfs_dbg_files(mci);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002300
Borislav Petkova135cef2010-11-26 19:24:44 +01002301 if (boot_cpu_data.x86 >= 0x10)
Mauro Carvalho Chehabc5608752012-03-21 14:00:44 -03002302 amd64_remove_sysfs_inject_files(mci);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002303}
2304
Borislav Petkovdf71a052011-01-19 18:15:10 +01002305static void setup_mci_misc_attrs(struct mem_ctl_info *mci,
2306 struct amd64_family_type *fam)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002307{
2308 struct amd64_pvt *pvt = mci->pvt_info;
2309
2310 mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
2311 mci->edac_ctl_cap = EDAC_FLAG_NONE;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002312
Borislav Petkov5980bb92011-01-07 16:26:49 +01002313 if (pvt->nbcap & NBCAP_SECDED)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002314 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
2315
Borislav Petkov5980bb92011-01-07 16:26:49 +01002316 if (pvt->nbcap & NBCAP_CHIPKILL)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002317 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
2318
2319 mci->edac_cap = amd64_determine_edac_cap(pvt);
2320 mci->mod_name = EDAC_MOD_STR;
2321 mci->mod_ver = EDAC_AMD64_VERSION;
Borislav Petkovdf71a052011-01-19 18:15:10 +01002322 mci->ctl_name = fam->ctl_name;
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002323 mci->dev_name = pci_name(pvt->F2);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002324 mci->ctl_page_to_phys = NULL;
2325
Doug Thompson7d6034d2009-04-27 20:01:01 +02002326 /* memory scrubber interface */
2327 mci->set_sdram_scrub_rate = amd64_set_scrub_rate;
2328 mci->get_sdram_scrub_rate = amd64_get_scrub_rate;
2329}
2330
Borislav Petkov0092b202010-10-01 19:20:05 +02002331/*
2332 * returns a pointer to the family descriptor on success, NULL otherwise.
2333 */
2334static struct amd64_family_type *amd64_per_family_init(struct amd64_pvt *pvt)
Borislav Petkov395ae782010-10-01 18:38:19 +02002335{
Borislav Petkov0092b202010-10-01 19:20:05 +02002336 u8 fam = boot_cpu_data.x86;
2337 struct amd64_family_type *fam_type = NULL;
2338
2339 switch (fam) {
Borislav Petkov395ae782010-10-01 18:38:19 +02002340 case 0xf:
Borislav Petkov0092b202010-10-01 19:20:05 +02002341 fam_type = &amd64_family_types[K8_CPUS];
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002342 pvt->ops = &amd64_family_types[K8_CPUS].ops;
Borislav Petkov395ae782010-10-01 18:38:19 +02002343 break;
Borislav Petkovdf71a052011-01-19 18:15:10 +01002344
Borislav Petkov395ae782010-10-01 18:38:19 +02002345 case 0x10:
Borislav Petkov0092b202010-10-01 19:20:05 +02002346 fam_type = &amd64_family_types[F10_CPUS];
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002347 pvt->ops = &amd64_family_types[F10_CPUS].ops;
Borislav Petkovdf71a052011-01-19 18:15:10 +01002348 break;
2349
2350 case 0x15:
2351 fam_type = &amd64_family_types[F15_CPUS];
2352 pvt->ops = &amd64_family_types[F15_CPUS].ops;
Borislav Petkov395ae782010-10-01 18:38:19 +02002353 break;
2354
2355 default:
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002356 amd64_err("Unsupported family!\n");
Borislav Petkov0092b202010-10-01 19:20:05 +02002357 return NULL;
Borislav Petkov395ae782010-10-01 18:38:19 +02002358 }
Borislav Petkov0092b202010-10-01 19:20:05 +02002359
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002360 pvt->ext_model = boot_cpu_data.x86_model >> 4;
2361
Borislav Petkovdf71a052011-01-19 18:15:10 +01002362 amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name,
Borislav Petkov0092b202010-10-01 19:20:05 +02002363 (fam == 0xf ?
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002364 (pvt->ext_model >= K8_REV_F ? "revF or later "
2365 : "revE or earlier ")
2366 : ""), pvt->mc_node_id);
Borislav Petkov0092b202010-10-01 19:20:05 +02002367 return fam_type;
Borislav Petkov395ae782010-10-01 18:38:19 +02002368}
2369
Borislav Petkov2299ef72010-10-15 17:44:04 +02002370static int amd64_init_one_instance(struct pci_dev *F2)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002371{
2372 struct amd64_pvt *pvt = NULL;
Borislav Petkov0092b202010-10-01 19:20:05 +02002373 struct amd64_family_type *fam_type = NULL;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002374 struct mem_ctl_info *mci = NULL;
Mauro Carvalho Chehabab5a5032012-04-16 15:03:50 -03002375 struct edac_mc_layer layers[2];
Doug Thompson7d6034d2009-04-27 20:01:01 +02002376 int err = 0, ret;
Daniel J Blueman772c3ff2012-11-27 14:32:09 +08002377 u16 nid = amd_get_node_id(F2);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002378
2379 ret = -ENOMEM;
2380 pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
2381 if (!pvt)
Borislav Petkov360b7f32010-10-15 19:25:38 +02002382 goto err_ret;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002383
Borislav Petkov360b7f32010-10-15 19:25:38 +02002384 pvt->mc_node_id = nid;
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002385 pvt->F2 = F2;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002386
Borislav Petkov395ae782010-10-01 18:38:19 +02002387 ret = -EINVAL;
Borislav Petkov0092b202010-10-01 19:20:05 +02002388 fam_type = amd64_per_family_init(pvt);
2389 if (!fam_type)
Borislav Petkov395ae782010-10-01 18:38:19 +02002390 goto err_free;
2391
Doug Thompson7d6034d2009-04-27 20:01:01 +02002392 ret = -ENODEV;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002393 err = reserve_mc_sibling_devs(pvt, fam_type->f1_id, fam_type->f3_id);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002394 if (err)
2395 goto err_free;
2396
Borislav Petkov360b7f32010-10-15 19:25:38 +02002397 read_mc_regs(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002398
Doug Thompson7d6034d2009-04-27 20:01:01 +02002399 /*
2400 * We need to determine how many memory channels there are. Then use
2401 * that information for calculating the size of the dynamic instance
Borislav Petkov360b7f32010-10-15 19:25:38 +02002402 * tables in the 'mci' structure.
Doug Thompson7d6034d2009-04-27 20:01:01 +02002403 */
Borislav Petkov360b7f32010-10-15 19:25:38 +02002404 ret = -EINVAL;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002405 pvt->channel_count = pvt->ops->early_channel_count(pvt);
2406 if (pvt->channel_count < 0)
Borislav Petkov360b7f32010-10-15 19:25:38 +02002407 goto err_siblings;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002408
2409 ret = -ENOMEM;
Mauro Carvalho Chehabab5a5032012-04-16 15:03:50 -03002410 layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
2411 layers[0].size = pvt->csels[0].b_cnt;
2412 layers[0].is_virt_csrow = true;
2413 layers[1].type = EDAC_MC_LAYER_CHANNEL;
2414 layers[1].size = pvt->channel_count;
2415 layers[1].is_virt_csrow = false;
Mauro Carvalho Chehabca0907b2012-05-02 14:37:00 -03002416 mci = edac_mc_alloc(nid, ARRAY_SIZE(layers), layers, 0);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002417 if (!mci)
Borislav Petkov360b7f32010-10-15 19:25:38 +02002418 goto err_siblings;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002419
2420 mci->pvt_info = pvt;
Mauro Carvalho Chehabfd687502012-03-16 07:44:18 -03002421 mci->pdev = &pvt->F2->dev;
Borislav Petkov11652762012-09-13 17:19:40 +02002422 mci->csbased = 1;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002423
Borislav Petkovdf71a052011-01-19 18:15:10 +01002424 setup_mci_misc_attrs(mci, fam_type);
Borislav Petkov360b7f32010-10-15 19:25:38 +02002425
2426 if (init_csrows(mci))
Doug Thompson7d6034d2009-04-27 20:01:01 +02002427 mci->edac_cap = EDAC_FLAG_NONE;
2428
Doug Thompson7d6034d2009-04-27 20:01:01 +02002429 ret = -ENODEV;
2430 if (edac_mc_add_mc(mci)) {
Joe Perches956b9ba2012-04-29 17:08:39 -03002431 edac_dbg(1, "failed edac_mc_add_mc()\n");
Doug Thompson7d6034d2009-04-27 20:01:01 +02002432 goto err_add_mc;
2433 }
Mauro Carvalho Chehabc5608752012-03-21 14:00:44 -03002434 if (set_mc_sysfs_attrs(mci)) {
Joe Perches956b9ba2012-04-29 17:08:39 -03002435 edac_dbg(1, "failed edac_mc_add_mc()\n");
Mauro Carvalho Chehabc5608752012-03-21 14:00:44 -03002436 goto err_add_sysfs;
2437 }
Doug Thompson7d6034d2009-04-27 20:01:01 +02002438
Borislav Petkov549d0422009-07-24 13:51:42 +02002439 /* register stuff with EDAC MCE */
2440 if (report_gart_errors)
2441 amd_report_gart_errors(true);
2442
2443 amd_register_ecc_decoder(amd64_decode_bus_error);
2444
Borislav Petkov360b7f32010-10-15 19:25:38 +02002445 mcis[nid] = mci;
2446
2447 atomic_inc(&drv_instances);
2448
Doug Thompson7d6034d2009-04-27 20:01:01 +02002449 return 0;
2450
Mauro Carvalho Chehabc5608752012-03-21 14:00:44 -03002451err_add_sysfs:
2452 edac_mc_del_mc(mci->pdev);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002453err_add_mc:
2454 edac_mc_free(mci);
2455
Borislav Petkov360b7f32010-10-15 19:25:38 +02002456err_siblings:
2457 free_mc_sibling_devs(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002458
Borislav Petkov360b7f32010-10-15 19:25:38 +02002459err_free:
2460 kfree(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002461
Borislav Petkov360b7f32010-10-15 19:25:38 +02002462err_ret:
Doug Thompson7d6034d2009-04-27 20:01:01 +02002463 return ret;
2464}
2465
Greg Kroah-Hartman9b3c6e82012-12-21 13:23:51 -08002466static int amd64_probe_one_instance(struct pci_dev *pdev,
2467 const struct pci_device_id *mc_type)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002468{
Daniel J Blueman772c3ff2012-11-27 14:32:09 +08002469 u16 nid = amd_get_node_id(pdev);
Borislav Petkov2299ef72010-10-15 17:44:04 +02002470 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002471 struct ecc_settings *s;
Borislav Petkov2299ef72010-10-15 17:44:04 +02002472 int ret = 0;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002473
Doug Thompson7d6034d2009-04-27 20:01:01 +02002474 ret = pci_enable_device(pdev);
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002475 if (ret < 0) {
Joe Perches956b9ba2012-04-29 17:08:39 -03002476 edac_dbg(0, "ret=%d\n", ret);
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002477 return -EIO;
2478 }
2479
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002480 ret = -ENOMEM;
2481 s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
2482 if (!s)
Borislav Petkov2299ef72010-10-15 17:44:04 +02002483 goto err_out;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002484
2485 ecc_stngs[nid] = s;
2486
Borislav Petkov2299ef72010-10-15 17:44:04 +02002487 if (!ecc_enabled(F3, nid)) {
2488 ret = -ENODEV;
2489
2490 if (!ecc_enable_override)
2491 goto err_enable;
2492
2493 amd64_warn("Forcing ECC on!\n");
2494
2495 if (!enable_ecc_error_reporting(s, nid, F3))
2496 goto err_enable;
2497 }
2498
2499 ret = amd64_init_one_instance(pdev);
Borislav Petkov360b7f32010-10-15 19:25:38 +02002500 if (ret < 0) {
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002501 amd64_err("Error probing instance: %d\n", nid);
Borislav Petkov360b7f32010-10-15 19:25:38 +02002502 restore_ecc_error_reporting(s, nid, F3);
2503 }
Doug Thompson7d6034d2009-04-27 20:01:01 +02002504
2505 return ret;
Borislav Petkov2299ef72010-10-15 17:44:04 +02002506
2507err_enable:
2508 kfree(s);
2509 ecc_stngs[nid] = NULL;
2510
2511err_out:
2512 return ret;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002513}
2514
Greg Kroah-Hartman9b3c6e82012-12-21 13:23:51 -08002515static void amd64_remove_one_instance(struct pci_dev *pdev)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002516{
2517 struct mem_ctl_info *mci;
2518 struct amd64_pvt *pvt;
Daniel J Blueman772c3ff2012-11-27 14:32:09 +08002519 u16 nid = amd_get_node_id(pdev);
Borislav Petkov360b7f32010-10-15 19:25:38 +02002520 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2521 struct ecc_settings *s = ecc_stngs[nid];
Doug Thompson7d6034d2009-04-27 20:01:01 +02002522
Mauro Carvalho Chehabc5608752012-03-21 14:00:44 -03002523 mci = find_mci_by_dev(&pdev->dev);
2524 del_mc_sysfs_attrs(mci);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002525 /* Remove from EDAC CORE tracking list */
2526 mci = edac_mc_del_mc(&pdev->dev);
2527 if (!mci)
2528 return;
2529
2530 pvt = mci->pvt_info;
2531
Borislav Petkov360b7f32010-10-15 19:25:38 +02002532 restore_ecc_error_reporting(s, nid, F3);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002533
Borislav Petkov360b7f32010-10-15 19:25:38 +02002534 free_mc_sibling_devs(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002535
Borislav Petkov549d0422009-07-24 13:51:42 +02002536 /* unregister from EDAC MCE */
2537 amd_report_gart_errors(false);
2538 amd_unregister_ecc_decoder(amd64_decode_bus_error);
2539
Borislav Petkov360b7f32010-10-15 19:25:38 +02002540 kfree(ecc_stngs[nid]);
2541 ecc_stngs[nid] = NULL;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002542
Doug Thompson7d6034d2009-04-27 20:01:01 +02002543 /* Free the EDAC CORE resources */
Borislav Petkov8f68ed92009-12-21 15:15:59 +01002544 mci->pvt_info = NULL;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002545 mcis[nid] = NULL;
Borislav Petkov8f68ed92009-12-21 15:15:59 +01002546
2547 kfree(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002548 edac_mc_free(mci);
2549}
2550
2551/*
2552 * This table is part of the interface for loading drivers for PCI devices. The
2553 * PCI core identifies what devices are on a system during boot, and then
2554 * inquiry this table to see if this driver is for a given device found.
2555 */
Lionel Debroux36c46f32012-02-27 07:41:47 +01002556static DEFINE_PCI_DEVICE_TABLE(amd64_pci_table) = {
Doug Thompson7d6034d2009-04-27 20:01:01 +02002557 {
2558 .vendor = PCI_VENDOR_ID_AMD,
2559 .device = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL,
2560 .subvendor = PCI_ANY_ID,
2561 .subdevice = PCI_ANY_ID,
2562 .class = 0,
2563 .class_mask = 0,
Doug Thompson7d6034d2009-04-27 20:01:01 +02002564 },
2565 {
2566 .vendor = PCI_VENDOR_ID_AMD,
2567 .device = PCI_DEVICE_ID_AMD_10H_NB_DRAM,
2568 .subvendor = PCI_ANY_ID,
2569 .subdevice = PCI_ANY_ID,
2570 .class = 0,
2571 .class_mask = 0,
Doug Thompson7d6034d2009-04-27 20:01:01 +02002572 },
Borislav Petkovdf71a052011-01-19 18:15:10 +01002573 {
2574 .vendor = PCI_VENDOR_ID_AMD,
2575 .device = PCI_DEVICE_ID_AMD_15H_NB_F2,
2576 .subvendor = PCI_ANY_ID,
2577 .subdevice = PCI_ANY_ID,
2578 .class = 0,
2579 .class_mask = 0,
2580 },
2581
Doug Thompson7d6034d2009-04-27 20:01:01 +02002582 {0, }
2583};
2584MODULE_DEVICE_TABLE(pci, amd64_pci_table);
2585
2586static struct pci_driver amd64_pci_driver = {
2587 .name = EDAC_MOD_STR,
Borislav Petkov2299ef72010-10-15 17:44:04 +02002588 .probe = amd64_probe_one_instance,
Greg Kroah-Hartman9b3c6e82012-12-21 13:23:51 -08002589 .remove = amd64_remove_one_instance,
Doug Thompson7d6034d2009-04-27 20:01:01 +02002590 .id_table = amd64_pci_table,
2591};
2592
Borislav Petkov360b7f32010-10-15 19:25:38 +02002593static void setup_pci_device(void)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002594{
2595 struct mem_ctl_info *mci;
2596 struct amd64_pvt *pvt;
2597
2598 if (amd64_ctl_pci)
2599 return;
2600
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002601 mci = mcis[0];
Doug Thompson7d6034d2009-04-27 20:01:01 +02002602 if (mci) {
2603
2604 pvt = mci->pvt_info;
2605 amd64_ctl_pci =
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002606 edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002607
2608 if (!amd64_ctl_pci) {
2609 pr_warning("%s(): Unable to create PCI control\n",
2610 __func__);
2611
2612 pr_warning("%s(): PCI error report via EDAC not set\n",
2613 __func__);
2614 }
2615 }
2616}
2617
2618static int __init amd64_edac_init(void)
2619{
Borislav Petkov360b7f32010-10-15 19:25:38 +02002620 int err = -ENODEV;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002621
Borislav Petkovdf71a052011-01-19 18:15:10 +01002622 printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002623
2624 opstate_init();
2625
Hans Rosenfeld9653a5c2010-10-29 17:14:31 +02002626 if (amd_cache_northbridges() < 0)
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01002627 goto err_ret;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002628
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002629 err = -ENOMEM;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002630 mcis = kzalloc(amd_nb_num() * sizeof(mcis[0]), GFP_KERNEL);
2631 ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL);
Borislav Petkov360b7f32010-10-15 19:25:38 +02002632 if (!(mcis && ecc_stngs))
Borislav Petkova9f0fbe2011-03-29 18:10:53 +02002633 goto err_free;
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002634
Borislav Petkov50542252009-12-11 18:14:40 +01002635 msrs = msrs_alloc();
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01002636 if (!msrs)
Borislav Petkov360b7f32010-10-15 19:25:38 +02002637 goto err_free;
Borislav Petkov50542252009-12-11 18:14:40 +01002638
Doug Thompson7d6034d2009-04-27 20:01:01 +02002639 err = pci_register_driver(&amd64_pci_driver);
2640 if (err)
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01002641 goto err_pci;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002642
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01002643 err = -ENODEV;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002644 if (!atomic_read(&drv_instances))
2645 goto err_no_instances;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002646
Borislav Petkov360b7f32010-10-15 19:25:38 +02002647 setup_pci_device();
2648 return 0;
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01002649
Borislav Petkov360b7f32010-10-15 19:25:38 +02002650err_no_instances:
Doug Thompson7d6034d2009-04-27 20:01:01 +02002651 pci_unregister_driver(&amd64_pci_driver);
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002652
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01002653err_pci:
2654 msrs_free(msrs);
2655 msrs = NULL;
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002656
Borislav Petkov360b7f32010-10-15 19:25:38 +02002657err_free:
2658 kfree(mcis);
2659 mcis = NULL;
2660
2661 kfree(ecc_stngs);
2662 ecc_stngs = NULL;
2663
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01002664err_ret:
Doug Thompson7d6034d2009-04-27 20:01:01 +02002665 return err;
2666}
2667
2668static void __exit amd64_edac_exit(void)
2669{
2670 if (amd64_ctl_pci)
2671 edac_pci_release_generic_ctl(amd64_ctl_pci);
2672
2673 pci_unregister_driver(&amd64_pci_driver);
Borislav Petkov50542252009-12-11 18:14:40 +01002674
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002675 kfree(ecc_stngs);
2676 ecc_stngs = NULL;
2677
Borislav Petkovcc4d8862010-10-13 16:11:59 +02002678 kfree(mcis);
2679 mcis = NULL;
2680
Borislav Petkov50542252009-12-11 18:14:40 +01002681 msrs_free(msrs);
2682 msrs = NULL;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002683}
2684
2685module_init(amd64_edac_init);
2686module_exit(amd64_edac_exit);
2687
2688MODULE_LICENSE("GPL");
2689MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
2690 "Dave Peterson, Thayne Harbaugh");
2691MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
2692 EDAC_AMD64_VERSION);
2693
2694module_param(edac_op_state, int, 0444);
2695MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");