blob: 9eee13ef83a560fd0c54153c61bc01a77cd3eb7f [file] [log] [blame]
Doug Thompson2bc65412009-05-04 20:11:14 +02001#include "amd64_edac.h"
Andreas Herrmann23ac4ae2010-09-17 18:03:43 +02002#include <asm/amd_nb.h>
Doug Thompson2bc65412009-05-04 20:11:14 +02003
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01004static struct edac_pci_ctl_info *pci_ctl;
Doug Thompson2bc65412009-05-04 20:11:14 +02005
6static int report_gart_errors;
7module_param(report_gart_errors, int, 0644);
8
9/*
10 * Set by command line parameter. If BIOS has enabled the ECC, this override is
11 * cleared to prevent re-enabling the hardware by this driver.
12 */
13static int ecc_enable_override;
14module_param(ecc_enable_override, int, 0644);
15
Tejun Heoa29d8b82010-02-02 14:39:15 +090016static struct msr __percpu *msrs;
Borislav Petkov50542252009-12-11 18:14:40 +010017
Borislav Petkov360b7f32010-10-15 19:25:38 +020018/*
19 * count successfully initialized driver instances for setup_pci_device()
20 */
21static atomic_t drv_instances = ATOMIC_INIT(0);
22
Borislav Petkov2ec591a2015-02-17 10:58:34 +010023/* Per-node stuff */
Borislav Petkovae7bb7c2010-10-14 16:01:30 +020024static struct ecc_settings **ecc_stngs;
Doug Thompson2bc65412009-05-04 20:11:14 +020025
26/*
Borislav Petkovb70ef012009-06-25 19:32:38 +020027 * Valid scrub rates for the K8 hardware memory scrubber. We map the scrubbing
28 * bandwidth to a valid bit pattern. The 'set' operation finds the 'matching-
29 * or higher value'.
30 *
31 *FIXME: Produce a better mapping/linearisation.
32 */
Daniel J Bluemanc7e53012012-11-30 16:44:20 +080033static const struct scrubrate {
Borislav Petkov39094442010-11-24 19:52:09 +010034 u32 scrubval; /* bit pattern for scrub rate */
35 u32 bandwidth; /* bandwidth consumed (bytes/sec) */
36} scrubrates[] = {
Borislav Petkovb70ef012009-06-25 19:32:38 +020037 { 0x01, 1600000000UL},
38 { 0x02, 800000000UL},
39 { 0x03, 400000000UL},
40 { 0x04, 200000000UL},
41 { 0x05, 100000000UL},
42 { 0x06, 50000000UL},
43 { 0x07, 25000000UL},
44 { 0x08, 12284069UL},
45 { 0x09, 6274509UL},
46 { 0x0A, 3121951UL},
47 { 0x0B, 1560975UL},
48 { 0x0C, 781440UL},
49 { 0x0D, 390720UL},
50 { 0x0E, 195300UL},
51 { 0x0F, 97650UL},
52 { 0x10, 48854UL},
53 { 0x11, 24427UL},
54 { 0x12, 12213UL},
55 { 0x13, 6101UL},
56 { 0x14, 3051UL},
57 { 0x15, 1523UL},
58 { 0x16, 761UL},
59 { 0x00, 0UL}, /* scrubbing off */
60};
61
Borislav Petkov66fed2d2012-08-09 18:41:07 +020062int __amd64_read_pci_cfg_dword(struct pci_dev *pdev, int offset,
63 u32 *val, const char *func)
Borislav Petkovb2b0c602010-10-08 18:32:29 +020064{
65 int err = 0;
66
67 err = pci_read_config_dword(pdev, offset, val);
68 if (err)
69 amd64_warn("%s: error reading F%dx%03x.\n",
70 func, PCI_FUNC(pdev->devfn), offset);
71
72 return err;
73}
74
75int __amd64_write_pci_cfg_dword(struct pci_dev *pdev, int offset,
76 u32 val, const char *func)
77{
78 int err = 0;
79
80 err = pci_write_config_dword(pdev, offset, val);
81 if (err)
82 amd64_warn("%s: error writing to F%dx%03x.\n",
83 func, PCI_FUNC(pdev->devfn), offset);
84
85 return err;
86}
87
88/*
Borislav Petkov73ba8592011-09-19 17:34:45 +020089 * Select DCT to which PCI cfg accesses are routed
90 */
91static void f15h_select_dct(struct amd64_pvt *pvt, u8 dct)
92{
93 u32 reg = 0;
94
95 amd64_read_pci_cfg(pvt->F1, DCT_CFG_SEL, &reg);
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -050096 reg &= (pvt->model == 0x30) ? ~3 : ~1;
Borislav Petkov73ba8592011-09-19 17:34:45 +020097 reg |= dct;
98 amd64_write_pci_cfg(pvt->F1, DCT_CFG_SEL, reg);
99}
100
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -0500101/*
102 *
103 * Depending on the family, F2 DCT reads need special handling:
104 *
105 * K8: has a single DCT only and no address offsets >= 0x100
106 *
107 * F10h: each DCT has its own set of regs
108 * DCT0 -> F2x040..
109 * DCT1 -> F2x140..
110 *
111 * F16h: has only 1 DCT
112 *
113 * F15h: we select which DCT we access using F1x10C[DctCfgSel]
114 */
115static inline int amd64_read_dct_pci_cfg(struct amd64_pvt *pvt, u8 dct,
116 int offset, u32 *val)
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200117{
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -0500118 switch (pvt->fam) {
119 case 0xf:
120 if (dct || offset >= 0x100)
121 return -EINVAL;
122 break;
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200123
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -0500124 case 0x10:
125 if (dct) {
126 /*
127 * Note: If ganging is enabled, barring the regs
128 * F2x[1,0]98 and F2x[1,0]9C; reads reads to F2x1xx
129 * return 0. (cf. Section 2.8.1 F10h BKDG)
130 */
131 if (dct_ganging_enabled(pvt))
132 return 0;
133
134 offset += 0x100;
135 }
136 break;
137
138 case 0x15:
139 /*
140 * F15h: F2x1xx addresses do not map explicitly to DCT1.
141 * We should select which DCT we access using F1x10C[DctCfgSel]
142 */
143 dct = (dct && pvt->model == 0x30) ? 3 : dct;
144 f15h_select_dct(pvt, dct);
145 break;
146
147 case 0x16:
148 if (dct)
149 return -EINVAL;
150 break;
151
152 default:
153 break;
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200154 }
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -0500155 return amd64_read_pci_cfg(pvt->F2, offset, val);
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200156}
157
Borislav Petkovb70ef012009-06-25 19:32:38 +0200158/*
Doug Thompson2bc65412009-05-04 20:11:14 +0200159 * Memory scrubber control interface. For K8, memory scrubbing is handled by
160 * hardware and can involve L2 cache, dcache as well as the main memory. With
161 * F10, this is extended to L3 cache scrubbing on CPU models sporting that
162 * functionality.
163 *
164 * This causes the "units" for the scrubbing speed to vary from 64 byte blocks
165 * (dram) over to cache lines. This is nasty, so we will use bandwidth in
166 * bytes/sec for the setting.
167 *
168 * Currently, we only do dram scrubbing. If the scrubbing is done in software on
169 * other archs, we might not have access to the caches directly.
170 */
171
172/*
173 * scan the scrub rate mapping table for a close or matching bandwidth value to
174 * issue. If requested is too big, then use last maximum value found.
175 */
Aravind Gopalakrishnanda921102015-09-28 06:43:12 -0500176static int __set_scrub_rate(struct amd64_pvt *pvt, u32 new_bw, u32 min_rate)
Doug Thompson2bc65412009-05-04 20:11:14 +0200177{
178 u32 scrubval;
179 int i;
180
181 /*
182 * map the configured rate (new_bw) to a value specific to the AMD64
183 * memory controller and apply to register. Search for the first
184 * bandwidth entry that is greater or equal than the setting requested
185 * and program that. If at last entry, turn off DRAM scrubbing.
Andrew Morton168bfee2012-10-23 14:09:39 -0700186 *
187 * If no suitable bandwidth is found, turn off DRAM scrubbing entirely
188 * by falling back to the last element in scrubrates[].
Doug Thompson2bc65412009-05-04 20:11:14 +0200189 */
Andrew Morton168bfee2012-10-23 14:09:39 -0700190 for (i = 0; i < ARRAY_SIZE(scrubrates) - 1; i++) {
Doug Thompson2bc65412009-05-04 20:11:14 +0200191 /*
192 * skip scrub rates which aren't recommended
193 * (see F10 BKDG, F3x58)
194 */
Borislav Petkov395ae782010-10-01 18:38:19 +0200195 if (scrubrates[i].scrubval < min_rate)
Doug Thompson2bc65412009-05-04 20:11:14 +0200196 continue;
197
198 if (scrubrates[i].bandwidth <= new_bw)
199 break;
Doug Thompson2bc65412009-05-04 20:11:14 +0200200 }
201
202 scrubval = scrubrates[i].scrubval;
Doug Thompson2bc65412009-05-04 20:11:14 +0200203
Aravind Gopalakrishnanda921102015-09-28 06:43:12 -0500204 if (pvt->fam == 0x15 && pvt->model == 0x60) {
205 f15h_select_dct(pvt, 0);
206 pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
207 f15h_select_dct(pvt, 1);
208 pci_write_bits32(pvt->F2, F15H_M60H_SCRCTRL, scrubval, 0x001F);
209 } else {
210 pci_write_bits32(pvt->F3, SCRCTRL, scrubval, 0x001F);
211 }
Doug Thompson2bc65412009-05-04 20:11:14 +0200212
Borislav Petkov39094442010-11-24 19:52:09 +0100213 if (scrubval)
214 return scrubrates[i].bandwidth;
215
Doug Thompson2bc65412009-05-04 20:11:14 +0200216 return 0;
217}
218
Borislav Petkovd1ea71c2013-12-15 17:54:27 +0100219static int set_scrub_rate(struct mem_ctl_info *mci, u32 bw)
Doug Thompson2bc65412009-05-04 20:11:14 +0200220{
221 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkov87b3e0e2011-01-19 20:02:38 +0100222 u32 min_scrubrate = 0x5;
Doug Thompson2bc65412009-05-04 20:11:14 +0200223
Borislav Petkova4b4bed2013-08-10 13:54:48 +0200224 if (pvt->fam == 0xf)
Borislav Petkov87b3e0e2011-01-19 20:02:38 +0100225 min_scrubrate = 0x0;
226
Aravind Gopalakrishnanda921102015-09-28 06:43:12 -0500227 if (pvt->fam == 0x15) {
228 /* Erratum #505 */
229 if (pvt->model < 0x10)
230 f15h_select_dct(pvt, 0);
Borislav Petkov73ba8592011-09-19 17:34:45 +0200231
Aravind Gopalakrishnanda921102015-09-28 06:43:12 -0500232 if (pvt->model == 0x60)
233 min_scrubrate = 0x6;
234 }
235 return __set_scrub_rate(pvt, bw, min_scrubrate);
Doug Thompson2bc65412009-05-04 20:11:14 +0200236}
237
Borislav Petkovd1ea71c2013-12-15 17:54:27 +0100238static int get_scrub_rate(struct mem_ctl_info *mci)
Doug Thompson2bc65412009-05-04 20:11:14 +0200239{
240 struct amd64_pvt *pvt = mci->pvt_info;
241 u32 scrubval = 0;
Borislav Petkov39094442010-11-24 19:52:09 +0100242 int i, retval = -EINVAL;
Doug Thompson2bc65412009-05-04 20:11:14 +0200243
Aravind Gopalakrishnanda921102015-09-28 06:43:12 -0500244 if (pvt->fam == 0x15) {
245 /* Erratum #505 */
246 if (pvt->model < 0x10)
247 f15h_select_dct(pvt, 0);
Borislav Petkov73ba8592011-09-19 17:34:45 +0200248
Aravind Gopalakrishnanda921102015-09-28 06:43:12 -0500249 if (pvt->model == 0x60)
250 amd64_read_pci_cfg(pvt->F2, F15H_M60H_SCRCTRL, &scrubval);
251 } else
252 amd64_read_pci_cfg(pvt->F3, SCRCTRL, &scrubval);
Doug Thompson2bc65412009-05-04 20:11:14 +0200253
254 scrubval = scrubval & 0x001F;
255
Roel Kluin926311f2010-01-11 20:58:21 +0100256 for (i = 0; i < ARRAY_SIZE(scrubrates); i++) {
Doug Thompson2bc65412009-05-04 20:11:14 +0200257 if (scrubrates[i].scrubval == scrubval) {
Borislav Petkov39094442010-11-24 19:52:09 +0100258 retval = scrubrates[i].bandwidth;
Doug Thompson2bc65412009-05-04 20:11:14 +0200259 break;
260 }
261 }
Borislav Petkov39094442010-11-24 19:52:09 +0100262 return retval;
Doug Thompson2bc65412009-05-04 20:11:14 +0200263}
264
Doug Thompson67757632009-04-27 15:53:22 +0200265/*
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200266 * returns true if the SysAddr given by sys_addr matches the
267 * DRAM base/limit associated with node_id
Doug Thompson67757632009-04-27 15:53:22 +0200268 */
Borislav Petkovd1ea71c2013-12-15 17:54:27 +0100269static bool base_limit_match(struct amd64_pvt *pvt, u64 sys_addr, u8 nid)
Doug Thompson67757632009-04-27 15:53:22 +0200270{
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200271 u64 addr;
Doug Thompson67757632009-04-27 15:53:22 +0200272
273 /* The K8 treats this as a 40-bit value. However, bits 63-40 will be
274 * all ones if the most significant implemented address bit is 1.
275 * Here we discard bits 63-40. See section 3.4.2 of AMD publication
276 * 24592: AMD x86-64 Architecture Programmer's Manual Volume 1
277 * Application Programming.
278 */
279 addr = sys_addr & 0x000000ffffffffffull;
280
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200281 return ((addr >= get_dram_base(pvt, nid)) &&
282 (addr <= get_dram_limit(pvt, nid)));
Doug Thompson67757632009-04-27 15:53:22 +0200283}
284
285/*
286 * Attempt to map a SysAddr to a node. On success, return a pointer to the
287 * mem_ctl_info structure for the node that the SysAddr maps to.
288 *
289 * On failure, return NULL.
290 */
291static struct mem_ctl_info *find_mc_by_sys_addr(struct mem_ctl_info *mci,
292 u64 sys_addr)
293{
294 struct amd64_pvt *pvt;
Daniel J Bluemanc7e53012012-11-30 16:44:20 +0800295 u8 node_id;
Doug Thompson67757632009-04-27 15:53:22 +0200296 u32 intlv_en, bits;
297
298 /*
299 * Here we use the DRAM Base (section 3.4.4.1) and DRAM Limit (section
300 * 3.4.4.2) registers to map the SysAddr to a node ID.
301 */
302 pvt = mci->pvt_info;
303
304 /*
305 * The value of this field should be the same for all DRAM Base
306 * registers. Therefore we arbitrarily choose to read it from the
307 * register for node 0.
308 */
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200309 intlv_en = dram_intlv_en(pvt, 0);
Doug Thompson67757632009-04-27 15:53:22 +0200310
311 if (intlv_en == 0) {
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200312 for (node_id = 0; node_id < DRAM_RANGES; node_id++) {
Borislav Petkovd1ea71c2013-12-15 17:54:27 +0100313 if (base_limit_match(pvt, sys_addr, node_id))
Borislav Petkov8edc5442009-09-18 12:39:19 +0200314 goto found;
Doug Thompson67757632009-04-27 15:53:22 +0200315 }
Borislav Petkov8edc5442009-09-18 12:39:19 +0200316 goto err_no_match;
Doug Thompson67757632009-04-27 15:53:22 +0200317 }
318
Borislav Petkov72f158f2009-09-18 12:27:27 +0200319 if (unlikely((intlv_en != 0x01) &&
320 (intlv_en != 0x03) &&
321 (intlv_en != 0x07))) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200322 amd64_warn("DRAM Base[IntlvEn] junk value: 0x%x, BIOS bug?\n", intlv_en);
Doug Thompson67757632009-04-27 15:53:22 +0200323 return NULL;
324 }
325
326 bits = (((u32) sys_addr) >> 12) & intlv_en;
327
328 for (node_id = 0; ; ) {
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200329 if ((dram_intlv_sel(pvt, node_id) & intlv_en) == bits)
Doug Thompson67757632009-04-27 15:53:22 +0200330 break; /* intlv_sel field matches */
331
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200332 if (++node_id >= DRAM_RANGES)
Doug Thompson67757632009-04-27 15:53:22 +0200333 goto err_no_match;
334 }
335
336 /* sanity test for sys_addr */
Borislav Petkovd1ea71c2013-12-15 17:54:27 +0100337 if (unlikely(!base_limit_match(pvt, sys_addr, node_id))) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200338 amd64_warn("%s: sys_addr 0x%llx falls outside base/limit address"
339 "range for node %d with node interleaving enabled.\n",
340 __func__, sys_addr, node_id);
Doug Thompson67757632009-04-27 15:53:22 +0200341 return NULL;
342 }
343
344found:
Borislav Petkovb487c332011-02-21 18:55:00 +0100345 return edac_mc_find((int)node_id);
Doug Thompson67757632009-04-27 15:53:22 +0200346
347err_no_match:
Joe Perches956b9ba2012-04-29 17:08:39 -0300348 edac_dbg(2, "sys_addr 0x%lx doesn't match any node\n",
349 (unsigned long)sys_addr);
Doug Thompson67757632009-04-27 15:53:22 +0200350
351 return NULL;
352}
Doug Thompsone2ce7252009-04-27 15:57:12 +0200353
354/*
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100355 * compute the CS base address of the @csrow on the DRAM controller @dct.
356 * For details see F2x[5C:40] in the processor's BKDG
Doug Thompsone2ce7252009-04-27 15:57:12 +0200357 */
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100358static void get_cs_base_and_mask(struct amd64_pvt *pvt, int csrow, u8 dct,
359 u64 *base, u64 *mask)
Doug Thompsone2ce7252009-04-27 15:57:12 +0200360{
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100361 u64 csbase, csmask, base_bits, mask_bits;
362 u8 addr_shift;
363
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -0500364 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100365 csbase = pvt->csels[dct].csbases[csrow];
366 csmask = pvt->csels[dct].csmasks[csrow];
Chen, Gong10ef6b02013-10-18 14:29:07 -0700367 base_bits = GENMASK_ULL(31, 21) | GENMASK_ULL(15, 9);
368 mask_bits = GENMASK_ULL(29, 21) | GENMASK_ULL(15, 9);
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100369 addr_shift = 4;
Aravind Gopalakrishnan94c1acf2013-04-17 14:57:13 -0500370
371 /*
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -0500372 * F16h and F15h, models 30h and later need two addr_shift values:
373 * 8 for high and 6 for low (cf. F16h BKDG).
374 */
375 } else if (pvt->fam == 0x16 ||
376 (pvt->fam == 0x15 && pvt->model >= 0x30)) {
Aravind Gopalakrishnan94c1acf2013-04-17 14:57:13 -0500377 csbase = pvt->csels[dct].csbases[csrow];
378 csmask = pvt->csels[dct].csmasks[csrow >> 1];
379
Chen, Gong10ef6b02013-10-18 14:29:07 -0700380 *base = (csbase & GENMASK_ULL(15, 5)) << 6;
381 *base |= (csbase & GENMASK_ULL(30, 19)) << 8;
Aravind Gopalakrishnan94c1acf2013-04-17 14:57:13 -0500382
383 *mask = ~0ULL;
384 /* poke holes for the csmask */
Chen, Gong10ef6b02013-10-18 14:29:07 -0700385 *mask &= ~((GENMASK_ULL(15, 5) << 6) |
386 (GENMASK_ULL(30, 19) << 8));
Aravind Gopalakrishnan94c1acf2013-04-17 14:57:13 -0500387
Chen, Gong10ef6b02013-10-18 14:29:07 -0700388 *mask |= (csmask & GENMASK_ULL(15, 5)) << 6;
389 *mask |= (csmask & GENMASK_ULL(30, 19)) << 8;
Aravind Gopalakrishnan94c1acf2013-04-17 14:57:13 -0500390
391 return;
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100392 } else {
393 csbase = pvt->csels[dct].csbases[csrow];
394 csmask = pvt->csels[dct].csmasks[csrow >> 1];
395 addr_shift = 8;
396
Borislav Petkova4b4bed2013-08-10 13:54:48 +0200397 if (pvt->fam == 0x15)
Chen, Gong10ef6b02013-10-18 14:29:07 -0700398 base_bits = mask_bits =
399 GENMASK_ULL(30,19) | GENMASK_ULL(13,5);
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100400 else
Chen, Gong10ef6b02013-10-18 14:29:07 -0700401 base_bits = mask_bits =
402 GENMASK_ULL(28,19) | GENMASK_ULL(13,5);
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100403 }
404
405 *base = (csbase & base_bits) << addr_shift;
406
407 *mask = ~0ULL;
408 /* poke holes for the csmask */
409 *mask &= ~(mask_bits << addr_shift);
410 /* OR them in */
411 *mask |= (csmask & mask_bits) << addr_shift;
Doug Thompsone2ce7252009-04-27 15:57:12 +0200412}
413
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100414#define for_each_chip_select(i, dct, pvt) \
415 for (i = 0; i < pvt->csels[dct].b_cnt; i++)
Doug Thompsone2ce7252009-04-27 15:57:12 +0200416
Borislav Petkov614ec9d2011-01-13 18:02:22 +0100417#define chip_select_base(i, dct, pvt) \
418 pvt->csels[dct].csbases[i]
419
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100420#define for_each_chip_select_mask(i, dct, pvt) \
421 for (i = 0; i < pvt->csels[dct].m_cnt; i++)
Doug Thompsone2ce7252009-04-27 15:57:12 +0200422
423/*
424 * @input_addr is an InputAddr associated with the node given by mci. Return the
425 * csrow that input_addr maps to, or -1 on failure (no csrow claims input_addr).
426 */
427static int input_addr_to_csrow(struct mem_ctl_info *mci, u64 input_addr)
428{
429 struct amd64_pvt *pvt;
430 int csrow;
431 u64 base, mask;
432
433 pvt = mci->pvt_info;
434
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100435 for_each_chip_select(csrow, 0, pvt) {
436 if (!csrow_enabled(csrow, 0, pvt))
Doug Thompsone2ce7252009-04-27 15:57:12 +0200437 continue;
438
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100439 get_cs_base_and_mask(pvt, csrow, 0, &base, &mask);
440
441 mask = ~mask;
Doug Thompsone2ce7252009-04-27 15:57:12 +0200442
443 if ((input_addr & mask) == (base & mask)) {
Joe Perches956b9ba2012-04-29 17:08:39 -0300444 edac_dbg(2, "InputAddr 0x%lx matches csrow %d (node %d)\n",
445 (unsigned long)input_addr, csrow,
446 pvt->mc_node_id);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200447
448 return csrow;
449 }
450 }
Joe Perches956b9ba2012-04-29 17:08:39 -0300451 edac_dbg(2, "no matching csrow for InputAddr 0x%lx (MC node %d)\n",
452 (unsigned long)input_addr, pvt->mc_node_id);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200453
454 return -1;
455}
456
457/*
Doug Thompsone2ce7252009-04-27 15:57:12 +0200458 * Obtain info from the DRAM Hole Address Register (section 3.4.8, pub #26094)
459 * for the node represented by mci. Info is passed back in *hole_base,
460 * *hole_offset, and *hole_size. Function returns 0 if info is valid or 1 if
461 * info is invalid. Info may be invalid for either of the following reasons:
462 *
463 * - The revision of the node is not E or greater. In this case, the DRAM Hole
464 * Address Register does not exist.
465 *
466 * - The DramHoleValid bit is cleared in the DRAM Hole Address Register,
467 * indicating that its contents are not valid.
468 *
469 * The values passed back in *hole_base, *hole_offset, and *hole_size are
470 * complete 32-bit values despite the fact that the bitfields in the DHAR
471 * only represent bits 31-24 of the base and offset values.
472 */
473int amd64_get_dram_hole_info(struct mem_ctl_info *mci, u64 *hole_base,
474 u64 *hole_offset, u64 *hole_size)
475{
476 struct amd64_pvt *pvt = mci->pvt_info;
Doug Thompsone2ce7252009-04-27 15:57:12 +0200477
478 /* only revE and later have the DRAM Hole Address Register */
Borislav Petkova4b4bed2013-08-10 13:54:48 +0200479 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_E) {
Joe Perches956b9ba2012-04-29 17:08:39 -0300480 edac_dbg(1, " revision %d for node %d does not support DHAR\n",
481 pvt->ext_model, pvt->mc_node_id);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200482 return 1;
483 }
484
Borislav Petkovbc21fa52010-11-11 17:29:13 +0100485 /* valid for Fam10h and above */
Borislav Petkova4b4bed2013-08-10 13:54:48 +0200486 if (pvt->fam >= 0x10 && !dhar_mem_hoist_valid(pvt)) {
Joe Perches956b9ba2012-04-29 17:08:39 -0300487 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this system\n");
Doug Thompsone2ce7252009-04-27 15:57:12 +0200488 return 1;
489 }
490
Borislav Petkovc8e518d2010-12-10 19:49:19 +0100491 if (!dhar_valid(pvt)) {
Joe Perches956b9ba2012-04-29 17:08:39 -0300492 edac_dbg(1, " Dram Memory Hoisting is DISABLED on this node %d\n",
493 pvt->mc_node_id);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200494 return 1;
495 }
496
497 /* This node has Memory Hoisting */
498
499 /* +------------------+--------------------+--------------------+-----
500 * | memory | DRAM hole | relocated |
501 * | [0, (x - 1)] | [x, 0xffffffff] | addresses from |
502 * | | | DRAM hole |
503 * | | | [0x100000000, |
504 * | | | (0x100000000+ |
505 * | | | (0xffffffff-x))] |
506 * +------------------+--------------------+--------------------+-----
507 *
508 * Above is a diagram of physical memory showing the DRAM hole and the
509 * relocated addresses from the DRAM hole. As shown, the DRAM hole
510 * starts at address x (the base address) and extends through address
511 * 0xffffffff. The DRAM Hole Address Register (DHAR) relocates the
512 * addresses in the hole so that they start at 0x100000000.
513 */
514
Borislav Petkov1f316772012-08-10 12:50:50 +0200515 *hole_base = dhar_base(pvt);
516 *hole_size = (1ULL << 32) - *hole_base;
Doug Thompsone2ce7252009-04-27 15:57:12 +0200517
Borislav Petkova4b4bed2013-08-10 13:54:48 +0200518 *hole_offset = (pvt->fam > 0xf) ? f10_dhar_offset(pvt)
519 : k8_dhar_offset(pvt);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200520
Joe Perches956b9ba2012-04-29 17:08:39 -0300521 edac_dbg(1, " DHAR info for node %d base 0x%lx offset 0x%lx size 0x%lx\n",
522 pvt->mc_node_id, (unsigned long)*hole_base,
523 (unsigned long)*hole_offset, (unsigned long)*hole_size);
Doug Thompsone2ce7252009-04-27 15:57:12 +0200524
525 return 0;
526}
527EXPORT_SYMBOL_GPL(amd64_get_dram_hole_info);
528
Doug Thompson93c2df52009-05-04 20:46:50 +0200529/*
530 * Return the DramAddr that the SysAddr given by @sys_addr maps to. It is
531 * assumed that sys_addr maps to the node given by mci.
532 *
533 * The first part of section 3.4.4 (p. 70) shows how the DRAM Base (section
534 * 3.4.4.1) and DRAM Limit (section 3.4.4.2) registers are used to translate a
535 * SysAddr to a DramAddr. If the DRAM Hole Address Register (DHAR) is enabled,
536 * then it is also involved in translating a SysAddr to a DramAddr. Sections
537 * 3.4.8 and 3.5.8.2 describe the DHAR and how it is used for memory hoisting.
538 * These parts of the documentation are unclear. I interpret them as follows:
539 *
540 * When node n receives a SysAddr, it processes the SysAddr as follows:
541 *
542 * 1. It extracts the DRAMBase and DRAMLimit values from the DRAM Base and DRAM
543 * Limit registers for node n. If the SysAddr is not within the range
544 * specified by the base and limit values, then node n ignores the Sysaddr
545 * (since it does not map to node n). Otherwise continue to step 2 below.
546 *
547 * 2. If the DramHoleValid bit of the DHAR for node n is clear, the DHAR is
548 * disabled so skip to step 3 below. Otherwise see if the SysAddr is within
549 * the range of relocated addresses (starting at 0x100000000) from the DRAM
550 * hole. If not, skip to step 3 below. Else get the value of the
551 * DramHoleOffset field from the DHAR. To obtain the DramAddr, subtract the
552 * offset defined by this value from the SysAddr.
553 *
554 * 3. Obtain the base address for node n from the DRAMBase field of the DRAM
555 * Base register for node n. To obtain the DramAddr, subtract the base
556 * address from the SysAddr, as shown near the start of section 3.4.4 (p.70).
557 */
558static u64 sys_addr_to_dram_addr(struct mem_ctl_info *mci, u64 sys_addr)
559{
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200560 struct amd64_pvt *pvt = mci->pvt_info;
Doug Thompson93c2df52009-05-04 20:46:50 +0200561 u64 dram_base, hole_base, hole_offset, hole_size, dram_addr;
Borislav Petkov1f316772012-08-10 12:50:50 +0200562 int ret;
Doug Thompson93c2df52009-05-04 20:46:50 +0200563
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200564 dram_base = get_dram_base(pvt, pvt->mc_node_id);
Doug Thompson93c2df52009-05-04 20:46:50 +0200565
566 ret = amd64_get_dram_hole_info(mci, &hole_base, &hole_offset,
567 &hole_size);
568 if (!ret) {
Borislav Petkov1f316772012-08-10 12:50:50 +0200569 if ((sys_addr >= (1ULL << 32)) &&
570 (sys_addr < ((1ULL << 32) + hole_size))) {
Doug Thompson93c2df52009-05-04 20:46:50 +0200571 /* use DHAR to translate SysAddr to DramAddr */
572 dram_addr = sys_addr - hole_offset;
573
Joe Perches956b9ba2012-04-29 17:08:39 -0300574 edac_dbg(2, "using DHAR to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
575 (unsigned long)sys_addr,
576 (unsigned long)dram_addr);
Doug Thompson93c2df52009-05-04 20:46:50 +0200577
578 return dram_addr;
579 }
580 }
581
582 /*
583 * Translate the SysAddr to a DramAddr as shown near the start of
584 * section 3.4.4 (p. 70). Although sys_addr is a 64-bit value, the k8
585 * only deals with 40-bit values. Therefore we discard bits 63-40 of
586 * sys_addr below. If bit 39 of sys_addr is 1 then the bits we
587 * discard are all 1s. Otherwise the bits we discard are all 0s. See
588 * section 3.4.2 of AMD publication 24592: AMD x86-64 Architecture
589 * Programmer's Manual Volume 1 Application Programming.
590 */
Chen, Gong10ef6b02013-10-18 14:29:07 -0700591 dram_addr = (sys_addr & GENMASK_ULL(39, 0)) - dram_base;
Doug Thompson93c2df52009-05-04 20:46:50 +0200592
Joe Perches956b9ba2012-04-29 17:08:39 -0300593 edac_dbg(2, "using DRAM Base register to translate SysAddr 0x%lx to DramAddr 0x%lx\n",
594 (unsigned long)sys_addr, (unsigned long)dram_addr);
Doug Thompson93c2df52009-05-04 20:46:50 +0200595 return dram_addr;
596}
597
598/*
599 * @intlv_en is the value of the IntlvEn field from a DRAM Base register
600 * (section 3.4.4.1). Return the number of bits from a SysAddr that are used
601 * for node interleaving.
602 */
603static int num_node_interleave_bits(unsigned intlv_en)
604{
605 static const int intlv_shift_table[] = { 0, 1, 0, 2, 0, 0, 0, 3 };
606 int n;
607
608 BUG_ON(intlv_en > 7);
609 n = intlv_shift_table[intlv_en];
610 return n;
611}
612
613/* Translate the DramAddr given by @dram_addr to an InputAddr. */
614static u64 dram_addr_to_input_addr(struct mem_ctl_info *mci, u64 dram_addr)
615{
616 struct amd64_pvt *pvt;
617 int intlv_shift;
618 u64 input_addr;
619
620 pvt = mci->pvt_info;
621
622 /*
623 * See the start of section 3.4.4 (p. 70, BKDG #26094, K8, revA-E)
624 * concerning translating a DramAddr to an InputAddr.
625 */
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200626 intlv_shift = num_node_interleave_bits(dram_intlv_en(pvt, 0));
Chen, Gong10ef6b02013-10-18 14:29:07 -0700627 input_addr = ((dram_addr >> intlv_shift) & GENMASK_ULL(35, 12)) +
Borislav Petkovf678b8c2010-12-13 19:21:07 +0100628 (dram_addr & 0xfff);
Doug Thompson93c2df52009-05-04 20:46:50 +0200629
Joe Perches956b9ba2012-04-29 17:08:39 -0300630 edac_dbg(2, " Intlv Shift=%d DramAddr=0x%lx maps to InputAddr=0x%lx\n",
631 intlv_shift, (unsigned long)dram_addr,
632 (unsigned long)input_addr);
Doug Thompson93c2df52009-05-04 20:46:50 +0200633
634 return input_addr;
635}
636
637/*
638 * Translate the SysAddr represented by @sys_addr to an InputAddr. It is
639 * assumed that @sys_addr maps to the node given by mci.
640 */
641static u64 sys_addr_to_input_addr(struct mem_ctl_info *mci, u64 sys_addr)
642{
643 u64 input_addr;
644
645 input_addr =
646 dram_addr_to_input_addr(mci, sys_addr_to_dram_addr(mci, sys_addr));
647
Joe Perches956b9ba2012-04-29 17:08:39 -0300648 edac_dbg(2, "SysAdddr 0x%lx translates to InputAddr 0x%lx\n",
649 (unsigned long)sys_addr, (unsigned long)input_addr);
Doug Thompson93c2df52009-05-04 20:46:50 +0200650
651 return input_addr;
652}
653
Doug Thompson93c2df52009-05-04 20:46:50 +0200654/* Map the Error address to a PAGE and PAGE OFFSET. */
655static inline void error_address_to_page_and_offset(u64 error_address,
Borislav Petkov33ca0642012-08-30 18:01:36 +0200656 struct err_info *err)
Doug Thompson93c2df52009-05-04 20:46:50 +0200657{
Borislav Petkov33ca0642012-08-30 18:01:36 +0200658 err->page = (u32) (error_address >> PAGE_SHIFT);
659 err->offset = ((u32) error_address) & ~PAGE_MASK;
Doug Thompson93c2df52009-05-04 20:46:50 +0200660}
661
662/*
663 * @sys_addr is an error address (a SysAddr) extracted from the MCA NB Address
664 * Low (section 3.6.4.5) and MCA NB Address High (section 3.6.4.6) registers
665 * of a node that detected an ECC memory error. mci represents the node that
666 * the error address maps to (possibly different from the node that detected
667 * the error). Return the number of the csrow that sys_addr maps to, or -1 on
668 * error.
669 */
670static int sys_addr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr)
671{
672 int csrow;
673
674 csrow = input_addr_to_csrow(mci, sys_addr_to_input_addr(mci, sys_addr));
675
676 if (csrow == -1)
Borislav Petkov24f9a7f2010-10-07 18:29:15 +0200677 amd64_mc_err(mci, "Failed to translate InputAddr to csrow for "
678 "address 0x%lx\n", (unsigned long)sys_addr);
Doug Thompson93c2df52009-05-04 20:46:50 +0200679 return csrow;
680}
Doug Thompsone2ce7252009-04-27 15:57:12 +0200681
Borislav Petkovbfc04ae2009-11-12 19:05:07 +0100682static int get_channel_from_ecc_syndrome(struct mem_ctl_info *, u16);
Doug Thompson2da11652009-04-27 16:09:09 +0200683
Doug Thompson2da11652009-04-27 16:09:09 +0200684/*
685 * Determine if the DIMMs have ECC enabled. ECC is enabled ONLY if all the DIMMs
686 * are ECC capable.
687 */
Borislav Petkovd1ea71c2013-12-15 17:54:27 +0100688static unsigned long determine_edac_cap(struct amd64_pvt *pvt)
Doug Thompson2da11652009-04-27 16:09:09 +0200689{
Borislav Petkovcb328502010-12-22 14:28:24 +0100690 u8 bit;
Dan Carpenter1f6189e2011-10-06 02:30:25 -0400691 unsigned long edac_cap = EDAC_FLAG_NONE;
Doug Thompson2da11652009-04-27 16:09:09 +0200692
Borislav Petkova4b4bed2013-08-10 13:54:48 +0200693 bit = (pvt->fam > 0xf || pvt->ext_model >= K8_REV_F)
Doug Thompson2da11652009-04-27 16:09:09 +0200694 ? 19
695 : 17;
696
Borislav Petkov584fcff2009-06-10 18:29:54 +0200697 if (pvt->dclr0 & BIT(bit))
Doug Thompson2da11652009-04-27 16:09:09 +0200698 edac_cap = EDAC_FLAG_SECDED;
699
700 return edac_cap;
701}
702
Borislav Petkovd1ea71c2013-12-15 17:54:27 +0100703static void debug_display_dimm_sizes(struct amd64_pvt *, u8);
Doug Thompson2da11652009-04-27 16:09:09 +0200704
Borislav Petkovd1ea71c2013-12-15 17:54:27 +0100705static void debug_dump_dramcfg_low(struct amd64_pvt *pvt, u32 dclr, int chan)
Borislav Petkov68798e12009-11-03 16:18:33 +0100706{
Joe Perches956b9ba2012-04-29 17:08:39 -0300707 edac_dbg(1, "F2x%d90 (DRAM Cfg Low): 0x%08x\n", chan, dclr);
Borislav Petkov68798e12009-11-03 16:18:33 +0100708
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +0100709 if (pvt->dram_type == MEM_LRDDR3) {
710 u32 dcsm = pvt->csels[chan].csmasks[0];
711 /*
712 * It's assumed all LRDIMMs in a DCT are going to be of
713 * same 'type' until proven otherwise. So, use a cs
714 * value of '0' here to get dcsm value.
715 */
716 edac_dbg(1, " LRDIMM %dx rank multiply\n", (dcsm & 0x3));
717 }
718
719 edac_dbg(1, "All DIMMs support ECC:%s\n",
720 (dclr & BIT(19)) ? "yes" : "no");
721
Borislav Petkov68798e12009-11-03 16:18:33 +0100722
Joe Perches956b9ba2012-04-29 17:08:39 -0300723 edac_dbg(1, " PAR/ERR parity: %s\n",
724 (dclr & BIT(8)) ? "enabled" : "disabled");
Borislav Petkov68798e12009-11-03 16:18:33 +0100725
Borislav Petkova4b4bed2013-08-10 13:54:48 +0200726 if (pvt->fam == 0x10)
Joe Perches956b9ba2012-04-29 17:08:39 -0300727 edac_dbg(1, " DCT 128bit mode width: %s\n",
728 (dclr & BIT(11)) ? "128b" : "64b");
Borislav Petkov68798e12009-11-03 16:18:33 +0100729
Joe Perches956b9ba2012-04-29 17:08:39 -0300730 edac_dbg(1, " x4 logical DIMMs present: L0: %s L1: %s L2: %s L3: %s\n",
731 (dclr & BIT(12)) ? "yes" : "no",
732 (dclr & BIT(13)) ? "yes" : "no",
733 (dclr & BIT(14)) ? "yes" : "no",
734 (dclr & BIT(15)) ? "yes" : "no");
Borislav Petkov68798e12009-11-03 16:18:33 +0100735}
736
Doug Thompson2da11652009-04-27 16:09:09 +0200737/* Display and decode various NB registers for debug purposes. */
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200738static void dump_misc_regs(struct amd64_pvt *pvt)
Doug Thompson2da11652009-04-27 16:09:09 +0200739{
Joe Perches956b9ba2012-04-29 17:08:39 -0300740 edac_dbg(1, "F3xE8 (NB Cap): 0x%08x\n", pvt->nbcap);
Doug Thompson2da11652009-04-27 16:09:09 +0200741
Joe Perches956b9ba2012-04-29 17:08:39 -0300742 edac_dbg(1, " NB two channel DRAM capable: %s\n",
743 (pvt->nbcap & NBCAP_DCT_DUAL) ? "yes" : "no");
Borislav Petkov68798e12009-11-03 16:18:33 +0100744
Joe Perches956b9ba2012-04-29 17:08:39 -0300745 edac_dbg(1, " ECC capable: %s, ChipKill ECC capable: %s\n",
746 (pvt->nbcap & NBCAP_SECDED) ? "yes" : "no",
747 (pvt->nbcap & NBCAP_CHIPKILL) ? "yes" : "no");
Borislav Petkov68798e12009-11-03 16:18:33 +0100748
Borislav Petkovd1ea71c2013-12-15 17:54:27 +0100749 debug_dump_dramcfg_low(pvt, pvt->dclr0, 0);
Doug Thompson2da11652009-04-27 16:09:09 +0200750
Joe Perches956b9ba2012-04-29 17:08:39 -0300751 edac_dbg(1, "F3xB0 (Online Spare): 0x%08x\n", pvt->online_spare);
Doug Thompson2da11652009-04-27 16:09:09 +0200752
Joe Perches956b9ba2012-04-29 17:08:39 -0300753 edac_dbg(1, "F1xF0 (DRAM Hole Address): 0x%08x, base: 0x%08x, offset: 0x%08x\n",
754 pvt->dhar, dhar_base(pvt),
Borislav Petkova4b4bed2013-08-10 13:54:48 +0200755 (pvt->fam == 0xf) ? k8_dhar_offset(pvt)
756 : f10_dhar_offset(pvt));
Doug Thompson2da11652009-04-27 16:09:09 +0200757
Joe Perches956b9ba2012-04-29 17:08:39 -0300758 edac_dbg(1, " DramHoleValid: %s\n", dhar_valid(pvt) ? "yes" : "no");
Doug Thompson2da11652009-04-27 16:09:09 +0200759
Borislav Petkovd1ea71c2013-12-15 17:54:27 +0100760 debug_display_dimm_sizes(pvt, 0);
Borislav Petkov4d796362011-02-03 15:59:57 +0100761
Borislav Petkov8de1d912009-10-16 13:39:30 +0200762 /* everything below this point is Fam10h and above */
Borislav Petkova4b4bed2013-08-10 13:54:48 +0200763 if (pvt->fam == 0xf)
Doug Thompson2da11652009-04-27 16:09:09 +0200764 return;
Borislav Petkov4d796362011-02-03 15:59:57 +0100765
Borislav Petkovd1ea71c2013-12-15 17:54:27 +0100766 debug_display_dimm_sizes(pvt, 1);
Doug Thompson2da11652009-04-27 16:09:09 +0200767
Borislav Petkova3b7db02011-01-19 20:35:12 +0100768 amd64_info("using %s syndromes.\n", ((pvt->ecc_sym_sz == 8) ? "x8" : "x4"));
Borislav Petkovad6a32e2010-03-09 12:46:00 +0100769
Borislav Petkov8de1d912009-10-16 13:39:30 +0200770 /* Only if NOT ganged does dclr1 have valid info */
Borislav Petkov68798e12009-11-03 16:18:33 +0100771 if (!dct_ganging_enabled(pvt))
Borislav Petkovd1ea71c2013-12-15 17:54:27 +0100772 debug_dump_dramcfg_low(pvt, pvt->dclr1, 1);
Doug Thompson2da11652009-04-27 16:09:09 +0200773}
774
Doug Thompson94be4bf2009-04-27 16:12:00 +0200775/*
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -0500776 * See BKDG, F2x[1,0][5C:40], F2[1,0][6C:60]
Doug Thompson94be4bf2009-04-27 16:12:00 +0200777 */
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100778static void prep_chip_selects(struct amd64_pvt *pvt)
Doug Thompson94be4bf2009-04-27 16:12:00 +0200779{
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -0500780 if (pvt->fam == 0xf && pvt->ext_model < K8_REV_F) {
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100781 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
782 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 8;
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +0100783 } else if (pvt->fam == 0x15 && pvt->model == 0x30) {
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -0500784 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 4;
785 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 2;
Borislav Petkov9d858bb2009-09-21 14:35:51 +0200786 } else {
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100787 pvt->csels[0].b_cnt = pvt->csels[1].b_cnt = 8;
788 pvt->csels[0].m_cnt = pvt->csels[1].m_cnt = 4;
Doug Thompson94be4bf2009-04-27 16:12:00 +0200789 }
790}
791
792/*
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100793 * Function 2 Offset F10_DCSB0; read in the DCS Base and DCS Mask registers
Doug Thompson94be4bf2009-04-27 16:12:00 +0200794 */
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200795static void read_dct_base_mask(struct amd64_pvt *pvt)
Doug Thompson94be4bf2009-04-27 16:12:00 +0200796{
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100797 int cs;
Doug Thompson94be4bf2009-04-27 16:12:00 +0200798
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100799 prep_chip_selects(pvt);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200800
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100801 for_each_chip_select(cs, 0, pvt) {
Borislav Petkov71d2a322011-02-21 19:37:24 +0100802 int reg0 = DCSB0 + (cs * 4);
803 int reg1 = DCSB1 + (cs * 4);
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100804 u32 *base0 = &pvt->csels[0].csbases[cs];
805 u32 *base1 = &pvt->csels[1].csbases[cs];
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200806
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -0500807 if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, base0))
Joe Perches956b9ba2012-04-29 17:08:39 -0300808 edac_dbg(0, " DCSB0[%d]=0x%08x reg: F2x%x\n",
809 cs, *base0, reg0);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200810
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -0500811 if (pvt->fam == 0xf)
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100812 continue;
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200813
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -0500814 if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, base1))
Joe Perches956b9ba2012-04-29 17:08:39 -0300815 edac_dbg(0, " DCSB1[%d]=0x%08x reg: F2x%x\n",
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -0500816 cs, *base1, (pvt->fam == 0x10) ? reg1
817 : reg0);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200818 }
819
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100820 for_each_chip_select_mask(cs, 0, pvt) {
Borislav Petkov71d2a322011-02-21 19:37:24 +0100821 int reg0 = DCSM0 + (cs * 4);
822 int reg1 = DCSM1 + (cs * 4);
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100823 u32 *mask0 = &pvt->csels[0].csmasks[cs];
824 u32 *mask1 = &pvt->csels[1].csmasks[cs];
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200825
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -0500826 if (!amd64_read_dct_pci_cfg(pvt, 0, reg0, mask0))
Joe Perches956b9ba2012-04-29 17:08:39 -0300827 edac_dbg(0, " DCSM0[%d]=0x%08x reg: F2x%x\n",
828 cs, *mask0, reg0);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200829
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -0500830 if (pvt->fam == 0xf)
Borislav Petkov11c75ea2010-11-29 19:49:02 +0100831 continue;
Borislav Petkovb2b0c602010-10-08 18:32:29 +0200832
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -0500833 if (!amd64_read_dct_pci_cfg(pvt, 1, reg0, mask1))
Joe Perches956b9ba2012-04-29 17:08:39 -0300834 edac_dbg(0, " DCSM1[%d]=0x%08x reg: F2x%x\n",
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -0500835 cs, *mask1, (pvt->fam == 0x10) ? reg1
836 : reg0);
Doug Thompson94be4bf2009-04-27 16:12:00 +0200837 }
838}
839
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +0100840static void determine_memory_type(struct amd64_pvt *pvt)
Doug Thompson94be4bf2009-04-27 16:12:00 +0200841{
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +0100842 u32 dram_ctrl, dcsm;
Doug Thompson94be4bf2009-04-27 16:12:00 +0200843
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +0100844 switch (pvt->fam) {
845 case 0xf:
846 if (pvt->ext_model >= K8_REV_F)
847 goto ddr3;
848
849 pvt->dram_type = (pvt->dclr0 & BIT(18)) ? MEM_DDR : MEM_RDDR;
850 return;
851
852 case 0x10:
Borislav Petkov6b4c0bd2009-11-12 15:37:57 +0100853 if (pvt->dchr0 & DDR3_MODE)
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +0100854 goto ddr3;
855
856 pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR2 : MEM_RDDR2;
857 return;
858
859 case 0x15:
860 if (pvt->model < 0x60)
861 goto ddr3;
862
863 /*
864 * Model 0x60h needs special handling:
865 *
866 * We use a Chip Select value of '0' to obtain dcsm.
867 * Theoretically, it is possible to populate LRDIMMs of different
868 * 'Rank' value on a DCT. But this is not the common case. So,
869 * it's reasonable to assume all DIMMs are going to be of same
870 * 'type' until proven otherwise.
871 */
872 amd64_read_dct_pci_cfg(pvt, 0, DRAM_CONTROL, &dram_ctrl);
873 dcsm = pvt->csels[0].csmasks[0];
874
875 if (((dram_ctrl >> 8) & 0x7) == 0x2)
876 pvt->dram_type = MEM_DDR4;
877 else if (pvt->dclr0 & BIT(16))
878 pvt->dram_type = MEM_DDR3;
879 else if (dcsm & 0x3)
880 pvt->dram_type = MEM_LRDDR3;
Borislav Petkov6b4c0bd2009-11-12 15:37:57 +0100881 else
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +0100882 pvt->dram_type = MEM_RDDR3;
883
884 return;
885
886 case 0x16:
887 goto ddr3;
888
889 default:
890 WARN(1, KERN_ERR "%s: Family??? 0x%x\n", __func__, pvt->fam);
891 pvt->dram_type = MEM_EMPTY;
Doug Thompson94be4bf2009-04-27 16:12:00 +0200892 }
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +0100893 return;
Doug Thompson94be4bf2009-04-27 16:12:00 +0200894
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +0100895ddr3:
896 pvt->dram_type = (pvt->dclr0 & BIT(16)) ? MEM_DDR3 : MEM_RDDR3;
Doug Thompson94be4bf2009-04-27 16:12:00 +0200897}
898
Borislav Petkovcb328502010-12-22 14:28:24 +0100899/* Get the number of DCT channels the memory controller is using. */
Doug Thompsonddff8762009-04-27 16:14:52 +0200900static int k8_early_channel_count(struct amd64_pvt *pvt)
901{
Borislav Petkovcb328502010-12-22 14:28:24 +0100902 int flag;
Doug Thompsonddff8762009-04-27 16:14:52 +0200903
Borislav Petkov9f56da02010-10-01 19:44:53 +0200904 if (pvt->ext_model >= K8_REV_F)
Doug Thompsonddff8762009-04-27 16:14:52 +0200905 /* RevF (NPT) and later */
Borislav Petkov41d8bfa2011-01-18 19:16:08 +0100906 flag = pvt->dclr0 & WIDTH_128;
Borislav Petkov9f56da02010-10-01 19:44:53 +0200907 else
Doug Thompsonddff8762009-04-27 16:14:52 +0200908 /* RevE and earlier */
909 flag = pvt->dclr0 & REVE_WIDTH_128;
Doug Thompsonddff8762009-04-27 16:14:52 +0200910
911 /* not used */
912 pvt->dclr1 = 0;
913
914 return (flag) ? 2 : 1;
915}
916
Borislav Petkov70046622011-01-10 14:37:27 +0100917/* On F10h and later ErrAddr is MC4_ADDR[47:1] */
Borislav Petkova4b4bed2013-08-10 13:54:48 +0200918static u64 get_error_address(struct amd64_pvt *pvt, struct mce *m)
Doug Thompsonddff8762009-04-27 16:14:52 +0200919{
Borislav Petkov2ec591a2015-02-17 10:58:34 +0100920 u16 mce_nid = amd_get_nb_id(m->extcpu);
921 struct mem_ctl_info *mci;
Borislav Petkov70046622011-01-10 14:37:27 +0100922 u8 start_bit = 1;
923 u8 end_bit = 47;
Borislav Petkov2ec591a2015-02-17 10:58:34 +0100924 u64 addr;
925
926 mci = edac_mc_find(mce_nid);
927 if (!mci)
928 return 0;
929
930 pvt = mci->pvt_info;
Borislav Petkov70046622011-01-10 14:37:27 +0100931
Borislav Petkova4b4bed2013-08-10 13:54:48 +0200932 if (pvt->fam == 0xf) {
Borislav Petkov70046622011-01-10 14:37:27 +0100933 start_bit = 3;
934 end_bit = 39;
935 }
936
Chen, Gong10ef6b02013-10-18 14:29:07 -0700937 addr = m->addr & GENMASK_ULL(end_bit, start_bit);
Borislav Petkovc1ae6832011-03-30 15:42:10 +0200938
939 /*
940 * Erratum 637 workaround
941 */
Borislav Petkova4b4bed2013-08-10 13:54:48 +0200942 if (pvt->fam == 0x15) {
Borislav Petkovc1ae6832011-03-30 15:42:10 +0200943 u64 cc6_base, tmp_addr;
944 u32 tmp;
Daniel J Blueman8b84c8d2012-11-27 14:32:10 +0800945 u8 intlv_en;
Borislav Petkovc1ae6832011-03-30 15:42:10 +0200946
Chen, Gong10ef6b02013-10-18 14:29:07 -0700947 if ((addr & GENMASK_ULL(47, 24)) >> 24 != 0x00fdf7)
Borislav Petkovc1ae6832011-03-30 15:42:10 +0200948 return addr;
949
Borislav Petkovc1ae6832011-03-30 15:42:10 +0200950
951 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_LIM, &tmp);
952 intlv_en = tmp >> 21 & 0x7;
953
954 /* add [47:27] + 3 trailing bits */
Chen, Gong10ef6b02013-10-18 14:29:07 -0700955 cc6_base = (tmp & GENMASK_ULL(20, 0)) << 3;
Borislav Petkovc1ae6832011-03-30 15:42:10 +0200956
957 /* reverse and add DramIntlvEn */
958 cc6_base |= intlv_en ^ 0x7;
959
960 /* pin at [47:24] */
961 cc6_base <<= 24;
962
963 if (!intlv_en)
Chen, Gong10ef6b02013-10-18 14:29:07 -0700964 return cc6_base | (addr & GENMASK_ULL(23, 0));
Borislav Petkovc1ae6832011-03-30 15:42:10 +0200965
966 amd64_read_pci_cfg(pvt->F1, DRAM_LOCAL_NODE_BASE, &tmp);
967
968 /* faster log2 */
Chen, Gong10ef6b02013-10-18 14:29:07 -0700969 tmp_addr = (addr & GENMASK_ULL(23, 12)) << __fls(intlv_en + 1);
Borislav Petkovc1ae6832011-03-30 15:42:10 +0200970
971 /* OR DramIntlvSel into bits [14:12] */
Chen, Gong10ef6b02013-10-18 14:29:07 -0700972 tmp_addr |= (tmp & GENMASK_ULL(23, 21)) >> 9;
Borislav Petkovc1ae6832011-03-30 15:42:10 +0200973
974 /* add remaining [11:0] bits from original MC4_ADDR */
Chen, Gong10ef6b02013-10-18 14:29:07 -0700975 tmp_addr |= addr & GENMASK_ULL(11, 0);
Borislav Petkovc1ae6832011-03-30 15:42:10 +0200976
977 return cc6_base | tmp_addr;
978 }
979
980 return addr;
Doug Thompsonddff8762009-04-27 16:14:52 +0200981}
982
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +0800983static struct pci_dev *pci_get_related_function(unsigned int vendor,
984 unsigned int device,
985 struct pci_dev *related)
986{
987 struct pci_dev *dev = NULL;
988
989 while ((dev = pci_get_device(vendor, device, dev))) {
990 if (pci_domain_nr(dev->bus) == pci_domain_nr(related->bus) &&
991 (dev->bus->number == related->bus->number) &&
992 (PCI_SLOT(dev->devfn) == PCI_SLOT(related->devfn)))
993 break;
994 }
995
996 return dev;
997}
998
Borislav Petkov7f19bf72010-10-21 18:52:53 +0200999static void read_dram_base_limit_regs(struct amd64_pvt *pvt, unsigned range)
Doug Thompsonddff8762009-04-27 16:14:52 +02001000{
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +08001001 struct amd_northbridge *nb;
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001002 struct pci_dev *f1 = NULL;
1003 unsigned int pci_func;
Borislav Petkov71d2a322011-02-21 19:37:24 +01001004 int off = range << 3;
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +08001005 u32 llim;
Doug Thompsonddff8762009-04-27 16:14:52 +02001006
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001007 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_LO + off, &pvt->ranges[range].base.lo);
1008 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_LO + off, &pvt->ranges[range].lim.lo);
Doug Thompsonddff8762009-04-27 16:14:52 +02001009
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001010 if (pvt->fam == 0xf)
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001011 return;
Doug Thompsonddff8762009-04-27 16:14:52 +02001012
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001013 if (!dram_rw(pvt, range))
1014 return;
Doug Thompsonddff8762009-04-27 16:14:52 +02001015
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001016 amd64_read_pci_cfg(pvt->F1, DRAM_BASE_HI + off, &pvt->ranges[range].base.hi);
1017 amd64_read_pci_cfg(pvt->F1, DRAM_LIMIT_HI + off, &pvt->ranges[range].lim.hi);
Borislav Petkovf08e4572011-03-21 20:45:06 +01001018
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +08001019 /* F15h: factor in CC6 save area by reading dst node's limit reg */
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001020 if (pvt->fam != 0x15)
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +08001021 return;
Borislav Petkovf08e4572011-03-21 20:45:06 +01001022
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +08001023 nb = node_to_amd_nb(dram_dst_node(pvt, range));
1024 if (WARN_ON(!nb))
1025 return;
Borislav Petkovf08e4572011-03-21 20:45:06 +01001026
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01001027 if (pvt->model == 0x60)
1028 pci_func = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1;
1029 else if (pvt->model == 0x30)
1030 pci_func = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1;
1031 else
1032 pci_func = PCI_DEVICE_ID_AMD_15H_NB_F1;
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001033
1034 f1 = pci_get_related_function(nb->misc->vendor, pci_func, nb->misc);
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +08001035 if (WARN_ON(!f1))
1036 return;
Borislav Petkovf08e4572011-03-21 20:45:06 +01001037
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +08001038 amd64_read_pci_cfg(f1, DRAM_LOCAL_NODE_LIM, &llim);
Borislav Petkovf08e4572011-03-21 20:45:06 +01001039
Chen, Gong10ef6b02013-10-18 14:29:07 -07001040 pvt->ranges[range].lim.lo &= GENMASK_ULL(15, 0);
Borislav Petkovf08e4572011-03-21 20:45:06 +01001041
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +08001042 /* {[39:27],111b} */
1043 pvt->ranges[range].lim.lo |= ((llim & 0x1fff) << 3 | 0x7) << 16;
Borislav Petkovf08e4572011-03-21 20:45:06 +01001044
Chen, Gong10ef6b02013-10-18 14:29:07 -07001045 pvt->ranges[range].lim.hi &= GENMASK_ULL(7, 0);
Borislav Petkovf08e4572011-03-21 20:45:06 +01001046
Daniel J Bluemane2c0bff2012-11-30 16:44:19 +08001047 /* [47:40] */
1048 pvt->ranges[range].lim.hi |= llim >> 13;
1049
1050 pci_dev_put(f1);
Doug Thompsonddff8762009-04-27 16:14:52 +02001051}
1052
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001053static void k8_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
Borislav Petkov33ca0642012-08-30 18:01:36 +02001054 struct err_info *err)
Doug Thompsonddff8762009-04-27 16:14:52 +02001055{
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001056 struct amd64_pvt *pvt = mci->pvt_info;
Doug Thompsonddff8762009-04-27 16:14:52 +02001057
Borislav Petkov33ca0642012-08-30 18:01:36 +02001058 error_address_to_page_and_offset(sys_addr, err);
Mauro Carvalho Chehabab5a5032012-04-16 15:03:50 -03001059
1060 /*
1061 * Find out which node the error address belongs to. This may be
1062 * different from the node that detected the error.
1063 */
Borislav Petkov33ca0642012-08-30 18:01:36 +02001064 err->src_mci = find_mc_by_sys_addr(mci, sys_addr);
1065 if (!err->src_mci) {
Mauro Carvalho Chehabab5a5032012-04-16 15:03:50 -03001066 amd64_mc_err(mci, "failed to map error addr 0x%lx to a node\n",
1067 (unsigned long)sys_addr);
Borislav Petkov33ca0642012-08-30 18:01:36 +02001068 err->err_code = ERR_NODE;
Mauro Carvalho Chehabab5a5032012-04-16 15:03:50 -03001069 return;
1070 }
1071
1072 /* Now map the sys_addr to a CSROW */
Borislav Petkov33ca0642012-08-30 18:01:36 +02001073 err->csrow = sys_addr_to_csrow(err->src_mci, sys_addr);
1074 if (err->csrow < 0) {
1075 err->err_code = ERR_CSROW;
Mauro Carvalho Chehabab5a5032012-04-16 15:03:50 -03001076 return;
1077 }
1078
Doug Thompsonddff8762009-04-27 16:14:52 +02001079 /* CHIPKILL enabled */
Borislav Petkovf192c7b2011-01-10 14:24:32 +01001080 if (pvt->nbcfg & NBCFG_CHIPKILL) {
Borislav Petkov33ca0642012-08-30 18:01:36 +02001081 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
1082 if (err->channel < 0) {
Doug Thompsonddff8762009-04-27 16:14:52 +02001083 /*
1084 * Syndrome didn't map, so we don't know which of the
1085 * 2 DIMMs is in error. So we need to ID 'both' of them
1086 * as suspect.
1087 */
Borislav Petkov33ca0642012-08-30 18:01:36 +02001088 amd64_mc_warn(err->src_mci, "unknown syndrome 0x%04x - "
Mauro Carvalho Chehabab5a5032012-04-16 15:03:50 -03001089 "possible error reporting race\n",
Borislav Petkov33ca0642012-08-30 18:01:36 +02001090 err->syndrome);
1091 err->err_code = ERR_CHANNEL;
Doug Thompsonddff8762009-04-27 16:14:52 +02001092 return;
1093 }
1094 } else {
1095 /*
1096 * non-chipkill ecc mode
1097 *
1098 * The k8 documentation is unclear about how to determine the
1099 * channel number when using non-chipkill memory. This method
1100 * was obtained from email communication with someone at AMD.
1101 * (Wish the email was placed in this comment - norsk)
1102 */
Borislav Petkov33ca0642012-08-30 18:01:36 +02001103 err->channel = ((sys_addr & BIT(3)) != 0);
Doug Thompsonddff8762009-04-27 16:14:52 +02001104 }
Doug Thompsonddff8762009-04-27 16:14:52 +02001105}
1106
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001107static int ddr2_cs_size(unsigned i, bool dct_width)
Doug Thompsonddff8762009-04-27 16:14:52 +02001108{
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001109 unsigned shift = 0;
Doug Thompsonddff8762009-04-27 16:14:52 +02001110
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001111 if (i <= 2)
1112 shift = i;
1113 else if (!(i & 0x1))
1114 shift = i >> 1;
Borislav Petkov1433eb92009-10-21 13:44:36 +02001115 else
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001116 shift = (i + 1) >> 1;
Doug Thompsonddff8762009-04-27 16:14:52 +02001117
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001118 return 128 << (shift + !!dct_width);
1119}
1120
1121static int k8_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01001122 unsigned cs_mode, int cs_mask_nr)
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001123{
1124 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1125
1126 if (pvt->ext_model >= K8_REV_F) {
1127 WARN_ON(cs_mode > 11);
1128 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1129 }
1130 else if (pvt->ext_model >= K8_REV_D) {
Borislav Petkov11b0a312011-11-09 21:28:43 +01001131 unsigned diff;
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001132 WARN_ON(cs_mode > 10);
1133
Borislav Petkov11b0a312011-11-09 21:28:43 +01001134 /*
1135 * the below calculation, besides trying to win an obfuscated C
1136 * contest, maps cs_mode values to DIMM chip select sizes. The
1137 * mappings are:
1138 *
1139 * cs_mode CS size (mb)
1140 * ======= ============
1141 * 0 32
1142 * 1 64
1143 * 2 128
1144 * 3 128
1145 * 4 256
1146 * 5 512
1147 * 6 256
1148 * 7 512
1149 * 8 1024
1150 * 9 1024
1151 * 10 2048
1152 *
1153 * Basically, it calculates a value with which to shift the
1154 * smallest CS size of 32MB.
1155 *
1156 * ddr[23]_cs_size have a similar purpose.
1157 */
1158 diff = cs_mode/3 + (unsigned)(cs_mode > 5);
1159
1160 return 32 << (cs_mode - diff);
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001161 }
1162 else {
1163 WARN_ON(cs_mode > 6);
1164 return 32 << cs_mode;
1165 }
Doug Thompsonddff8762009-04-27 16:14:52 +02001166}
1167
Doug Thompson1afd3c92009-04-27 16:16:50 +02001168/*
1169 * Get the number of DCT channels in use.
1170 *
1171 * Return:
1172 * number of Memory Channels in operation
1173 * Pass back:
1174 * contents of the DCL0_LOW register
1175 */
Borislav Petkov7d20d142011-01-07 17:58:04 +01001176static int f1x_early_channel_count(struct amd64_pvt *pvt)
Doug Thompson1afd3c92009-04-27 16:16:50 +02001177{
Borislav Petkov6ba5dcd2009-10-13 19:26:55 +02001178 int i, j, channels = 0;
Doug Thompsonddff8762009-04-27 16:14:52 +02001179
Borislav Petkov7d20d142011-01-07 17:58:04 +01001180 /* On F10h, if we are in 128 bit mode, then we are using 2 channels */
Borislav Petkova4b4bed2013-08-10 13:54:48 +02001181 if (pvt->fam == 0x10 && (pvt->dclr0 & WIDTH_128))
Borislav Petkov7d20d142011-01-07 17:58:04 +01001182 return 2;
Doug Thompson1afd3c92009-04-27 16:16:50 +02001183
1184 /*
Borislav Petkovd16149e2009-10-16 19:55:49 +02001185 * Need to check if in unganged mode: In such, there are 2 channels,
1186 * but they are not in 128 bit mode and thus the above 'dclr0' status
1187 * bit will be OFF.
Doug Thompson1afd3c92009-04-27 16:16:50 +02001188 *
1189 * Need to check DCT0[0] and DCT1[0] to see if only one of them has
1190 * their CSEnable bit on. If so, then SINGLE DIMM case.
1191 */
Joe Perches956b9ba2012-04-29 17:08:39 -03001192 edac_dbg(0, "Data width is not 128 bits - need more decoding\n");
Doug Thompson1afd3c92009-04-27 16:16:50 +02001193
1194 /*
1195 * Check DRAM Bank Address Mapping values for each DIMM to see if there
1196 * is more than just one DIMM present in unganged mode. Need to check
1197 * both controllers since DIMMs can be placed in either one.
1198 */
Borislav Petkov525a1b22010-12-21 15:53:27 +01001199 for (i = 0; i < 2; i++) {
1200 u32 dbam = (i ? pvt->dbam1 : pvt->dbam0);
Doug Thompson1afd3c92009-04-27 16:16:50 +02001201
Wan Wei57a30852009-08-07 17:04:49 +02001202 for (j = 0; j < 4; j++) {
1203 if (DBAM_DIMM(j, dbam) > 0) {
1204 channels++;
1205 break;
1206 }
1207 }
Doug Thompson1afd3c92009-04-27 16:16:50 +02001208 }
1209
Borislav Petkovd16149e2009-10-16 19:55:49 +02001210 if (channels > 2)
1211 channels = 2;
1212
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001213 amd64_info("MCT channel count: %d\n", channels);
Doug Thompson1afd3c92009-04-27 16:16:50 +02001214
1215 return channels;
Doug Thompson1afd3c92009-04-27 16:16:50 +02001216}
1217
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001218static int ddr3_cs_size(unsigned i, bool dct_width)
Doug Thompson1afd3c92009-04-27 16:16:50 +02001219{
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001220 unsigned shift = 0;
1221 int cs_size = 0;
1222
1223 if (i == 0 || i == 3 || i == 4)
1224 cs_size = -1;
1225 else if (i <= 2)
1226 shift = i;
1227 else if (i == 12)
1228 shift = 7;
1229 else if (!(i & 0x1))
1230 shift = i >> 1;
1231 else
1232 shift = (i + 1) >> 1;
1233
1234 if (cs_size != -1)
1235 cs_size = (128 * (1 << !!dct_width)) << shift;
1236
1237 return cs_size;
1238}
1239
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01001240static int ddr3_lrdimm_cs_size(unsigned i, unsigned rank_multiply)
1241{
1242 unsigned shift = 0;
1243 int cs_size = 0;
1244
1245 if (i < 4 || i == 6)
1246 cs_size = -1;
1247 else if (i == 12)
1248 shift = 7;
1249 else if (!(i & 0x1))
1250 shift = i >> 1;
1251 else
1252 shift = (i + 1) >> 1;
1253
1254 if (cs_size != -1)
1255 cs_size = rank_multiply * (128 << shift);
1256
1257 return cs_size;
1258}
1259
1260static int ddr4_cs_size(unsigned i)
1261{
1262 int cs_size = 0;
1263
1264 if (i == 0)
1265 cs_size = -1;
1266 else if (i == 1)
1267 cs_size = 1024;
1268 else
1269 /* Min cs_size = 1G */
1270 cs_size = 1024 * (1 << (i >> 1));
1271
1272 return cs_size;
1273}
1274
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001275static int f10_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01001276 unsigned cs_mode, int cs_mask_nr)
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001277{
1278 u32 dclr = dct ? pvt->dclr1 : pvt->dclr0;
1279
1280 WARN_ON(cs_mode > 11);
Borislav Petkov1433eb92009-10-21 13:44:36 +02001281
1282 if (pvt->dchr0 & DDR3_MODE || pvt->dchr1 & DDR3_MODE)
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001283 return ddr3_cs_size(cs_mode, dclr & WIDTH_128);
Borislav Petkov1433eb92009-10-21 13:44:36 +02001284 else
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001285 return ddr2_cs_size(cs_mode, dclr & WIDTH_128);
1286}
Borislav Petkov1433eb92009-10-21 13:44:36 +02001287
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001288/*
1289 * F15h supports only 64bit DCT interfaces
1290 */
1291static int f15_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01001292 unsigned cs_mode, int cs_mask_nr)
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001293{
1294 WARN_ON(cs_mode > 12);
1295
1296 return ddr3_cs_size(cs_mode, false);
Doug Thompson1afd3c92009-04-27 16:16:50 +02001297}
1298
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01001299/* F15h M60h supports DDR4 mapping as well.. */
1300static int f15_m60h_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
1301 unsigned cs_mode, int cs_mask_nr)
1302{
1303 int cs_size;
1304 u32 dcsm = pvt->csels[dct].csmasks[cs_mask_nr];
1305
1306 WARN_ON(cs_mode > 12);
1307
1308 if (pvt->dram_type == MEM_DDR4) {
1309 if (cs_mode > 9)
1310 return -1;
1311
1312 cs_size = ddr4_cs_size(cs_mode);
1313 } else if (pvt->dram_type == MEM_LRDDR3) {
1314 unsigned rank_multiply = dcsm & 0xf;
1315
1316 if (rank_multiply == 3)
1317 rank_multiply = 4;
1318 cs_size = ddr3_lrdimm_cs_size(cs_mode, rank_multiply);
1319 } else {
1320 /* Minimum cs size is 512mb for F15hM60h*/
1321 if (cs_mode == 0x1)
1322 return -1;
1323
1324 cs_size = ddr3_cs_size(cs_mode, false);
1325 }
1326
1327 return cs_size;
1328}
1329
Aravind Gopalakrishnan94c1acf2013-04-17 14:57:13 -05001330/*
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001331 * F16h and F15h model 30h have only limited cs_modes.
Aravind Gopalakrishnan94c1acf2013-04-17 14:57:13 -05001332 */
1333static int f16_dbam_to_chip_select(struct amd64_pvt *pvt, u8 dct,
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01001334 unsigned cs_mode, int cs_mask_nr)
Aravind Gopalakrishnan94c1acf2013-04-17 14:57:13 -05001335{
1336 WARN_ON(cs_mode > 12);
1337
1338 if (cs_mode == 6 || cs_mode == 8 ||
1339 cs_mode == 9 || cs_mode == 12)
1340 return -1;
1341 else
1342 return ddr3_cs_size(cs_mode, false);
1343}
1344
Borislav Petkov5a5d2372011-01-17 17:52:57 +01001345static void read_dram_ctl_register(struct amd64_pvt *pvt)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001346{
Doug Thompson6163b5d2009-04-27 16:20:17 +02001347
Borislav Petkova4b4bed2013-08-10 13:54:48 +02001348 if (pvt->fam == 0xf)
Borislav Petkov5a5d2372011-01-17 17:52:57 +01001349 return;
1350
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -05001351 if (!amd64_read_pci_cfg(pvt->F2, DCT_SEL_LO, &pvt->dct_sel_lo)) {
Joe Perches956b9ba2012-04-29 17:08:39 -03001352 edac_dbg(0, "F2x110 (DCTSelLow): 0x%08x, High range addrs at: 0x%x\n",
1353 pvt->dct_sel_lo, dct_sel_baseaddr(pvt));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001354
Joe Perches956b9ba2012-04-29 17:08:39 -03001355 edac_dbg(0, " DCTs operate in %s mode\n",
1356 (dct_ganging_enabled(pvt) ? "ganged" : "unganged"));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001357
Borislav Petkov72381bd2009-10-09 19:14:43 +02001358 if (!dct_ganging_enabled(pvt))
Joe Perches956b9ba2012-04-29 17:08:39 -03001359 edac_dbg(0, " Address range split per DCT: %s\n",
1360 (dct_high_range_enabled(pvt) ? "yes" : "no"));
Borislav Petkov72381bd2009-10-09 19:14:43 +02001361
Joe Perches956b9ba2012-04-29 17:08:39 -03001362 edac_dbg(0, " data interleave for ECC: %s, DRAM cleared since last warm reset: %s\n",
1363 (dct_data_intlv_enabled(pvt) ? "enabled" : "disabled"),
1364 (dct_memory_cleared(pvt) ? "yes" : "no"));
Borislav Petkov72381bd2009-10-09 19:14:43 +02001365
Joe Perches956b9ba2012-04-29 17:08:39 -03001366 edac_dbg(0, " channel interleave: %s, "
1367 "interleave bits selector: 0x%x\n",
1368 (dct_interleave_enabled(pvt) ? "enabled" : "disabled"),
1369 dct_sel_interleave_addr(pvt));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001370 }
1371
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -05001372 amd64_read_pci_cfg(pvt->F2, DCT_SEL_HI, &pvt->dct_sel_hi);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001373}
1374
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001375/*
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001376 * Determine channel (DCT) based on the interleaving mode (see F15h M30h BKDG,
1377 * 2.10.12 Memory Interleaving Modes).
1378 */
1379static u8 f15_m30h_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
1380 u8 intlv_en, int num_dcts_intlv,
1381 u32 dct_sel)
1382{
1383 u8 channel = 0;
1384 u8 select;
1385
1386 if (!(intlv_en))
1387 return (u8)(dct_sel);
1388
1389 if (num_dcts_intlv == 2) {
1390 select = (sys_addr >> 8) & 0x3;
1391 channel = select ? 0x3 : 0;
Aravind Gopalakrishnan9d0e8d82014-01-21 15:03:36 -06001392 } else if (num_dcts_intlv == 4) {
1393 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1394 switch (intlv_addr) {
1395 case 0x4:
1396 channel = (sys_addr >> 8) & 0x3;
1397 break;
1398 case 0x5:
1399 channel = (sys_addr >> 9) & 0x3;
1400 break;
1401 }
1402 }
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001403 return channel;
1404}
1405
1406/*
Borislav Petkov229a7a12010-12-09 18:57:54 +01001407 * Determine channel (DCT) based on the interleaving mode: F10h BKDG, 2.8.9 Memory
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001408 * Interleaving Modes.
1409 */
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001410static u8 f1x_determine_channel(struct amd64_pvt *pvt, u64 sys_addr,
Borislav Petkov229a7a12010-12-09 18:57:54 +01001411 bool hi_range_sel, u8 intlv_en)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001412{
Borislav Petkov151fa712011-02-21 19:33:10 +01001413 u8 dct_sel_high = (pvt->dct_sel_lo >> 1) & 1;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001414
1415 if (dct_ganging_enabled(pvt))
Borislav Petkov229a7a12010-12-09 18:57:54 +01001416 return 0;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001417
Borislav Petkov229a7a12010-12-09 18:57:54 +01001418 if (hi_range_sel)
1419 return dct_sel_high;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001420
Borislav Petkov229a7a12010-12-09 18:57:54 +01001421 /*
1422 * see F2x110[DctSelIntLvAddr] - channel interleave mode
1423 */
1424 if (dct_interleave_enabled(pvt)) {
1425 u8 intlv_addr = dct_sel_interleave_addr(pvt);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001426
Borislav Petkov229a7a12010-12-09 18:57:54 +01001427 /* return DCT select function: 0=DCT0, 1=DCT1 */
1428 if (!intlv_addr)
1429 return sys_addr >> 6 & 1;
1430
1431 if (intlv_addr & 0x2) {
1432 u8 shift = intlv_addr & 0x1 ? 9 : 6;
1433 u32 temp = hweight_long((u32) ((sys_addr >> 16) & 0x1F)) % 2;
1434
1435 return ((sys_addr >> shift) & 1) ^ temp;
1436 }
1437
1438 return (sys_addr >> (12 + hweight8(intlv_en))) & 1;
1439 }
1440
1441 if (dct_high_range_enabled(pvt))
1442 return ~dct_sel_high & 1;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001443
1444 return 0;
1445}
1446
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001447/* Convert the sys_addr to the normalized DCT address */
Daniel J Bluemanc7e53012012-11-30 16:44:20 +08001448static u64 f1x_get_norm_dct_addr(struct amd64_pvt *pvt, u8 range,
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001449 u64 sys_addr, bool hi_rng,
1450 u32 dct_sel_base_addr)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001451{
1452 u64 chan_off;
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001453 u64 dram_base = get_dram_base(pvt, range);
1454 u64 hole_off = f10_dhar_offset(pvt);
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001455 u64 dct_sel_base_off = (pvt->dct_sel_hi & 0xFFFFFC00) << 16;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001456
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001457 if (hi_rng) {
1458 /*
1459 * if
1460 * base address of high range is below 4Gb
1461 * (bits [47:27] at [31:11])
1462 * DRAM address space on this DCT is hoisted above 4Gb &&
1463 * sys_addr > 4Gb
1464 *
1465 * remove hole offset from sys_addr
1466 * else
1467 * remove high range offset from sys_addr
1468 */
1469 if ((!(dct_sel_base_addr >> 16) ||
1470 dct_sel_base_addr < dhar_base(pvt)) &&
Borislav Petkov972ea172011-02-21 19:43:02 +01001471 dhar_valid(pvt) &&
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001472 (sys_addr >= BIT_64(32)))
Borislav Petkovbc21fa52010-11-11 17:29:13 +01001473 chan_off = hole_off;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001474 else
1475 chan_off = dct_sel_base_off;
1476 } else {
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001477 /*
1478 * if
1479 * we have a valid hole &&
1480 * sys_addr > 4Gb
1481 *
1482 * remove hole
1483 * else
1484 * remove dram base to normalize to DCT address
1485 */
Borislav Petkov972ea172011-02-21 19:43:02 +01001486 if (dhar_valid(pvt) && (sys_addr >= BIT_64(32)))
Borislav Petkovbc21fa52010-11-11 17:29:13 +01001487 chan_off = hole_off;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001488 else
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001489 chan_off = dram_base;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001490 }
1491
Chen, Gong10ef6b02013-10-18 14:29:07 -07001492 return (sys_addr & GENMASK_ULL(47,6)) - (chan_off & GENMASK_ULL(47,23));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001493}
1494
Doug Thompson6163b5d2009-04-27 16:20:17 +02001495/*
1496 * checks if the csrow passed in is marked as SPARED, if so returns the new
1497 * spare row
1498 */
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001499static int f10_process_possible_spare(struct amd64_pvt *pvt, u8 dct, int csrow)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001500{
Borislav Petkov614ec9d2011-01-13 18:02:22 +01001501 int tmp_cs;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001502
Borislav Petkov614ec9d2011-01-13 18:02:22 +01001503 if (online_spare_swap_done(pvt, dct) &&
1504 csrow == online_spare_bad_dramcs(pvt, dct)) {
1505
1506 for_each_chip_select(tmp_cs, dct, pvt) {
1507 if (chip_select_base(tmp_cs, dct, pvt) & 0x2) {
1508 csrow = tmp_cs;
1509 break;
1510 }
1511 }
Doug Thompson6163b5d2009-04-27 16:20:17 +02001512 }
1513 return csrow;
1514}
1515
1516/*
1517 * Iterate over the DRAM DCT "base" and "mask" registers looking for a
1518 * SystemAddr match on the specified 'ChannelSelect' and 'NodeID'
1519 *
1520 * Return:
1521 * -EINVAL: NOT FOUND
1522 * 0..csrow = Chip-Select Row
1523 */
Daniel J Bluemanc7e53012012-11-30 16:44:20 +08001524static int f1x_lookup_addr_in_dct(u64 in_addr, u8 nid, u8 dct)
Doug Thompson6163b5d2009-04-27 16:20:17 +02001525{
1526 struct mem_ctl_info *mci;
1527 struct amd64_pvt *pvt;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001528 u64 cs_base, cs_mask;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001529 int cs_found = -EINVAL;
1530 int csrow;
1531
Borislav Petkov2ec591a2015-02-17 10:58:34 +01001532 mci = edac_mc_find(nid);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001533 if (!mci)
1534 return cs_found;
1535
1536 pvt = mci->pvt_info;
1537
Joe Perches956b9ba2012-04-29 17:08:39 -03001538 edac_dbg(1, "input addr: 0x%llx, DCT: %d\n", in_addr, dct);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001539
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001540 for_each_chip_select(csrow, dct, pvt) {
1541 if (!csrow_enabled(csrow, dct, pvt))
Doug Thompson6163b5d2009-04-27 16:20:17 +02001542 continue;
1543
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001544 get_cs_base_and_mask(pvt, csrow, dct, &cs_base, &cs_mask);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001545
Joe Perches956b9ba2012-04-29 17:08:39 -03001546 edac_dbg(1, " CSROW=%d CSBase=0x%llx CSMask=0x%llx\n",
1547 csrow, cs_base, cs_mask);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001548
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001549 cs_mask = ~cs_mask;
Doug Thompson6163b5d2009-04-27 16:20:17 +02001550
Joe Perches956b9ba2012-04-29 17:08:39 -03001551 edac_dbg(1, " (InputAddr & ~CSMask)=0x%llx (CSBase & ~CSMask)=0x%llx\n",
1552 (in_addr & cs_mask), (cs_base & cs_mask));
Doug Thompson6163b5d2009-04-27 16:20:17 +02001553
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001554 if ((in_addr & cs_mask) == (cs_base & cs_mask)) {
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001555 if (pvt->fam == 0x15 && pvt->model >= 0x30) {
1556 cs_found = csrow;
1557 break;
1558 }
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001559 cs_found = f10_process_possible_spare(pvt, dct, csrow);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001560
Joe Perches956b9ba2012-04-29 17:08:39 -03001561 edac_dbg(1, " MATCH csrow=%d\n", cs_found);
Doug Thompson6163b5d2009-04-27 16:20:17 +02001562 break;
1563 }
1564 }
1565 return cs_found;
1566}
1567
Borislav Petkov95b0ef52011-01-11 22:08:07 +01001568/*
1569 * See F2x10C. Non-interleaved graphics framebuffer memory under the 16G is
1570 * swapped with a region located at the bottom of memory so that the GPU can use
1571 * the interleaved region and thus two channels.
1572 */
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001573static u64 f1x_swap_interleaved_region(struct amd64_pvt *pvt, u64 sys_addr)
Borislav Petkov95b0ef52011-01-11 22:08:07 +01001574{
1575 u32 swap_reg, swap_base, swap_limit, rgn_size, tmp_addr;
1576
Borislav Petkova4b4bed2013-08-10 13:54:48 +02001577 if (pvt->fam == 0x10) {
Borislav Petkov95b0ef52011-01-11 22:08:07 +01001578 /* only revC3 and revE have that feature */
Borislav Petkova4b4bed2013-08-10 13:54:48 +02001579 if (pvt->model < 4 || (pvt->model < 0xa && pvt->stepping < 3))
Borislav Petkov95b0ef52011-01-11 22:08:07 +01001580 return sys_addr;
1581 }
1582
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -05001583 amd64_read_pci_cfg(pvt->F2, SWAP_INTLV_REG, &swap_reg);
Borislav Petkov95b0ef52011-01-11 22:08:07 +01001584
1585 if (!(swap_reg & 0x1))
1586 return sys_addr;
1587
1588 swap_base = (swap_reg >> 3) & 0x7f;
1589 swap_limit = (swap_reg >> 11) & 0x7f;
1590 rgn_size = (swap_reg >> 20) & 0x7f;
1591 tmp_addr = sys_addr >> 27;
1592
1593 if (!(sys_addr >> 34) &&
1594 (((tmp_addr >= swap_base) &&
1595 (tmp_addr <= swap_limit)) ||
1596 (tmp_addr < rgn_size)))
1597 return sys_addr ^ (u64)swap_base << 27;
1598
1599 return sys_addr;
1600}
1601
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001602/* For a given @dram_range, check if @sys_addr falls within it. */
Borislav Petkove7613592011-02-21 19:49:01 +01001603static int f1x_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
Borislav Petkov33ca0642012-08-30 18:01:36 +02001604 u64 sys_addr, int *chan_sel)
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001605{
Borislav Petkov229a7a12010-12-09 18:57:54 +01001606 int cs_found = -EINVAL;
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001607 u64 chan_addr;
Borislav Petkov5d4b58e2011-01-13 16:01:13 +01001608 u32 dct_sel_base;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001609 u8 channel;
Borislav Petkov229a7a12010-12-09 18:57:54 +01001610 bool high_range = false;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001611
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001612 u8 node_id = dram_dst_node(pvt, range);
Borislav Petkov229a7a12010-12-09 18:57:54 +01001613 u8 intlv_en = dram_intlv_en(pvt, range);
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001614 u32 intlv_sel = dram_intlv_sel(pvt, range);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001615
Joe Perches956b9ba2012-04-29 17:08:39 -03001616 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1617 range, sys_addr, get_dram_limit(pvt, range));
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001618
Borislav Petkov355fba62011-01-17 13:03:26 +01001619 if (dhar_valid(pvt) &&
1620 dhar_base(pvt) <= sys_addr &&
1621 sys_addr < BIT_64(32)) {
1622 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1623 sys_addr);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001624 return -EINVAL;
Borislav Petkov355fba62011-01-17 13:03:26 +01001625 }
1626
Borislav Petkovf030ddf2011-04-08 15:05:21 +02001627 if (intlv_en && (intlv_sel != ((sys_addr >> 12) & intlv_en)))
Borislav Petkov355fba62011-01-17 13:03:26 +01001628 return -EINVAL;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001629
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001630 sys_addr = f1x_swap_interleaved_region(pvt, sys_addr);
Borislav Petkov95b0ef52011-01-11 22:08:07 +01001631
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001632 dct_sel_base = dct_sel_baseaddr(pvt);
1633
1634 /*
1635 * check whether addresses >= DctSelBaseAddr[47:27] are to be used to
1636 * select between DCT0 and DCT1.
1637 */
1638 if (dct_high_range_enabled(pvt) &&
1639 !dct_ganging_enabled(pvt) &&
1640 ((sys_addr >> 27) >= (dct_sel_base >> 11)))
Borislav Petkov229a7a12010-12-09 18:57:54 +01001641 high_range = true;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001642
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001643 channel = f1x_determine_channel(pvt, sys_addr, high_range, intlv_en);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001644
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001645 chan_addr = f1x_get_norm_dct_addr(pvt, range, sys_addr,
Borislav Petkovc8e518d2010-12-10 19:49:19 +01001646 high_range, dct_sel_base);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001647
Borislav Petkove2f79db2011-01-13 14:57:34 +01001648 /* Remove node interleaving, see F1x120 */
1649 if (intlv_en)
1650 chan_addr = ((chan_addr >> (12 + hweight8(intlv_en))) << 12) |
1651 (chan_addr & 0xfff);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001652
Borislav Petkov5d4b58e2011-01-13 16:01:13 +01001653 /* remove channel interleave */
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001654 if (dct_interleave_enabled(pvt) &&
1655 !dct_high_range_enabled(pvt) &&
1656 !dct_ganging_enabled(pvt)) {
Borislav Petkov5d4b58e2011-01-13 16:01:13 +01001657
1658 if (dct_sel_interleave_addr(pvt) != 1) {
1659 if (dct_sel_interleave_addr(pvt) == 0x3)
1660 /* hash 9 */
1661 chan_addr = ((chan_addr >> 10) << 9) |
1662 (chan_addr & 0x1ff);
1663 else
1664 /* A[6] or hash 6 */
1665 chan_addr = ((chan_addr >> 7) << 6) |
1666 (chan_addr & 0x3f);
1667 } else
1668 /* A[12] */
1669 chan_addr = ((chan_addr >> 13) << 12) |
1670 (chan_addr & 0xfff);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001671 }
1672
Joe Perches956b9ba2012-04-29 17:08:39 -03001673 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001674
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001675 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, channel);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001676
Borislav Petkov33ca0642012-08-30 18:01:36 +02001677 if (cs_found >= 0)
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001678 *chan_sel = channel;
Borislav Petkov33ca0642012-08-30 18:01:36 +02001679
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001680 return cs_found;
1681}
1682
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001683static int f15_m30h_match_to_this_node(struct amd64_pvt *pvt, unsigned range,
1684 u64 sys_addr, int *chan_sel)
1685{
1686 int cs_found = -EINVAL;
1687 int num_dcts_intlv = 0;
1688 u64 chan_addr, chan_offset;
1689 u64 dct_base, dct_limit;
1690 u32 dct_cont_base_reg, dct_cont_limit_reg, tmp;
1691 u8 channel, alias_channel, leg_mmio_hole, dct_sel, dct_offset_en;
1692
1693 u64 dhar_offset = f10_dhar_offset(pvt);
1694 u8 intlv_addr = dct_sel_interleave_addr(pvt);
1695 u8 node_id = dram_dst_node(pvt, range);
1696 u8 intlv_en = dram_intlv_en(pvt, range);
1697
1698 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_BASE, &dct_cont_base_reg);
1699 amd64_read_pci_cfg(pvt->F1, DRAM_CONT_LIMIT, &dct_cont_limit_reg);
1700
1701 dct_offset_en = (u8) ((dct_cont_base_reg >> 3) & BIT(0));
1702 dct_sel = (u8) ((dct_cont_base_reg >> 4) & 0x7);
1703
1704 edac_dbg(1, "(range %d) SystemAddr= 0x%llx Limit=0x%llx\n",
1705 range, sys_addr, get_dram_limit(pvt, range));
1706
1707 if (!(get_dram_base(pvt, range) <= sys_addr) &&
1708 !(get_dram_limit(pvt, range) >= sys_addr))
1709 return -EINVAL;
1710
1711 if (dhar_valid(pvt) &&
1712 dhar_base(pvt) <= sys_addr &&
1713 sys_addr < BIT_64(32)) {
1714 amd64_warn("Huh? Address is in the MMIO hole: 0x%016llx\n",
1715 sys_addr);
1716 return -EINVAL;
1717 }
1718
1719 /* Verify sys_addr is within DCT Range. */
Aravind Gopalakrishnan4fc06b32013-08-24 10:47:48 -05001720 dct_base = (u64) dct_sel_baseaddr(pvt);
1721 dct_limit = (dct_cont_limit_reg >> 11) & 0x1FFF;
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001722
1723 if (!(dct_cont_base_reg & BIT(0)) &&
Aravind Gopalakrishnan4fc06b32013-08-24 10:47:48 -05001724 !(dct_base <= (sys_addr >> 27) &&
1725 dct_limit >= (sys_addr >> 27)))
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001726 return -EINVAL;
1727
1728 /* Verify number of dct's that participate in channel interleaving. */
1729 num_dcts_intlv = (int) hweight8(intlv_en);
1730
1731 if (!(num_dcts_intlv % 2 == 0) || (num_dcts_intlv > 4))
1732 return -EINVAL;
1733
1734 channel = f15_m30h_determine_channel(pvt, sys_addr, intlv_en,
1735 num_dcts_intlv, dct_sel);
1736
1737 /* Verify we stay within the MAX number of channels allowed */
Aravind Gopalakrishnan7f3f5242013-12-04 11:40:11 -06001738 if (channel > 3)
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001739 return -EINVAL;
1740
1741 leg_mmio_hole = (u8) (dct_cont_base_reg >> 1 & BIT(0));
1742
1743 /* Get normalized DCT addr */
1744 if (leg_mmio_hole && (sys_addr >= BIT_64(32)))
1745 chan_offset = dhar_offset;
1746 else
Aravind Gopalakrishnan4fc06b32013-08-24 10:47:48 -05001747 chan_offset = dct_base << 27;
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001748
1749 chan_addr = sys_addr - chan_offset;
1750
1751 /* remove channel interleave */
1752 if (num_dcts_intlv == 2) {
1753 if (intlv_addr == 0x4)
1754 chan_addr = ((chan_addr >> 9) << 8) |
1755 (chan_addr & 0xff);
1756 else if (intlv_addr == 0x5)
1757 chan_addr = ((chan_addr >> 10) << 9) |
1758 (chan_addr & 0x1ff);
1759 else
1760 return -EINVAL;
1761
1762 } else if (num_dcts_intlv == 4) {
1763 if (intlv_addr == 0x4)
1764 chan_addr = ((chan_addr >> 10) << 8) |
1765 (chan_addr & 0xff);
1766 else if (intlv_addr == 0x5)
1767 chan_addr = ((chan_addr >> 11) << 9) |
1768 (chan_addr & 0x1ff);
1769 else
1770 return -EINVAL;
1771 }
1772
1773 if (dct_offset_en) {
1774 amd64_read_pci_cfg(pvt->F1,
1775 DRAM_CONT_HIGH_OFF + (int) channel * 4,
1776 &tmp);
Aravind Gopalakrishnan4fc06b32013-08-24 10:47:48 -05001777 chan_addr += (u64) ((tmp >> 11) & 0xfff) << 27;
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001778 }
1779
1780 f15h_select_dct(pvt, channel);
1781
1782 edac_dbg(1, " Normalized DCT addr: 0x%llx\n", chan_addr);
1783
1784 /*
1785 * Find Chip select:
1786 * if channel = 3, then alias it to 1. This is because, in F15 M30h,
1787 * there is support for 4 DCT's, but only 2 are currently functional.
1788 * They are DCT0 and DCT3. But we have read all registers of DCT3 into
1789 * pvt->csels[1]. So we need to use '1' here to get correct info.
1790 * Refer F15 M30h BKDG Section 2.10 and 2.10.3 for clarifications.
1791 */
1792 alias_channel = (channel == 3) ? 1 : channel;
1793
1794 cs_found = f1x_lookup_addr_in_dct(chan_addr, node_id, alias_channel);
1795
1796 if (cs_found >= 0)
1797 *chan_sel = alias_channel;
1798
1799 return cs_found;
1800}
1801
1802static int f1x_translate_sysaddr_to_cs(struct amd64_pvt *pvt,
1803 u64 sys_addr,
1804 int *chan_sel)
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001805{
Borislav Petkove7613592011-02-21 19:49:01 +01001806 int cs_found = -EINVAL;
1807 unsigned range;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001808
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001809 for (range = 0; range < DRAM_RANGES; range++) {
Borislav Petkov7f19bf72010-10-21 18:52:53 +02001810 if (!dram_rw(pvt, range))
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001811 continue;
1812
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001813 if (pvt->fam == 0x15 && pvt->model >= 0x30)
1814 cs_found = f15_m30h_match_to_this_node(pvt, range,
1815 sys_addr,
1816 chan_sel);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001817
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001818 else if ((get_dram_base(pvt, range) <= sys_addr) &&
1819 (get_dram_limit(pvt, range) >= sys_addr)) {
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001820 cs_found = f1x_match_to_this_node(pvt, range,
Borislav Petkov33ca0642012-08-30 18:01:36 +02001821 sys_addr, chan_sel);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001822 if (cs_found >= 0)
1823 break;
1824 }
1825 }
1826 return cs_found;
1827}
1828
1829/*
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001830 * For reference see "2.8.5 Routing DRAM Requests" in F10 BKDG. This code maps
1831 * a @sys_addr to NodeID, DCT (channel) and chip select (CSROW).
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001832 *
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001833 * The @sys_addr is usually an error address received from the hardware
1834 * (MCX_ADDR).
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001835 */
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001836static void f1x_map_sysaddr_to_csrow(struct mem_ctl_info *mci, u64 sys_addr,
Borislav Petkov33ca0642012-08-30 18:01:36 +02001837 struct err_info *err)
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001838{
1839 struct amd64_pvt *pvt = mci->pvt_info;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001840
Borislav Petkov33ca0642012-08-30 18:01:36 +02001841 error_address_to_page_and_offset(sys_addr, err);
Mauro Carvalho Chehabab5a5032012-04-16 15:03:50 -03001842
Borislav Petkov33ca0642012-08-30 18:01:36 +02001843 err->csrow = f1x_translate_sysaddr_to_cs(pvt, sys_addr, &err->channel);
1844 if (err->csrow < 0) {
1845 err->err_code = ERR_CSROW;
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001846 return;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001847 }
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001848
Borislav Petkovbdc30a02009-11-13 15:10:43 +01001849 /*
1850 * We need the syndromes for channel detection only when we're
1851 * ganged. Otherwise @chan should already contain the channel at
1852 * this point.
1853 */
Borislav Petkova97fa682010-12-23 14:07:18 +01001854 if (dct_ganging_enabled(pvt))
Borislav Petkov33ca0642012-08-30 18:01:36 +02001855 err->channel = get_channel_from_ecc_syndrome(mci, err->syndrome);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001856}
1857
1858/*
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001859 * debug routine to display the memory sizes of all logical DIMMs and its
Borislav Petkovcb328502010-12-22 14:28:24 +01001860 * CSROWs
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001861 */
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01001862static void debug_display_dimm_sizes(struct amd64_pvt *pvt, u8 ctrl)
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001863{
Borislav Petkovbb89f5a2012-09-12 18:06:00 +02001864 int dimm, size0, size1;
Borislav Petkov525a1b22010-12-21 15:53:27 +01001865 u32 *dcsb = ctrl ? pvt->csels[1].csbases : pvt->csels[0].csbases;
1866 u32 dbam = ctrl ? pvt->dbam1 : pvt->dbam0;
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001867
Borislav Petkova4b4bed2013-08-10 13:54:48 +02001868 if (pvt->fam == 0xf) {
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001869 /* K8 families < revF not supported yet */
Borislav Petkov1433eb92009-10-21 13:44:36 +02001870 if (pvt->ext_model < K8_REV_F)
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001871 return;
1872 else
1873 WARN_ON(ctrl != 0);
1874 }
1875
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -05001876 if (pvt->fam == 0x10) {
1877 dbam = (ctrl && !dct_ganging_enabled(pvt)) ? pvt->dbam1
1878 : pvt->dbam0;
1879 dcsb = (ctrl && !dct_ganging_enabled(pvt)) ?
1880 pvt->csels[1].csbases :
1881 pvt->csels[0].csbases;
1882 } else if (ctrl) {
1883 dbam = pvt->dbam0;
1884 dcsb = pvt->csels[1].csbases;
1885 }
Joe Perches956b9ba2012-04-29 17:08:39 -03001886 edac_dbg(1, "F2x%d80 (DRAM Bank Address Mapping): 0x%08x\n",
1887 ctrl, dbam);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001888
Borislav Petkov8566c4d2009-10-16 13:48:28 +02001889 edac_printk(KERN_DEBUG, EDAC_MC, "DCT%d chip selects:\n", ctrl);
1890
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001891 /* Dump memory sizes for DIMM and its CSROWs */
1892 for (dimm = 0; dimm < 4; dimm++) {
1893
1894 size0 = 0;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001895 if (dcsb[dimm*2] & DCSB_CS_ENABLE)
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01001896 /* For f15m60h, need multiplier for LRDIMM cs_size
1897 * calculation. We pass 'dimm' value to the dbam_to_cs
1898 * mapper so we can find the multiplier from the
1899 * corresponding DCSM.
1900 */
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001901 size0 = pvt->ops->dbam_to_cs(pvt, ctrl,
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01001902 DBAM_DIMM(dimm, dbam),
1903 dimm);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001904
1905 size1 = 0;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01001906 if (dcsb[dimm*2 + 1] & DCSB_CS_ENABLE)
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001907 size1 = pvt->ops->dbam_to_cs(pvt, ctrl,
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01001908 DBAM_DIMM(dimm, dbam),
1909 dimm);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001910
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02001911 amd64_info(EDAC_MC ": %d: %5dMB %d: %5dMB\n",
Borislav Petkovbb89f5a2012-09-12 18:06:00 +02001912 dimm * 2, size0,
1913 dimm * 2 + 1, size1);
Doug Thompsonf71d0a02009-04-27 16:22:43 +02001914 }
1915}
1916
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01001917static struct amd64_family_type family_types[] = {
Doug Thompson4d376072009-04-27 16:25:05 +02001918 [K8_CPUS] = {
Borislav Petkov0092b202010-10-01 19:20:05 +02001919 .ctl_name = "K8",
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001920 .f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP,
1921 .f3_id = PCI_DEVICE_ID_AMD_K8_NB_MISC,
Doug Thompson4d376072009-04-27 16:25:05 +02001922 .ops = {
Borislav Petkov1433eb92009-10-21 13:44:36 +02001923 .early_channel_count = k8_early_channel_count,
Borislav Petkov1433eb92009-10-21 13:44:36 +02001924 .map_sysaddr_to_csrow = k8_map_sysaddr_to_csrow,
1925 .dbam_to_cs = k8_dbam_to_chip_select,
Doug Thompson4d376072009-04-27 16:25:05 +02001926 }
1927 },
1928 [F10_CPUS] = {
Borislav Petkov0092b202010-10-01 19:20:05 +02001929 .ctl_name = "F10h",
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02001930 .f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP,
1931 .f3_id = PCI_DEVICE_ID_AMD_10H_NB_MISC,
Doug Thompson4d376072009-04-27 16:25:05 +02001932 .ops = {
Borislav Petkov7d20d142011-01-07 17:58:04 +01001933 .early_channel_count = f1x_early_channel_count,
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001934 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
Borislav Petkov1433eb92009-10-21 13:44:36 +02001935 .dbam_to_cs = f10_dbam_to_chip_select,
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001936 }
1937 },
1938 [F15_CPUS] = {
1939 .ctl_name = "F15h",
Borislav Petkovdf71a052011-01-19 18:15:10 +01001940 .f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1,
1941 .f3_id = PCI_DEVICE_ID_AMD_15H_NB_F3,
Borislav Petkovb2b0c602010-10-08 18:32:29 +02001942 .ops = {
Borislav Petkov7d20d142011-01-07 17:58:04 +01001943 .early_channel_count = f1x_early_channel_count,
Borislav Petkovb15f0fc2011-01-17 15:59:58 +01001944 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
Borislav Petkov41d8bfa2011-01-18 19:16:08 +01001945 .dbam_to_cs = f15_dbam_to_chip_select,
Doug Thompson4d376072009-04-27 16:25:05 +02001946 }
1947 },
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001948 [F15_M30H_CPUS] = {
1949 .ctl_name = "F15h_M30h",
1950 .f1_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1,
1951 .f3_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F3,
1952 .ops = {
1953 .early_channel_count = f1x_early_channel_count,
1954 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1955 .dbam_to_cs = f16_dbam_to_chip_select,
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05001956 }
1957 },
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01001958 [F15_M60H_CPUS] = {
1959 .ctl_name = "F15h_M60h",
1960 .f1_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1,
1961 .f3_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F3,
1962 .ops = {
1963 .early_channel_count = f1x_early_channel_count,
1964 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1965 .dbam_to_cs = f15_m60h_dbam_to_chip_select,
1966 }
1967 },
Aravind Gopalakrishnan94c1acf2013-04-17 14:57:13 -05001968 [F16_CPUS] = {
1969 .ctl_name = "F16h",
1970 .f1_id = PCI_DEVICE_ID_AMD_16H_NB_F1,
1971 .f3_id = PCI_DEVICE_ID_AMD_16H_NB_F3,
1972 .ops = {
1973 .early_channel_count = f1x_early_channel_count,
1974 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1975 .dbam_to_cs = f16_dbam_to_chip_select,
Aravind Gopalakrishnan94c1acf2013-04-17 14:57:13 -05001976 }
1977 },
Aravind Gopalakrishnan85a88852014-02-20 10:28:46 -06001978 [F16_M30H_CPUS] = {
1979 .ctl_name = "F16h_M30h",
1980 .f1_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F1,
1981 .f3_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F3,
1982 .ops = {
1983 .early_channel_count = f1x_early_channel_count,
1984 .map_sysaddr_to_csrow = f1x_map_sysaddr_to_csrow,
1985 .dbam_to_cs = f16_dbam_to_chip_select,
Aravind Gopalakrishnan85a88852014-02-20 10:28:46 -06001986 }
1987 },
Doug Thompson4d376072009-04-27 16:25:05 +02001988};
1989
Doug Thompsonb1289d62009-04-27 16:37:05 +02001990/*
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001991 * These are tables of eigenvectors (one per line) which can be used for the
1992 * construction of the syndrome tables. The modified syndrome search algorithm
1993 * uses those to find the symbol in error and thus the DIMM.
Doug Thompsonb1289d62009-04-27 16:37:05 +02001994 *
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001995 * Algorithm courtesy of Ross LaFetra from AMD.
Doug Thompsonb1289d62009-04-27 16:37:05 +02001996 */
Daniel J Bluemanc7e53012012-11-30 16:44:20 +08001997static const u16 x4_vectors[] = {
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01001998 0x2f57, 0x1afe, 0x66cc, 0xdd88,
1999 0x11eb, 0x3396, 0x7f4c, 0xeac8,
2000 0x0001, 0x0002, 0x0004, 0x0008,
2001 0x1013, 0x3032, 0x4044, 0x8088,
2002 0x106b, 0x30d6, 0x70fc, 0xe0a8,
2003 0x4857, 0xc4fe, 0x13cc, 0x3288,
2004 0x1ac5, 0x2f4a, 0x5394, 0xa1e8,
2005 0x1f39, 0x251e, 0xbd6c, 0x6bd8,
2006 0x15c1, 0x2a42, 0x89ac, 0x4758,
2007 0x2b03, 0x1602, 0x4f0c, 0xca08,
2008 0x1f07, 0x3a0e, 0x6b04, 0xbd08,
2009 0x8ba7, 0x465e, 0x244c, 0x1cc8,
2010 0x2b87, 0x164e, 0x642c, 0xdc18,
2011 0x40b9, 0x80de, 0x1094, 0x20e8,
2012 0x27db, 0x1eb6, 0x9dac, 0x7b58,
2013 0x11c1, 0x2242, 0x84ac, 0x4c58,
2014 0x1be5, 0x2d7a, 0x5e34, 0xa718,
2015 0x4b39, 0x8d1e, 0x14b4, 0x28d8,
2016 0x4c97, 0xc87e, 0x11fc, 0x33a8,
2017 0x8e97, 0x497e, 0x2ffc, 0x1aa8,
2018 0x16b3, 0x3d62, 0x4f34, 0x8518,
2019 0x1e2f, 0x391a, 0x5cac, 0xf858,
2020 0x1d9f, 0x3b7a, 0x572c, 0xfe18,
2021 0x15f5, 0x2a5a, 0x5264, 0xa3b8,
2022 0x1dbb, 0x3b66, 0x715c, 0xe3f8,
2023 0x4397, 0xc27e, 0x17fc, 0x3ea8,
2024 0x1617, 0x3d3e, 0x6464, 0xb8b8,
2025 0x23ff, 0x12aa, 0xab6c, 0x56d8,
2026 0x2dfb, 0x1ba6, 0x913c, 0x7328,
2027 0x185d, 0x2ca6, 0x7914, 0x9e28,
2028 0x171b, 0x3e36, 0x7d7c, 0xebe8,
2029 0x4199, 0x82ee, 0x19f4, 0x2e58,
2030 0x4807, 0xc40e, 0x130c, 0x3208,
2031 0x1905, 0x2e0a, 0x5804, 0xac08,
2032 0x213f, 0x132a, 0xadfc, 0x5ba8,
2033 0x19a9, 0x2efe, 0xb5cc, 0x6f88,
Doug Thompsonb1289d62009-04-27 16:37:05 +02002034};
2035
Daniel J Bluemanc7e53012012-11-30 16:44:20 +08002036static const u16 x8_vectors[] = {
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01002037 0x0145, 0x028a, 0x2374, 0x43c8, 0xa1f0, 0x0520, 0x0a40, 0x1480,
2038 0x0211, 0x0422, 0x0844, 0x1088, 0x01b0, 0x44e0, 0x23c0, 0xed80,
2039 0x1011, 0x0116, 0x022c, 0x0458, 0x08b0, 0x8c60, 0x2740, 0x4e80,
2040 0x0411, 0x0822, 0x1044, 0x0158, 0x02b0, 0x2360, 0x46c0, 0xab80,
2041 0x0811, 0x1022, 0x012c, 0x0258, 0x04b0, 0x4660, 0x8cc0, 0x2780,
2042 0x2071, 0x40e2, 0xa0c4, 0x0108, 0x0210, 0x0420, 0x0840, 0x1080,
2043 0x4071, 0x80e2, 0x0104, 0x0208, 0x0410, 0x0820, 0x1040, 0x2080,
2044 0x8071, 0x0102, 0x0204, 0x0408, 0x0810, 0x1020, 0x2040, 0x4080,
2045 0x019d, 0x03d6, 0x136c, 0x2198, 0x50b0, 0xb2e0, 0x0740, 0x0e80,
2046 0x0189, 0x03ea, 0x072c, 0x0e58, 0x1cb0, 0x56e0, 0x37c0, 0xf580,
2047 0x01fd, 0x0376, 0x06ec, 0x0bb8, 0x1110, 0x2220, 0x4440, 0x8880,
2048 0x0163, 0x02c6, 0x1104, 0x0758, 0x0eb0, 0x2be0, 0x6140, 0xc280,
2049 0x02fd, 0x01c6, 0x0b5c, 0x1108, 0x07b0, 0x25a0, 0x8840, 0x6180,
2050 0x0801, 0x012e, 0x025c, 0x04b8, 0x1370, 0x26e0, 0x57c0, 0xb580,
2051 0x0401, 0x0802, 0x015c, 0x02b8, 0x22b0, 0x13e0, 0x7140, 0xe280,
2052 0x0201, 0x0402, 0x0804, 0x01b8, 0x11b0, 0x31a0, 0x8040, 0x7180,
2053 0x0101, 0x0202, 0x0404, 0x0808, 0x1010, 0x2020, 0x4040, 0x8080,
2054 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080,
2055 0x0100, 0x0200, 0x0400, 0x0800, 0x1000, 0x2000, 0x4000, 0x8000,
2056};
2057
Daniel J Bluemanc7e53012012-11-30 16:44:20 +08002058static int decode_syndrome(u16 syndrome, const u16 *vectors, unsigned num_vecs,
Borislav Petkovd34a6ec2011-02-23 17:41:50 +01002059 unsigned v_dim)
Doug Thompsonb1289d62009-04-27 16:37:05 +02002060{
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01002061 unsigned int i, err_sym;
Doug Thompsonb1289d62009-04-27 16:37:05 +02002062
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01002063 for (err_sym = 0; err_sym < num_vecs / v_dim; err_sym++) {
2064 u16 s = syndrome;
Borislav Petkovd34a6ec2011-02-23 17:41:50 +01002065 unsigned v_idx = err_sym * v_dim;
2066 unsigned v_end = (err_sym + 1) * v_dim;
Doug Thompsonb1289d62009-04-27 16:37:05 +02002067
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01002068 /* walk over all 16 bits of the syndrome */
2069 for (i = 1; i < (1U << 16); i <<= 1) {
2070
2071 /* if bit is set in that eigenvector... */
2072 if (v_idx < v_end && vectors[v_idx] & i) {
2073 u16 ev_comp = vectors[v_idx++];
2074
2075 /* ... and bit set in the modified syndrome, */
2076 if (s & i) {
2077 /* remove it. */
2078 s ^= ev_comp;
2079
2080 if (!s)
2081 return err_sym;
2082 }
2083
2084 } else if (s & i)
2085 /* can't get to zero, move to next symbol */
2086 break;
2087 }
Doug Thompsonb1289d62009-04-27 16:37:05 +02002088 }
2089
Joe Perches956b9ba2012-04-29 17:08:39 -03002090 edac_dbg(0, "syndrome(%x) not found\n", syndrome);
Doug Thompsonb1289d62009-04-27 16:37:05 +02002091 return -1;
2092}
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002093
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01002094static int map_err_sym_to_channel(int err_sym, int sym_size)
2095{
2096 if (sym_size == 4)
2097 switch (err_sym) {
2098 case 0x20:
2099 case 0x21:
2100 return 0;
2101 break;
2102 case 0x22:
2103 case 0x23:
2104 return 1;
2105 break;
2106 default:
2107 return err_sym >> 4;
2108 break;
2109 }
2110 /* x8 symbols */
2111 else
2112 switch (err_sym) {
2113 /* imaginary bits not in a DIMM */
2114 case 0x10:
2115 WARN(1, KERN_ERR "Invalid error symbol: 0x%x\n",
2116 err_sym);
2117 return -1;
2118 break;
2119
2120 case 0x11:
2121 return 0;
2122 break;
2123 case 0x12:
2124 return 1;
2125 break;
2126 default:
2127 return err_sym >> 3;
2128 break;
2129 }
2130 return -1;
2131}
2132
2133static int get_channel_from_ecc_syndrome(struct mem_ctl_info *mci, u16 syndrome)
2134{
2135 struct amd64_pvt *pvt = mci->pvt_info;
Borislav Petkovad6a32e2010-03-09 12:46:00 +01002136 int err_sym = -1;
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01002137
Borislav Petkova3b7db02011-01-19 20:35:12 +01002138 if (pvt->ecc_sym_sz == 8)
Borislav Petkovad6a32e2010-03-09 12:46:00 +01002139 err_sym = decode_syndrome(syndrome, x8_vectors,
2140 ARRAY_SIZE(x8_vectors),
Borislav Petkova3b7db02011-01-19 20:35:12 +01002141 pvt->ecc_sym_sz);
2142 else if (pvt->ecc_sym_sz == 4)
Borislav Petkovad6a32e2010-03-09 12:46:00 +01002143 err_sym = decode_syndrome(syndrome, x4_vectors,
2144 ARRAY_SIZE(x4_vectors),
Borislav Petkova3b7db02011-01-19 20:35:12 +01002145 pvt->ecc_sym_sz);
Borislav Petkovad6a32e2010-03-09 12:46:00 +01002146 else {
Borislav Petkova3b7db02011-01-19 20:35:12 +01002147 amd64_warn("Illegal syndrome type: %u\n", pvt->ecc_sym_sz);
Borislav Petkovad6a32e2010-03-09 12:46:00 +01002148 return err_sym;
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01002149 }
Borislav Petkovad6a32e2010-03-09 12:46:00 +01002150
Borislav Petkova3b7db02011-01-19 20:35:12 +01002151 return map_err_sym_to_channel(err_sym, pvt->ecc_sym_sz);
Borislav Petkovbfc04ae2009-11-12 19:05:07 +01002152}
2153
Borislav Petkov33ca0642012-08-30 18:01:36 +02002154static void __log_bus_error(struct mem_ctl_info *mci, struct err_info *err,
2155 u8 ecc_type)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002156{
Borislav Petkov33ca0642012-08-30 18:01:36 +02002157 enum hw_event_mc_err_type err_type;
2158 const char *string;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002159
Borislav Petkov33ca0642012-08-30 18:01:36 +02002160 if (ecc_type == 2)
2161 err_type = HW_EVENT_ERR_CORRECTED;
2162 else if (ecc_type == 1)
2163 err_type = HW_EVENT_ERR_UNCORRECTED;
2164 else {
2165 WARN(1, "Something is rotten in the state of Denmark.\n");
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002166 return;
2167 }
2168
Borislav Petkov33ca0642012-08-30 18:01:36 +02002169 switch (err->err_code) {
2170 case DECODE_OK:
2171 string = "";
2172 break;
2173 case ERR_NODE:
2174 string = "Failed to map error addr to a node";
2175 break;
2176 case ERR_CSROW:
2177 string = "Failed to map error addr to a csrow";
2178 break;
2179 case ERR_CHANNEL:
2180 string = "unknown syndrome - possible error reporting race";
2181 break;
2182 default:
2183 string = "WTF error";
2184 break;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002185 }
Borislav Petkov33ca0642012-08-30 18:01:36 +02002186
2187 edac_mc_handle_error(err_type, mci, 1,
2188 err->page, err->offset, err->syndrome,
2189 err->csrow, err->channel, -1,
2190 string, "");
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002191}
2192
Borislav Petkovdf781d02013-12-15 17:29:44 +01002193static inline void decode_bus_error(int node_id, struct mce *m)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002194{
Daniel J Blueman0c510cc2015-02-17 11:34:38 +08002195 struct mem_ctl_info *mci;
2196 struct amd64_pvt *pvt;
Borislav Petkovf192c7b2011-01-10 14:24:32 +01002197 u8 ecc_type = (m->status >> 45) & 0x3;
Borislav Petkov66fed2d2012-08-09 18:41:07 +02002198 u8 xec = XEC(m->status, 0x1f);
2199 u16 ec = EC(m->status);
Borislav Petkov33ca0642012-08-30 18:01:36 +02002200 u64 sys_addr;
2201 struct err_info err;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002202
Daniel J Blueman0c510cc2015-02-17 11:34:38 +08002203 mci = edac_mc_find(node_id);
2204 if (!mci)
2205 return;
2206
2207 pvt = mci->pvt_info;
2208
Borislav Petkov66fed2d2012-08-09 18:41:07 +02002209 /* Bail out early if this was an 'observed' error */
Borislav Petkov5980bb92011-01-07 16:26:49 +01002210 if (PP(ec) == NBSL_PP_OBS)
Borislav Petkovb70ef012009-06-25 19:32:38 +02002211 return;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002212
Borislav Petkovecaf5602009-07-23 16:32:01 +02002213 /* Do only ECC errors */
2214 if (xec && xec != F10_NBSL_EXT_ERR_ECC)
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002215 return;
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002216
Borislav Petkov33ca0642012-08-30 18:01:36 +02002217 memset(&err, 0, sizeof(err));
2218
Borislav Petkova4b4bed2013-08-10 13:54:48 +02002219 sys_addr = get_error_address(pvt, m);
Borislav Petkov33ca0642012-08-30 18:01:36 +02002220
Borislav Petkovecaf5602009-07-23 16:32:01 +02002221 if (ecc_type == 2)
Borislav Petkov33ca0642012-08-30 18:01:36 +02002222 err.syndrome = extract_syndrome(m->status);
2223
2224 pvt->ops->map_sysaddr_to_csrow(mci, sys_addr, &err);
2225
2226 __log_bus_error(mci, &err, ecc_type);
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002227}
2228
Doug Thompson0ec449e2009-04-27 19:41:25 +02002229/*
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002230 * Use pvt->F2 which contains the F2 CPU PCI device to get the related
Borislav Petkovbbd0c1f2010-10-01 19:27:58 +02002231 * F1 (AddrMap) and F3 (Misc) devices. Return negative value on error.
Doug Thompson0ec449e2009-04-27 19:41:25 +02002232 */
Borislav Petkov360b7f32010-10-15 19:25:38 +02002233static int reserve_mc_sibling_devs(struct amd64_pvt *pvt, u16 f1_id, u16 f3_id)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002234{
Doug Thompson0ec449e2009-04-27 19:41:25 +02002235 /* Reserve the ADDRESS MAP Device */
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002236 pvt->F1 = pci_get_related_function(pvt->F2->vendor, f1_id, pvt->F2);
2237 if (!pvt->F1) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002238 amd64_err("error address map device not found: "
2239 "vendor %x device 0x%x (broken BIOS?)\n",
2240 PCI_VENDOR_ID_AMD, f1_id);
Borislav Petkovbbd0c1f2010-10-01 19:27:58 +02002241 return -ENODEV;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002242 }
2243
2244 /* Reserve the MISC Device */
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002245 pvt->F3 = pci_get_related_function(pvt->F2->vendor, f3_id, pvt->F2);
2246 if (!pvt->F3) {
2247 pci_dev_put(pvt->F1);
2248 pvt->F1 = NULL;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002249
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002250 amd64_err("error F3 device not found: "
2251 "vendor %x device 0x%x (broken BIOS?)\n",
2252 PCI_VENDOR_ID_AMD, f3_id);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002253
Borislav Petkovbbd0c1f2010-10-01 19:27:58 +02002254 return -ENODEV;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002255 }
Joe Perches956b9ba2012-04-29 17:08:39 -03002256 edac_dbg(1, "F1: %s\n", pci_name(pvt->F1));
2257 edac_dbg(1, "F2: %s\n", pci_name(pvt->F2));
2258 edac_dbg(1, "F3: %s\n", pci_name(pvt->F3));
Doug Thompson0ec449e2009-04-27 19:41:25 +02002259
2260 return 0;
2261}
2262
Borislav Petkov360b7f32010-10-15 19:25:38 +02002263static void free_mc_sibling_devs(struct amd64_pvt *pvt)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002264{
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002265 pci_dev_put(pvt->F1);
2266 pci_dev_put(pvt->F3);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002267}
2268
2269/*
2270 * Retrieve the hardware registers of the memory controller (this includes the
2271 * 'Address Map' and 'Misc' device regs)
2272 */
Borislav Petkov360b7f32010-10-15 19:25:38 +02002273static void read_mc_regs(struct amd64_pvt *pvt)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002274{
Borislav Petkova4b4bed2013-08-10 13:54:48 +02002275 unsigned range;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002276 u64 msr_val;
Borislav Petkovad6a32e2010-03-09 12:46:00 +01002277 u32 tmp;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002278
2279 /*
2280 * Retrieve TOP_MEM and TOP_MEM2; no masking off of reserved bits since
2281 * those are Read-As-Zero
2282 */
Borislav Petkove97f8bb2009-10-12 15:27:45 +02002283 rdmsrl(MSR_K8_TOP_MEM1, pvt->top_mem);
Joe Perches956b9ba2012-04-29 17:08:39 -03002284 edac_dbg(0, " TOP_MEM: 0x%016llx\n", pvt->top_mem);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002285
2286 /* check first whether TOP_MEM2 is enabled */
2287 rdmsrl(MSR_K8_SYSCFG, msr_val);
2288 if (msr_val & (1U << 21)) {
Borislav Petkove97f8bb2009-10-12 15:27:45 +02002289 rdmsrl(MSR_K8_TOP_MEM2, pvt->top_mem2);
Joe Perches956b9ba2012-04-29 17:08:39 -03002290 edac_dbg(0, " TOP_MEM2: 0x%016llx\n", pvt->top_mem2);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002291 } else
Joe Perches956b9ba2012-04-29 17:08:39 -03002292 edac_dbg(0, " TOP_MEM2 disabled\n");
Doug Thompson0ec449e2009-04-27 19:41:25 +02002293
Borislav Petkov5980bb92011-01-07 16:26:49 +01002294 amd64_read_pci_cfg(pvt->F3, NBCAP, &pvt->nbcap);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002295
Borislav Petkov5a5d2372011-01-17 17:52:57 +01002296 read_dram_ctl_register(pvt);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002297
Borislav Petkov7f19bf72010-10-21 18:52:53 +02002298 for (range = 0; range < DRAM_RANGES; range++) {
2299 u8 rw;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002300
Borislav Petkov7f19bf72010-10-21 18:52:53 +02002301 /* read settings for this DRAM range */
2302 read_dram_base_limit_regs(pvt, range);
Borislav Petkove97f8bb2009-10-12 15:27:45 +02002303
Borislav Petkov7f19bf72010-10-21 18:52:53 +02002304 rw = dram_rw(pvt, range);
2305 if (!rw)
2306 continue;
2307
Joe Perches956b9ba2012-04-29 17:08:39 -03002308 edac_dbg(1, " DRAM range[%d], base: 0x%016llx; limit: 0x%016llx\n",
2309 range,
2310 get_dram_base(pvt, range),
2311 get_dram_limit(pvt, range));
Borislav Petkov7f19bf72010-10-21 18:52:53 +02002312
Joe Perches956b9ba2012-04-29 17:08:39 -03002313 edac_dbg(1, " IntlvEn=%s; Range access: %s%s IntlvSel=%d DstNode=%d\n",
2314 dram_intlv_en(pvt, range) ? "Enabled" : "Disabled",
2315 (rw & 0x1) ? "R" : "-",
2316 (rw & 0x2) ? "W" : "-",
2317 dram_intlv_sel(pvt, range),
2318 dram_dst_node(pvt, range));
Doug Thompson0ec449e2009-04-27 19:41:25 +02002319 }
2320
Borislav Petkovb2b0c602010-10-08 18:32:29 +02002321 read_dct_base_mask(pvt);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002322
Borislav Petkovbc21fa52010-11-11 17:29:13 +01002323 amd64_read_pci_cfg(pvt->F1, DHAR, &pvt->dhar);
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -05002324 amd64_read_dct_pci_cfg(pvt, 0, DBAM0, &pvt->dbam0);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002325
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002326 amd64_read_pci_cfg(pvt->F3, F10_ONLINE_SPARE, &pvt->online_spare);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002327
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -05002328 amd64_read_dct_pci_cfg(pvt, 0, DCLR0, &pvt->dclr0);
2329 amd64_read_dct_pci_cfg(pvt, 0, DCHR0, &pvt->dchr0);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002330
Borislav Petkov78da1212010-12-22 19:31:45 +01002331 if (!dct_ganging_enabled(pvt)) {
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -05002332 amd64_read_dct_pci_cfg(pvt, 1, DCLR0, &pvt->dclr1);
2333 amd64_read_dct_pci_cfg(pvt, 1, DCHR0, &pvt->dchr1);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002334 }
Borislav Petkovad6a32e2010-03-09 12:46:00 +01002335
Borislav Petkova3b7db02011-01-19 20:35:12 +01002336 pvt->ecc_sym_sz = 4;
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01002337 determine_memory_type(pvt);
2338 edac_dbg(1, " DIMM type: %s\n", edac_mem_types[pvt->dram_type]);
Borislav Petkova3b7db02011-01-19 20:35:12 +01002339
Borislav Petkova4b4bed2013-08-10 13:54:48 +02002340 if (pvt->fam >= 0x10) {
Borislav Petkovb2b0c602010-10-08 18:32:29 +02002341 amd64_read_pci_cfg(pvt->F3, EXT_NB_MCA_CFG, &tmp);
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -05002342 /* F16h has only DCT0, so no need to read dbam1 */
Borislav Petkova4b4bed2013-08-10 13:54:48 +02002343 if (pvt->fam != 0x16)
Aravind Gopalakrishnan7981a282014-09-15 11:37:38 -05002344 amd64_read_dct_pci_cfg(pvt, 1, DBAM0, &pvt->dbam1);
Borislav Petkova3b7db02011-01-19 20:35:12 +01002345
2346 /* F10h, revD and later can do x8 ECC too */
Borislav Petkova4b4bed2013-08-10 13:54:48 +02002347 if ((pvt->fam > 0x10 || pvt->model > 7) && tmp & BIT(25))
Borislav Petkova3b7db02011-01-19 20:35:12 +01002348 pvt->ecc_sym_sz = 8;
Borislav Petkov525a1b22010-12-21 15:53:27 +01002349 }
Borislav Petkovb2b0c602010-10-08 18:32:29 +02002350 dump_misc_regs(pvt);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002351}
2352
2353/*
2354 * NOTE: CPU Revision Dependent code
2355 *
2356 * Input:
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002357 * @csrow_nr ChipSelect Row Number (0..NUM_CHIPSELECTS-1)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002358 * k8 private pointer to -->
2359 * DRAM Bank Address mapping register
2360 * node_id
2361 * DCL register where dual_channel_active is
2362 *
2363 * The DBAM register consists of 4 sets of 4 bits each definitions:
2364 *
2365 * Bits: CSROWs
2366 * 0-3 CSROWs 0 and 1
2367 * 4-7 CSROWs 2 and 3
2368 * 8-11 CSROWs 4 and 5
2369 * 12-15 CSROWs 6 and 7
2370 *
2371 * Values range from: 0 to 15
2372 * The meaning of the values depends on CPU revision and dual-channel state,
2373 * see relevant BKDG more info.
2374 *
2375 * The memory controller provides for total of only 8 CSROWs in its current
2376 * architecture. Each "pair" of CSROWs normally represents just one DIMM in
2377 * single channel or two (2) DIMMs in dual channel mode.
2378 *
2379 * The following code logic collapses the various tables for CSROW based on CPU
2380 * revision.
2381 *
2382 * Returns:
2383 * The number of PAGE_SIZE pages on the specified CSROW number it
2384 * encompasses
2385 *
2386 */
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01002387static u32 get_csrow_nr_pages(struct amd64_pvt *pvt, u8 dct, int csrow_nr)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002388{
Borislav Petkov1433eb92009-10-21 13:44:36 +02002389 u32 cs_mode, nr_pages;
Ashish Shenoyf92cae42012-02-22 17:20:38 -08002390 u32 dbam = dct ? pvt->dbam1 : pvt->dbam0;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002391
Borislav Petkov10de6492012-09-12 19:00:38 +02002392
Doug Thompson0ec449e2009-04-27 19:41:25 +02002393 /*
2394 * The math on this doesn't look right on the surface because x/2*4 can
2395 * be simplified to x*2 but this expression makes use of the fact that
2396 * it is integral math where 1/2=0. This intermediate value becomes the
2397 * number of bits to shift the DBAM register to extract the proper CSROW
2398 * field.
2399 */
Borislav Petkov0a5dfc32012-09-12 18:16:01 +02002400 cs_mode = DBAM_DIMM(csrow_nr / 2, dbam);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002401
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01002402 nr_pages = pvt->ops->dbam_to_cs(pvt, dct, cs_mode, (csrow_nr / 2))
2403 << (20 - PAGE_SHIFT);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002404
Borislav Petkov10de6492012-09-12 19:00:38 +02002405 edac_dbg(0, "csrow: %d, channel: %d, DBAM idx: %d\n",
2406 csrow_nr, dct, cs_mode);
2407 edac_dbg(0, "nr_pages/channel: %u\n", nr_pages);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002408
2409 return nr_pages;
2410}
2411
2412/*
2413 * Initialize the array of csrow attribute instances, based on the values
2414 * from pci config hardware registers.
2415 */
Borislav Petkov360b7f32010-10-15 19:25:38 +02002416static int init_csrows(struct mem_ctl_info *mci)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002417{
Borislav Petkov10de6492012-09-12 19:00:38 +02002418 struct amd64_pvt *pvt = mci->pvt_info;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002419 struct csrow_info *csrow;
Mauro Carvalho Chehabde3910eb2012-04-24 15:05:43 -03002420 struct dimm_info *dimm;
Mauro Carvalho Chehab084a4fc2012-01-27 18:38:08 -03002421 enum edac_type edac_mode;
Borislav Petkov10de6492012-09-12 19:00:38 +02002422 int i, j, empty = 1;
Mauro Carvalho Chehaba895bf82012-01-28 09:09:38 -03002423 int nr_pages = 0;
Borislav Petkov10de6492012-09-12 19:00:38 +02002424 u32 val;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002425
Borislav Petkova97fa682010-12-23 14:07:18 +01002426 amd64_read_pci_cfg(pvt->F3, NBCFG, &val);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002427
Borislav Petkov2299ef72010-10-15 17:44:04 +02002428 pvt->nbcfg = val;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002429
Joe Perches956b9ba2012-04-29 17:08:39 -03002430 edac_dbg(0, "node %d, NBCFG=0x%08x[ChipKillEccCap: %d|DramEccEn: %d]\n",
2431 pvt->mc_node_id, val,
2432 !!(val & NBCFG_CHIPKILL), !!(val & NBCFG_ECC_ENABLE));
Doug Thompson0ec449e2009-04-27 19:41:25 +02002433
Borislav Petkov10de6492012-09-12 19:00:38 +02002434 /*
2435 * We iterate over DCT0 here but we look at DCT1 in parallel, if needed.
2436 */
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002437 for_each_chip_select(i, 0, pvt) {
Borislav Petkov10de6492012-09-12 19:00:38 +02002438 bool row_dct0 = !!csrow_enabled(i, 0, pvt);
2439 bool row_dct1 = false;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002440
Borislav Petkova4b4bed2013-08-10 13:54:48 +02002441 if (pvt->fam != 0xf)
Borislav Petkov10de6492012-09-12 19:00:38 +02002442 row_dct1 = !!csrow_enabled(i, 1, pvt);
2443
2444 if (!row_dct0 && !row_dct1)
Doug Thompson0ec449e2009-04-27 19:41:25 +02002445 continue;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002446
Borislav Petkov10de6492012-09-12 19:00:38 +02002447 csrow = mci->csrows[i];
Doug Thompson0ec449e2009-04-27 19:41:25 +02002448 empty = 0;
Borislav Petkov11c75ea2010-11-29 19:49:02 +01002449
Borislav Petkov10de6492012-09-12 19:00:38 +02002450 edac_dbg(1, "MC node: %d, csrow: %d\n",
2451 pvt->mc_node_id, i);
2452
Mauro Carvalho Chehab1eef1282013-03-11 09:07:46 -03002453 if (row_dct0) {
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01002454 nr_pages = get_csrow_nr_pages(pvt, 0, i);
Mauro Carvalho Chehab1eef1282013-03-11 09:07:46 -03002455 csrow->channels[0]->dimm->nr_pages = nr_pages;
2456 }
Borislav Petkov10de6492012-09-12 19:00:38 +02002457
2458 /* K8 has only one DCT */
Borislav Petkova4b4bed2013-08-10 13:54:48 +02002459 if (pvt->fam != 0xf && row_dct1) {
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01002460 int row_dct1_pages = get_csrow_nr_pages(pvt, 1, i);
Mauro Carvalho Chehab1eef1282013-03-11 09:07:46 -03002461
2462 csrow->channels[1]->dimm->nr_pages = row_dct1_pages;
2463 nr_pages += row_dct1_pages;
2464 }
Doug Thompson0ec449e2009-04-27 19:41:25 +02002465
Borislav Petkov10de6492012-09-12 19:00:38 +02002466 edac_dbg(1, "Total csrow%d pages: %u\n", i, nr_pages);
Doug Thompson0ec449e2009-04-27 19:41:25 +02002467
2468 /*
2469 * determine whether CHIPKILL or JUST ECC or NO ECC is operating
2470 */
Borislav Petkova97fa682010-12-23 14:07:18 +01002471 if (pvt->nbcfg & NBCFG_ECC_ENABLE)
Mauro Carvalho Chehab084a4fc2012-01-27 18:38:08 -03002472 edac_mode = (pvt->nbcfg & NBCFG_CHIPKILL) ?
2473 EDAC_S4ECD4ED : EDAC_SECDED;
Doug Thompson0ec449e2009-04-27 19:41:25 +02002474 else
Mauro Carvalho Chehab084a4fc2012-01-27 18:38:08 -03002475 edac_mode = EDAC_NONE;
2476
2477 for (j = 0; j < pvt->channel_count; j++) {
Mauro Carvalho Chehabde3910eb2012-04-24 15:05:43 -03002478 dimm = csrow->channels[j]->dimm;
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01002479 dimm->mtype = pvt->dram_type;
Mauro Carvalho Chehabde3910eb2012-04-24 15:05:43 -03002480 dimm->edac_mode = edac_mode;
Mauro Carvalho Chehab084a4fc2012-01-27 18:38:08 -03002481 }
Doug Thompson0ec449e2009-04-27 19:41:25 +02002482 }
2483
2484 return empty;
2485}
Doug Thompsond27bf6f2009-05-06 17:55:27 +02002486
Borislav Petkov06724532009-09-16 13:05:46 +02002487/* get all cores on this DCT */
Daniel J Blueman8b84c8d2012-11-27 14:32:10 +08002488static void get_cpus_on_this_dct_cpumask(struct cpumask *mask, u16 nid)
Doug Thompsonf9431992009-04-27 19:46:08 +02002489{
Borislav Petkov06724532009-09-16 13:05:46 +02002490 int cpu;
Doug Thompsonf9431992009-04-27 19:46:08 +02002491
Borislav Petkov06724532009-09-16 13:05:46 +02002492 for_each_online_cpu(cpu)
2493 if (amd_get_nb_id(cpu) == nid)
2494 cpumask_set_cpu(cpu, mask);
Doug Thompsonf9431992009-04-27 19:46:08 +02002495}
2496
2497/* check MCG_CTL on all the cpus on this node */
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01002498static bool nb_mce_bank_enabled_on_node(u16 nid)
Doug Thompsonf9431992009-04-27 19:46:08 +02002499{
Rusty Russellba578cb2009-11-03 14:56:35 +10302500 cpumask_var_t mask;
Borislav Petkov50542252009-12-11 18:14:40 +01002501 int cpu, nbe;
Borislav Petkov06724532009-09-16 13:05:46 +02002502 bool ret = false;
Doug Thompsonf9431992009-04-27 19:46:08 +02002503
Rusty Russellba578cb2009-11-03 14:56:35 +10302504 if (!zalloc_cpumask_var(&mask, GFP_KERNEL)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002505 amd64_warn("%s: Error allocating mask\n", __func__);
Rusty Russellba578cb2009-11-03 14:56:35 +10302506 return false;
2507 }
Borislav Petkov06724532009-09-16 13:05:46 +02002508
Rusty Russellba578cb2009-11-03 14:56:35 +10302509 get_cpus_on_this_dct_cpumask(mask, nid);
Borislav Petkov06724532009-09-16 13:05:46 +02002510
Rusty Russellba578cb2009-11-03 14:56:35 +10302511 rdmsr_on_cpus(mask, MSR_IA32_MCG_CTL, msrs);
Borislav Petkov06724532009-09-16 13:05:46 +02002512
Rusty Russellba578cb2009-11-03 14:56:35 +10302513 for_each_cpu(cpu, mask) {
Borislav Petkov50542252009-12-11 18:14:40 +01002514 struct msr *reg = per_cpu_ptr(msrs, cpu);
Borislav Petkov5980bb92011-01-07 16:26:49 +01002515 nbe = reg->l & MSR_MCGCTL_NBE;
Borislav Petkov06724532009-09-16 13:05:46 +02002516
Joe Perches956b9ba2012-04-29 17:08:39 -03002517 edac_dbg(0, "core: %u, MCG_CTL: 0x%llx, NB MSR is %s\n",
2518 cpu, reg->q,
2519 (nbe ? "enabled" : "disabled"));
Borislav Petkov06724532009-09-16 13:05:46 +02002520
2521 if (!nbe)
2522 goto out;
Borislav Petkov06724532009-09-16 13:05:46 +02002523 }
2524 ret = true;
2525
2526out:
Rusty Russellba578cb2009-11-03 14:56:35 +10302527 free_cpumask_var(mask);
Doug Thompsonf9431992009-04-27 19:46:08 +02002528 return ret;
2529}
2530
Daniel J Bluemanc7e53012012-11-30 16:44:20 +08002531static int toggle_ecc_err_reporting(struct ecc_settings *s, u16 nid, bool on)
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002532{
2533 cpumask_var_t cmask;
Borislav Petkov50542252009-12-11 18:14:40 +01002534 int cpu;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002535
2536 if (!zalloc_cpumask_var(&cmask, GFP_KERNEL)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002537 amd64_warn("%s: error allocating mask\n", __func__);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002538 return false;
2539 }
2540
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002541 get_cpus_on_this_dct_cpumask(cmask, nid);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002542
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002543 rdmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2544
2545 for_each_cpu(cpu, cmask) {
2546
Borislav Petkov50542252009-12-11 18:14:40 +01002547 struct msr *reg = per_cpu_ptr(msrs, cpu);
2548
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002549 if (on) {
Borislav Petkov5980bb92011-01-07 16:26:49 +01002550 if (reg->l & MSR_MCGCTL_NBE)
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002551 s->flags.nb_mce_enable = 1;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002552
Borislav Petkov5980bb92011-01-07 16:26:49 +01002553 reg->l |= MSR_MCGCTL_NBE;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002554 } else {
2555 /*
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002556 * Turn off NB MCE reporting only when it was off before
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002557 */
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002558 if (!s->flags.nb_mce_enable)
Borislav Petkov5980bb92011-01-07 16:26:49 +01002559 reg->l &= ~MSR_MCGCTL_NBE;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002560 }
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002561 }
2562 wrmsr_on_cpus(cmask, MSR_IA32_MCG_CTL, msrs);
2563
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002564 free_cpumask_var(cmask);
2565
2566 return 0;
2567}
2568
Daniel J Bluemanc7e53012012-11-30 16:44:20 +08002569static bool enable_ecc_error_reporting(struct ecc_settings *s, u16 nid,
Borislav Petkov2299ef72010-10-15 17:44:04 +02002570 struct pci_dev *F3)
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002571{
Borislav Petkov2299ef72010-10-15 17:44:04 +02002572 bool ret = true;
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002573 u32 value, mask = 0x3; /* UECC/CECC enable */
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002574
Borislav Petkov2299ef72010-10-15 17:44:04 +02002575 if (toggle_ecc_err_reporting(s, nid, ON)) {
2576 amd64_warn("Error enabling ECC reporting over MCGCTL!\n");
2577 return false;
2578 }
2579
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002580 amd64_read_pci_cfg(F3, NBCTL, &value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002581
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002582 s->old_nbctl = value & mask;
2583 s->nbctl_valid = true;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002584
2585 value |= mask;
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002586 amd64_write_pci_cfg(F3, NBCTL, value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002587
Borislav Petkova97fa682010-12-23 14:07:18 +01002588 amd64_read_pci_cfg(F3, NBCFG, &value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002589
Joe Perches956b9ba2012-04-29 17:08:39 -03002590 edac_dbg(0, "1: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2591 nid, value, !!(value & NBCFG_ECC_ENABLE));
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002592
Borislav Petkova97fa682010-12-23 14:07:18 +01002593 if (!(value & NBCFG_ECC_ENABLE)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002594 amd64_warn("DRAM ECC disabled on this node, enabling...\n");
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002595
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002596 s->flags.nb_ecc_prev = 0;
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002597
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002598 /* Attempt to turn on DRAM ECC Enable */
Borislav Petkova97fa682010-12-23 14:07:18 +01002599 value |= NBCFG_ECC_ENABLE;
2600 amd64_write_pci_cfg(F3, NBCFG, value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002601
Borislav Petkova97fa682010-12-23 14:07:18 +01002602 amd64_read_pci_cfg(F3, NBCFG, &value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002603
Borislav Petkova97fa682010-12-23 14:07:18 +01002604 if (!(value & NBCFG_ECC_ENABLE)) {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002605 amd64_warn("Hardware rejected DRAM ECC enable,"
2606 "check memory DIMM configuration.\n");
Borislav Petkov2299ef72010-10-15 17:44:04 +02002607 ret = false;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002608 } else {
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002609 amd64_info("Hardware accepted DRAM ECC Enable\n");
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002610 }
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002611 } else {
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002612 s->flags.nb_ecc_prev = 1;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002613 }
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002614
Joe Perches956b9ba2012-04-29 17:08:39 -03002615 edac_dbg(0, "2: node %d, NBCFG=0x%08x[DramEccEn: %d]\n",
2616 nid, value, !!(value & NBCFG_ECC_ENABLE));
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002617
Borislav Petkov2299ef72010-10-15 17:44:04 +02002618 return ret;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002619}
2620
Daniel J Bluemanc7e53012012-11-30 16:44:20 +08002621static void restore_ecc_error_reporting(struct ecc_settings *s, u16 nid,
Borislav Petkov360b7f32010-10-15 19:25:38 +02002622 struct pci_dev *F3)
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002623{
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002624 u32 value, mask = 0x3; /* UECC/CECC enable */
2625
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002626
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002627 if (!s->nbctl_valid)
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002628 return;
2629
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002630 amd64_read_pci_cfg(F3, NBCTL, &value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002631 value &= ~mask;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002632 value |= s->old_nbctl;
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002633
Borislav Petkovc9f4f262010-12-22 19:48:20 +01002634 amd64_write_pci_cfg(F3, NBCTL, value);
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002635
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002636 /* restore previous BIOS DRAM ECC "off" setting we force-enabled */
2637 if (!s->flags.nb_ecc_prev) {
Borislav Petkova97fa682010-12-23 14:07:18 +01002638 amd64_read_pci_cfg(F3, NBCFG, &value);
2639 value &= ~NBCFG_ECC_ENABLE;
2640 amd64_write_pci_cfg(F3, NBCFG, value);
Borislav Petkovd95cf4d2010-02-24 14:49:47 +01002641 }
2642
2643 /* restore the NB Enable MCGCTL bit */
Borislav Petkov2299ef72010-10-15 17:44:04 +02002644 if (toggle_ecc_err_reporting(s, nid, OFF))
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002645 amd64_warn("Error restoring NB MCGCTL settings!\n");
Borislav Petkovf6d6ae92009-11-03 15:29:26 +01002646}
2647
Doug Thompsonf9431992009-04-27 19:46:08 +02002648/*
Borislav Petkov2299ef72010-10-15 17:44:04 +02002649 * EDAC requires that the BIOS have ECC enabled before
2650 * taking over the processing of ECC errors. A command line
2651 * option allows to force-enable hardware ECC later in
2652 * enable_ecc_error_reporting().
Doug Thompsonf9431992009-04-27 19:46:08 +02002653 */
Borislav Petkovcab4d272010-02-11 17:15:57 +01002654static const char *ecc_msg =
2655 "ECC disabled in the BIOS or no ECC capability, module will not load.\n"
2656 " Either enable ECC checking or force module loading by setting "
2657 "'ecc_enable_override'.\n"
2658 " (Note that use of the override may cause unknown side effects.)\n";
Borislav Petkovbe3468e2009-08-05 15:47:22 +02002659
Daniel J Bluemanc7e53012012-11-30 16:44:20 +08002660static bool ecc_enabled(struct pci_dev *F3, u16 nid)
Doug Thompsonf9431992009-04-27 19:46:08 +02002661{
2662 u32 value;
Borislav Petkov2299ef72010-10-15 17:44:04 +02002663 u8 ecc_en = 0;
Borislav Petkov06724532009-09-16 13:05:46 +02002664 bool nb_mce_en = false;
Doug Thompsonf9431992009-04-27 19:46:08 +02002665
Borislav Petkova97fa682010-12-23 14:07:18 +01002666 amd64_read_pci_cfg(F3, NBCFG, &value);
Doug Thompsonf9431992009-04-27 19:46:08 +02002667
Borislav Petkova97fa682010-12-23 14:07:18 +01002668 ecc_en = !!(value & NBCFG_ECC_ENABLE);
Borislav Petkov2299ef72010-10-15 17:44:04 +02002669 amd64_info("DRAM ECC %s.\n", (ecc_en ? "enabled" : "disabled"));
Doug Thompsonf9431992009-04-27 19:46:08 +02002670
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01002671 nb_mce_en = nb_mce_bank_enabled_on_node(nid);
Borislav Petkov06724532009-09-16 13:05:46 +02002672 if (!nb_mce_en)
Borislav Petkov2299ef72010-10-15 17:44:04 +02002673 amd64_notice("NB MCE bank disabled, set MSR "
2674 "0x%08x[4] on node %d to enable.\n",
2675 MSR_IA32_MCG_CTL, nid);
Doug Thompsonf9431992009-04-27 19:46:08 +02002676
Borislav Petkov2299ef72010-10-15 17:44:04 +02002677 if (!ecc_en || !nb_mce_en) {
2678 amd64_notice("%s", ecc_msg);
2679 return false;
Borislav Petkov43f5e682009-12-21 18:55:18 +01002680 }
Borislav Petkov2299ef72010-10-15 17:44:04 +02002681 return true;
Doug Thompsonf9431992009-04-27 19:46:08 +02002682}
2683
Borislav Petkovdf71a052011-01-19 18:15:10 +01002684static void setup_mci_misc_attrs(struct mem_ctl_info *mci,
2685 struct amd64_family_type *fam)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002686{
2687 struct amd64_pvt *pvt = mci->pvt_info;
2688
2689 mci->mtype_cap = MEM_FLAG_DDR2 | MEM_FLAG_RDDR2;
2690 mci->edac_ctl_cap = EDAC_FLAG_NONE;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002691
Borislav Petkov5980bb92011-01-07 16:26:49 +01002692 if (pvt->nbcap & NBCAP_SECDED)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002693 mci->edac_ctl_cap |= EDAC_FLAG_SECDED;
2694
Borislav Petkov5980bb92011-01-07 16:26:49 +01002695 if (pvt->nbcap & NBCAP_CHIPKILL)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002696 mci->edac_ctl_cap |= EDAC_FLAG_S4ECD4ED;
2697
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01002698 mci->edac_cap = determine_edac_cap(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002699 mci->mod_name = EDAC_MOD_STR;
2700 mci->mod_ver = EDAC_AMD64_VERSION;
Borislav Petkovdf71a052011-01-19 18:15:10 +01002701 mci->ctl_name = fam->ctl_name;
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002702 mci->dev_name = pci_name(pvt->F2);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002703 mci->ctl_page_to_phys = NULL;
2704
Doug Thompson7d6034d2009-04-27 20:01:01 +02002705 /* memory scrubber interface */
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01002706 mci->set_sdram_scrub_rate = set_scrub_rate;
2707 mci->get_sdram_scrub_rate = get_scrub_rate;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002708}
2709
Borislav Petkov0092b202010-10-01 19:20:05 +02002710/*
2711 * returns a pointer to the family descriptor on success, NULL otherwise.
2712 */
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01002713static struct amd64_family_type *per_family_init(struct amd64_pvt *pvt)
Borislav Petkov395ae782010-10-01 18:38:19 +02002714{
Borislav Petkov0092b202010-10-01 19:20:05 +02002715 struct amd64_family_type *fam_type = NULL;
2716
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05002717 pvt->ext_model = boot_cpu_data.x86_model >> 4;
Borislav Petkova4b4bed2013-08-10 13:54:48 +02002718 pvt->stepping = boot_cpu_data.x86_mask;
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05002719 pvt->model = boot_cpu_data.x86_model;
2720 pvt->fam = boot_cpu_data.x86;
2721
2722 switch (pvt->fam) {
Borislav Petkov395ae782010-10-01 18:38:19 +02002723 case 0xf:
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01002724 fam_type = &family_types[K8_CPUS];
2725 pvt->ops = &family_types[K8_CPUS].ops;
Borislav Petkov395ae782010-10-01 18:38:19 +02002726 break;
Borislav Petkovdf71a052011-01-19 18:15:10 +01002727
Borislav Petkov395ae782010-10-01 18:38:19 +02002728 case 0x10:
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01002729 fam_type = &family_types[F10_CPUS];
2730 pvt->ops = &family_types[F10_CPUS].ops;
Borislav Petkovdf71a052011-01-19 18:15:10 +01002731 break;
2732
2733 case 0x15:
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05002734 if (pvt->model == 0x30) {
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01002735 fam_type = &family_types[F15_M30H_CPUS];
2736 pvt->ops = &family_types[F15_M30H_CPUS].ops;
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05002737 break;
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01002738 } else if (pvt->model == 0x60) {
2739 fam_type = &family_types[F15_M60H_CPUS];
2740 pvt->ops = &family_types[F15_M60H_CPUS].ops;
2741 break;
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05002742 }
2743
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01002744 fam_type = &family_types[F15_CPUS];
2745 pvt->ops = &family_types[F15_CPUS].ops;
Borislav Petkov395ae782010-10-01 18:38:19 +02002746 break;
2747
Aravind Gopalakrishnan94c1acf2013-04-17 14:57:13 -05002748 case 0x16:
Aravind Gopalakrishnan85a88852014-02-20 10:28:46 -06002749 if (pvt->model == 0x30) {
2750 fam_type = &family_types[F16_M30H_CPUS];
2751 pvt->ops = &family_types[F16_M30H_CPUS].ops;
2752 break;
2753 }
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01002754 fam_type = &family_types[F16_CPUS];
2755 pvt->ops = &family_types[F16_CPUS].ops;
Aravind Gopalakrishnan94c1acf2013-04-17 14:57:13 -05002756 break;
2757
Borislav Petkov395ae782010-10-01 18:38:19 +02002758 default:
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002759 amd64_err("Unsupported family!\n");
Borislav Petkov0092b202010-10-01 19:20:05 +02002760 return NULL;
Borislav Petkov395ae782010-10-01 18:38:19 +02002761 }
Borislav Petkov0092b202010-10-01 19:20:05 +02002762
Borislav Petkovdf71a052011-01-19 18:15:10 +01002763 amd64_info("%s %sdetected (node %d).\n", fam_type->ctl_name,
Aravind Gopalakrishnan18b94f62013-08-09 11:54:49 -05002764 (pvt->fam == 0xf ?
Borislav Petkov24f9a7f2010-10-07 18:29:15 +02002765 (pvt->ext_model >= K8_REV_F ? "revF or later "
2766 : "revE or earlier ")
2767 : ""), pvt->mc_node_id);
Borislav Petkov0092b202010-10-01 19:20:05 +02002768 return fam_type;
Borislav Petkov395ae782010-10-01 18:38:19 +02002769}
2770
Takashi Iwaie339f1e2015-02-04 11:48:53 +01002771static const struct attribute_group *amd64_edac_attr_groups[] = {
2772#ifdef CONFIG_EDAC_DEBUG
2773 &amd64_edac_dbg_group,
2774#endif
2775#ifdef CONFIG_EDAC_AMD64_ERROR_INJECTION
2776 &amd64_edac_inj_group,
2777#endif
2778 NULL
2779};
2780
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01002781static int init_one_instance(struct pci_dev *F2)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002782{
2783 struct amd64_pvt *pvt = NULL;
Borislav Petkov0092b202010-10-01 19:20:05 +02002784 struct amd64_family_type *fam_type = NULL;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002785 struct mem_ctl_info *mci = NULL;
Mauro Carvalho Chehabab5a5032012-04-16 15:03:50 -03002786 struct edac_mc_layer layers[2];
Doug Thompson7d6034d2009-04-27 20:01:01 +02002787 int err = 0, ret;
Aravind Gopalakrishnan1a6775c2015-10-19 11:17:42 +02002788 u16 nid = amd_pci_dev_to_node_id(F2);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002789
2790 ret = -ENOMEM;
2791 pvt = kzalloc(sizeof(struct amd64_pvt), GFP_KERNEL);
2792 if (!pvt)
Borislav Petkov360b7f32010-10-15 19:25:38 +02002793 goto err_ret;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002794
Borislav Petkov360b7f32010-10-15 19:25:38 +02002795 pvt->mc_node_id = nid;
Borislav Petkov8d5b5d92010-10-01 20:11:07 +02002796 pvt->F2 = F2;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002797
Borislav Petkov395ae782010-10-01 18:38:19 +02002798 ret = -EINVAL;
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01002799 fam_type = per_family_init(pvt);
Borislav Petkov0092b202010-10-01 19:20:05 +02002800 if (!fam_type)
Borislav Petkov395ae782010-10-01 18:38:19 +02002801 goto err_free;
2802
Doug Thompson7d6034d2009-04-27 20:01:01 +02002803 ret = -ENODEV;
Borislav Petkov360b7f32010-10-15 19:25:38 +02002804 err = reserve_mc_sibling_devs(pvt, fam_type->f1_id, fam_type->f3_id);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002805 if (err)
2806 goto err_free;
2807
Borislav Petkov360b7f32010-10-15 19:25:38 +02002808 read_mc_regs(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002809
Doug Thompson7d6034d2009-04-27 20:01:01 +02002810 /*
2811 * We need to determine how many memory channels there are. Then use
2812 * that information for calculating the size of the dynamic instance
Borislav Petkov360b7f32010-10-15 19:25:38 +02002813 * tables in the 'mci' structure.
Doug Thompson7d6034d2009-04-27 20:01:01 +02002814 */
Borislav Petkov360b7f32010-10-15 19:25:38 +02002815 ret = -EINVAL;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002816 pvt->channel_count = pvt->ops->early_channel_count(pvt);
2817 if (pvt->channel_count < 0)
Borislav Petkov360b7f32010-10-15 19:25:38 +02002818 goto err_siblings;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002819
2820 ret = -ENOMEM;
Mauro Carvalho Chehabab5a5032012-04-16 15:03:50 -03002821 layers[0].type = EDAC_MC_LAYER_CHIP_SELECT;
2822 layers[0].size = pvt->csels[0].b_cnt;
2823 layers[0].is_virt_csrow = true;
2824 layers[1].type = EDAC_MC_LAYER_CHANNEL;
Borislav Petkovf0a56c42013-07-23 20:01:23 +02002825
2826 /*
2827 * Always allocate two channels since we can have setups with DIMMs on
2828 * only one channel. Also, this simplifies handling later for the price
2829 * of a couple of KBs tops.
2830 */
2831 layers[1].size = 2;
Mauro Carvalho Chehabab5a5032012-04-16 15:03:50 -03002832 layers[1].is_virt_csrow = false;
Borislav Petkovf0a56c42013-07-23 20:01:23 +02002833
Mauro Carvalho Chehabca0907b2012-05-02 14:37:00 -03002834 mci = edac_mc_alloc(nid, ARRAY_SIZE(layers), layers, 0);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002835 if (!mci)
Borislav Petkov360b7f32010-10-15 19:25:38 +02002836 goto err_siblings;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002837
2838 mci->pvt_info = pvt;
Mauro Carvalho Chehabfd687502012-03-16 07:44:18 -03002839 mci->pdev = &pvt->F2->dev;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002840
Borislav Petkovdf71a052011-01-19 18:15:10 +01002841 setup_mci_misc_attrs(mci, fam_type);
Borislav Petkov360b7f32010-10-15 19:25:38 +02002842
2843 if (init_csrows(mci))
Doug Thompson7d6034d2009-04-27 20:01:01 +02002844 mci->edac_cap = EDAC_FLAG_NONE;
2845
Doug Thompson7d6034d2009-04-27 20:01:01 +02002846 ret = -ENODEV;
Takashi Iwaie339f1e2015-02-04 11:48:53 +01002847 if (edac_mc_add_mc_with_groups(mci, amd64_edac_attr_groups)) {
Joe Perches956b9ba2012-04-29 17:08:39 -03002848 edac_dbg(1, "failed edac_mc_add_mc()\n");
Doug Thompson7d6034d2009-04-27 20:01:01 +02002849 goto err_add_mc;
2850 }
2851
Borislav Petkov549d0422009-07-24 13:51:42 +02002852 /* register stuff with EDAC MCE */
2853 if (report_gart_errors)
2854 amd_report_gart_errors(true);
2855
Borislav Petkovdf781d02013-12-15 17:29:44 +01002856 amd_register_ecc_decoder(decode_bus_error);
Borislav Petkov549d0422009-07-24 13:51:42 +02002857
Borislav Petkov360b7f32010-10-15 19:25:38 +02002858 atomic_inc(&drv_instances);
2859
Doug Thompson7d6034d2009-04-27 20:01:01 +02002860 return 0;
2861
2862err_add_mc:
2863 edac_mc_free(mci);
2864
Borislav Petkov360b7f32010-10-15 19:25:38 +02002865err_siblings:
2866 free_mc_sibling_devs(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002867
Borislav Petkov360b7f32010-10-15 19:25:38 +02002868err_free:
2869 kfree(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002870
Borislav Petkov360b7f32010-10-15 19:25:38 +02002871err_ret:
Doug Thompson7d6034d2009-04-27 20:01:01 +02002872 return ret;
2873}
2874
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01002875static int probe_one_instance(struct pci_dev *pdev,
2876 const struct pci_device_id *mc_type)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002877{
Aravind Gopalakrishnan1a6775c2015-10-19 11:17:42 +02002878 u16 nid = amd_pci_dev_to_node_id(pdev);
Borislav Petkov2299ef72010-10-15 17:44:04 +02002879 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002880 struct ecc_settings *s;
Borislav Petkov2299ef72010-10-15 17:44:04 +02002881 int ret = 0;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002882
Doug Thompson7d6034d2009-04-27 20:01:01 +02002883 ret = pci_enable_device(pdev);
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002884 if (ret < 0) {
Joe Perches956b9ba2012-04-29 17:08:39 -03002885 edac_dbg(0, "ret=%d\n", ret);
Borislav Petkovb8cfa022010-10-01 19:35:38 +02002886 return -EIO;
2887 }
2888
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002889 ret = -ENOMEM;
2890 s = kzalloc(sizeof(struct ecc_settings), GFP_KERNEL);
2891 if (!s)
Borislav Petkov2299ef72010-10-15 17:44:04 +02002892 goto err_out;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002893
2894 ecc_stngs[nid] = s;
2895
Borislav Petkov2299ef72010-10-15 17:44:04 +02002896 if (!ecc_enabled(F3, nid)) {
2897 ret = -ENODEV;
2898
2899 if (!ecc_enable_override)
2900 goto err_enable;
2901
2902 amd64_warn("Forcing ECC on!\n");
2903
2904 if (!enable_ecc_error_reporting(s, nid, F3))
2905 goto err_enable;
2906 }
2907
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01002908 ret = init_one_instance(pdev);
Borislav Petkov360b7f32010-10-15 19:25:38 +02002909 if (ret < 0) {
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002910 amd64_err("Error probing instance: %d\n", nid);
Borislav Petkov360b7f32010-10-15 19:25:38 +02002911 restore_ecc_error_reporting(s, nid, F3);
2912 }
Doug Thompson7d6034d2009-04-27 20:01:01 +02002913
2914 return ret;
Borislav Petkov2299ef72010-10-15 17:44:04 +02002915
2916err_enable:
2917 kfree(s);
2918 ecc_stngs[nid] = NULL;
2919
2920err_out:
2921 return ret;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002922}
2923
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01002924static void remove_one_instance(struct pci_dev *pdev)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002925{
2926 struct mem_ctl_info *mci;
2927 struct amd64_pvt *pvt;
Aravind Gopalakrishnan1a6775c2015-10-19 11:17:42 +02002928 u16 nid = amd_pci_dev_to_node_id(pdev);
Borislav Petkov360b7f32010-10-15 19:25:38 +02002929 struct pci_dev *F3 = node_to_amd_nb(nid)->misc;
2930 struct ecc_settings *s = ecc_stngs[nid];
Doug Thompson7d6034d2009-04-27 20:01:01 +02002931
Mauro Carvalho Chehabc5608752012-03-21 14:00:44 -03002932 mci = find_mci_by_dev(&pdev->dev);
Borislav Petkova4b4bed2013-08-10 13:54:48 +02002933 WARN_ON(!mci);
2934
Doug Thompson7d6034d2009-04-27 20:01:01 +02002935 /* Remove from EDAC CORE tracking list */
2936 mci = edac_mc_del_mc(&pdev->dev);
2937 if (!mci)
2938 return;
2939
2940 pvt = mci->pvt_info;
2941
Borislav Petkov360b7f32010-10-15 19:25:38 +02002942 restore_ecc_error_reporting(s, nid, F3);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002943
Borislav Petkov360b7f32010-10-15 19:25:38 +02002944 free_mc_sibling_devs(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002945
Borislav Petkov549d0422009-07-24 13:51:42 +02002946 /* unregister from EDAC MCE */
2947 amd_report_gart_errors(false);
Borislav Petkovdf781d02013-12-15 17:29:44 +01002948 amd_unregister_ecc_decoder(decode_bus_error);
Borislav Petkov549d0422009-07-24 13:51:42 +02002949
Borislav Petkov360b7f32010-10-15 19:25:38 +02002950 kfree(ecc_stngs[nid]);
2951 ecc_stngs[nid] = NULL;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02002952
Doug Thompson7d6034d2009-04-27 20:01:01 +02002953 /* Free the EDAC CORE resources */
Borislav Petkov8f68ed92009-12-21 15:15:59 +01002954 mci->pvt_info = NULL;
Borislav Petkov8f68ed92009-12-21 15:15:59 +01002955
2956 kfree(pvt);
Doug Thompson7d6034d2009-04-27 20:01:01 +02002957 edac_mc_free(mci);
2958}
2959
2960/*
2961 * This table is part of the interface for loading drivers for PCI devices. The
2962 * PCI core identifies what devices are on a system during boot, and then
2963 * inquiry this table to see if this driver is for a given device found.
2964 */
Jingoo Hanba935f42013-12-06 10:23:08 +01002965static const struct pci_device_id amd64_pci_table[] = {
Aravind Gopalakrishnana597d2a2014-10-30 12:16:09 +01002966 { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_K8_NB_MEMCTL) },
2967 { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_10H_NB_DRAM) },
2968 { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_15H_NB_F2) },
2969 { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_15H_M30H_NB_F2) },
2970 { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_15H_M60H_NB_F2) },
2971 { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_16H_NB_F2) },
2972 { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_16H_M30H_NB_F2) },
Doug Thompson7d6034d2009-04-27 20:01:01 +02002973 {0, }
2974};
2975MODULE_DEVICE_TABLE(pci, amd64_pci_table);
2976
2977static struct pci_driver amd64_pci_driver = {
2978 .name = EDAC_MOD_STR,
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01002979 .probe = probe_one_instance,
2980 .remove = remove_one_instance,
Doug Thompson7d6034d2009-04-27 20:01:01 +02002981 .id_table = amd64_pci_table,
Luis R. Rodriguez735c0f82015-03-30 16:20:08 -07002982 .driver.probe_type = PROBE_FORCE_SYNCHRONOUS,
Doug Thompson7d6034d2009-04-27 20:01:01 +02002983};
2984
Borislav Petkov360b7f32010-10-15 19:25:38 +02002985static void setup_pci_device(void)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002986{
2987 struct mem_ctl_info *mci;
2988 struct amd64_pvt *pvt;
2989
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01002990 if (pci_ctl)
Doug Thompson7d6034d2009-04-27 20:01:01 +02002991 return;
2992
Borislav Petkov2ec591a2015-02-17 10:58:34 +01002993 mci = edac_mc_find(0);
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01002994 if (!mci)
2995 return;
Doug Thompson7d6034d2009-04-27 20:01:01 +02002996
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01002997 pvt = mci->pvt_info;
2998 pci_ctl = edac_pci_create_generic_ctl(&pvt->F2->dev, EDAC_MOD_STR);
2999 if (!pci_ctl) {
3000 pr_warn("%s(): Unable to create PCI control\n", __func__);
3001 pr_warn("%s(): PCI error report via EDAC not set\n", __func__);
Doug Thompson7d6034d2009-04-27 20:01:01 +02003002 }
3003}
3004
3005static int __init amd64_edac_init(void)
3006{
Borislav Petkov360b7f32010-10-15 19:25:38 +02003007 int err = -ENODEV;
Doug Thompson7d6034d2009-04-27 20:01:01 +02003008
Borislav Petkovdf71a052011-01-19 18:15:10 +01003009 printk(KERN_INFO "AMD64 EDAC driver v%s\n", EDAC_AMD64_VERSION);
Doug Thompson7d6034d2009-04-27 20:01:01 +02003010
3011 opstate_init();
3012
Hans Rosenfeld9653a5c2010-10-29 17:14:31 +02003013 if (amd_cache_northbridges() < 0)
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01003014 goto err_ret;
Doug Thompson7d6034d2009-04-27 20:01:01 +02003015
Borislav Petkovcc4d8862010-10-13 16:11:59 +02003016 err = -ENOMEM;
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02003017 ecc_stngs = kzalloc(amd_nb_num() * sizeof(ecc_stngs[0]), GFP_KERNEL);
Borislav Petkov2ec591a2015-02-17 10:58:34 +01003018 if (!ecc_stngs)
Borislav Petkova9f0fbe2011-03-29 18:10:53 +02003019 goto err_free;
Borislav Petkovcc4d8862010-10-13 16:11:59 +02003020
Borislav Petkov50542252009-12-11 18:14:40 +01003021 msrs = msrs_alloc();
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01003022 if (!msrs)
Borislav Petkov360b7f32010-10-15 19:25:38 +02003023 goto err_free;
Borislav Petkov50542252009-12-11 18:14:40 +01003024
Doug Thompson7d6034d2009-04-27 20:01:01 +02003025 err = pci_register_driver(&amd64_pci_driver);
3026 if (err)
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01003027 goto err_pci;
Doug Thompson7d6034d2009-04-27 20:01:01 +02003028
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01003029 err = -ENODEV;
Borislav Petkov360b7f32010-10-15 19:25:38 +02003030 if (!atomic_read(&drv_instances))
3031 goto err_no_instances;
Doug Thompson7d6034d2009-04-27 20:01:01 +02003032
Borislav Petkov360b7f32010-10-15 19:25:38 +02003033 setup_pci_device();
Tomasz Palaf5b10c42014-11-02 11:22:12 +01003034
3035#ifdef CONFIG_X86_32
3036 amd64_err("%s on 32-bit is unsupported. USE AT YOUR OWN RISK!\n", EDAC_MOD_STR);
3037#endif
3038
Borislav Petkov360b7f32010-10-15 19:25:38 +02003039 return 0;
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01003040
Borislav Petkov360b7f32010-10-15 19:25:38 +02003041err_no_instances:
Doug Thompson7d6034d2009-04-27 20:01:01 +02003042 pci_unregister_driver(&amd64_pci_driver);
Borislav Petkovcc4d8862010-10-13 16:11:59 +02003043
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01003044err_pci:
3045 msrs_free(msrs);
3046 msrs = NULL;
Borislav Petkovcc4d8862010-10-13 16:11:59 +02003047
Borislav Petkov360b7f32010-10-15 19:25:38 +02003048err_free:
Borislav Petkov360b7f32010-10-15 19:25:38 +02003049 kfree(ecc_stngs);
3050 ecc_stngs = NULL;
3051
Borislav Petkov56b34b91e2009-12-21 18:13:01 +01003052err_ret:
Doug Thompson7d6034d2009-04-27 20:01:01 +02003053 return err;
3054}
3055
3056static void __exit amd64_edac_exit(void)
3057{
Borislav Petkovd1ea71c2013-12-15 17:54:27 +01003058 if (pci_ctl)
3059 edac_pci_release_generic_ctl(pci_ctl);
Doug Thompson7d6034d2009-04-27 20:01:01 +02003060
3061 pci_unregister_driver(&amd64_pci_driver);
Borislav Petkov50542252009-12-11 18:14:40 +01003062
Borislav Petkovae7bb7c2010-10-14 16:01:30 +02003063 kfree(ecc_stngs);
3064 ecc_stngs = NULL;
3065
Borislav Petkov50542252009-12-11 18:14:40 +01003066 msrs_free(msrs);
3067 msrs = NULL;
Doug Thompson7d6034d2009-04-27 20:01:01 +02003068}
3069
3070module_init(amd64_edac_init);
3071module_exit(amd64_edac_exit);
3072
3073MODULE_LICENSE("GPL");
3074MODULE_AUTHOR("SoftwareBitMaker: Doug Thompson, "
3075 "Dave Peterson, Thayne Harbaugh");
3076MODULE_DESCRIPTION("MC support for AMD64 memory controllers - "
3077 EDAC_AMD64_VERSION);
3078
3079module_param(edac_op_state, int, 0444);
3080MODULE_PARM_DESC(edac_op_state, "EDAC Error Reporting state: 0=Poll,1=NMI");