From: AMEET M. PARANJAPE <aparanja@redhat.com> Date: Wed, 22 Oct 2008 13:29:24 -0500 Subject: [ppc64] EEH PCI-E: recovery fails E1000; support MSI Message-id: 48FF7104.8040109@REDHAT.COM O-Subject: Re: [PATCH 1/2 RHEL5.3 bz445299] EEH recovery fails E1000 PCI-E; support for EEH on PCI-E using MSI Bugzilla: 445299 RH-Acked-by: David Howells <dhowells@redhat.com> RH-Acked-by: David Howells <dhowells@redhat.com> RH-Acked-by: Pete Zaitcev <zaitcev@redhat.com> RH-Acked-by: David Howells <dhowells@redhat.com> RHBZ#: ====== https://bugzilla.redhat.com/show_bug.cgi?id=445299 Description: =========== EEH recovery fails after injects EEH error to 4 ports E1000 PCI-E card(coleman) on Power 6 machine The following is the shown message after EEH injection. [root@devl4e-shoal-lp1 ~]# errinjct ioa-bus-error -f 17 -s net/eth3 -m 0 Injecting an ioa-bus-error... Call to RTAS errinjct succeeded! The device then fails to recover, as can be seen here: # Call Trace: [C000000002A6FAC0] [C00000000000FFDC] .show_stack+0x68/0x1b0 (unreliable) [C000000002A6FB60] [C00000000004EFA4] .eeh_dn_check_failure+0x25c/0x29c [C000000002A6FC10] [C00000000004F1F0] .eeh_check_failure+0xdc/0x104 [C000000002A6FC90] [D00000000037ABF0] .e1000_watchdog_task+0xb8/0xb00 [e1000] [C000000002A6FD50] [C00000000007E66C] .run_workqueue+0xdc/0x168 [C000000002A6FDF0] [C00000000007F3D8] .worker_thread+0x128/0x198 [C000000002A6FEE0] [C000000000083EB8] .kthread+0x128/0x178 [C000000002A6FF90] [C000000000026A14] .kernel_thread+0x4c/0x68 EEH: This PCI device has failed 1 times since last reboot: location=U789D.001.DQDMLZZ-P1-C3 driver=e1000 pci addr=0002:03:00.1 e1000: eth3: e1000_reset: Hardware Error Badness in disable_msi_mode at arch/powerpc/kernel/msi.c:749 This problem prevents e1000e devices using MSI on powerpc from automatically recovering after a PCI error is detected. Automatic I/O error recovery is a vital feature in customer environments where availability is critical. RHEL Version Found: ================ RHEL 5.1 kABI Status: ============ Will test once brew is back up Brew: ===== Will test once brew is back up Upstream Status: ================ The commit to mainline appears here: http://git.kernel.org/gitweb.cgi?p=linux/kernel/git/torvalds/linux-2.6.git;a=commitdiff;h=b56a5a23bfecd9cac9187164a9d5f22d287c48b9. I kept these in the core eeh code to avoid touching the pci subsystem directly. The commit to export pci_restore_msi_state appears here: http://git.kernel.org/gitweb.cgi?p=linux/kernel/git/torvalds/linux-2.6.git;a=commit;h=94688cf2454986309fbcd495233ba2423786a14a The commit to increase EEH_MAX_FAILS appears here: http://git.kernel.org/gitweb.cgi?p=linux/kernel/git/torvalds/linux-2.6.git;a=commitdiff;h=2fd30be8dae25386fc5167c34c6d73201334a8d4 A commit similar to what's posted here for e1000e is here: http://git.kernel.org/gitweb.cgi?p=linux/kernel/git/torvalds/linux-2.6.git;a=commitdiff;h=aad32739641d3a75818fbe653d4b0d530e965f2f Test Status: ============ Was thoroughly tested on the RH 2.6.18-88 kernel in 5.2, but awaiting brew to confirm on 5.3 diff --git a/arch/powerpc/kernel/msi-rtas.c b/arch/powerpc/kernel/msi-rtas.c index e2d952b..6229b0e 100644 --- a/arch/powerpc/kernel/msi-rtas.c +++ b/arch/powerpc/kernel/msi-rtas.c @@ -165,6 +165,7 @@ static int rtas_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) struct pci_dn *pdn; int hwirq, virq, i, rc; struct msi_desc *entry; + struct msi_msg msg; pdn = get_pdn(pdev); if (!pdn) @@ -214,6 +215,11 @@ static int rtas_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) dev_dbg(&pdev->dev, "rtas_msi: allocated virq %d\n", virq); set_irq_msi(virq, entry); + + /* Read config space back so we can restore after reset*/ + read_msi_msg(virq, &msg); + entry->msg = msg; + unmask_msi_irq(virq); } diff --git a/arch/powerpc/kernel/msi.c b/arch/powerpc/kernel/msi.c index 4e5a3e9..e543d3b 100644 --- a/arch/powerpc/kernel/msi.c +++ b/arch/powerpc/kernel/msi.c @@ -262,7 +262,6 @@ static struct msi_desc* alloc_msi_entry(void) return entry; } -#ifdef CONFIG_PM int pci_save_msi_state(struct pci_dev *dev) { return 0; @@ -325,7 +324,7 @@ void pci_restore_msix_state(struct pci_dev *dev) control |= PCI_MSIX_FLAGS_ENABLE; pci_write_config_word(dev, pos + PCI_MSIX_FLAGS, control); } -#endif /* CONFIG_PM */ +EXPORT_SYMBOL_GPL(pci_restore_msi_state); /** * msi_capability_init - configure device's MSI capability structure diff --git a/arch/powerpc/platforms/pseries/eeh.c b/arch/powerpc/platforms/pseries/eeh.c index 34b87a6..778d2b5 100644 --- a/arch/powerpc/platforms/pseries/eeh.c +++ b/arch/powerpc/platforms/pseries/eeh.c @@ -128,6 +128,59 @@ static unsigned long slot_resets; /* --------------------------------------------------------------- */ /* Below lies the EEH event infrastructure */ +int save_pcie_reg(struct pci_dev *dev) +{ + int pos, i = 0; + struct pci_cap_saved_state *save_state; + u16 *cap; + + pos = pci_find_capability(dev, PCI_CAP_ID_EXP); + if (pos <= 0) + return 0; + + save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP); + if (save_state) + return 0; + + save_state = kzalloc(sizeof(*save_state) + sizeof(u16) * 4, GFP_KERNEL); + if (!save_state) { + dev_err(&dev->dev, "Out of memory in save_pcie_reg\n"); + return -ENOMEM; + } + + cap = (u16 *)&save_state->data[0]; + + pci_read_config_word(dev, pos + PCI_EXP_DEVCTL, &cap[i++]); + pci_read_config_word(dev, pos + PCI_EXP_LNKCTL, &cap[i++]); + pci_read_config_word(dev, pos + PCI_EXP_SLTCTL, &cap[i++]); + pci_read_config_word(dev, pos + PCI_EXP_RTCTL, &cap[i++]); + save_state->cap_nr = PCI_CAP_ID_EXP; + pci_add_saved_cap(dev, save_state); + + return 0; +} + +void restore_pcie_reg(struct pci_dev *dev) +{ + int i = 0, pos; + struct pci_cap_saved_state *save_state; + u16 *cap; + + save_state = pci_find_saved_cap(dev, PCI_CAP_ID_EXP); + pos = pci_find_capability(dev, PCI_CAP_ID_EXP); + if (!save_state || pos <= 0) + return; + cap = (u16 *)&save_state->data[0]; + + pci_write_config_word(dev, pos + PCI_EXP_DEVCTL, cap[i++]); + pci_write_config_word(dev, pos + PCI_EXP_LNKCTL, cap[i++]); + pci_write_config_word(dev, pos + PCI_EXP_SLTCTL, cap[i++]); + pci_write_config_word(dev, pos + PCI_EXP_RTCTL, cap[i++]); +} + +EXPORT_SYMBOL_GPL(save_pcie_reg); +EXPORT_SYMBOL_GPL(restore_pcie_reg); + static void rtas_slot_error_detail(struct pci_dn *pdn, int severity, char *driver_log, size_t loglen) { diff --git a/drivers/pci/msi.c b/drivers/pci/msi.c index 2a63a74..96bec55 100644 --- a/drivers/pci/msi.c +++ b/drivers/pci/msi.c @@ -574,6 +574,7 @@ void pci_restore_msi_state(struct pci_dev *dev) pci_remove_saved_cap(save_state); kfree(save_state); } +EXPORT_SYMBOL_GPL(pci_restore_msi_state); int pci_save_msix_state(struct pci_dev *dev) { diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index 9550591..61cb569 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -41,26 +41,22 @@ extern void pci_remove_legacy_files(struct pci_bus *bus); /* Lock for read/write access to pci device and bus lists */ extern struct rw_semaphore pci_bus_sem; - extern unsigned int pci_pm_d3_delay; + #ifdef CONFIG_PCI_MSI void disable_msi_mode(struct pci_dev *dev, int pos, int type); void pci_no_msi(void); -#else -static inline void disable_msi_mode(struct pci_dev *dev, int pos, int type) { } -static inline void pci_no_msi(void) { } -#endif -#if defined(CONFIG_PCI_MSI) && defined(CONFIG_PM) int pci_save_msi_state(struct pci_dev *dev); int pci_save_msix_state(struct pci_dev *dev); -void pci_restore_msi_state(struct pci_dev *dev); void pci_restore_msix_state(struct pci_dev *dev); #else +static inline void disable_msi_mode(struct pci_dev *dev, int pos, int type) { } +static inline void pci_no_msi(void) { } static inline int pci_save_msi_state(struct pci_dev *dev) { return 0; } static inline int pci_save_msix_state(struct pci_dev *dev) { return 0; } -static inline void pci_restore_msi_state(struct pci_dev *dev) {} static inline void pci_restore_msix_state(struct pci_dev *dev) {} #endif + static inline int pci_no_d1d2(struct pci_dev *dev) { unsigned int parent_dstates = 0; diff --git a/include/linux/pci.h b/include/linux/pci.h index 68ac37b..42d636a 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -657,6 +657,7 @@ static inline int pci_enable_msix(struct pci_dev* dev, struct msix_entry *entries, int nvec) {return -1;} static inline void pci_disable_msix(struct pci_dev *dev) {} static inline void msi_remove_pci_irq_vectors(struct pci_dev *dev) {} +static inline void pci_restore_msi_state(struct pci_dev *dev) {} #else extern void pci_scan_msi_device(struct pci_dev *dev); extern int pci_enable_msi(struct pci_dev *dev); @@ -665,6 +666,15 @@ extern int pci_enable_msix(struct pci_dev* dev, struct msix_entry *entries, int nvec); extern void pci_disable_msix(struct pci_dev *dev); extern void msi_remove_pci_irq_vectors(struct pci_dev *dev); +extern void pci_restore_msi_state(struct pci_dev *dev); +#endif + +#ifndef CONFIG_EEH +static inline int save_pcie_reg(struct pci_dev *dev) {return -1;} +static inline void restore_pcie_reg(struct pci_dev *dev) {} +#else +extern int save_pcie_reg(struct pci_dev *dev); +extern void restore_pcie_reg(struct pci_dev *dev); #endif extern void pci_block_user_cfg_access(struct pci_dev *dev);