From 3b5fac37e4de4e6ba3a43f4abfa015b49b0b8ff3 Mon Sep 17 00:00:00 2001 From: Huacai Chen Date: Wed, 1 Feb 2023 12:30:17 +0800 Subject: [PATCH] PCI/portdrv: Prevent LS7A Bus Master clearing on shutdown stable inclusion from stable-v6.3-rc1 commit 62b6dee1b44aa23b3935543aff7df80399ec726b category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I7BC4C Reference: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/commit/drivers/pci/pcie/portdrv.c?h=v6.3.6&id=62b6dee1b44aa23b3935543aff7df80399ec726b -------------------------------- After cc27b735ad3a ("PCI/portdrv: Turn off PCIe services during shutdown") we observe hangs during poweroff/reboot on systems with LS7A chipset. This happens because the portdrv .shutdown() method (pcie_portdrv_remove()) clears PCI_COMMAND_MASTER via pci_disable_device(), which prevents bridges from forwarding memory or I/O Requests in the upstream direction (PCIe r6.0, sec 7.5.1.1.3). LS7A Root Ports have a hardware defect: clearing PCI_COMMAND_MASTER *also* prevents the bridge from forwarding CPU MMIO requests in the downstream direction, and these MMIO accesses to devices below the bridge happen even after .shutdown(), e.g., to print console messages. LS7A neither forwards the requests nor sends an unsuccessful completion to the CPU, so the CPU waits forever, resulting in the hang. The purpose of .shutdown() is to disable interrupts and DMA from the device. PCIe ports may generate interrupts (either MSI/MSI-X or INTx) for AER, DPC, PME, hotplug, etc., but they never perform DMA except MSI/MSI-X. Clearing PCI_COMMAND_MASTER effectively disables MSI/MSI-X, but not INTx. The port service driver .remove() methods clear the interrupt enables in PCI_ERR_ROOT_COMMAND, PCI_EXP_DPC_CTL, PCI_EXP_SLTCTL, and PCI_EXP_RTCTL, etc., which disables interrupts regardless of whether they are MSI/MSI-X or INTx. Add a pcie_portdrv_shutdown() method that calls all the port service driver .remove() methods to clear the interrupt enables for each service but does not clear Bus Mastering on the port itself. [bhelgaas: commit log] Link: https://lore.kernel.org/r/20230201043018.778499-2-chenhuacai@loongson.cn Signed-off-by: Huacai Chen Signed-off-by: Bjorn Helgaas Signed-off-by: Hongchen Zhang (cherry picked from commit cb7f300fcc9eb75612680212d2bb241ff13ca82a) --- drivers/pci/pcie/portdrv_core.c | 1 - drivers/pci/pcie/portdrv_pci.c | 15 ++++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/drivers/pci/pcie/portdrv_core.c b/drivers/pci/pcie/portdrv_core.c index 3028307a9b14..c0a49fa20866 100644 --- a/drivers/pci/pcie/portdrv_core.c +++ b/drivers/pci/pcie/portdrv_core.c @@ -486,7 +486,6 @@ void pcie_port_device_remove(struct pci_dev *dev) { device_for_each_child(&dev->dev, NULL, remove_iter); pci_free_irq_vectors(dev); - pci_disable_device(dev); } /** diff --git a/drivers/pci/pcie/portdrv_pci.c b/drivers/pci/pcie/portdrv_pci.c index 8bd4992a4f32..8d282cdcb5f8 100644 --- a/drivers/pci/pcie/portdrv_pci.c +++ b/drivers/pci/pcie/portdrv_pci.c @@ -148,6 +148,19 @@ static void pcie_portdrv_remove(struct pci_dev *dev) } pcie_port_device_remove(dev); + + pci_disable_device(dev); +} + +static void pcie_portdrv_shutdown(struct pci_dev *dev) +{ + if (pci_bridge_d3_possible(dev)) { + pm_runtime_forbid(&dev->dev); + pm_runtime_get_noresume(&dev->dev); + pm_runtime_dont_use_autosuspend(&dev->dev); + } + + pcie_port_device_remove(dev); } static pci_ers_result_t pcie_portdrv_error_detected(struct pci_dev *dev, @@ -218,7 +231,7 @@ static struct pci_driver pcie_portdriver = { .probe = pcie_portdrv_probe, .remove = pcie_portdrv_remove, - .shutdown = pcie_portdrv_remove, + .shutdown = pcie_portdrv_shutdown, .err_handler = &pcie_portdrv_err_handler, -- GitLab