s390/pci: fix PF/VF linking on hot plug

Currently there are four places in which a PCI function is scanned
and made available to drivers:
 1. In pci_scan_root_bus() as part of the initial zbus
    creation.
 2. In zpci_bus_add_devices() when registering
    a device in configured state on a zbus that has already been
    scanned.
 3. When a function is already known to zPCI (in reserved/standby state)
    and configuration is triggered through firmware by PEC 0x301.
 4. When a device is already known to zPCI (in standby/reserved state)
    and configuration is triggered from within Linux using
    enable_slot().

The PF/VF linking step and setting of pdev->is_virtfn introduced with
commit e5794cf ("s390/pci: create links between PFs and VFs") was
only triggered for the second case, which is where VFs created through
sriov_numvfs usually land. However unlike some other platforms but like
POWER VFs can be individually enabled/disabled through
/sys/bus/pci/slots.

Fix this by doing VF setup as part of pcibios_bus_add_device() which is
called in all of the above cases.

Finally to remove the PF/VF links call the common code
pci_iov_remove_virtfn() function to remove linked VFs.
This takes care of the necessary sysfs cleanup.

Fixes: e5794cf ("s390/pci: create links between PFs and VFs")
Cc: <stable@vger.kernel.org> # 5.8: 2f0230b: s390/pci: re-introduce zpci_remove_device()
Cc: <stable@vger.kernel.org> # 5.8
Acked-by: Pierre Morel <pmorel@linux.ibm.com>
Signed-off-by: Niklas Schnelle <schnelle@linux.ibm.com>
Signed-off-by: Heiko Carstens <hca@linux.ibm.com>

Author: niklas88

arch/s390/pci/pci.c

@@ -678,8 +678,11 @@ void zpci_remove_device(struct zpci_dev *zdev)
 	struct pci_dev *pdev;
 
 	pdev = pci_get_slot(zbus->bus, zdev->devfn);
-	if (pdev)
+	if (pdev) {
+		if (pdev->is_virtfn)
+			return zpci_remove_virtfn(pdev, zdev->vfn);
 		pci_stop_and_remove_bus_device_locked(pdev);
+	}
 }
 
 int zpci_create_device(struct zpci_dev *zdev)

arch/s390/pci/pci_bus.c

@@ -189,6 +189,19 @@ static inline int zpci_bus_setup_virtfn(struct zpci_bus *zbus,
 }
 #endif
 
+void pcibios_bus_add_device(struct pci_dev *pdev)
+{
+	struct zpci_dev *zdev = to_zpci(pdev);
+
+	/*
+	 * With pdev->no_vf_scan the common PCI probing code does not
+	 * perform PF/VF linking.
+	 */
+	if (zdev->vfn)
+		zpci_bus_setup_virtfn(zdev->zbus, pdev, zdev->vfn);
+
+}
+
 static int zpci_bus_add_device(struct zpci_bus *zbus, struct zpci_dev *zdev)
 {
 	struct pci_bus *bus;
@@ -219,20 +232,10 @@ static int zpci_bus_add_device(struct zpci_bus *zbus, struct zpci_dev *zdev)
 	}
 
 	pdev = pci_scan_single_device(bus, zdev->devfn);
-	if (pdev) {
-		if (!zdev->is_physfn) {
-			rc = zpci_bus_setup_virtfn(zbus, pdev, zdev->vfn);
-			if (rc)
-				goto failed_with_pdev;
-		}
+	if (pdev)
 		pci_bus_add_device(pdev);
-	}
-	return 0;
 
-failed_with_pdev:
-	pci_stop_and_remove_bus_device(pdev);
-	pci_dev_put(pdev);
-	return rc;
+	return 0;
 }
 
 static void zpci_bus_add_devices(struct zpci_bus *zbus)

arch/s390/pci/pci_bus.h

@@ -29,3 +29,16 @@ static inline struct zpci_dev *get_zdev_by_bus(struct pci_bus *bus,
 
 	return (devfn >= ZPCI_FUNCTIONS_PER_BUS) ? NULL : zbus->function[devfn];
 }
+
+#ifdef CONFIG_PCI_IOV
+static inline void zpci_remove_virtfn(struct pci_dev *pdev, int vfn)
+{
+
+	pci_lock_rescan_remove();
+	/* Linux' vfid's start at 0 vfn at 1 */
+	pci_iov_remove_virtfn(pdev->physfn, vfn - 1);
+	pci_unlock_rescan_remove();
+}
+#else /* CONFIG_PCI_IOV */
+static inline void zpci_remove_virtfn(struct pci_dev *pdev, int vfn) {}
+#endif /* CONFIG_PCI_IOV */