diff --git a/internal/configs/config_bootloaders.go b/internal/configs/config_bootloaders.go index 281cc4d..c7f6cc7 100644 --- a/internal/configs/config_bootloaders.go +++ b/internal/configs/config_bootloaders.go @@ -51,7 +51,7 @@ func getBootloader(config *Config) { // This function adds the default kernel arguments we want to the config/cmdline file // This gives us a file we can read all the kernel arguments this system needs // in case of an unknown bootloader -func Set_Cmdline(gpu_IDs []string) { +func Set_Cmdline(gpu_IDs []string, includeDeviceIdsForVfio bool) { // Get the system info cpuinfo := cpuid.CPU @@ -69,8 +69,10 @@ func Set_Cmdline(gpu_IDs []string) { fileio.AppendContent(" intel_iommu=on", config.Path.CMDLINE) } - // Add the GPU ids for vfio to the kernel arguments - fileio.AppendContent(fmt.Sprintf(" vfio_pci.ids=%s", strings.Join(gpu_IDs, ",")), config.Path.CMDLINE) + if includeDeviceIdsForVfio { + // Add the GPU ids for vfio to the kernel arguments + fileio.AppendContent(fmt.Sprintf(" vfio_pci.ids=%s", strings.Join(gpu_IDs, ",")), config.Path.CMDLINE) + } } // Set_KernelStub configures systemd-boot using kernelstub. diff --git a/internal/configs/config_dracut.go b/internal/configs/config_dracut.go index bc0b503..c902fdc 100644 --- a/internal/configs/config_dracut.go +++ b/internal/configs/config_dracut.go @@ -3,16 +3,17 @@ package configs import ( "fmt" "os" + "path" + "regexp" "strings" + "github.com/HikariKnight/quickpassthrough/internal/common" "github.com/HikariKnight/quickpassthrough/internal/logger" "github.com/HikariKnight/quickpassthrough/pkg/fileio" ) // Set_Dracut writes a dracut configuration file for `/etc/dracut.conf.d/`. -func Set_Dracut() { - config := GetConfig() - +func Set_Dracut(config *Config) { // Set the dracut config file dracutConf := fmt.Sprintf("%s/vfio.conf", config.Path.DRACUT) @@ -38,4 +39,61 @@ func Set_Dracut() { // Make a backup of dracutConf if there is one there backupFile(strings.Replace(dracutConf, "config", "", 1)) + + if config.HasDuplicateDeviceIds { + setDracutEarlyBinds(config) + } +} + +func setDracutEarlyBinds(config *Config) { + err := os.MkdirAll(config.Path.DRACUTMODULE, os.ModePerm) + common.ErrorCheck(err, "Error, could not create dracut module config directory") + confToSystemPathRe := regexp.MustCompile(`^config`) + + earlyBindScriptConfigPath := path.Join(config.Path.DRACUTMODULE, "early-vfio-bind.sh") + earlyBindScriptSysPath := confToSystemPathRe.ReplaceAllString(earlyBindScriptConfigPath, "") + config.EarlyBindFilePaths[earlyBindScriptConfigPath] = earlyBindScriptSysPath + if exists, _ := fileio.FileExist(earlyBindScriptConfigPath); exists { + _ = os.Remove(earlyBindScriptConfigPath) + } + + logger.Printf("Writing to early bind script to %s", earlyBindScriptConfigPath) + vfioBindScript := fmt.Sprintf(`#!/bin/bash +DEVS="%s" + +for DEV in $DEVS; do + echo "vfio-pci" > /sys/bus/pci/devices/$DEV/driver_override +done + +# Load the vfio-pci module +modprobe -i vfio-pci`, strings.Join(config.Gpu_Addresses, " ")) + + fileio.AppendContent(vfioBindScript, earlyBindScriptConfigPath) + err = os.Chmod(earlyBindScriptConfigPath, 0755) + common.ErrorCheck(err, fmt.Sprintf("Error, could not chmod %s", earlyBindScriptConfigPath)) + + dracutModuleConfigPath := path.Join(config.Path.DRACUTMODULE, "module-setup.sh") + dracutModuleSysPath := confToSystemPathRe.ReplaceAllString(dracutModuleConfigPath, "") + config.EarlyBindFilePaths[dracutModuleConfigPath] = dracutModuleSysPath + if exists, _ := fileio.FileExist(dracutModuleConfigPath); exists { + _ = os.Remove(dracutModuleConfigPath) + } + + logger.Printf("Writing to dracut early bind config to %s", dracutModuleConfigPath) + dracutConfig := fmt.Sprintf(`#!/bin/bash +check() { + return 0 +} + +depends() { + return 0 +} + +install() { + inst_hook pre-trigger 90 "$moddir/%s" +}`, path.Base(earlyBindScriptSysPath)) + + fileio.AppendContent(dracutConfig, dracutModuleConfigPath) + err = os.Chmod(dracutModuleConfigPath, 0755) + common.ErrorCheck(err, fmt.Sprintf("Error, could not chmod %s", dracutModuleConfigPath)) } diff --git a/internal/configs/configs.go b/internal/configs/configs.go index c9b851b..cd9b0b7 100644 --- a/internal/configs/configs.go +++ b/internal/configs/configs.go @@ -17,36 +17,41 @@ import ( ) type Path struct { - CMDLINE string - MODPROBE string - INITRAMFS string - ETCMODULES string - DEFAULT string - QEMU string - DRACUT string - MKINITCPIO string + CMDLINE string + MODPROBE string + INITRAMFS string + ETCMODULES string + DEFAULT string + QEMU string + DRACUT string + DRACUTMODULE string + MKINITCPIO string } type Config struct { - Bootloader string - Cpuvendor string - Path *Path - Gpu_Group string - Gpu_IDs []string - IsRoot bool + Bootloader string + Cpuvendor string + Path *Path + Gpu_Group string + Gpu_IDs []string + Gpu_Addresses []string + EarlyBindFilePaths map[string]string + IsRoot bool + HasDuplicateDeviceIds bool } // GetConfigPaths retrieves the path to all the config files. func GetConfigPaths() *Path { Paths := &Path{ - CMDLINE: "config/kernel_args", - MODPROBE: "config/etc/modprobe.d", - INITRAMFS: "config/etc/initramfs-tools", - ETCMODULES: "config/etc/modules", - DEFAULT: "config/etc/default", - QEMU: "config/qemu", - DRACUT: "config/etc/dracut.conf.d", - MKINITCPIO: "config/etc/mkinitcpio.conf", + CMDLINE: "config/kernel_args", + MODPROBE: "config/etc/modprobe.d", + INITRAMFS: "config/etc/initramfs-tools", + ETCMODULES: "config/etc/modules", + DEFAULT: "config/etc/default", + QEMU: "config/qemu", + DRACUT: "config/etc/dracut.conf.d", + DRACUTMODULE: "config/usr/lib/dracut/modules.d/90early-vfio-bind", + MKINITCPIO: "config/etc/mkinitcpio.conf", } return Paths @@ -55,11 +60,14 @@ func GetConfigPaths() *Path { // GetConfig retrieves all the configs and returns the struct. func GetConfig() *Config { config := &Config{ - Bootloader: "unknown", - Cpuvendor: cpuid.CPU.VendorString, - Path: GetConfigPaths(), - Gpu_Group: "", - Gpu_IDs: []string{}, + Bootloader: "unknown", + Cpuvendor: cpuid.CPU.VendorString, + Path: GetConfigPaths(), + Gpu_Group: "", + Gpu_IDs: []string{}, + Gpu_Addresses: []string{}, + EarlyBindFilePaths: map[string]string{}, + HasDuplicateDeviceIds: false, } // Detect the bootloader we are using @@ -78,6 +86,7 @@ func InitConfigs() { config.Path.INITRAMFS, config.Path.DEFAULT, config.Path.DRACUT, + config.Path.DRACUTMODULE, } // Remove old config diff --git a/internal/pages/02_select_gpu.go b/internal/pages/02_select_gpu.go index a035f65..e4be1bb 100644 --- a/internal/pages/02_select_gpu.go +++ b/internal/pages/02_select_gpu.go @@ -3,11 +3,13 @@ package pages import ( "fmt" "os" + "regexp" "github.com/gookit/color" "github.com/HikariKnight/quickpassthrough/internal/common" "github.com/HikariKnight/quickpassthrough/internal/configs" + "github.com/HikariKnight/quickpassthrough/internal/logger" "github.com/HikariKnight/quickpassthrough/internal/lsiommu" "github.com/HikariKnight/quickpassthrough/pkg/command" "github.com/HikariKnight/quickpassthrough/pkg/fileio" @@ -94,37 +96,67 @@ func viewGPU(config *configs.Config, ext ...int) { // Get the device ids for the selected gpu using ls-iommu config.Gpu_IDs = lsiommu.GetIOMMU("-g", mode, "-i", config.Gpu_Group, "--id") - // If the kernel_args file already exists - if exists, _ := fileio.FileExist(config.Path.CMDLINE); exists { - // Delete it as we will have to make a new one anyway - err := os.Remove(config.Path.CMDLINE) - common.ErrorCheck(err, fmt.Sprintf("Could not remove %s", config.Path.CMDLINE)) - } - - // Write initial kernel_arg file - configs.Set_Cmdline(config.Gpu_IDs) - - // Go to the vbios dumper page - genVBIOS_dumper(config) - case "manual": config.Gpu_IDs = menu.ManualInput( "Please manually enter the vendorID:deviceID for every device to use except PCI Express Switches\n"+ "NOTE: All devices sharing the same IOMMU group will still get pulled into the VM!", "xxxx:yyyy,xxxx:yyyy,xxxx:yyyy", ) + } - // If the kernel_args file already exists - if exists, _ := fileio.FileExist(config.Path.CMDLINE); exists { - // Delete it as we will have to make a new one anyway - err := os.Remove(config.Path.CMDLINE) - common.ErrorCheck(err, fmt.Sprintf("Could not remove %s", config.Path.CMDLINE)) + logger.Printf("Checking for duplicate device Ids") + hasDuplicateDeviceIds := detectDuplicateDeviceIds(config.Gpu_Group, config.Gpu_IDs) + + if hasDuplicateDeviceIds { + config.HasDuplicateDeviceIds = true + config.Gpu_Addresses = lsiommu.GetIOMMU("-g", mode, "-i", config.Gpu_Group, "--pciaddr") + } + + // If the kernel_args file already exists + if exists, _ := fileio.FileExist(config.Path.CMDLINE); exists { + // Delete it as we will have to make a new one anyway + err := os.Remove(config.Path.CMDLINE) + common.ErrorCheck(err, fmt.Sprintf("Could not remove %s", config.Path.CMDLINE)) + } + + // Write initial kernel_arg file + configs.Set_Cmdline(config.Gpu_IDs, !config.HasDuplicateDeviceIds) + + // Go to the vbios dumper page + genVBIOS_dumper(config) +} + +func detectDuplicateDeviceIds(selectedGpuGroup string, selectedDeviceIds []string) bool { + // TODO: this would be made much simpler if ls-iommu allowed using the --id flag without + // the "-i" flag. + gpus := lsiommu.GetIOMMU("-g", "-F", "vendor:,prod_name,optional_revision:,device_id") + iommu_group_regex := regexp.MustCompile(`(\d{1,3})`) + iommuGroups := []string{} + for _, gpu := range gpus { + iommuGroup := iommu_group_regex.FindString(gpu) + iommuGroups = append(iommuGroups, iommuGroup) + } + + allDeviceIds := []string{} + for _, group := range iommuGroups { + if group == selectedGpuGroup { + continue } - // Write initial kernel_arg file - configs.Set_Cmdline(config.Gpu_IDs) - - // Go to the vbios dumper page - genVBIOS_dumper(config) + deviceIds := lsiommu.GetIOMMU("-g", "-r", "-i", group, "--id") + for _, deviceId := range deviceIds { + allDeviceIds = append(allDeviceIds, deviceId) + } } + + for _, deviceId := range allDeviceIds { + for _, selectedDeviceId := range selectedDeviceIds { + if deviceId == selectedDeviceId { + logger.Printf("Found duplicate device id: %s", deviceId) + return true + } + } + } + + return false } diff --git a/internal/pages/06_finalize.go b/internal/pages/06_finalize.go index a1ab656..1817764 100644 --- a/internal/pages/06_finalize.go +++ b/internal/pages/06_finalize.go @@ -6,6 +6,7 @@ import ( "log" "os" "os/user" + "strings" "syscall" "github.com/gookit/color" @@ -29,7 +30,7 @@ func prepModules(config *configs.Config) { // If we have a folder for dracut if exists, _ := fileio.FileExist(config.Path.DRACUT); exists { // Configure dracut - configs.Set_Dracut() + configs.Set_Dracut(config) } // If we have a mkinitcpio.conf file @@ -209,6 +210,17 @@ func installPassthrough(config *configs.Config) { // Copy dracut config to /etc/dracut.conf.d/vfio configs.CopyToSystem(config.IsRoot, dracutFile, "/etc/dracut.conf.d/vfio") + if config.HasDuplicateDeviceIds { + moduleSysPath := strings.Replace(config.Path.DRACUTMODULE, "config", "", 1) + if err := command.ExecAndLogSudo(config.IsRoot, false, "mkdir", "-p", moduleSysPath); err != nil { + log.Fatalf("Failed to create dracut module directory: %s", err) + } + + for configPath, sysPath := range config.EarlyBindFilePaths { + configs.CopyToSystem(config.IsRoot, configPath, sysPath) + } + } + // Get systeminfo sysinfo := uname.New()