diff --git a/CHANGELOG.md b/CHANGELOG.md index 5752e034..c6a35203 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,12 @@ That said, these are more guidelines rather than hard rules, though the project --- +## 0.15.0 - Unreleased + +### Bug Fixes + +- [#2225](https://github.com/ClementTsang/bottom/pull/2225): Fix waking up NVIDIA GPUs when getting stats on Linux. + ## 0.14.9 - 2026-08-27 ### Bug Fixes diff --git a/src/collection.rs b/src/collection.rs index 3b158941..16fc21c7 100644 --- a/src/collection.rs +++ b/src/collection.rs @@ -190,6 +190,9 @@ pub struct DataCollector { gpu_pids: Option>>, #[cfg(feature = "gpu")] gpus_total_mem: Option, + #[cfg(all(target_os = "linux", feature = "gpu", feature = "nvidia"))] + nvidia_gpu_list_cache: Option<(Vec, Instant)>, + #[cfg(feature = "zfs")] free_arc_mem: bool, @@ -238,6 +241,8 @@ impl DataCollector { gpu_pids: None, #[cfg(feature = "gpu")] gpus_total_mem: None, + #[cfg(all(target_os = "linux", feature = "gpu", feature = "nvidia"))] + nvidia_gpu_list_cache: None, #[cfg(feature = "zfs")] free_arc_mem: false, last_list_collection_time: last_collection_time, @@ -422,11 +427,7 @@ impl DataCollector { let mut local_gpu_total_mem: u64 = 0; #[cfg(feature = "nvidia")] - if let Some(data) = nvidia::get_nvidia_vecs( - &self.filters.temp_filter, - &self.filters.temp_graph_filter, - &self.widgets_to_harvest, - ) { + if let Some(data) = nvidia::get_nvidia_gpu_data(self) { if let Some(mut temp) = data.temperature { if let Some(sensors) = &mut self.data.temperature_sensors { sensors.append(&mut temp); diff --git a/src/collection/linux/utils.rs b/src/collection/linux/utils.rs index 8ec0b2a4..a87cf1a2 100644 --- a/src/collection/linux/utils.rs +++ b/src/collection/linux/utils.rs @@ -2,10 +2,9 @@ use std::{fs, path::Path}; /// Whether the temperature should *actually* be read during enumeration. /// Will return false if the state is not D0/unknown, or if it does not support -/// `device/power_state`. +/// `device/power_state` (e.g. the path does not exist). /// -/// `path` is a path to the device itself (e.g. -/// `/sys/class/hwmon/hwmon1/device`). +/// `path` is a path to the device itself (e.g. `/sys/class/hwmon/hwmon1/device`). #[inline] pub fn is_device_awake(device: &Path) -> bool { // Whether the temperature should *actually* be read during enumeration. diff --git a/src/collection/nvidia.rs b/src/collection/nvidia.rs index f971aa1b..f8a204b4 100644 --- a/src/collection/nvidia.rs +++ b/src/collection/nvidia.rs @@ -5,8 +5,8 @@ use nvml_wrapper::{ }; use crate::{ - app::{filter::Filter, layout_manager::UsedWidgets}, - collection::{memory::MemData, processes::Pid, temperature::TempSensorData}, + app::filter::Filter, + collection::{DataCollector, memory::MemData, processes::Pid, temperature::TempSensorData}, utils::int_hash::IntHashMap, }; @@ -42,142 +42,261 @@ fn init_nvml() -> Result { } } +/// Returns whether the vendor ID passed in is NVIDIA's vendor ID. +/// +/// See for details +/// (search for `PCI_VENDOR_ID_NVIDIA`). +#[cfg(target_os = "linux")] +#[inline] +fn is_nvidia_vendor(vendor_id: &str) -> bool { + const NVIDIA_VENDOR: &str = "0x10de"; + vendor_id == NVIDIA_VENDOR +} + +/// Returns whether the PCI code is a GPU. +/// +/// See for details +/// (search for `PCI_BASE_CLASS_DISPLAY`). +#[cfg(target_os = "linux")] +#[inline] +fn is_gpu_class(class_code: &str) -> bool { + const PCI_BASE_CLASS_DISPLAY: &str = "0x03"; + class_code.starts_with(PCI_BASE_CLASS_DISPLAY) +} + +/// Get a list of PCI bus IDs for Linux. This will handle whether the device is awake or not. +/// We do this separately to avoid the possibility of NVML waking up the device at all; +/// this is particularly useful for things like laptops with hybrid graphics (e.g. NVIDIA Optimus). +/// +/// Note this is somewhat expensive, so it may be worth caching this result. +/// +/// --- +/// +/// For more information, see: +/// - +/// - +#[cfg(target_os = "linux")] +fn get_active_pci_bus_ids() -> Vec { + use crate::collection::linux::utils::is_device_awake; + use std::fs; + + let Ok(entries) = fs::read_dir("/sys/bus/pci/devices") else { + return Vec::new(); + }; + + let mut result: Vec = entries + .flatten() + .filter_map(|entry| { + let path = entry.path(); + + let is_nvidia = fs::read_to_string(path.join("vendor")) + .is_ok_and(|vendor| is_nvidia_vendor(vendor.trim())); + if !is_nvidia { + return None; + } + + let is_gpu = fs::read_to_string(path.join("class")) + .is_ok_and(|class| is_gpu_class(class.trim())); + if !is_gpu { + return None; + } + + let is_awake = is_device_awake(&path); + + // This returns values in the "shape" of "0000:01:00.0" (domain:bus:device.function). + // + // Just as an FYI: + // The "0th" function is the GPU itself - from the NVIDIA power management docs + // (https://us.download.nvidia.com/XFree86/Linux-x86_64/525.89.02/README/dynamicpowermanagement.html): + // > The NVIDIA GPU may have one, two or four PCI functions: + // > - Function 0: VGA controller / 3D controller + // > - Function 1: Audio device + // > - Function 2: USB xHCI Host controller + // > - Function 3: USB Type-C UCSI controller + // + // We also know the "shape" of the path from aforementioned docs (ignore what it's trying to do): + // > For pre-Ampere notebooks, runtime D3 power management can be enabled for each PCI function using the following command. + // > echo auto > /sys/bus/pci/devices/::./power/control + // > For example: + // > echo auto > /sys/bus/pci/devices/0000:01:00.0/power/control + if is_awake { + // Note that NVML expects an eight-digit bus ID at the front, so we prepend the current device name + // with `0000`. + entry + .file_name() + .into_string() + .ok() + .map(|name| concat_string::concat_string!("0000", name)) + } else { + None + } + }) + .collect(); + + result.sort_unstable(); + result +} + /// Returns the GPU data from NVIDIA cards. #[inline] -pub fn get_nvidia_vecs( - filter: &Option, graph_filter: &Option, widgets_to_harvest: &UsedWidgets, -) -> Option { - if let Ok(nvml) = NVML_DATA.get_or_init(init_nvml) { - if let Ok(num_gpu) = nvml.device_count() { - let mut temp_vec = Vec::with_capacity(num_gpu as usize); - let mut mem_vec = Vec::with_capacity(num_gpu as usize); - let mut proc_vec = Vec::with_capacity(num_gpu as usize); - let mut total_mem = 0; +pub fn get_nvidia_gpu_data(collector: &mut DataCollector) -> Option { + let filter = &collector.filters.temp_filter; + let graph_filter = &collector.filters.temp_graph_filter; + let widgets_to_harvest = &collector.widgets_to_harvest; - for i in 0..num_gpu { - if let Ok(device) = nvml.device_by_index(i) { - if let Ok(name) = device.name() { - if widgets_to_harvest.use_mem - && let Ok(mem) = device.memory_info() - && let Some(total_bytes) = NonZeroU64::new(mem.total) - { - mem_vec.push(( - name.clone(), - MemData { - total_bytes, - used_bytes: mem.used, - }, - )); - } + let Ok(nvml) = NVML_DATA.get_or_init(init_nvml) else { + return None; + }; - if (widgets_to_harvest.use_temp || widgets_to_harvest.use_temp_graph) - && (Filter::optional_should_keep(filter, &name) - || Filter::optional_should_keep(graph_filter, &name)) - { - if let Ok(temperature) = device.temperature(TemperatureSensor::Gpu) { - temp_vec.push(TempSensorData { - name, - temperature: Some(temperature as f32), - }); - } else { - temp_vec.push(TempSensorData { - name, - temperature: None, - }); - } - } - } + let (gpu_iter, max_num_gpus): (_, usize) = { + cfg_select! { + target_os = "linux" => { + use itertools::Either; - if widgets_to_harvest.use_proc { - let mut procs = IntHashMap::default(); + // Refresh every ~10 seconds. + if let Some((cached_list, cached_time)) = &collector.nvidia_gpu_list_cache && cached_time.elapsed().as_secs() < 10 { + let devices = Either::Left(cached_list.iter().filter_map(|id| nvml.device_by_pci_bus_id(id.as_str()).ok())); + (devices, cached_list.len()) + } + else { + let pci_bus_ids = get_active_pci_bus_ids(); + let num_gpus = pci_bus_ids.len(); + collector.nvidia_gpu_list_cache = Some((pci_bus_ids.clone(), std::time::Instant::now())); - if let Ok(gpu_procs) = device.process_utilization_stats(None) { - for proc in gpu_procs { - let pid = proc.pid as Pid; - let gpu_util = proc.sm_util + proc.enc_util + proc.dec_util; - procs.insert(pid, (0, gpu_util)); - } - } + let devices = Either::Right(pci_bus_ids.into_iter().filter_map(|id| nvml.device_by_pci_bus_id(id).ok())); + (devices, num_gpus) + } + }, + _ => { + // The fallback behaviour (the old one) is to just list all nvml devices blindly. + // Note this has the risk of waking up sleeping devices. + let num_gpus = nvml.device_count().ok()?; + ((0..num_gpus).flat_map(|i| nvml.device_by_index(i)), num_gpus as usize) + } + } + }; - if let Ok(compute_procs) = device.running_compute_processes() { - for proc in compute_procs { - let pid = proc.pid as Pid; - let gpu_mem = match proc.used_gpu_memory { - UsedGpuMemory::Used(val) => val, - UsedGpuMemory::Unavailable => 0, - }; - if let Some(prev) = procs.get(&pid) { - procs.insert(pid, (gpu_mem, prev.1)); - } else { - procs.insert(pid, (gpu_mem, 0)); - } - } - } + let mut temp_vec = Vec::with_capacity(max_num_gpus); + let mut mem_vec = Vec::with_capacity(max_num_gpus); + let mut proc_vec = Vec::with_capacity(max_num_gpus); + let mut total_mem = 0; - // Use the legacy API too but prefer newer API results - if let Ok(graphics_procs) = device.running_graphics_processes_v2() { - for proc in graphics_procs { - let pid = proc.pid as Pid; - let gpu_mem = match proc.used_gpu_memory { - UsedGpuMemory::Used(val) => val, - UsedGpuMemory::Unavailable => 0, - }; - if let Some(prev) = procs.get(&pid) { - procs.insert(pid, (gpu_mem, prev.1)); - } else { - procs.insert(pid, (gpu_mem, 0)); - } - } - } + for device in gpu_iter { + if let Ok(name) = device.name() { + if widgets_to_harvest.use_mem + && let Ok(mem) = device.memory_info() + && let Some(total_bytes) = NonZeroU64::new(mem.total) + { + mem_vec.push(( + name.clone(), + MemData { + total_bytes, + used_bytes: mem.used, + }, + )); + } - if let Ok(graphics_procs) = device.running_graphics_processes() { - for proc in graphics_procs { - let pid = proc.pid as Pid; - let gpu_mem = match proc.used_gpu_memory { - UsedGpuMemory::Used(val) => val, - UsedGpuMemory::Unavailable => 0, - }; - if let Some(prev) = procs.get(&pid) { - procs.insert(pid, (gpu_mem, prev.1)); - } else { - procs.insert(pid, (gpu_mem, 0)); - } - } - } + if (widgets_to_harvest.use_temp || widgets_to_harvest.use_temp_graph) + && (Filter::optional_should_keep(filter, &name) + || Filter::optional_should_keep(graph_filter, &name)) + { + if let Ok(temperature) = device.temperature(TemperatureSensor::Gpu) { + temp_vec.push(TempSensorData { + name, + temperature: Some(temperature as f32), + }); + } else { + temp_vec.push(TempSensorData { + name, + temperature: None, + }); + } + } + } - if !procs.is_empty() { - proc_vec.push(procs); - } + if widgets_to_harvest.use_proc { + let mut procs = IntHashMap::default(); - // running total for proc % - if let Ok(mem) = device.memory_info() { - total_mem += mem.total; - } + if let Ok(gpu_procs) = device.process_utilization_stats(None) { + for proc in gpu_procs { + let pid = proc.pid as Pid; + let gpu_util = proc.sm_util + proc.enc_util + proc.dec_util; + procs.insert(pid, (0, gpu_util)); + } + } + + if let Ok(compute_procs) = device.running_compute_processes() { + for proc in compute_procs { + let pid = proc.pid as Pid; + let gpu_mem = match proc.used_gpu_memory { + UsedGpuMemory::Used(val) => val, + UsedGpuMemory::Unavailable => 0, + }; + if let Some(prev) = procs.get(&pid) { + procs.insert(pid, (gpu_mem, prev.1)); + } else { + procs.insert(pid, (gpu_mem, 0)); } } } - Some(GpusData { - memory: if !mem_vec.is_empty() { - Some(mem_vec) - } else { - None - }, - temperature: if !temp_vec.is_empty() { - Some(temp_vec) - } else { - None - }, - procs: if !proc_vec.is_empty() { - Some((total_mem, proc_vec)) - } else { - None - }, - }) + // Use the legacy API too but prefer newer API results + if let Ok(graphics_procs) = device.running_graphics_processes_v2() { + for proc in graphics_procs { + let pid = proc.pid as Pid; + let gpu_mem = match proc.used_gpu_memory { + UsedGpuMemory::Used(val) => val, + UsedGpuMemory::Unavailable => 0, + }; + if let Some(prev) = procs.get(&pid) { + procs.insert(pid, (gpu_mem, prev.1)); + } else { + procs.insert(pid, (gpu_mem, 0)); + } + } + } + + if let Ok(graphics_procs) = device.running_graphics_processes() { + for proc in graphics_procs { + let pid = proc.pid as Pid; + let gpu_mem = match proc.used_gpu_memory { + UsedGpuMemory::Used(val) => val, + UsedGpuMemory::Unavailable => 0, + }; + if let Some(prev) = procs.get(&pid) { + procs.insert(pid, (gpu_mem, prev.1)); + } else { + procs.insert(pid, (gpu_mem, 0)); + } + } + } + + if !procs.is_empty() { + proc_vec.push(procs); + } + + // running total for proc % + if let Ok(mem) = device.memory_info() { + total_mem += mem.total; + } + } + } + + Some(GpusData { + memory: if !mem_vec.is_empty() { + Some(mem_vec) } else { None - } - } else { - None - } + }, + temperature: if !temp_vec.is_empty() { + Some(temp_vec) + } else { + None + }, + procs: if !proc_vec.is_empty() { + Some((total_mem, proc_vec)) + } else { + None + }, + }) }