Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 15 additions & 11 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -174,17 +174,21 @@ successor should work fine. For more information about supported GPUs please
take a look at the [NVML documentation](http://docs.nvidia.com/deploy/nvml-api/nvml-api-reference.html#nvml-api-reference).

Unified-memory systems such as DGX Spark (GB10) and RTX Spark N1X are supported
through the same NVML backend. When NVML reports no dedicated framebuffer,
memory capacity comes from Linux `MemTotal`, available memory from
`MemAvailable`, and GPU usage from the visible GPU processes' allocations.
These are different counters: total minus GPU usage is not the remaining
allocatable system memory. Processes using both graphics and compute contexts
are counted once. In containers, make host GPU processes visible (for example
with `--pid=host`) to obtain complete per-process accounting.

PCIe link generation and width are left unavailable when NVML reports an active
NVLink-C2C attachment. Driver-unsupported sensors, including memory clocks, fan
speed and link throughput, remain `N/A` rather than being estimated.
through the same NVML backend. When NVML reports no dedicated framebuffer, the
GPU has no memory pool of its own, so the header reports the system pool the
host shares with it: capacity from Linux `MemTotal`, free from `MemAvailable`,
and used as the difference between the two. Used therefore covers allocations by
host processes too, which is the point: they consume the same memory and leave
less for the GPU. Per-process GPU allocations are still listed separately. In
containers, make host GPU processes visible (for example with `--pid=host`) to
obtain complete per-process accounting.

These devices are also integrated with the host, so the header shows `CPU-FAN`
instead of a GPU fan speed, and `C2C LINK <n>x` instead of the PCIe generation and
width for a device attached over NVLink-C2C. NVML answers the PCIe link queries
on those SoCs with placeholder values, so the link is left unreported rather
than shown. Driver-unsupported sensors, including memory clocks, power limits
and PCIe throughput, remain `N/A` rather than being estimated.

The extra GPU info bar (`nvtop -i`) shows the dynamic parameters NVML does not
report, such as the XBAR, SYS and NVD clocks, and `nvtop -X` also shows the
Expand Down
2 changes: 2 additions & 0 deletions include/nvtop/extract_gpuinfo_common.h
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ enum gpuinfo_static_info_valid {
gpuinfo_n_exec_engines_valid,
gpuinfo_engine_count_valid,
gpuinfo_memory_type_valid,
gpuinfo_c2c_link_count_valid,
gpuinfo_static_info_count,
};

Expand All @@ -79,6 +80,7 @@ struct gpuinfo_static_info {
unsigned l2cache_size;
unsigned n_exec_engines;
unsigned engine_count;
unsigned c2c_link_count; // Number of NVLink-C2C links, 0 when the device has none
bool integrated_graphics;
bool encode_decode_shared;
bool memory_shared_with_host; // True if device memory is shared with the host (e.g. UMA)
Expand Down
18 changes: 6 additions & 12 deletions src/extract_gpuinfo.c
Original file line number Diff line number Diff line change
Expand Up @@ -141,15 +141,6 @@ bool gpuinfo_fix_dynamic_info_from_process_info(struct list_head *devices) {
bool needGpuDecode = !GPUINFO_DYNAMIC_FIELD_VALID(dynamic_info, decoder_rate);
bool needGPUMemory = !GPUINFO_DYNAMIC_FIELD_VALID(dynamic_info, used_memory) &&
GPUINFO_DYNAMIC_FIELD_VALID(dynamic_info, total_memory);
// A device whose memory is shared with the host (e.g. UMA) reports the
// total and free amounts but cannot report the used amount itself; it is
// reconstructed from the process list below. Start from zero so an idle
// device reports 0. Devices that only cannot measure used (e.g. Apple
// without per-process accounting) do not set the flag and keep reporting it
// as unknown.
bool sharedHostMemory = needGPUMemory && device->static_info.memory_shared_with_host;
if (sharedHostMemory)
SET_GPUINFO_DYNAMIC(dynamic_info, used_memory, 0);
if (needGpuRate || needGpuEncode || needGpuDecode || needGPUMemory) {
for (unsigned processIdx = 0; processIdx < device->processes_count; ++processIdx) {
struct gpu_process *process_info = &device->processes[processIdx];
Expand Down Expand Up @@ -196,9 +187,12 @@ bool gpuinfo_fix_dynamic_info_from_process_info(struct list_head *devices) {
unsigned long long free = dynamic_info->total_memory - dynamic_info->used_memory;
SET_GPUINFO_DYNAMIC(dynamic_info, free_memory, free);
}
// A shared-memory device cannot report the used amount itself, so derive the
// utilization rate for the graph from the reconstructed sum.
if (sharedHostMemory && !GPUINFO_DYNAMIC_FIELD_VALID(dynamic_info, mem_util_rate) &&
// When a backend reports the pool's total but not its used amount, the
// process list is the only source for used memory. Backends that do report
// used themselves (unified memory backed by system counters included) keep
// that value, since it also covers allocations the process list cannot see.
// The graph's memory rate is derived from whatever used memory is reported.
if (needGPUMemory && !GPUINFO_DYNAMIC_FIELD_VALID(dynamic_info, mem_util_rate) &&
GPUINFO_DYNAMIC_FIELD_VALID(dynamic_info, used_memory) && dynamic_info->total_memory > 0) {
SET_GPUINFO_DYNAMIC(dynamic_info, mem_util_rate, dynamic_info->used_memory * 100 / dynamic_info->total_memory);
}
Expand Down
105 changes: 81 additions & 24 deletions src/extract_gpuinfo_nvidia.c
Original file line number Diff line number Diff line change
Expand Up @@ -207,6 +207,11 @@ typedef struct {

static nvmlReturn_t (*nvmlDeviceGetC2cModeInfoV)(nvmlDevice_t device, nvmlC2cModeInfo_v1_t *info);

// NVLink-C2C link count from nvml.h, queried through nvmlDeviceGetFieldValues.
// A C2C link replaces the PCIe link between the GPU and the CPU on SoCs such
// as GB10, where the PCIe link queries answer with placeholder values.
#define NVML_FI_DEV_C2C_LINK_COUNT 170

typedef enum {
NVML_PCIE_UTIL_TX_BYTES = 0,
NVML_PCIE_UTIL_RX_BYTES = 1,
Expand Down Expand Up @@ -585,12 +590,16 @@ static const char *local_error_string = didnt_call_gpuinfo_init;

// On UMA platforms the GPU has no dedicated framebuffer and shares the system
// memory pool with the host, so there is no framebuffer size to query. Report
// the whole system RAM as the total capacity and what the kernel says is still
// allocatable as free. The used memory is deliberately left unset here: the
// generic process accounting in
// gpuinfo_fix_dynamic_info_from_process_info() fills it with the sum of the
// running processes' GPU allocations, which is the only per-device figure NVML
// exposes on these platforms.
// the whole system RAM as the total capacity, what the kernel says is still
// allocatable as free, and the difference as used.
//
// There is only one memory pool, so "used" has to mean the same thing as it
// does on a discrete GPU: memory that is not available to new allocations.
// That includes host processes, which is why used is not limited to the GPU
// processes' own allocations (those are still shown per process). Reporting
// only the GPU allocations against a MemTotal-sized capacity would advertise
// headroom the machine does not have: on DGX Spark / GB10 a loaded system can
// show tens of GiB of host memory held while the bar claims it is free.
static bool set_unified_system_memory_info(struct gpuinfo_dynamic_info *dynamic_info) {
FILE *meminfo = fopen("/proc/meminfo", "r");
if (!meminfo)
Expand Down Expand Up @@ -621,20 +630,58 @@ static bool set_unified_system_memory_info(struct gpuinfo_dynamic_info *dynamic_

unsigned long long total_memory = total_memory_kb * 1024;
unsigned long long free_memory = available_memory_kb * 1024;
unsigned long long used_memory = total_memory - free_memory;

SET_GPUINFO_DYNAMIC(dynamic_info, total_memory, total_memory);
SET_GPUINFO_DYNAMIC(dynamic_info, free_memory, free_memory);

// The used memory is recomputed from the process list every refresh by the
// generic pass, so invalidate it (and the derived rate) for it to fill in.
RESET_GPUINFO_DYNAMIC(dynamic_info, used_memory);
RESET_GPUINFO_DYNAMIC(dynamic_info, mem_util_rate);
SET_GPUINFO_DYNAMIC(dynamic_info, used_memory, used_memory);
SET_GPUINFO_DYNAMIC(dynamic_info, mem_util_rate, (unsigned)(used_memory * 100 / total_memory));

return true;
}

// Processes GPU Utilization

// Reads a single NVML field value of the unsigned long long kind. Returns false
// when the entry point is missing, the call fails, or the field itself reports
// an error, so callers never see a partially valid value.
static bool gpuinfo_nvidia_get_field_value_u64(nvmlDevice_t device, unsigned int field_id, unsigned int scope_id,
unsigned long long *value) {
if (!nvmlDeviceGetFieldValues)
return false;
nvmlFieldValue_t field = {0};
field.fieldId = field_id;
field.scopeId = scope_id;
if (nvmlDeviceGetFieldValues(device, 1, &field) != NVML_SUCCESS || field.nvmlReturn != NVML_SUCCESS)
return false;
*value = field.value.ullVal;
return true;
}

// Number of NVLink-C2C links the GPU is attached over. The C2C mode query only
// says whether the attachment is enabled; the link count is what the header can
// display in place of a PCIe link. Devices without a C2C attach report either a
// failed query or a count of zero, and both mean "no links".
static unsigned gpuinfo_nvidia_c2c_link_count(nvmlDevice_t device) {
unsigned long long count = 0;
if (!gpuinfo_nvidia_get_field_value_u64(device, NVML_FI_DEV_C2C_LINK_COUNT, 0, &count))
return 0;
return count > UINT_MAX ? UINT_MAX : (unsigned)count;
}

// True when the GPU is attached over NVLink-C2C rather than PCIe. The link
// count is the direct statement of the attachment, but some drivers expose the
// older boolean instead, so both are consulted. A failed query is never read as
// an attach, which keeps discrete GPUs on the PCIe path.
static bool gpuinfo_nvidia_uses_c2c(nvmlDevice_t device, unsigned link_count) {
if (link_count > 0)
return true;
if (!nvmlDeviceGetC2cModeInfoV)
return false;
nvmlC2cModeInfo_v1_t c2c_mode = {0};
return nvmlDeviceGetC2cModeInfoV(device, &c2c_mode) == NVML_SUCCESS && c2c_mode.isC2cEnabled != 0;
}

typedef struct {
unsigned int pid;
unsigned long long timeStamp;
Expand Down Expand Up @@ -1039,8 +1086,16 @@ static void gpuinfo_nvidia_populate_static_info(struct gpu_info *_gpu_info) {
static_info->memory_shared_with_host =
ret == NVML_ERROR_NOT_SUPPORTED || (ret == NVML_SUCCESS && memory_info.total == 0);
}
// A device sharing the system memory pool with the host is by definition
// integrated with it: there is no add-in board and therefore no GPU fan,
// which the interface renders as "CPU-FAN" like the other integrated GPUs.
static_info->integrated_graphics = static_info->memory_shared_with_host;
static_info->c2c_link_count = gpuinfo_nvidia_c2c_link_count(device);
RESET_ALL(static_info->valid);

if (static_info->c2c_link_count > 0)
SET_VALID(gpuinfo_c2c_link_count_valid, static_info->valid);

last_nvml_return_status = nvmlDeviceGetName(device, static_info->device_name, MAX_DEVICE_NAME);
if (last_nvml_return_status == NVML_SUCCESS)
SET_VALID(gpuinfo_device_name_valid, static_info->valid);
Expand Down Expand Up @@ -1102,13 +1157,18 @@ static void gpuinfo_nvidia_populate_static_info(struct gpu_info *_gpu_info) {
}
}

last_nvml_return_status = nvmlDeviceGetMaxPcieLinkGeneration(device, &static_info->max_pcie_gen);
if (last_nvml_return_status == NVML_SUCCESS)
SET_VALID(gpuinfo_max_pcie_gen_valid, static_info->valid);
// The PCIe link queries answer with placeholder values on a C2C-attached
// device (GB10 reports "Max Gen 1 / Max Width 16x" while the actual link is an
// NVLink-C2C with no PCIe topology behind it), so leave them unknown there.
if (!gpuinfo_nvidia_uses_c2c(device, static_info->c2c_link_count)) {
last_nvml_return_status = nvmlDeviceGetMaxPcieLinkGeneration(device, &static_info->max_pcie_gen);
if (last_nvml_return_status == NVML_SUCCESS)
SET_VALID(gpuinfo_max_pcie_gen_valid, static_info->valid);

last_nvml_return_status = nvmlDeviceGetMaxPcieLinkWidth(device, &static_info->max_pcie_link_width);
if (last_nvml_return_status == NVML_SUCCESS)
SET_VALID(gpuinfo_max_pcie_link_width_valid, static_info->valid);
last_nvml_return_status = nvmlDeviceGetMaxPcieLinkWidth(device, &static_info->max_pcie_link_width);
if (last_nvml_return_status == NVML_SUCCESS)
SET_VALID(gpuinfo_max_pcie_link_width_valid, static_info->valid);
}

last_nvml_return_status = nvmlDeviceGetTemperatureThreshold(device, NVML_TEMPERATURE_THRESHOLD_SHUTDOWN,
&static_info->temperature_shutdown_threshold);
Expand Down Expand Up @@ -1230,19 +1290,16 @@ static void gpuinfo_nvidia_refresh_dynamic_info(struct gpu_info *_gpu_info) {
// Pcie generation and width used by the device.
// Only report these for PCIe devices without an active NVLink-C2C attach.
// DGX Spark / GB10 and RTX Spark N1X report PCIe with placeholder link
// values even though C2C is enabled. Do not infer the link from UMA alone.
// values even though C2C is enabled. Do not infer the link from UMA alone:
// the C2C link count is a direct statement of the attachment.
bool device_on_pcie_bus = nvmlDeviceGetBusType == NULL; // Older drivers: assume PCIe as before.
if (nvmlDeviceGetBusType) {
unsigned int bus_type;
if (nvmlDeviceGetBusType(device, &bus_type) == NVML_SUCCESS)
device_on_pcie_bus = bus_type == NVML_BUS_TYPE_PCIE;
}

if (device_on_pcie_bus && nvmlDeviceGetC2cModeInfoV) {
nvmlC2cModeInfo_v1_t c2c_mode = {0};
if (nvmlDeviceGetC2cModeInfoV(device, &c2c_mode) == NVML_SUCCESS && c2c_mode.isC2cEnabled)
device_on_pcie_bus = false;
}
if (device_on_pcie_bus && gpuinfo_nvidia_uses_c2c(device, gpu_info->base.static_info.c2c_link_count))
device_on_pcie_bus = false;

if (device_on_pcie_bus) {
last_nvml_return_status = nvmlDeviceGetCurrPcieLinkGeneration(device, &dynamic_info->pcie_link_gen);
Expand Down
49 changes: 34 additions & 15 deletions src/interface.c
Original file line number Diff line number Diff line change
Expand Up @@ -1227,7 +1227,18 @@ static void draw_devices(struct list_head *devices, struct nvtop_interface *inte

// PICe throughput
werase(dev->pcie_info);
if (device->static_info.integrated_graphics) {
if (GPUINFO_STATIC_FIELD_VALID(&device->static_info, c2c_link_count)) {
// The device is attached over NVLink-C2C instead of PCIe, so reporting a
// PCIe generation and width would describe a link it does not use. NVML
// exposes no byte throughput counter for C2C either, so only the link
// count is shown, styled like the PCIe link it replaces.
wcolor_set(dev->pcie_info, cyan_color, NULL);
mvwprintw(dev->pcie_info, 0, 0, "C2C ");
wcolor_set(dev->pcie_info, magenta_color, NULL);
wprintw(dev->pcie_info, "LINK ");
wstandend(dev->pcie_info);
wprintw(dev->pcie_info, "%ux", device->static_info.c2c_link_count);
} else if (device->static_info.integrated_graphics) {
wcolor_set(dev->pcie_info, cyan_color, NULL);
mvwprintw(dev->pcie_info, 0, 0, "Integrated %s", DEVICE_UNIT_NAME(device));
} else {
Expand All @@ -1243,20 +1254,22 @@ static void draw_devices(struct list_head *devices, struct nvtop_interface *inte
wprintw(dev->pcie_info, "N/A");
}

wcolor_set(dev->pcie_info, magenta_color, NULL);
wprintw(dev->pcie_info, " RX: ");
wstandend(dev->pcie_info);
if (GPUINFO_DYNAMIC_FIELD_VALID(&device->dynamic_info, pcie_rx))
print_data_at_scale(dev->pcie_info, device->dynamic_info.pcie_rx);
else
wprintw(dev->pcie_info, "N/A");
wcolor_set(dev->pcie_info, magenta_color, NULL);
wprintw(dev->pcie_info, " TX: ");
wstandend(dev->pcie_info);
if (GPUINFO_DYNAMIC_FIELD_VALID(&device->dynamic_info, pcie_tx))
print_data_at_scale(dev->pcie_info, device->dynamic_info.pcie_tx);
else
wprintw(dev->pcie_info, "N/A");
if (!GPUINFO_STATIC_FIELD_VALID(&device->static_info, c2c_link_count)) {
wcolor_set(dev->pcie_info, magenta_color, NULL);
wprintw(dev->pcie_info, " RX: ");
wstandend(dev->pcie_info);
if (GPUINFO_DYNAMIC_FIELD_VALID(&device->dynamic_info, pcie_rx))
print_data_at_scale(dev->pcie_info, device->dynamic_info.pcie_rx);
else
wprintw(dev->pcie_info, "N/A");
wcolor_set(dev->pcie_info, magenta_color, NULL);
wprintw(dev->pcie_info, " TX: ");
wstandend(dev->pcie_info);
if (GPUINFO_DYNAMIC_FIELD_VALID(&device->dynamic_info, pcie_tx))
print_data_at_scale(dev->pcie_info, device->dynamic_info.pcie_tx);
else
wprintw(dev->pcie_info, "N/A");
}

wnoutrefresh(dev->pcie_info);

Expand Down Expand Up @@ -2990,6 +3003,12 @@ void print_snapshot(struct list_head *devices, bool use_fahrenheit_option, bool
else
printf("%s\"pcie_tx\": null,\n", indent_level_four);

// NVLink-C2C attachment, when the device uses one instead of a PCIe link
if (GPUINFO_STATIC_FIELD_VALID(&device->static_info, c2c_link_count))
printf("%s\"c2c_link_count\": %u,\n", indent_level_four, device->static_info.c2c_link_count);
else
printf("%s\"c2c_link_count\": null,\n", indent_level_four);

// Effective load rate
if (GPUINFO_DYNAMIC_FIELD_VALID(&device->dynamic_info, effective_load_rate))
printf("%s\"effective_load_rate\": \"%u%%\",\n", indent_level_four, device->dynamic_info.effective_load_rate);
Expand Down
2 changes: 1 addition & 1 deletion tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ if (BUILD_TESTING AND NVIDIA_SUPPORT AND CMAKE_SYSTEM_NAME STREQUAL "Linux")
set_property(TARGET nvidiaTests PROPERTY C_STANDARD 11)
target_link_libraries(nvidiaTests PRIVATE m ${CMAKE_DL_LIBS})
add_sanitizers(nvidiaTests)
foreach (test_case uma_version legacy_fallback mixed_processes c2c_link unavailable_memory
foreach (test_case uma_version legacy_fallback mixed_processes c2c_link c2c_link_count unavailable_memory
memory_probes pcie_preserved uma_idle
utilization_probe_not_found utilization_buffered_not_found utilization_samples)
add_test(NAME Nvidia.${test_case} COMMAND nvidiaTests ${test_case})
Expand Down
Loading
Loading