From 913cfe3726c1eacb11d324ef68fafcc5e43a50fd Mon Sep 17 00:00:00 2001 From: Owen Rummage Date: Mon, 20 Jul 2026 22:36:13 -0500 Subject: [PATCH 1/4] Record CPU telemetry during benchmarks --- src/app/benchmark.c | 220 +++++++++++++++++++++++++++++++++++++++++++- src/app/upload.c | 28 +++++- 2 files changed, 244 insertions(+), 4 deletions(-) diff --git a/src/app/benchmark.c b/src/app/benchmark.c index 7b1992f..78ac4a4 100644 --- a/src/app/benchmark.c +++ b/src/app/benchmark.c @@ -220,6 +220,219 @@ struct background_metrics { int samples, available; }; +#define MAX_TELEMETRY_SAMPLES 1800 + +struct telemetry_sample { + uint64_t elapsed_ms; + double temperature_c, clock_mhz; +}; + +struct telemetry_monitor { + struct telemetry_sample samples[MAX_TELEMETRY_SAMPLES]; + size_t count; + double started; + volatile int stop; +#if defined(_WIN32) + HANDLE thread; +#else + pthread_t thread; +#endif +}; + +static int read_number_file(const char *path, double *value) +{ + FILE *f = fopen(path, "r"); + int ok; + if (!f) return 0; + ok = fscanf(f, "%lf", value) == 1; + fclose(f); + return ok; +} + +#if defined(__linux__) +static int read_text_file(const char *path, char *value, size_t cap) +{ + FILE *f = fopen(path, "r"); + if (!f || !fgets(value, (int)cap, f)) { if (f) fclose(f); return 0; } + fclose(f); + value[strcspn(value, "\r\n")] = '\0'; + return 1; +} + +static int cpu_sensor_name(const char *name) +{ + return strstr(name, "cpu") || strstr(name, "CPU") || strstr(name, "package") || + strstr(name, "Package") || strstr(name, "coretemp") || strstr(name, "k10temp") || + strstr(name, "zenpower") || strstr(name, "x86_pkg_temp") || strstr(name, "soc_thermal"); +} +#endif + +static double sample_cpu_temperature(void) +{ +#if defined(__linux__) + DIR *dir = opendir("/sys/class/thermal"); + struct dirent *entry; + double hottest = -1.0; + if (!dir) return -1.0; + while ((entry = readdir(dir)) != NULL) { + char path[512], type[128]; + double value; + if (strncmp(entry->d_name, "thermal_zone", 12) != 0) continue; + snprintf(path, sizeof(path), "/sys/class/thermal/%s/type", entry->d_name); + if (!read_text_file(path, type, sizeof(type)) || !cpu_sensor_name(type)) continue; + snprintf(path, sizeof(path), "/sys/class/thermal/%s/temp", entry->d_name); + if (read_number_file(path, &value)) { + if (value > 1000.0) value /= 1000.0; + if (value >= 0.0 && value <= 150.0 && value > hottest) hottest = value; + } + } + closedir(dir); + if (hottest < 0.0) { + dir = opendir("/sys/class/hwmon"); + if (!dir) return -1.0; + while ((entry = readdir(dir)) != NULL) { + char base[512], path[512], name[128]; + DIR *sensor_dir; + struct dirent *sensor; + if (strncmp(entry->d_name, "hwmon", 5) != 0) continue; + snprintf(base, sizeof(base), "/sys/class/hwmon/%s", entry->d_name); + snprintf(path, sizeof(path), "%s/name", base); + if (!read_text_file(path, name, sizeof(name)) || !cpu_sensor_name(name)) continue; + sensor_dir = opendir(base); + if (!sensor_dir) continue; + while ((sensor = readdir(sensor_dir)) != NULL) { + size_t length = strlen(sensor->d_name); + double value; + if (strncmp(sensor->d_name, "temp", 4) != 0 || length < 7 || strcmp(sensor->d_name + length - 6, "_input") != 0) continue; + snprintf(path, sizeof(path), "%s/%s", base, sensor->d_name); + if (read_number_file(path, &value)) { + if (value > 1000.0) value /= 1000.0; + if (value >= 0.0 && value <= 150.0 && value > hottest) hottest = value; + } + } + closedir(sensor_dir); + } + closedir(dir); + } + return hottest; +#else + return -1.0; +#endif +} + +static double sample_cpu_clock_mhz(void) +{ +#if defined(__linux__) + DIR *dir = opendir("/sys/devices/system/cpu"); + struct dirent *entry; + double total = 0.0; + long count = 0; + if (dir) { + while ((entry = readdir(dir)) != NULL) { + char path[512], *end; + double value; + if (strncmp(entry->d_name, "cpu", 3) != 0 || !isdigit((unsigned char)entry->d_name[3])) continue; + strtol(entry->d_name + 3, &end, 10); + if (*end) continue; + snprintf(path, sizeof(path), "/sys/devices/system/cpu/%s/cpufreq/scaling_cur_freq", entry->d_name); + if (read_number_file(path, &value) && value > 0.0) { total += value / 1000.0; count++; } + } + closedir(dir); + } + if (count) return total / (double)count; + { + FILE *f = fopen("/proc/cpuinfo", "r"); + char line[256]; + if (!f) return -1.0; + while (fgets(line, sizeof(line), f)) { + double value; + if (sscanf(line, "cpu MHz%*[^:]: %lf", &value) == 1 && value > 0.0) { total += value; count++; } + } + fclose(f); + } + return count ? total / (double)count : -1.0; +#elif defined(__APPLE__) + { + uint64_t hz = 0; size_t size = sizeof(hz); + return sysctlbyname("hw.cpufrequency", &hz, &size, NULL, 0) == 0 && hz ? (double)hz / 1000000.0 : -1.0; + } +#elif defined(_WIN32) + { + HKEY key; DWORD mhz = 0, size = sizeof(mhz), type = 0; + if (RegOpenKeyExA(HKEY_LOCAL_MACHINE, "HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0", 0, KEY_READ, &key) != ERROR_SUCCESS) return -1.0; + if (RegQueryValueExA(key, "~MHz", NULL, &type, (LPBYTE)&mhz, &size) != ERROR_SUCCESS || type != REG_DWORD) mhz = 0; + RegCloseKey(key); + return mhz ? (double)mhz : -1.0; + } +#else + return -1.0; +#endif +} + +static void record_telemetry_sample(struct telemetry_monitor *monitor) +{ + struct telemetry_sample *sample; + if (monitor->count >= MAX_TELEMETRY_SAMPLES) return; + sample = &monitor->samples[monitor->count++]; + sample->elapsed_ms = (uint64_t)((now_seconds() - monitor->started) * 1000.0); + sample->temperature_c = sample_cpu_temperature(); + sample->clock_mhz = sample_cpu_clock_mhz(); +} + +static void telemetry_sleep(void) +{ +#if defined(_WIN32) + Sleep(100); +#else + usleep(100000); +#endif +} + +#if defined(_WIN32) +static DWORD WINAPI telemetry_entry(LPVOID arg) +#else +static void *telemetry_entry(void *arg) +#endif +{ + struct telemetry_monitor *monitor = (struct telemetry_monitor *)arg; + double next = monitor->started; + while (!monitor->stop) { + double current = now_seconds(); + if (current >= next) { record_telemetry_sample(monitor); next += 1.0; } + telemetry_sleep(); + } + record_telemetry_sample(monitor); +#if defined(_WIN32) + return 0; +#else + return NULL; +#endif +} + +static int start_telemetry_monitor(struct telemetry_monitor *monitor, double started) +{ + memset(monitor, 0, sizeof(*monitor)); + monitor->started = started; +#if defined(_WIN32) + monitor->thread = CreateThread(NULL, 0, telemetry_entry, monitor, 0, NULL); + return monitor->thread != NULL; +#else + return pthread_create(&monitor->thread, NULL, telemetry_entry, monitor) == 0; +#endif +} + +static void stop_telemetry_monitor(struct telemetry_monitor *monitor, int started) +{ + if (!started) return; + monitor->stop = 1; +#if defined(_WIN32) + WaitForSingleObject(monitor->thread, INFINITE); + CloseHandle(monitor->thread); +#else + pthread_join(monitor->thread, NULL); +#endif +} + struct cpu_snapshot { uint64_t total, idle; }; static int take_cpu_snapshot(struct cpu_snapshot *s) @@ -789,7 +1002,9 @@ int fossbench_run(int verbose, int upload_mode, int system_check) struct result real_multi[NTESTS], real_single[NTESTS]; struct system_info system_info; struct background_metrics background; + struct telemetry_monitor telemetry; double benchmark_started; + int telemetry_started; uint64_t duration_ms; size_t i; @@ -810,6 +1025,7 @@ int fossbench_run(int verbose, int upload_mode, int system_check) } } benchmark_started = now_seconds(); + telemetry_started = start_telemetry_monitor(&telemetry, benchmark_started); printf("\n preparing workloads..."); fflush(stdout); @@ -860,6 +1076,7 @@ int fossbench_run(int verbose, int upload_mode, int system_check) } printf(" --------------------------------------------------------------------------\n"); + stop_telemetry_monitor(&telemetry, telemetry_started); duration_ms = (uint64_t)((now_seconds() - benchmark_started) * 1000.0); printf(" %-24s %41.2fs\n", "TOTAL DURATION", (double)duration_ms / 1000.0); @@ -890,7 +1107,8 @@ int fossbench_run(int verbose, int upload_mode, int system_check) fprintf(stderr, " Upload support is disabled in this build.\n"); #else upload_results(&system_info, raw_multi, raw_single, - real_multi, real_single, duration_ms, &background); + real_multi, real_single, duration_ms, &background, + &telemetry); #endif } } diff --git a/src/app/upload.c b/src/app/upload.c index 4e2e01c..f1098f2 100644 --- a/src/app/upload.c +++ b/src/app/upload.c @@ -157,15 +157,16 @@ static int upload_results(const struct system_info *info, const struct result *raw_single, const struct result *real_multi, const struct result *real_single, uint64_t duration_ms, - const struct background_metrics *background) + const struct background_metrics *background, + const struct telemetry_monitor *telemetry) { - char host[256], port[16], path[512], payload[32768]; + char host[256], port[16], path[512], payload[131072]; char auth_header[600], response_body[2048], claim_url[1024], result_url[1024]; char cpu[512], model[512], os[512], compiler[256], kernel[256]; const char *base = FB_API_BASE_URL, *p, *slash, *colon; int status = 0, payload_len; #if !defined(_WIN32) - char request[40000], response[4096]; + char request[140000], response[4096]; struct addrinfo hints, *addresses = NULL, *a; int fd = -1, request_len; #endif @@ -225,6 +226,27 @@ static int upload_results(const struct system_info *info, used += (size_t)n; if (!append_result_tests(payload, sizeof payload, &used, real_multi, real_single)) return 0; if (used + 3 >= sizeof payload) return 0; + n = snprintf(payload + used, sizeof payload - used, "],\"telemetry\":["); + if (n < 0 || (size_t)n >= sizeof payload - used) return 0; + used += (size_t)n; + { + size_t i; + for (i = 0; i < telemetry->count; i++) { + const struct telemetry_sample *sample = &telemetry->samples[i]; + char temperature[32], clock[32]; + if (sample->temperature_c < 0.0) strcpy(temperature, "null"); + else snprintf(temperature, sizeof(temperature), "%.2f", sample->temperature_c); + if (sample->clock_mhz < 0.0) strcpy(clock, "null"); + else snprintf(clock, sizeof(clock), "%.2f", sample->clock_mhz); + n = snprintf(payload + used, sizeof payload - used, + "%s{\"elapsed_ms\":%llu,\"temperature_c\":%s,\"clock_mhz\":%s}", + i ? "," : "", (unsigned long long)sample->elapsed_ms, + temperature, clock); + if (n < 0 || (size_t)n >= sizeof payload - used) return 0; + used += (size_t)n; + } + } + if (used + 4 >= sizeof payload) return 0; memcpy(payload + used, "]}}", 4); payload_len = (int)(used + 3); } From 920092333f22adb502ad2f9481016bcaa82e7e7f Mon Sep 17 00:00:00 2001 From: Owen Rummage Date: Mon, 20 Jul 2026 22:47:39 -0500 Subject: [PATCH 2/4] Sample CPU telemetry every quarter second --- src/app/benchmark.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/app/benchmark.c b/src/app/benchmark.c index 78ac4a4..65f8b5a 100644 --- a/src/app/benchmark.c +++ b/src/app/benchmark.c @@ -382,9 +382,9 @@ static void record_telemetry_sample(struct telemetry_monitor *monitor) static void telemetry_sleep(void) { #if defined(_WIN32) - Sleep(100); + Sleep(25); #else - usleep(100000); + usleep(25000); #endif } @@ -398,7 +398,7 @@ static void *telemetry_entry(void *arg) double next = monitor->started; while (!monitor->stop) { double current = now_seconds(); - if (current >= next) { record_telemetry_sample(monitor); next += 1.0; } + if (current >= next) { record_telemetry_sample(monitor); next += 0.25; } telemetry_sleep(); } record_telemetry_sample(monitor); From 042c83975123c499d3803971e88852af4a82b910 Mon Sep 17 00:00:00 2001 From: Owen Rummage Date: Mon, 20 Jul 2026 22:54:57 -0500 Subject: [PATCH 3/4] Record benchmark workload phases --- src/app/benchmark.c | 26 ++++++++++++++++++++++++++ src/app/upload.c | 15 +++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/src/app/benchmark.c b/src/app/benchmark.c index 65f8b5a..a50a8be 100644 --- a/src/app/benchmark.c +++ b/src/app/benchmark.c @@ -221,15 +221,24 @@ struct background_metrics { }; #define MAX_TELEMETRY_SAMPLES 1800 +#define MAX_TELEMETRY_PHASES 44 struct telemetry_sample { uint64_t elapsed_ms; double temperature_c, clock_mhz; }; +struct telemetry_phase { + uint64_t elapsed_ms; + const char *suite, *mode; + int workload_index; +}; + struct telemetry_monitor { struct telemetry_sample samples[MAX_TELEMETRY_SAMPLES]; size_t count; + struct telemetry_phase phases[MAX_TELEMETRY_PHASES]; + size_t phase_count; double started; volatile int stop; #if defined(_WIN32) @@ -239,6 +248,19 @@ struct telemetry_monitor { #endif }; +static void record_telemetry_phase(struct telemetry_monitor *monitor, + const char *suite, int workload_index, + const char *mode) +{ + struct telemetry_phase *phase; + if (monitor->phase_count >= MAX_TELEMETRY_PHASES) return; + phase = &monitor->phases[monitor->phase_count++]; + phase->elapsed_ms = (uint64_t)((now_seconds() - monitor->started) * 1000.0); + phase->suite = suite; + phase->workload_index = workload_index; + phase->mode = mode; +} + static int read_number_file(const char *path, double *value) { FILE *f = fopen(path, "r"); @@ -1041,7 +1063,9 @@ int fossbench_run(int verbose, int upload_mode, int system_check) fflush(stdout); /* Run every test with all cores and one core. */ + record_telemetry_phase(&telemetry, "asm", (int)i, "multicore"); raw_multi[i] = run_test(&tests[i], (int)g_ncores); + record_telemetry_phase(&telemetry, "asm", (int)i, "singlecore"); raw_single[i] = run_test(&tests[i], 1); printf(" %12.1f %-11s %7.2fs\n", @@ -1062,7 +1086,9 @@ int fossbench_run(int verbose, int upload_mode, int system_check) for (i = 0; i < NTESTS; i++) { printf(" %-24s", tests[i].name); fflush(stdout); + record_telemetry_phase(&telemetry, "c", (int)i, "multicore"); real_multi[i] = run_test(&tests[i], (int)g_ncores); + record_telemetry_phase(&telemetry, "c", (int)i, "singlecore"); real_single[i] = run_test(&tests[i], 1); printf(" %12.1f %-11s %7.2fs\n", display_metric(&tests[i], &real_multi[i]), tests[i].unit, diff --git a/src/app/upload.c b/src/app/upload.c index f1098f2..17ed4b1 100644 --- a/src/app/upload.c +++ b/src/app/upload.c @@ -246,6 +246,21 @@ static int upload_results(const struct system_info *info, used += (size_t)n; } } + n = snprintf(payload + used, sizeof payload - used, "],\"telemetry_phases\":["); + if (n < 0 || (size_t)n >= sizeof payload - used) return 0; + used += (size_t)n; + { + size_t i; + for (i = 0; i < telemetry->phase_count; i++) { + const struct telemetry_phase *phase = &telemetry->phases[i]; + n = snprintf(payload + used, sizeof payload - used, + "%s{\"elapsed_ms\":%llu,\"suite\":\"%s\",\"workload_index\":%d,\"mode\":\"%s\"}", + i ? "," : "", (unsigned long long)phase->elapsed_ms, + phase->suite, phase->workload_index, phase->mode); + if (n < 0 || (size_t)n >= sizeof payload - used) return 0; + used += (size_t)n; + } + } if (used + 4 >= sizeof payload) return 0; memcpy(payload + used, "]}}", 4); payload_len = (int)(used + 3); From 31ec0f67e0520136457de5128b845b1680def13a Mon Sep 17 00:00:00 2001 From: Owen Rummage Date: Tue, 21 Jul 2026 00:29:39 -0500 Subject: [PATCH 4/4] Drop workload phase recording and slow telemetry sampling to 2/sec The telemetry thread's per-core /sys scan was contending with the timed benchmark passes, and could corrupt the workload-size calibration step, causing run-to-run swings of up to 4x. Removing phase recording and dropping the sample rate from 4/sec to 2/sec brings runs back to <5% variance. --- src/app/benchmark.c | 31 ++++--------------------------- src/app/upload.c | 15 --------------- 2 files changed, 4 insertions(+), 42 deletions(-) diff --git a/src/app/benchmark.c b/src/app/benchmark.c index a50a8be..87bee02 100644 --- a/src/app/benchmark.c +++ b/src/app/benchmark.c @@ -173,6 +173,8 @@ extern uint64_t fb_c_chase(void **ptrs, uint64_t steps); # define REPEATS 3 /* Number of tries. */ #endif +#define TELEMETRY_REFRESH_INTERVAL 0.5 + /* Simple repeatable random numbers. */ static uint64_t rng_state = 0x853c49e6748fea9bULL; @@ -220,25 +222,17 @@ struct background_metrics { int samples, available; }; + #define MAX_TELEMETRY_SAMPLES 1800 -#define MAX_TELEMETRY_PHASES 44 struct telemetry_sample { uint64_t elapsed_ms; double temperature_c, clock_mhz; }; -struct telemetry_phase { - uint64_t elapsed_ms; - const char *suite, *mode; - int workload_index; -}; - struct telemetry_monitor { struct telemetry_sample samples[MAX_TELEMETRY_SAMPLES]; size_t count; - struct telemetry_phase phases[MAX_TELEMETRY_PHASES]; - size_t phase_count; double started; volatile int stop; #if defined(_WIN32) @@ -248,19 +242,6 @@ struct telemetry_monitor { #endif }; -static void record_telemetry_phase(struct telemetry_monitor *monitor, - const char *suite, int workload_index, - const char *mode) -{ - struct telemetry_phase *phase; - if (monitor->phase_count >= MAX_TELEMETRY_PHASES) return; - phase = &monitor->phases[monitor->phase_count++]; - phase->elapsed_ms = (uint64_t)((now_seconds() - monitor->started) * 1000.0); - phase->suite = suite; - phase->workload_index = workload_index; - phase->mode = mode; -} - static int read_number_file(const char *path, double *value) { FILE *f = fopen(path, "r"); @@ -420,7 +401,7 @@ static void *telemetry_entry(void *arg) double next = monitor->started; while (!monitor->stop) { double current = now_seconds(); - if (current >= next) { record_telemetry_sample(monitor); next += 0.25; } + if (current >= next) { record_telemetry_sample(monitor); next += TELEMETRY_REFRESH_INTERVAL; } telemetry_sleep(); } record_telemetry_sample(monitor); @@ -1063,9 +1044,7 @@ int fossbench_run(int verbose, int upload_mode, int system_check) fflush(stdout); /* Run every test with all cores and one core. */ - record_telemetry_phase(&telemetry, "asm", (int)i, "multicore"); raw_multi[i] = run_test(&tests[i], (int)g_ncores); - record_telemetry_phase(&telemetry, "asm", (int)i, "singlecore"); raw_single[i] = run_test(&tests[i], 1); printf(" %12.1f %-11s %7.2fs\n", @@ -1086,9 +1065,7 @@ int fossbench_run(int verbose, int upload_mode, int system_check) for (i = 0; i < NTESTS; i++) { printf(" %-24s", tests[i].name); fflush(stdout); - record_telemetry_phase(&telemetry, "c", (int)i, "multicore"); real_multi[i] = run_test(&tests[i], (int)g_ncores); - record_telemetry_phase(&telemetry, "c", (int)i, "singlecore"); real_single[i] = run_test(&tests[i], 1); printf(" %12.1f %-11s %7.2fs\n", display_metric(&tests[i], &real_multi[i]), tests[i].unit, diff --git a/src/app/upload.c b/src/app/upload.c index 17ed4b1..f1098f2 100644 --- a/src/app/upload.c +++ b/src/app/upload.c @@ -246,21 +246,6 @@ static int upload_results(const struct system_info *info, used += (size_t)n; } } - n = snprintf(payload + used, sizeof payload - used, "],\"telemetry_phases\":["); - if (n < 0 || (size_t)n >= sizeof payload - used) return 0; - used += (size_t)n; - { - size_t i; - for (i = 0; i < telemetry->phase_count; i++) { - const struct telemetry_phase *phase = &telemetry->phases[i]; - n = snprintf(payload + used, sizeof payload - used, - "%s{\"elapsed_ms\":%llu,\"suite\":\"%s\",\"workload_index\":%d,\"mode\":\"%s\"}", - i ? "," : "", (unsigned long long)phase->elapsed_ms, - phase->suite, phase->workload_index, phase->mode); - if (n < 0 || (size_t)n >= sizeof payload - used) return 0; - used += (size_t)n; - } - } if (used + 4 >= sizeof payload) return 0; memcpy(payload + used, "]}}", 4); payload_len = (int)(used + 3);