FOSSbench/app
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move benchmark scoring to the server

Browse Source
This commit is contained in:
owen
2026-07-19 19:22:21 -05:00
parent 290344ae5b
commit 59b158b1c9
2 changed files with 44 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/app/benchmark.c
+22 -88
View File
@@ -4,7 +4,6 @@
#include <stdlib.h> #include <stdlib.h>
#include <string.h> #include <string.h>
#include <stdint.h> #include <stdint.h>
#include <math.h>
#include <pthread.h> #include <pthread.h>
#include <unistd.h> #include <unistd.h>
#include <ctype.h> #include <ctype.h>
@@ -30,7 +29,7 @@
#ifndef FB_API_BASE_URL #ifndef FB_API_BASE_URL
# define FB_API_BASE_URL "http://fossbench.net" # define FB_API_BASE_URL "http://fossbench.net"
#endif #endif
#define FB_VERSION "0.2.2" #define FB_VERSION "0.3.0"
#if defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) #if defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64)
# define D_INT "64-bit ALU: madd, umulh, udiv, bitops" # define D_INT "64-bit ALU: madd, umulh, udiv, bitops"
@@ -136,36 +135,6 @@ extern uint64_t fb_chase(void **ptrs, uint64_t steps);
# define REPEATS 3 /* Number of tries. */ # define REPEATS 3 /* Number of tries. */
#endif #endif
/* Numbers used to calculate scores. */
#define FB_TARGET_SCORE 10000.0 /* reference-machine overall. */
/* Rates from the reference machine. */
#define FB_REF_INT 3086.0 /* Reference rate. */
#define FB_REF_INT32 3086.0 /* Common 32-bit integer reference. */
#define FB_REF_FP 1682.0 /* Reference rate. */
#define FB_REF_PRIMES 812.0 /* Reference rate. */
#define FB_REF_SIMD 6576.0 /* Reference rate. */
#define FB_REF_COMPRESS 674.0 /* Reference rate. */
#define FB_REF_CRYPTO 406.0 /* Reference rate. */
#define FB_REF_PHYSICS 631.0 /* Reference rate. */
#define FB_REF_SORT 363.0 /* Reference rate. */
#define FB_REF_CHASE 79.0 /* Memory test reference. */
#define FB_REF_STREAM 3200.0 /* STREAM triad reference rate. */
/* How much each test counts. */
#define FB_WEIGHT_INT 3.0 /* Wide integer mix. */
#define FB_WEIGHT_INT32 10.0 /* Common native-width integer work. */
#define FB_WEIGHT_CHASE 10.0 /* Random-access latency. */
#define FB_WEIGHT_STREAM 15.0 /* Sustained memory bandwidth. */
#define FB_WEIGHT_COMPRESS 12.0 /* Score weight. */
#define FB_WEIGHT_SORT 8.0 /* Score weight. */
#define FB_WEIGHT_SIMD 16.0 /* Vector and packed arithmetic. */
#define FB_WEIGHT_FP 10.0 /* Score weight. */
#define FB_WEIGHT_CRYPTO 8.0 /* Score weight. */
#define FB_WEIGHT_PRIMES 5.0 /* Score weight. */
#define FB_WEIGHT_PHYSICS 3.0 /* Divide/square-root-heavy. */
/* Simple repeatable random numbers. */ /* Simple repeatable random numbers. */
static uint64_t rng_state = 0x853c49e6748fea9bULL; static uint64_t rng_state = 0x853c49e6748fea9bULL;
@@ -464,10 +433,8 @@ struct test {
const char *detail; const char *detail;
run_fn run; run_fn run;
uint64_t start_n; uint64_t start_n;
double work_per_n; /* Amount of work used for scoring. */ double work_per_n; /* Amount of work used to calculate the raw rate. */
const char *unit; const char *unit;
double ref_rate; /* Rate used as the score reference. */
double weight; /* How much this test counts. */
}; };
static uint64_t run_int(uint64_t n, struct workspace *ws) static uint64_t run_int(uint64_t n, struct workspace *ws)
@@ -566,45 +533,33 @@ static uint64_t run_stream(uint64_t n, struct workspace *ws)
static const struct test tests[] = { static const struct test tests[] = {
{ "Native Integer Math", "common 32-bit multiply/add/rotate mix", { "Native Integer Math", "common 32-bit multiply/add/rotate mix",
run_int32, 20, 100000.0 * 16, "Mops/s", run_int32, 20, 100000.0 * 16, "Mops/s" },
FB_REF_INT32, FB_WEIGHT_INT32 },
{ "Wide Integer Math", D_INT, { "Wide Integer Math", D_INT,
run_int, 20, 100000.0 * 24, "Mops/s", run_int, 20, 100000.0 * 24, "Mops/s" },
FB_REF_INT, FB_WEIGHT_INT },
{ "Floating Point Math", D_FP, { "Floating Point Math", D_FP,
run_fp, 20, 100000.0 * 20, "Mops/s", run_fp, 20, 100000.0 * 20, "Mops/s" },
FB_REF_FP, FB_WEIGHT_FP },
{ "Prime Numbers", "sieve of Eratosthenes to 2M", { "Prime Numbers", "sieve of Eratosthenes to 2M",
run_primes, 1, (double)PRIME_LIMIT, "Mcand/s", run_primes, 1, (double)PRIME_LIMIT, "Mcand/s" },
FB_REF_PRIMES, FB_WEIGHT_PRIMES },
{ "Extended Instructions",D_SIMD, { "Extended Instructions",D_SIMD,
run_simd, 10, 100000.0 * 32, "Mops/s", run_simd, 10, 100000.0 * 32, "Mops/s" },
FB_REF_SIMD, FB_WEIGHT_SIMD },
{ "Compression", "LZ77 match finder, 4 MiB corpus", { "Compression", "LZ77 match finder, 4 MiB corpus",
run_compress, 1, (double)COMPRESS_LEN, "MB/s", run_compress, 1, (double)COMPRESS_LEN, "MB/s" },
FB_REF_COMPRESS, FB_WEIGHT_COMPRESS },
{ "Encryption", "ChaCha20, 20 rounds, 1 MiB", { "Encryption", "ChaCha20, 20 rounds, 1 MiB",
run_crypto, 4, (double)CIPHER_LEN, "MB/s", run_crypto, 4, (double)CIPHER_LEN, "MB/s" },
FB_REF_CRYPTO, FB_WEIGHT_CRYPTO },
{ "Physics", "512-body direct-sum gravity", { "Physics", "512-body direct-sum gravity",
run_physics, 4, (double)NBODY_N * NBODY_N, "Mpair/s", run_physics, 4, (double)NBODY_N * NBODY_N, "Mpair/s" },
FB_REF_PHYSICS, FB_WEIGHT_PHYSICS },
{ "Sorting", "heapsort, 1M uint32", { "Sorting", "heapsort, 1M uint32",
run_sort, 1, (double)SORT_N * 20, "Mkey-cmp/s", run_sort, 1, (double)SORT_N * 20, "Mkey-cmp/s" },
FB_REF_SORT, FB_WEIGHT_SORT },
{ "Memory Latency", CHASE_DETAIL, { "Memory Latency", CHASE_DETAIL,
run_chase, 1, 1000000.0, "ns/access", run_chase, 1, 1000000.0, "ns/access" },
FB_REF_CHASE, FB_WEIGHT_CHASE },
{ "Memory Bandwidth", "STREAM triad, 12 MiB working set per thread", { "Memory Bandwidth", "STREAM triad, 12 MiB working set per thread",
run_stream, 16, (double)STREAM_N * 12.0, "MB/s", run_stream, 16, (double)STREAM_N * 12.0, "MB/s" },
FB_REF_STREAM, FB_WEIGHT_STREAM },
}; };
#define NTESTS (sizeof(tests) / sizeof(tests[0])) #define NTESTS (sizeof(tests) / sizeof(tests[0]))
struct result { struct result {
double rate; /* Measured speed. */ double rate; /* Measured speed. */
double score;
uint64_t checksum; uint64_t checksum;
double seconds; double seconds;
uint64_t iters; uint64_t iters;
@@ -712,8 +667,6 @@ static struct result run_test(const struct test *t, int threads)
r.threads = threads; r.threads = threads;
/* Calculate the total speed. */ /* Calculate the total speed. */
r.rate = ((double)threads * (double)n * t->work_per_n) / best / 1e6; r.rate = ((double)threads * (double)n * t->work_per_n) / best / 1e6;
/* Turn the speed into a score. */
r.score = FB_TARGET_SCORE * (r.rate / t->ref_rate);
return r; return r;
} }
@@ -745,8 +698,8 @@ static void print_header(const struct system_info *info)
printf(" kernel: %s\n", info->kernel[0] ? info->kernel : "unknown"); printf(" kernel: %s\n", info->kernel[0] ? info->kernel : "unknown");
printf(" compiler: %s\n", info->compiler); printf(" compiler: %s\n", info->compiler);
printf("\n"); printf("\n");
printf(" %-24s %12s %-11s %8s %9s\n", printf(" %-24s %12s %-11s %8s\n",
"TEST", "RATE", "UNIT", "TIME", "SCORE"); "TEST", "RATE", "UNIT", "TIME");
printf(" --------------------------------------------------------------------------\n"); printf(" --------------------------------------------------------------------------\n");
fflush(stdout); fflush(stdout);
} }
@@ -756,9 +709,7 @@ int fossbench_run(int verbose, int upload_mode, int system_check)
struct result multi[NTESTS], single[NTESTS]; struct result multi[NTESTS], single[NTESTS];
struct system_info system_info; struct system_info system_info;
struct background_metrics background; struct background_metrics background;
double multi_log_sum = 0.0, single_log_sum = 0.0; double benchmark_started;
double weight_sum = 0.0;
double benchmark_started, multicore_score, singlecore_score;
uint64_t duration_ms; uint64_t duration_ms;
size_t i; size_t i;
@@ -788,8 +739,6 @@ int fossbench_run(int verbose, int upload_mode, int system_check)
print_header(&system_info); print_header(&system_info);
for (i = 0; i < NTESTS; i++) { for (i = 0; i < NTESTS; i++) {
double sm, ss;
printf(" %-24s", tests[i].name); printf(" %-24s", tests[i].name);
fflush(stdout); fflush(stdout);
@@ -797,35 +746,21 @@ int fossbench_run(int verbose, int upload_mode, int system_check)
multi[i] = run_test(&tests[i], (int)g_ncores); multi[i] = run_test(&tests[i], (int)g_ncores);
single[i] = run_test(&tests[i], 1); single[i] = run_test(&tests[i], 1);
printf(" %12.1f %-11s %7.2fs %9.0f\n", printf(" %12.1f %-11s %7.2fs\n",
display_metric(&tests[i], &multi[i]), tests[i].unit, display_metric(&tests[i], &multi[i]), tests[i].unit,
multi[i].seconds, multi[i].score); multi[i].seconds);
if (verbose) if (verbose)
printf(" %-24s %s\n" printf(" %-24s %s\n"
" %-24s weight=%.0f%% 1-core: %.1f %s / %.0f %ld-core: %.1f %s / %.0f\n", " %-24s 1-core: %.1f %s %ld-core: %.1f %s\n",
"", tests[i].detail, "", tests[i].weight, "", tests[i].detail, "",
display_metric(&tests[i], &single[i]), tests[i].unit, display_metric(&tests[i], &single[i]), tests[i].unit,
single[i].score, g_ncores, g_ncores, display_metric(&tests[i], &multi[i]), tests[i].unit);
display_metric(&tests[i], &multi[i]), tests[i].unit,
multi[i].score);
fflush(stdout); fflush(stdout);
/* Add this test to both total scores. */
sm = multi[i].score > 0.0 ? multi[i].score : 1e-9;
ss = single[i].score > 0.0 ? single[i].score : 1e-9;
multi_log_sum += tests[i].weight * log(sm);
single_log_sum += tests[i].weight * log(ss);
weight_sum += tests[i].weight;
} }
printf(" --------------------------------------------------------------------------\n"); printf(" --------------------------------------------------------------------------\n");
/* Add this test to both total scores. */
multicore_score = exp(multi_log_sum / weight_sum);
singlecore_score = exp(single_log_sum / weight_sum);
duration_ms = (uint64_t)((now_seconds() - benchmark_started) * 1000.0); duration_ms = (uint64_t)((now_seconds() - benchmark_started) * 1000.0);
printf(" %-24s %44.0f\n", "MULTICORE SCORE", multicore_score);
printf(" %-24s %44.0f\n", "SINGLECORE SCORE", singlecore_score);
printf(" %-24s %41.2fs\n", "TOTAL DURATION", (double)duration_ms / 1000.0); printf(" %-24s %41.2fs\n", "TOTAL DURATION", (double)duration_ms / 1000.0);
printf("\n"); printf("\n");
@@ -853,8 +788,7 @@ int fossbench_run(int verbose, int upload_mode, int system_check)
#if defined(FB_NO_UPLOAD) #if defined(FB_NO_UPLOAD)
fprintf(stderr, " Upload support is disabled in this build.\n"); fprintf(stderr, " Upload support is disabled in this build.\n");
#else #else
upload_results(&system_info, multicore_score, singlecore_score, upload_results(&system_info, multi, single, duration_ms, &background);
multi, single, duration_ms, &background);
#endif #endif
} }
} }
src/app/upload.c
+22 -20
View File
@@ -120,13 +120,13 @@ done:
} }
#endif #endif
static int upload_results(const struct system_info *info, double score, static int upload_results(const struct system_info *info,
double singlecore_score, const struct result *multi, const struct result *multi,
const struct result *single, uint64_t duration_ms, const struct result *single, uint64_t duration_ms,
const struct background_metrics *background) const struct background_metrics *background)
{ {
char host[256], port[16], path[512], payload[16384]; char host[256], port[16], path[512], payload[16384];
char auth_header[600], response_body[2048], claim_url[1024]; char auth_header[600], response_body[2048], claim_url[1024], result_url[1024];
char cpu[512], model[512], os[512], compiler[256], kernel[256]; char cpu[512], model[512], os[512], compiler[256], kernel[256];
const char *base = FB_API_BASE_URL, *p, *slash, *colon; const char *base = FB_API_BASE_URL, *p, *slash, *colon;
int status = 0, payload_len; int status = 0, payload_len;
@@ -168,16 +168,14 @@ static int upload_results(const struct system_info *info, double score,
payload_len = snprintf(payload, sizeof(payload), payload_len = snprintf(payload, sizeof(payload),
"{\"cpu\":\"%s\",\"model\":\"%s\",\"cpu_cores\":%ld,\"cpu_threads\":%ld," "{\"cpu\":\"%s\",\"model\":\"%s\",\"cpu_cores\":%ld,\"cpu_threads\":%ld,"
"\"memory_mb\":%ld,\"operating_system\":\"%s\",\"compiler\":\"%s\"," "\"memory_mb\":%ld,\"operating_system\":\"%s\",\"compiler\":\"%s\","
"\"fossmark_version\":\"%s\",\"score\":%.17g,\"duration_ms\":%llu," "\"fossmark_version\":\"%s\",\"workload_suite\":\"fossbench-cpu-v1\","
"\"score_details\":{\"scoring_method\":\"weighted_geometric_mean\"," "\"duration_ms\":%llu,\"score_details\":{"
"\"target_score\":%.17g,\"multicore_score\":%.17g,\"singlecore_score\":%.17g,"
"\"minimum_test_seconds\":%.17g,\"repeats\":%d," "\"minimum_test_seconds\":%.17g,\"repeats\":%d,"
"\"system_environment\":{\"kernel\":\"%s\",\"sample_seconds\":%d," "\"system_environment\":{\"kernel\":\"%s\",\"sample_seconds\":%d,"
"\"background_cpu_average_percent\":%.17g,\"background_cpu_peak_percent\":%.17g," "\"background_cpu_average_percent\":%.17g,\"background_cpu_peak_percent\":%.17g,"
"\"available_memory_mb\":%ld,\"process_count\":%ld},\"tests\":[", "\"available_memory_mb\":%ld,\"process_count\":%ld},\"tests\":[",
cpu, model, info->cpu_cores, info->cpu_threads, info->memory_mb, os, compiler, cpu, model, info->cpu_cores, info->cpu_threads, info->memory_mb, os, compiler,
FB_VERSION, score, (unsigned long long)duration_ms, FB_TARGET_SCORE, score, FB_VERSION, (unsigned long long)duration_ms, MIN_SECONDS, REPEATS, kernel, background->samples,
singlecore_score, MIN_SECONDS, REPEATS, kernel, background->samples,
background->average_cpu_percent, background->peak_cpu_percent, background->average_cpu_percent, background->peak_cpu_percent,
background->available_memory_mb, background->process_count); background->available_memory_mb, background->process_count);
if (payload_len < 0 || (size_t)payload_len >= sizeof(payload)) return 0; if (payload_len < 0 || (size_t)payload_len >= sizeof(payload)) return 0;
@@ -185,21 +183,24 @@ static int upload_results(const struct system_info *info, double score,
size_t used = (size_t)payload_len; size_t used = (size_t)payload_len;
size_t i; size_t i;
for (i = 0; i < NTESTS; i++) { for (i = 0; i < NTESTS; i++) {
static const char *ids[] = {
"native_integer", "wide_integer", "floating_point", "primes",
"extended_instructions", "compression", "encryption", "physics",
"sorting", "memory_latency", "memory_bandwidth"
};
int n = snprintf(payload + used, sizeof(payload) - used, int n = snprintf(payload + used, sizeof(payload) - used,
"%s{\"name\":\"%s\",\"detail\":\"%s\",\"unit\":\"%s\"," "%s{\"id\":\"%s\",\"name\":\"%s\",\"detail\":\"%s\",\"unit\":\"%s\","
"\"start_iterations\":%llu,\"work_per_iteration\":%.17g," "\"start_iterations\":%llu,\"work_per_iteration\":%.17g,"
"\"reference_rate\":%.17g,\"weight\":%.17g," "\"multicore\":{\"display_metric\":%.17g,\"rate\":%.17g,"
"\"multicore\":{\"display_metric\":%.17g,\"rate\":%.17g,\"score\":%.17g,"
"\"seconds\":%.17g,\"iterations\":%llu,\"threads\":%d,\"checksum\":\"%llu\"}," "\"seconds\":%.17g,\"iterations\":%llu,\"threads\":%d,\"checksum\":\"%llu\"},"
"\"singlecore\":{\"display_metric\":%.17g,\"rate\":%.17g,\"score\":%.17g," "\"singlecore\":{\"display_metric\":%.17g,\"rate\":%.17g,"
"\"seconds\":%.17g,\"iterations\":%llu,\"threads\":%d,\"checksum\":\"%llu\"}}", "\"seconds\":%.17g,\"iterations\":%llu,\"threads\":%d,\"checksum\":\"%llu\"}}",
i ? "," : "", tests[i].name, tests[i].detail, tests[i].unit, i ? "," : "", ids[i], tests[i].name, tests[i].detail, tests[i].unit,
(unsigned long long)tests[i].start_n, tests[i].work_per_n, (unsigned long long)tests[i].start_n, tests[i].work_per_n,
tests[i].ref_rate, tests[i].weight, display_metric(&tests[i], &multi[i]), multi[i].rate,
display_metric(&tests[i], &multi[i]), multi[i].rate, multi[i].score,
multi[i].seconds, (unsigned long long)multi[i].iters, multi[i].threads, multi[i].seconds, (unsigned long long)multi[i].iters, multi[i].threads,
(unsigned long long)multi[i].checksum, (unsigned long long)multi[i].checksum,
display_metric(&tests[i], &single[i]), single[i].rate, single[i].score, display_metric(&tests[i], &single[i]), single[i].rate,
single[i].seconds, (unsigned long long)single[i].iters, single[i].threads, single[i].seconds, (unsigned long long)single[i].iters, single[i].threads,
(unsigned long long)single[i].checksum); (unsigned long long)single[i].checksum);
if (n < 0 || (size_t)n >= sizeof(payload) - used) return 0; if (n < 0 || (size_t)n >= sizeof(payload) - used) return 0;
@@ -268,16 +269,17 @@ static int upload_results(const struct system_info *info, double score,
return 0; return 0;
} }
if (status < 200 || status >= 300) { fprintf(stderr, " upload failed: server returned HTTP %d\n", status); return 0; } if (status < 200 || status >= 300) { fprintf(stderr, " upload failed: server returned HTTP %d\n", status); return 0; }
if (!json_string_field(response_body, "claim_url", claim_url, sizeof(claim_url))) { if (!json_string_field(response_body, "result_url", result_url, sizeof(result_url))) {
fprintf(stderr, " upload failed: server did not return a claim link\n"); fprintf(stderr, " upload failed: server did not return a result link\n");
return 0; return 0;
} }
printf(" Results uploaded (HTTP %d).\n", status); printf(" Results uploaded (HTTP %d).\n", status);
{ printf(" View your result: %s\n", result_url);
if (json_string_field(response_body, "claim_url", claim_url, sizeof(claim_url))) {
const char *claim_code = strrchr(claim_url, '/'); const char *claim_code = strrchr(claim_url, '/');
printf(" Claim code: %s\n", claim_code && claim_code[1] ? claim_code + 1 : claim_url); printf(" Claim code: %s\n", claim_code && claim_code[1] ? claim_code + 1 : claim_url);
printf(" Claim your result: %s\n", claim_url);
} }
printf(" Claim your result: %s\n", claim_url);
return 1; return 1;
#if !defined(_WIN32) #if !defined(_WIN32)