From a8f62164b1d0ea6583c9c5d05aa4be9c946c2004 Mon Sep 17 00:00:00 2001 From: Noah Goldstein Date: Fri, 27 May 2022 18:25:40 -0500 Subject: [PATCH] benchtests: Improve benchtests for strstr, memmem, and memchr 1. Use json_ctx for output to help standardize format across all benchtests. 2. Add some additional tests to strstr and memchr expanding alignments and adding more small values. Reviewed-by: H.J. Lu --- benchtests/bench-memchr.c | 108 ++++++++++++++-------- benchtests/bench-memmem.c | 82 +++++++++++++---- benchtests/bench-strstr.c | 187 ++++++++++++++++++++++++++++++-------- 3 files changed, 284 insertions(+), 93 deletions(-) diff --git a/benchtests/bench-memchr.c b/benchtests/bench-memchr.c index fb0284f523..4d7212332f 100644 --- a/benchtests/bench-memchr.c +++ b/benchtests/bench-memchr.c @@ -53,8 +53,11 @@ SIMPLE_MEMCHR (const CHAR *s, int c, size_t n) } #endif /* !USE_AS_MEMRCHR */ +#include "json-lib.h" + static void -do_one_test (impl_t *impl, const CHAR *s, int c, size_t n) +do_one_test (json_ctx_t *json_ctx, impl_t *impl, const CHAR *s, int c, + size_t n) { size_t i, iters = INNER_LOOP_ITERS_LARGE; timing_t start, stop, cur; @@ -68,15 +71,16 @@ do_one_test (impl_t *impl, const CHAR *s, int c, size_t n) TIMING_DIFF (cur, start, stop); - TIMING_PRINT_MEAN ((double) cur, (double) iters); + json_element_double (json_ctx, (double) cur / (double) iters); } static void -do_test (size_t align, size_t pos, size_t len, int seek_char) +do_test (json_ctx_t *json_ctx, size_t align, size_t pos, size_t len, + int seek_char) { size_t i; - align &= 7; + align &= getpagesize () - 1; if ((align + len) * sizeof (CHAR) >= page_size) return; @@ -100,67 +104,99 @@ do_test (size_t align, size_t pos, size_t len, int seek_char) buf[align + len] = seek_char; } - printf ("Length %4zd, position %4zd, alignment %2zd:", - len, pos, align); + json_element_object_begin (json_ctx); + json_attr_uint (json_ctx, "align", align); + json_attr_uint (json_ctx, "pos", pos); + json_attr_uint (json_ctx, "len", len); + json_attr_uint (json_ctx, "seek_char", seek_char); + + json_array_begin (json_ctx, "timings"); FOR_EACH_IMPL (impl, 0) - do_one_test (impl, (CHAR *) (buf + align), seek_char, len); + do_one_test (json_ctx, impl, (CHAR *) (buf + align), seek_char, len); - putchar ('\n'); + json_array_end (json_ctx); + json_element_object_end (json_ctx); } int test_main (void) { size_t i; - + json_ctx_t json_ctx; test_init (); - printf ("%20s", ""); + json_init (&json_ctx, 0, stdout); + + json_document_begin (&json_ctx); + json_attr_string (&json_ctx, "timing_type", TIMING_TYPE); + + json_attr_object_begin (&json_ctx, "functions"); + json_attr_object_begin (&json_ctx, TEST_NAME); + json_attr_string (&json_ctx, "bench-variant", ""); + + json_array_begin (&json_ctx, "ifuncs"); FOR_EACH_IMPL (impl, 0) - printf ("\t%s", impl->name); - putchar ('\n'); + json_element_string (&json_ctx, impl->name); + json_array_end (&json_ctx); + + json_array_begin (&json_ctx, "results"); for (i = 1; i < 8; ++i) { - do_test (0, 16 << i, 2048, 23); - do_test (i, 64, 256, 23); - do_test (0, 16 << i, 2048, 0); - do_test (i, 64, 256, 0); + do_test (&json_ctx, 0, 16 << i, 2048, 23); + do_test (&json_ctx, i, 64, 256, 23); + do_test (&json_ctx, 0, 16 << i, 2048, 0); + do_test (&json_ctx, i, 64, 256, 0); + + do_test (&json_ctx, getpagesize () - 15, 64, 256, 0); #ifdef USE_AS_MEMRCHR /* Also test the position close to the beginning for memrchr. */ - do_test (0, i, 256, 23); - do_test (0, i, 256, 0); - do_test (i, i, 256, 23); - do_test (i, i, 256, 0); + do_test (&json_ctx, 0, i, 256, 23); + do_test (&json_ctx, 0, i, 256, 0); + do_test (&json_ctx, i, i, 256, 23); + do_test (&json_ctx, i, i, 256, 0); #endif } for (i = 1; i < 8; ++i) { - do_test (i, i << 5, 192, 23); - do_test (i, i << 5, 192, 0); - do_test (i, i << 5, 256, 23); - do_test (i, i << 5, 256, 0); - do_test (i, i << 5, 512, 23); - do_test (i, i << 5, 512, 0); + do_test (&json_ctx, i, i << 5, 192, 23); + do_test (&json_ctx, i, i << 5, 192, 0); + do_test (&json_ctx, i, i << 5, 256, 23); + do_test (&json_ctx, i, i << 5, 256, 0); + do_test (&json_ctx, i, i << 5, 512, 23); + do_test (&json_ctx, i, i << 5, 512, 0); + + do_test (&json_ctx, getpagesize () - 15, i << 5, 256, 23); } for (i = 1; i < 32; ++i) { - do_test (0, i, i + 1, 23); - do_test (0, i, i + 1, 0); - do_test (i, i, i + 1, 23); - do_test (i, i, i + 1, 0); - do_test (0, i, i - 1, 23); - do_test (0, i, i - 1, 0); - do_test (i, i, i - 1, 23); - do_test (i, i, i - 1, 0); + do_test (&json_ctx, 0, i, i + 1, 23); + do_test (&json_ctx, 0, i, i + 1, 0); + do_test (&json_ctx, i, i, i + 1, 23); + do_test (&json_ctx, i, i, i + 1, 0); + do_test (&json_ctx, 0, i, i - 1, 23); + do_test (&json_ctx, 0, i, i - 1, 0); + do_test (&json_ctx, i, i, i - 1, 23); + do_test (&json_ctx, i, i, i - 1, 0); + + do_test (&json_ctx, getpagesize () - 15, i, i - 1, 23); + do_test (&json_ctx, getpagesize () - 15, i, i - 1, 0); + + do_test (&json_ctx, getpagesize () - 15, i, i + 1, 23); + do_test (&json_ctx, getpagesize () - 15, i, i + 1, 0); #ifdef USE_AS_MEMRCHR /* Also test the position close to the beginning for memrchr. */ - do_test (0, 1, i + 1, 23); - do_test (0, 2, i + 1, 0); + do_test (&json_ctx, 0, 1, i + 1, 23); + do_test (&json_ctx, 0, 2, i + 1, 0); #endif } + json_array_end (&json_ctx); + json_attr_object_end (&json_ctx); + json_attr_object_end (&json_ctx); + json_document_end (&json_ctx); + return ret; } diff --git a/benchtests/bench-memmem.c b/benchtests/bench-memmem.c index 0fb87309c0..c2b92405dc 100644 --- a/benchtests/bench-memmem.c +++ b/benchtests/bench-memmem.c @@ -21,6 +21,7 @@ #define BUF1PAGES 20 #define ITERATIONS 100 #include "bench-string.h" +#include "json-lib.h" typedef char *(*proto_t) (const void *, size_t, const void *, size_t); @@ -102,44 +103,62 @@ IMPL (twoway_memmem, 0) IMPL (basic_memmem, 0) static void -do_one_test (impl_t *impl, const void *haystack, size_t haystack_len, - const void *needle, size_t needle_len, const void *expected) +do_one_test (json_ctx_t *json_ctx, impl_t *impl, const void *haystack, + size_t haystack_len, const void *needle, size_t needle_len, + const void *expected) { size_t i, iters = INNER_LOOP_ITERS_SMALL; timing_t start, stop, cur; - + void *res; TIMING_NOW (start); for (i = 0; i < iters; ++i) { - CALL (impl, haystack, haystack_len, needle, needle_len); + res = CALL (impl, haystack, haystack_len, needle, needle_len); } TIMING_NOW (stop); TIMING_DIFF (cur, start, stop); - TIMING_PRINT_MEAN ((double) cur, (double) iters); + json_element_double (json_ctx, (double) cur / (double) iters); + + if (res != expected) + { + error (0, 0, "Wrong result in function (%p != %p) %s(%p, %zu, %p, %zu)", + res, expected, impl->name, haystack, haystack_len, needle, + needle_len); + ret = 1; + } } static void -do_test (const char *str, size_t len, size_t idx) +do_test (json_ctx_t *json_ctx, const char *str, size_t len, size_t idx) { char tmpbuf[len]; memcpy (tmpbuf, buf1 + idx, len); memcpy (buf1 + idx, str, len); - printf ("String %s, offset %zd:", str, idx); + json_element_object_begin (json_ctx); + json_attr_uint (json_ctx, "len_haystack", BUF1PAGES * page_size); + json_attr_uint (json_ctx, "len_needle", len); + json_attr_uint (json_ctx, "haystack_ptr", (uintptr_t) buf1); + json_attr_uint (json_ctx, "needle_ptr", (uintptr_t) str); + json_attr_uint (json_ctx, "fail", 0); + + json_array_begin (json_ctx, "timings"); FOR_EACH_IMPL (impl, 0) - do_one_test (impl, buf1, BUF1PAGES * page_size, str, len, buf1 + idx); + do_one_test (json_ctx, impl, buf1, BUF1PAGES * page_size, str, len, + buf1 + idx); memcpy (buf1 + idx, tmpbuf, len); - putchar ('\n'); + json_array_end (json_ctx); + json_element_object_end (json_ctx); } static void -do_random_tests (void) +do_random_tests (json_ctx_t *json_ctx) { for (size_t n = 0; n < ITERATIONS; ++n) { @@ -159,13 +178,21 @@ do_random_tests (void) buf1[idx + off] = ch; } - printf ("String %.*s, offset %zd:", (int) len, buf1 + idx, idx); + json_element_object_begin (json_ctx); + json_attr_uint (json_ctx, "len_haystack", BUF1PAGES * page_size); + json_attr_uint (json_ctx, "len_needle", len); + json_attr_uint (json_ctx, "haystack_ptr", (uintptr_t) buf1); + json_attr_uint (json_ctx, "needle_ptr", (uintptr_t) (buf1 + idx)); + json_attr_uint (json_ctx, "fail", 0); + + json_array_begin (json_ctx, "timings"); FOR_EACH_IMPL (impl, 0) - do_one_test (impl, buf1, BUF1PAGES * page_size, buf1 + idx, len, - buf1 + idx); + do_one_test (json_ctx, impl, buf1, BUF1PAGES * page_size, buf1 + idx, + len, buf1 + idx); - putchar ('\n'); + json_array_end (json_ctx); + json_element_object_end (json_ctx); memcpy (buf1 + idx, tmpbuf, len); } @@ -178,18 +205,28 @@ static const char *const strs[] = "abc0", "aaaa0", "abcabc0" }; - int test_main (void) { + json_ctx_t json_ctx; size_t i; test_init (); + json_init (&json_ctx, 0, stdout); - printf ("%23s", ""); + json_document_begin (&json_ctx); + json_attr_string (&json_ctx, "timing_type", TIMING_TYPE); + + json_attr_object_begin (&json_ctx, "functions"); + json_attr_object_begin (&json_ctx, TEST_NAME); + json_attr_string (&json_ctx, "bench-variant", ""); + + json_array_begin (&json_ctx, "ifuncs"); FOR_EACH_IMPL (impl, 0) - printf ("\t%s", impl->name); - putchar ('\n'); + json_element_string (&json_ctx, impl->name); + json_array_end (&json_ctx); + + json_array_begin (&json_ctx, "results"); for (i = 0; i < BUF1PAGES * page_size; ++i) buf1[i] = 60 + random () % 32; @@ -199,10 +236,15 @@ test_main (void) { size_t len = strlen (strs[i]); - do_test (strs[i], len, j); + do_test (&json_ctx, strs[i], len, j); } - do_random_tests (); + do_random_tests (&json_ctx); + + json_array_end (&json_ctx); + json_attr_object_end (&json_ctx); + json_attr_object_end (&json_ctx); + json_document_end (&json_ctx); return ret; } diff --git a/benchtests/bench-strstr.c b/benchtests/bench-strstr.c index 467418b12c..feb7832837 100644 --- a/benchtests/bench-strstr.c +++ b/benchtests/bench-strstr.c @@ -21,6 +21,8 @@ #define TEST_NAME "strstr" #include "bench-string.h" +#include "json-lib.h" + static const char input[] = "This manual is written with the assumption that you are at least " "somewhat familiar with the C programming language and basic programming " @@ -129,7 +131,8 @@ IMPL (twoway_strstr, 0) IMPL (basic_strstr, 0) static void -do_one_test (impl_t *impl, const char *s1, const char *s2, char *exp_result) +do_one_test (json_ctx_t *json_ctx, impl_t *impl, const char *s1, + const char *s2, char *exp_result) { size_t i, iters = INNER_LOOP_ITERS_SMALL / 8; timing_t start, stop, cur; @@ -142,7 +145,7 @@ do_one_test (impl_t *impl, const char *s1, const char *s2, char *exp_result) TIMING_DIFF (cur, start, stop); - TIMING_PRINT_MEAN ((double) cur, (double) iters); + json_element_double (json_ctx, (double) cur / (double) iters); if (res != exp_result) { @@ -153,10 +156,9 @@ do_one_test (impl_t *impl, const char *s1, const char *s2, char *exp_result) } } - static void -do_test (size_t align1, size_t align2, size_t len1, size_t len2, - int fail) +do_test (json_ctx_t *json_ctx, size_t align1, size_t align2, size_t len1, + size_t len2, int fail) { char *s1 = (char *) (buf1 + align1); char *s2 = (char *) (buf2 + align2); @@ -195,13 +197,21 @@ do_test (size_t align1, size_t align2, size_t len1, size_t len2, if (fail || ss1 != s1 + len1 - len2) ++ss1[len2 / 2]; - printf ("Length %4zd/%3zd, alignment %2zd/%2zd, %s:", - len1, len2, align1, align2, fail ? "fail " : "found"); + json_element_object_begin (json_ctx); + json_attr_uint (json_ctx, "len_haystack", len1); + json_attr_uint (json_ctx, "len_needle", len2); + json_attr_uint (json_ctx, "align_haystack", align1); + json_attr_uint (json_ctx, "align_needle", align2); + json_attr_uint (json_ctx, "fail", fail); + + json_array_begin (json_ctx, "timings"); FOR_EACH_IMPL (impl, 0) - do_one_test (impl, s1, s2, fail ? NULL : s1 + len1 - len2); + do_one_test (json_ctx, impl, s1, s2, fail ? NULL : s1 + len1 - len2); + + json_array_end (json_ctx); + json_element_object_end (json_ctx); - putchar ('\n'); } /* Test needles which exhibit worst-case performance. This shows that @@ -211,7 +221,7 @@ do_test (size_t align1, size_t align2, size_t len1, size_t len2, within a factor of 2 on several different microarchitectures. */ static void -test_hard_needle (size_t ne_len, size_t hs_len) +test_hard_needle (json_ctx_t *json_ctx, size_t ne_len, size_t hs_len) { char *ne = (char *) buf1; char *hs = (char *) buf2; @@ -226,15 +236,25 @@ test_hard_needle (size_t ne_len, size_t hs_len) memset (hs, 'a', hs_len); for (size_t i = ne_len; i <= hs_len; i += ne_len) { - hs[i-5] = 'b'; - hs[i-62] = 'b'; + hs[i - 5] = 'b'; + hs[i - 62] = 'b'; } - printf ("Length %4zd/%3zd, complex needle 1:", hs_len, ne_len); + json_element_object_begin (json_ctx); + json_attr_uint (json_ctx, "len_haystack", hs_len); + json_attr_uint (json_ctx, "len_needle", ne_len); + json_attr_uint (json_ctx, "align_haystack", 0); + json_attr_uint (json_ctx, "align_needle", 0); + json_attr_uint (json_ctx, "fail", 1); + json_attr_string (json_ctx, "desc", "Difficult skiptable(0)"); + + json_array_begin (json_ctx, "timings"); FOR_EACH_IMPL (impl, 0) - do_one_test (impl, hs, ne, NULL); - putchar ('\n'); + do_one_test (json_ctx, impl, hs, ne, NULL); + + json_array_end (json_ctx); + json_element_object_end (json_ctx); } /* 2nd hard needle for strstr algorithm using skip table. This results in @@ -247,15 +267,25 @@ test_hard_needle (size_t ne_len, size_t hs_len) memset (hs, 'a', hs_len); for (size_t i = ne_len; i <= hs_len; i += ne_len) { - hs[i-5] = 'b'; - hs[i-6] = 'b'; + hs[i - 5] = 'b'; + hs[i - 6] = 'b'; } - printf ("Length %4zd/%3zd, complex needle 2:", hs_len, ne_len); + json_element_object_begin (json_ctx); + json_attr_uint (json_ctx, "len_haystack", hs_len); + json_attr_uint (json_ctx, "len_needle", ne_len); + json_attr_uint (json_ctx, "align_haystack", 0); + json_attr_uint (json_ctx, "align_needle", 0); + json_attr_uint (json_ctx, "fail", 1); + json_attr_string (json_ctx, "desc", "Difficult skiptable(1)"); + + json_array_begin (json_ctx, "timings"); FOR_EACH_IMPL (impl, 0) - do_one_test (impl, hs, ne, NULL); - putchar ('\n'); + do_one_test (json_ctx, impl, hs, ne, NULL); + + json_array_end (json_ctx); + json_element_object_end (json_ctx); } /* Hard needle for Two-way algorithm - the random input causes a large number @@ -267,45 +297,128 @@ test_hard_needle (size_t ne_len, size_t hs_len) hs[hs_len] = 0; memset (ne, 'a', ne_len); - ne[ne_len-2] = 'b'; + ne[ne_len - 2] = 'b'; ne[0] = 'b'; ne[ne_len] = 0; - printf ("Length %4zd/%3zd, complex needle 3:", hs_len, ne_len); + json_element_object_begin (json_ctx); + json_attr_uint (json_ctx, "len_haystack", hs_len); + json_attr_uint (json_ctx, "len_needle", ne_len); + json_attr_uint (json_ctx, "align_haystack", 0); + json_attr_uint (json_ctx, "align_needle", 0); + json_attr_uint (json_ctx, "fail", 1); + json_attr_string (json_ctx, "desc", "Difficult 2-way"); + + json_array_begin (json_ctx, "timings"); FOR_EACH_IMPL (impl, 0) - do_one_test (impl, hs, ne, NULL); - putchar ('\n'); + do_one_test (json_ctx, impl, hs, ne, NULL); + + json_array_end (json_ctx); + json_element_object_end (json_ctx); + } + + /* Hard needle for standard algorithm testing first few characters of + * needle. */ + { + for (int i = 0; i < hs_len; i++) + hs[i] = (rand () & 255) >= 128 ? 'a' : 'b'; + hs[hs_len] = 0; + + for (int i = 0; i < ne_len; i++) + { + if (i % 3 == 0) + ne[i] = 'a'; + else if (i % 3 == 1) + ne[i] = 'b'; + else + ne[i] = 'c'; + } + ne[ne_len] = 0; + + json_element_object_begin (json_ctx); + json_attr_uint (json_ctx, "len_haystack", hs_len); + json_attr_uint (json_ctx, "len_needle", ne_len); + json_attr_uint (json_ctx, "align_haystack", 0); + json_attr_uint (json_ctx, "align_needle", 0); + json_attr_uint (json_ctx, "fail", 1); + json_attr_string (json_ctx, "desc", "Difficult testing first 2"); + + json_array_begin (json_ctx, "timings"); + + FOR_EACH_IMPL (impl, 0) + do_one_test (json_ctx, impl, hs, ne, NULL); + + json_array_end (json_ctx); + json_element_object_end (json_ctx); } } static int test_main (void) { + json_ctx_t json_ctx; test_init (); - printf ("%23s", ""); - FOR_EACH_IMPL (impl, 0) - printf ("\t%s", impl->name); - putchar ('\n'); + json_init (&json_ctx, 0, stdout); - for (size_t hlen = 64; hlen <= 256; hlen += 32) + json_document_begin (&json_ctx); + json_attr_string (&json_ctx, "timing_type", TIMING_TYPE); + + json_attr_object_begin (&json_ctx, "functions"); + json_attr_object_begin (&json_ctx, TEST_NAME); + json_attr_string (&json_ctx, "bench-variant", ""); + + json_array_begin (&json_ctx, "ifuncs"); + FOR_EACH_IMPL (impl, 0) + json_element_string (&json_ctx, impl->name); + json_array_end (&json_ctx); + + json_array_begin (&json_ctx, "results"); + + for (size_t hlen = 8; hlen <= 256;) for (size_t klen = 1; klen <= 16; klen++) { - do_test (1, 3, hlen, klen, 0); - do_test (0, 9, hlen, klen, 1); + do_test (&json_ctx, 1, 3, hlen, klen, 0); + do_test (&json_ctx, 0, 9, hlen, klen, 1); + + do_test (&json_ctx, 1, 3, hlen + 1, klen, 0); + do_test (&json_ctx, 0, 9, hlen + 1, klen, 1); + + do_test (&json_ctx, getpagesize () - 15, 9, hlen, klen, 1); + if (hlen < 64) + { + hlen += 8; + } + else + { + hlen += 32; + } } for (size_t hlen = 256; hlen <= 65536; hlen *= 2) - for (size_t klen = 16; klen <= 256; klen *= 2) + for (size_t klen = 4; klen <= 256; klen *= 2) { - do_test (1, 11, hlen, klen, 0); - do_test (14, 5, hlen, klen, 1); + do_test (&json_ctx, 1, 11, hlen, klen, 0); + do_test (&json_ctx, 14, 5, hlen, klen, 1); + + do_test (&json_ctx, 1, 11, hlen + 1, klen + 1, 0); + do_test (&json_ctx, 14, 5, hlen + 1, klen + 1, 1); + + do_test (&json_ctx, 1, 11, hlen + 1, klen, 0); + do_test (&json_ctx, 14, 5, hlen + 1, klen, 1); + + do_test (&json_ctx, getpagesize () - 15, 5, hlen + 1, klen, 1); } - test_hard_needle (64, 65536); - test_hard_needle (256, 65536); - test_hard_needle (1024, 65536); + test_hard_needle (&json_ctx, 64, 65536); + test_hard_needle (&json_ctx, 256, 65536); + test_hard_needle (&json_ctx, 1024, 65536); + + json_array_end (&json_ctx); + json_attr_object_end (&json_ctx); + json_attr_object_end (&json_ctx); + json_document_end (&json_ctx); return ret; }