tools/perf/bench/mem-memcpy.c

   1 /*
   2  * mem-memcpy.c
   3  *
   4  * Simple memcpy() and memset() benchmarks
   5  *
   6  * Written by Hitoshi Mitake <mitake@dcl.info.waseda.ac.jp>
   7  */
   8
   9 #include "../perf.h"
  10 #include "../util/util.h"
  11 #include "../util/parse-options.h"
  12 #include "../util/header.h"
  13 #include "../util/cloexec.h"
  14 #include "bench.h"
  15 #include "mem-memcpy-arch.h"
  16 #include "mem-memset-arch.h"
  17
  18 #include <stdio.h>
  19 #include <stdlib.h>
  20 #include <string.h>
  21 #include <sys/time.h>
  22 #include <errno.h>
  23
  24 #define K 1024
  25
  26 static const char       *length_str     = "1MB";
  27 static const char       *routine        = "all";
  28 static int              iterations      = 1;
  29 static bool             use_cycle;
  30 static int              cycle_fd;
  31 static bool             only_prefault;
  32 static bool             no_prefault;
  33
  34 static const struct option options[] = {
  35         OPT_STRING('l', "length", &length_str, "1MB",
  36                     "Specify length of memory to copy. "
  37                     "Available units: B, KB, MB, GB and TB (upper and lower)"),
  38         OPT_STRING('r', "routine", &routine, "all",
  39                     "Specify routine to copy, \"all\" runs all available routines"),
  40         OPT_INTEGER('i', "iterations", &iterations,
  41                     "repeat memcpy() invocation this number of times"),
  42         OPT_BOOLEAN('c', "cycle", &use_cycle,
  43                     "Use cycles event instead of gettimeofday() for measuring"),
  44         OPT_BOOLEAN('o', "only-prefault", &only_prefault,
  45                     "Show only the result with page faults before memcpy()"),
  46         OPT_BOOLEAN('n', "no-prefault", &no_prefault,
  47                     "Show only the result without page faults before memcpy()"),
  48         OPT_END()
  49 };
  50
  51 typedef void *(*memcpy_t)(void *, const void *, size_t);
  52 typedef void *(*memset_t)(void *, int, size_t);
  53
  54 struct routine {
  55         const char *name;
  56         const char *desc;
  57         union {
  58                 memcpy_t memcpy;
  59                 memset_t memset;
  60         } fn;
  61 };
  62
  63 struct routine memcpy_routines[] = {
  64         { .name         = "default",
  65           .desc         = "Default memcpy() provided by glibc",
  66           .fn.memcpy    = memcpy },
  67
  68 #ifdef HAVE_ARCH_X86_64_SUPPORT
  69 # define MEMCPY_FN(_fn, _name, _desc) {.name = _name, .desc = _desc, .fn.memcpy = _fn},
  70 # include "mem-memcpy-x86-64-asm-def.h"
  71 # undef MEMCPY_FN
  72 #endif
  73
  74         { NULL, }
  75 };
  76
  77 static const char * const bench_mem_memcpy_usage[] = {
  78         "perf bench mem memcpy <options>",
  79         NULL
  80 };
  81
  82 static struct perf_event_attr cycle_attr = {
  83         .type           = PERF_TYPE_HARDWARE,
  84         .config         = PERF_COUNT_HW_CPU_CYCLES
  85 };
  86
  87 static void init_cycle(void)
  88 {
  89         cycle_fd = sys_perf_event_open(&cycle_attr, getpid(), -1, -1, perf_event_open_cloexec_flag());
  90
  91         if (cycle_fd < 0 && errno == ENOSYS)
  92                 die("No CONFIG_PERF_EVENTS=y kernel support configured?\n");
  93         else
  94                 BUG_ON(cycle_fd < 0);
  95 }
  96
  97 static u64 get_cycle(void)
  98 {
  99         int ret;
 100         u64 clk;
 101
 102         ret = read(cycle_fd, &clk, sizeof(u64));
 103         BUG_ON(ret != sizeof(u64));
 104
 105         return clk;
 106 }
 107
 108 static double timeval2double(struct timeval *ts)
 109 {
 110         return (double)ts->tv_sec + (double)ts->tv_usec / (double)1000000;
 111 }
 112
 113 #define print_bps(x) do {                                       \
 114                 if (x < K)                                      \
 115                         printf(" %14lf B/Sec", x);              \
 116                 else if (x < K * K)                             \
 117                         printf(" %14lfd KB/Sec", x / K);        \
 118                 else if (x < K * K * K)                         \
 119                         printf(" %14lf MB/Sec", x / K / K);     \
 120                 else                                            \
 121                         printf(" %14lf GB/Sec", x / K / K / K); \
 122         } while (0)
 123
 124 struct bench_mem_info {
 125         const struct routine *routines;
 126         u64 (*do_cycle)(const struct routine *r, size_t len, bool prefault);
 127         double (*do_gettimeofday)(const struct routine *r, size_t len, bool prefault);
 128         const char *const *usage;
 129 };
 130
 131 static void __bench_mem_routine(struct bench_mem_info *info, int r_idx, size_t len, double totallen)
 132 {
 133         const struct routine *r = &info->routines[r_idx];
 134         double result_bps[2];
 135         u64 result_cycle[2];
 136         int prefault = no_prefault ? 0 : 1;
 137
 138         result_cycle[0] = result_cycle[1] = 0ULL;
 139         result_bps[0] = result_bps[1] = 0.0;
 140
 141         printf("Routine %s (%s)\n", r->name, r->desc);
 142
 143         if (bench_format == BENCH_FORMAT_DEFAULT)
 144                 printf("# Copying %s Bytes ...\n\n", length_str);
 145
 146         if (!only_prefault && prefault) {
 147                 /* Show both results: */
 148                 if (use_cycle) {
 149                         result_cycle[0] = info->do_cycle(r, len, false);
 150                         result_cycle[1] = info->do_cycle(r, len, true);
 151                 } else {
 152                         result_bps[0]   = info->do_gettimeofday(r, len, false);
 153                         result_bps[1]   = info->do_gettimeofday(r, len, true);
 154                 }
 155         } else {
 156                 if (use_cycle)
 157                         result_cycle[prefault] = info->do_cycle(r, len, only_prefault);
 158                 else
 159                         result_bps[prefault] = info->do_gettimeofday(r, len, only_prefault);
 160         }
 161
 162         switch (bench_format) {
 163         case BENCH_FORMAT_DEFAULT:
 164                 if (!only_prefault && prefault) {
 165                         if (use_cycle) {
 166                                 printf(" %14lf Cycle/Byte\n",
 167                                         (double)result_cycle[0]
 168                                         / totallen);
 169                                 printf(" %14lf Cycle/Byte (with prefault)\n",
 170                                         (double)result_cycle[1]
 171                                         / totallen);
 172                         } else {
 173                                 print_bps(result_bps[0]);
 174                                 printf("\n");
 175                                 print_bps(result_bps[1]);
 176                                 printf(" (with prefault)\n");
 177                         }
 178                 } else {
 179                         if (use_cycle) {
 180                                 printf(" %14lf Cycle/Byte",
 181                                         (double)result_cycle[prefault]
 182                                         / totallen);
 183                         } else
 184                                 print_bps(result_bps[prefault]);
 185
 186                         printf("%s\n", only_prefault ? " (with prefault)" : "");
 187                 }
 188                 break;
 189         case BENCH_FORMAT_SIMPLE:
 190                 if (!only_prefault && prefault) {
 191                         if (use_cycle) {
 192                                 printf("%lf %lf\n",
 193                                         (double)result_cycle[0] / totallen,
 194                                         (double)result_cycle[1] / totallen);
 195                         } else {
 196                                 printf("%lf %lf\n",
 197                                         result_bps[0], result_bps[1]);
 198                         }
 199                 } else {
 200                         if (use_cycle) {
 201                                 printf("%lf\n", (double)result_cycle[prefault]
 202                                         / totallen);
 203                         } else
 204                                 printf("%lf\n", result_bps[prefault]);
 205                 }
 206                 break;
 207         default:
 208                 /* Reaching this means there's some disaster: */
 209                 die("unknown format: %d\n", bench_format);
 210                 break;
 211         }
 212 }
 213
 214 static int bench_mem_common(int argc, const char **argv,
 215                      const char *prefix __maybe_unused,
 216                      struct bench_mem_info *info)
 217 {
 218         int i;
 219         size_t len;
 220         double totallen;
 221
 222         argc = parse_options(argc, argv, options, info->usage, 0);
 223
 224         if (no_prefault && only_prefault) {
 225                 fprintf(stderr, "Invalid options: -o and -n are mutually exclusive\n");
 226                 return 1;
 227         }
 228
 229         if (use_cycle)
 230                 init_cycle();
 231
 232         len = (size_t)perf_atoll((char *)length_str);
 233         totallen = (double)len * iterations;
 234
 235         if ((s64)len <= 0) {
 236                 fprintf(stderr, "Invalid length:%s\n", length_str);
 237                 return 1;
 238         }
 239
 240         /* Same as without specifying either of prefault and no-prefault: */
 241         if (only_prefault && no_prefault)
 242                 only_prefault = no_prefault = false;
 243
 244         if (!strncmp(routine, "all", 3)) {
 245                 for (i = 0; info->routines[i].name; i++)
 246                         __bench_mem_routine(info, i, len, totallen);
 247                 return 0;
 248         }
 249
 250         for (i = 0; info->routines[i].name; i++) {
 251                 if (!strcmp(info->routines[i].name, routine))
 252                         break;
 253         }
 254         if (!info->routines[i].name) {
 255                 printf("Unknown routine:%s\n", routine);
 256                 printf("Available routines...\n");
 257                 for (i = 0; info->routines[i].name; i++) {
 258                         printf("\t%s ... %s\n",
 259                                info->routines[i].name, info->routines[i].desc);
 260                 }
 261                 return 1;
 262         }
 263
 264         __bench_mem_routine(info, i, len, totallen);
 265
 266         return 0;
 267 }
 268
 269 static void memcpy_alloc_mem(void **dst, void **src, size_t length)
 270 {
 271         *dst = zalloc(length);
 272         if (!*dst)
 273                 die("memory allocation failed - maybe length is too large?\n");
 274
 275         *src = zalloc(length);
 276         if (!*src)
 277                 die("memory allocation failed - maybe length is too large?\n");
 278
 279         /* Make sure to always prefault zero pages even if MMAP_THRESH is crossed: */
 280         memset(*src, 0, length);
 281 }
 282
 283 static u64 do_memcpy_cycle(const struct routine *r, size_t len, bool prefault)
 284 {
 285         u64 cycle_start = 0ULL, cycle_end = 0ULL;
 286         void *src = NULL, *dst = NULL;
 287         memcpy_t fn = r->fn.memcpy;
 288         int i;
 289
 290         memcpy_alloc_mem(&dst, &src, len);
 291
 292         if (prefault)
 293                 fn(dst, src, len);
 294
 295         cycle_start = get_cycle();
 296         for (i = 0; i < iterations; ++i)
 297                 fn(dst, src, len);
 298         cycle_end = get_cycle();
 299
 300         free(src);
 301         free(dst);
 302         return cycle_end - cycle_start;
 303 }
 304
 305 static double do_memcpy_gettimeofday(const struct routine *r, size_t len, bool prefault)
 306 {
 307         struct timeval tv_start, tv_end, tv_diff;
 308         memcpy_t fn = r->fn.memcpy;
 309         void *src = NULL, *dst = NULL;
 310         int i;
 311
 312         memcpy_alloc_mem(&dst, &src, len);
 313
 314         if (prefault)
 315                 fn(dst, src, len);
 316
 317         BUG_ON(gettimeofday(&tv_start, NULL));
 318         for (i = 0; i < iterations; ++i)
 319                 fn(dst, src, len);
 320         BUG_ON(gettimeofday(&tv_end, NULL));
 321
 322         timersub(&tv_end, &tv_start, &tv_diff);
 323
 324         free(src);
 325         free(dst);
 326         return (double)(((double)len * iterations) / timeval2double(&tv_diff));
 327 }
 328
 329 int bench_mem_memcpy(int argc, const char **argv,
 330                      const char *prefix __maybe_unused)
 331 {
 332         struct bench_mem_info info = {
 333                 .routines               = memcpy_routines,
 334                 .do_cycle               = do_memcpy_cycle,
 335                 .do_gettimeofday        = do_memcpy_gettimeofday,
 336                 .usage                  = bench_mem_memcpy_usage,
 337         };
 338
 339         return bench_mem_common(argc, argv, prefix, &info);
 340 }
 341
 342 static void memset_alloc_mem(void **dst, size_t length)
 343 {
 344         *dst = zalloc(length);
 345         if (!*dst)
 346                 die("memory allocation failed - maybe length is too large?\n");
 347 }
 348
 349 static u64 do_memset_cycle(const struct routine *r, size_t len, bool prefault)
 350 {
 351         u64 cycle_start = 0ULL, cycle_end = 0ULL;
 352         memset_t fn = r->fn.memset;
 353         void *dst = NULL;
 354         int i;
 355
 356         memset_alloc_mem(&dst, len);
 357
 358         if (prefault)
 359                 fn(dst, -1, len);
 360
 361         cycle_start = get_cycle();
 362         for (i = 0; i < iterations; ++i)
 363                 fn(dst, i, len);
 364         cycle_end = get_cycle();
 365
 366         free(dst);
 367         return cycle_end - cycle_start;
 368 }
 369
 370 static double do_memset_gettimeofday(const struct routine *r, size_t len,
 371                                      bool prefault)
 372 {
 373         struct timeval tv_start, tv_end, tv_diff;
 374         memset_t fn = r->fn.memset;
 375         void *dst = NULL;
 376         int i;
 377
 378         memset_alloc_mem(&dst, len);
 379
 380         if (prefault)
 381                 fn(dst, -1, len);
 382
 383         BUG_ON(gettimeofday(&tv_start, NULL));
 384         for (i = 0; i < iterations; ++i)
 385                 fn(dst, i, len);
 386         BUG_ON(gettimeofday(&tv_end, NULL));
 387
 388         timersub(&tv_end, &tv_start, &tv_diff);
 389
 390         free(dst);
 391         return (double)(((double)len * iterations) / timeval2double(&tv_diff));
 392 }
 393
 394 static const char * const bench_mem_memset_usage[] = {
 395         "perf bench mem memset <options>",
 396         NULL
 397 };
 398
 399 static const struct routine memset_routines[] = {
 400         { .name         = "default",
 401           .desc         = "Default memset() provided by glibc",
 402           .fn.memset    = memset },
 403
 404 #ifdef HAVE_ARCH_X86_64_SUPPORT
 405 # define MEMSET_FN(_fn, _name, _desc) { .name = _name, .desc = _desc, .fn.memset = _fn },
 406 # include "mem-memset-x86-64-asm-def.h"
 407 # undef MEMSET_FN
 408 #endif
 409
 410         { NULL, }
 411 };
 412
 413 int bench_mem_memset(int argc, const char **argv, const char *prefix __maybe_unused)
 414 {
 415         struct bench_mem_info info = {
 416                 .routines               = memset_routines,
 417                 .do_cycle               = do_memset_cycle,
 418                 .do_gettimeofday        = do_memset_gettimeofday,
 419                 .usage                  = bench_mem_memset_usage,
 420         };
 421
 422         return bench_mem_common(argc, argv, prefix, &info);
 423 }