From 310dd41438484bc9a332ff9d1e097ab1b37ff12f Mon Sep 17 00:00:00 2001 From: David Sommerseth Date: Tue, 23 Feb 2010 10:22:14 +0100 Subject: [PATCH 1/7] Updated hackbench to lastest version available Downloaded from http://people.redhat.com/mingo/cfs-scheduler/tools/hackbench.c February 19 2010. Signed-off-by: David Sommerseth --- src/hackbench/Makefile | 2 +- src/hackbench/hackbench.c | 490 +++++++++++++++++++++++++------------- 2 files changed, 320 insertions(+), 172 deletions(-) diff --git a/src/hackbench/Makefile b/src/hackbench/Makefile index d3672b5..55dd436 100644 --- a/src/hackbench/Makefile +++ b/src/hackbench/Makefile @@ -1,2 +1,2 @@ hackbench: hackbench.c - $(CC) $(CFLAGS) -o hackbench hackbench.c -g -Wall + $(CC) $(CFLAGS) -o hackbench hackbench.c -g -Wall -lpthread diff --git a/src/hackbench/hackbench.c b/src/hackbench/hackbench.c index dfc71d8..cd4d471 100644 --- a/src/hackbench/hackbench.c +++ b/src/hackbench/hackbench.c @@ -1,236 +1,384 @@ + +/* + * This is the latest version of hackbench.c, that tests scheduler and + * unix-socket (or pipe) performance. + * + * Usage: hackbench [-pipe] [process|thread] [loops] + * + * Build it with: + * gcc -g -Wall -O2 -o hackbench hackbench.c -lpthread + */ +#if 0 + +Date: Fri, 04 Jan 2008 14:06:26 +0800 +From: "Zhang, Yanmin" +To: LKML +Subject: Improve hackbench +Cc: Ingo Molnar , Arjan van de Ven + +hackbench tests the Linux scheduler. The original program is at +http://devresources.linux-foundation.org/craiger/hackbench/src/hackbench.c +Based on this multi-process version, a nice person created a multi-thread +version. Pls. see +http://www.bullopensource.org/posix/pi-futex/hackbench_pth.c + +When I integrated them into my automation testing system, I found +a couple of issues and did some improvements. + +1) Merge hackbench: I integrated hackbench_pth.c into hackbench and added a +new parameter which can be used to choose process mode or thread mode. The +default mode is process. + +2) It runs too fast and ends in a couple of seconds. Sometimes it's too hard to debug +the issues. On my ia64 Montecito machines, the result looks weird when comparing +process mode and thread mode. +I want a stable result and hope the testing could run for a stable longer time, so I +might use performance tools to debug issues. +I added another new parameter,`loops`, which can be used to change variable loops, +so more messages will be passed from writers to receivers. Parameter 'loops' is equal to +100 by default. + +For example on my 8-core x86_64: +[ymzhang@lkp-st01-x8664 hackbench]$ uname -a +Linux lkp-st01-x8664 2.6.24-rc6 #1 SMP Fri Dec 21 08:32:31 CST 2007 x86_64 x86_64 x86_64 GNU/Linux +[ymzhang@lkp-st01-x8664 hackbench]$ ./hackbench +Usage: hackbench [-pipe] [process|thread] [loops] +[ymzhang@lkp-st01-x8664 hackbench]$ ./hackbench 150 process 1000 +Time: 151.533 +[ymzhang@lkp-st01-x8664 hackbench]$ ./hackbench 150 thread 1000 +Time: 153.666 + + +With the same new parameters, I did captured the SLUB issue discussed on LKML recently. + +3) hackbench_pth.c will fail on ia64 machine because pthread_attr_setstacksize always +fails if the stack size is less than 196*1024. I moved this statement within a __ia64__ check. + + +This new program could be compiled with command line: +#gcc -g -Wall -o hackbench hackbench.c -lpthread + + +Thank Ingo for his great comments! + +-yanmin + +--- + +* Nathan Lynch wrote: + +> Here's a fixlet for the hackbench program found at +> +> http://people.redhat.com/mingo/cfs-scheduler/tools/hackbench.c +> +> When redirecting hackbench output I am seeing multiple copies of the +> "Running with %d*40 (== %d) tasks" line. Need to flush the buffered +> output before forking. + +#endif + /* Test groups of 20 processes spraying to 20 receivers */ +#include #include #include #include -#include #include +#include #include #include #include #include #include +#include #define DATASIZE 100 static unsigned int loops = 100; +/* + * 0 means thread mode and others mean process (default) + */ +static unsigned int process_mode = 1; + static int use_pipes = 0; +struct sender_context { + unsigned int num_fds; + int ready_out; + int wakefd; + int out_fds[0]; +}; + +struct receiver_context { + unsigned int num_packets; + int in_fds[2]; + int ready_out; + int wakefd; +}; + + static void barf(const char *msg) { - fprintf(stderr, "%s (error: %s)\n", msg, strerror(errno)); - exit(1); + fprintf(stderr, "%s (error: %s)\n", msg, strerror(errno)); + exit(1); +} + +static void print_usage_exit() +{ + printf("Usage: hackbench [-pipe] [process|thread] [loops]\n"); + exit(1); } static void fdpair(int fds[2]) { - if (use_pipes) { - if (pipe(fds) == 0) - return; - } else { - if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) == 0) - return; - } - barf("Creating fdpair"); + if (use_pipes) { + if (pipe(fds) == 0) + return; + } else { + if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) == 0) + return; + } + barf("Creating fdpair"); } /* Block until we're ready to go */ static void ready(int ready_out, int wakefd) { - char dummy = '*'; - struct pollfd pollfd = { .fd = wakefd, .events = POLLIN }; + char dummy; + struct pollfd pollfd = { .fd = wakefd, .events = POLLIN }; - /* Tell them we're ready. */ - if (write(ready_out, &dummy, 1) != 1) - barf("CLIENT: ready write"); + /* Tell them we're ready. */ + if (write(ready_out, &dummy, 1) != 1) + barf("CLIENT: ready write"); - /* Wait for "GO" signal */ - if (poll(&pollfd, 1, -1) != 1) - barf("poll"); + /* Wait for "GO" signal */ + if (poll(&pollfd, 1, -1) != 1) + barf("poll"); } /* Sender sprays loops messages down each file descriptor */ -static void sender(unsigned int num_fds, - int out_fd[num_fds], - int ready_out, - int wakefd) +static void *sender(struct sender_context *ctx) { - char data[DATASIZE]; - unsigned int i, j; + char data[DATASIZE]; + unsigned int i, j; - ready(ready_out, wakefd); - memset(&data, '-', DATASIZE); + ready(ctx->ready_out, ctx->wakefd); - /* Now pump to every receiver. */ - for (i = 0; i < loops; i++) { - for (j = 0; j < num_fds; j++) { - int ret, done = 0; + /* Now pump to every receiver. */ + for (i = 0; i < loops; i++) { + for (j = 0; j < ctx->num_fds; j++) { + int ret, done = 0; - again: - ret = write(out_fd[j], data + done, sizeof(data)-done); - if (ret < 0) - barf("SENDER: write"); - done += ret; - if (done < sizeof(data)) - goto again; - } - } +again: + ret = write(ctx->out_fds[j], data + done, sizeof(data)-done); + if (ret < 0) + barf("SENDER: write"); + done += ret; + if (done < sizeof(data)) + goto again; + } + } + + return NULL; } + /* One receiver per fd */ -static void receiver(unsigned int num_packets, - int in_fd, - int ready_out, - int wakefd) +static void *receiver(struct receiver_context* ctx) { - unsigned int i; + unsigned int i; - /* Wait for start... */ - ready(ready_out, wakefd); + if (process_mode) + close(ctx->in_fds[1]); - /* Receive them all */ - for (i = 0; i < num_packets; i++) { - char data[DATASIZE]; - int ret, done = 0; + /* Wait for start... */ + ready(ctx->ready_out, ctx->wakefd); - again: - ret = read(in_fd, data + done, DATASIZE - done); - if (ret < 0) - barf("SERVER: read"); - done += ret; - if (done < DATASIZE) - goto again; - } + /* Receive them all */ + for (i = 0; i < ctx->num_packets; i++) { + char data[DATASIZE]; + int ret, done = 0; + +again: + ret = read(ctx->in_fds[0], data + done, DATASIZE - done); + if (ret < 0) + barf("SERVER: read"); + done += ret; + if (done < DATASIZE) + goto again; + } + + return NULL; +} + +pthread_t create_worker(void *ctx, void *(*func)(void *)) +{ + pthread_attr_t attr; + pthread_t childid; + int err; + + if (process_mode) { + /* process mode */ + /* Fork the receiver. */ + switch (fork()) { + case -1: barf("fork()"); + case 0: + (*func) (ctx); + exit(0); + } + + return (pthread_t) 0; + } + + if (pthread_attr_init(&attr) != 0) + barf("pthread_attr_init:"); + +#ifndef __ia64__ + if (pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN) != 0) + barf("pthread_attr_setstacksize"); +#endif + + if ((err=pthread_create(&childid, &attr, func, ctx)) != 0) { + fprintf(stderr, "pthread_create failed: %s (%d)\n", strerror(err), err); + exit(-1); + } + return (childid); +} + +void reap_worker(pthread_t id) +{ + int status; + + if (process_mode) { + /* process mode */ + wait(&status); + if (!WIFEXITED(status)) + exit(1); + } else { + void *status; + + pthread_join(id, &status); + } } /* One group of senders and receivers */ -static unsigned int group(unsigned int num_fds, - int ready_out, - int wakefd, - pid_t *childpids, - unsigned int childp_offset) +static unsigned int group(pthread_t *pth, + unsigned int num_fds, + int ready_out, + int wakefd) { - unsigned int i; - int out_fds[num_fds]; - unsigned int children_started = 0; + unsigned int i; + struct sender_context* snd_ctx = malloc (sizeof(struct sender_context) + +num_fds*sizeof(int)); - for (i = 0; i < num_fds ; i++) { - int fds[2]; - pid_t pid = -1; + for (i = 0; i < num_fds; i++) { + int fds[2]; + struct receiver_context* ctx = malloc (sizeof(*ctx)); - /* Create the pipe between client and server */ - fdpair(fds); + if (!ctx) + barf("malloc()"); - /* Fork the receiver. */ - switch ((pid = fork())) { - case -1: - return children_started; - case 0: - close(fds[1]); - receiver(num_fds*loops, fds[0], ready_out, wakefd); - exit(0); - } - childpids[childp_offset + children_started] = pid; - children_started++; - out_fds[i] = fds[1]; - close(fds[0]); - } - /* Now we have all the fds, fork the senders */ - for (i = 0; i < num_fds; i++) { - pid_t pid = -1; + /* Create the pipe between client and server */ + fdpair(fds); - switch ((pid = fork())) { - case -1: - return children_started; - case 0: - sender(num_fds, out_fds, ready_out, wakefd); - exit(0); - } - childpids[childp_offset + children_started] = pid; - children_started++; - } + ctx->num_packets = num_fds*loops; + ctx->in_fds[0] = fds[0]; + ctx->in_fds[1] = fds[1]; + ctx->ready_out = ready_out; + ctx->wakefd = wakefd; - /* Close the fds we have left */ - for (i = 0; i < num_fds; i++) - close(out_fds[i]); + pth[i] = create_worker(ctx, (void *)(void *)receiver); - /* Return number of children to reap */ - return num_fds * 2; -} + snd_ctx->out_fds[i] = fds[1]; + if (process_mode) + close(fds[0]); + } -static unsigned int reap_children(pid_t *children, unsigned int num_childs, unsigned int dokill) { - unsigned int i, rc = 0; + /* Now we have all the fds, fork the senders */ + for (i = 0; i < num_fds; i++) { + snd_ctx->ready_out = ready_out; + snd_ctx->wakefd = wakefd; + snd_ctx->num_fds = num_fds; - for (i = 0; i < num_childs; i++) { - int status; - if( dokill ) { - kill(children[i], SIGTERM); - } - waitpid(children[i], &status, 0); - if (!WIFEXITED(status)) - rc++; /* count how many children not exiting "correctly" */ - } - return rc; + pth[num_fds+i] = create_worker(snd_ctx, (void *)(void *)sender); + } + + /* Close the fds we have left */ + if (process_mode) + for (i = 0; i < num_fds; i++) + close(snd_ctx->out_fds[i]); + + /* Return number of children to reap */ + return num_fds * 2; } int main(int argc, char *argv[]) { - unsigned int i, num_groups, total_children, rc; - struct timeval start, stop, diff; - unsigned int num_fds = 20; - int readyfds[2], wakefds[2]; - char dummy; - pid_t *children = NULL; + unsigned int i, num_groups = 10, total_children; + struct timeval start, stop, diff; + unsigned int num_fds = 20; + int readyfds[2], wakefds[2]; + char dummy; + pthread_t *pth_tab; - if (argv[1] && strcmp(argv[1], "-pipe") == 0) { - use_pipes = 1; - argc--; - argv++; - } - - if (argc != 2 || (num_groups = atoi(argv[1])) == 0) - barf("Usage: hackbench [-pipe] \n"); - - fdpair(readyfds); - fdpair(wakefds); - - children = calloc((num_groups * num_fds * 2)+1, sizeof(pid_t)+2); - if( !children ) - barf("calloc() for children array"); - - /* Start groups of children (num_fds * 2 - sender and receiver processes) */ - total_children = 0; - for (i = 0; i < num_groups; i++) { - int c = group(num_fds, readyfds[1], wakefds[0], children, total_children); - if( c < (num_fds*2) ) { - /* Not all expected children started - kill all children which are alive */ - reap_children(children, total_children + c, 1); - barf("fork()"); - } - total_children += c; - } - - /* Wait for everyone to be ready */ - for (i = 0; i < total_children; i++) - if (read(readyfds[0], &dummy, 1) != 1) - barf("Reading for readyfds"); - - gettimeofday(&start, NULL); - - /* Kick them off */ - if (write(wakefds[1], &dummy, 1) != 1) - barf("Writing to start them"); - - /* Reap them all */ - if ((rc = reap_children(children, total_children, 0)) != 0) { - printf("%i children did not exit correctly", rc); + if (argv[1] && strcmp(argv[1], "-pipe") == 0) { + use_pipes = 1; + argc--; + argv++; } - gettimeofday(&stop, NULL); + if (argc >= 2 && (num_groups = atoi(argv[1])) == 0) + print_usage_exit(); - /* Print time... */ - timersub(&stop, &start, &diff); - printf("Time: %lu.%03lu\n", diff.tv_sec, diff.tv_usec/1000); + printf("Running with %d*40 (== %d) tasks.\n", + num_groups, num_groups*40); - free(children); - exit(rc > 0 ? 1 : 0); + fflush(NULL); + + if (argc > 2) { + if ( !strcmp(argv[2], "process") ) + process_mode = 1; + else if ( !strcmp(argv[2], "thread") ) + process_mode = 0; + else + print_usage_exit(); + } + + if (argc > 3) + loops = atoi(argv[3]); + + pth_tab = malloc(num_fds * 2 * num_groups * sizeof(pthread_t)); + + if (!pth_tab) + barf("main:malloc()"); + + fdpair(readyfds); + fdpair(wakefds); + + total_children = 0; + for (i = 0; i < num_groups; i++) + total_children += group(pth_tab+total_children, num_fds, readyfds[1], wakefds[0]); + + /* Wait for everyone to be ready */ + for (i = 0; i < total_children; i++) + if (read(readyfds[0], &dummy, 1) != 1) + barf("Reading for readyfds"); + + gettimeofday(&start, NULL); + + /* Kick them off */ + if (write(wakefds[1], &dummy, 1) != 1) + barf("Writing to start them"); + + /* Reap them all */ + for (i = 0; i < total_children; i++) + reap_worker(pth_tab[i]); + + gettimeofday(&stop, NULL); + + /* Print time... */ + timersub(&stop, &start, &diff); + printf("Time: %lu.%03lu\n", diff.tv_sec, diff.tv_usec/1000); + exit(0); } + + From ad27df7b940e04e0ee6064e77e603c87e2dd70ee Mon Sep 17 00:00:00 2001 From: David Sommerseth Date: Tue, 23 Feb 2010 10:35:48 +0100 Subject: [PATCH 2/7] Reimplement better child tracking and improve error handling This does much what earlier commits did before hackbench got updated to the latest version in the previous commit. Consider the following commits being "forward ported", feature wise: 4c39eff2136c39b3c2746ca293eed5b5242aea52 0a72fcaade064b70b698aab676217f28681280ff fbd80c495bd861545713279d5f96f1e4770d6911 bd588c92b8bae59e3404fc3c90283e98c9f0a96e Signed-off-by: David Sommerseth --- src/hackbench/Makefile | 5 +- src/hackbench/hackbench.c | 236 +++++++++++++++++++------------------- 2 files changed, 121 insertions(+), 120 deletions(-) diff --git a/src/hackbench/Makefile b/src/hackbench/Makefile index 55dd436..469cf25 100644 --- a/src/hackbench/Makefile +++ b/src/hackbench/Makefile @@ -1,2 +1,5 @@ hackbench: hackbench.c - $(CC) $(CFLAGS) -o hackbench hackbench.c -g -Wall -lpthread + $(CC) $(CFLAGS) -o hackbench hackbench.c -g -Wall -O2 -lpthread + +clean : + rm -f hackbench diff --git a/src/hackbench/hackbench.c b/src/hackbench/hackbench.c index cd4d471..2fccecf 100644 --- a/src/hackbench/hackbench.c +++ b/src/hackbench/hackbench.c @@ -1,4 +1,3 @@ - /* * This is the latest version of hackbench.c, that tests scheduler and * unix-socket (or pipe) performance. @@ -7,75 +6,11 @@ * * Build it with: * gcc -g -Wall -O2 -o hackbench hackbench.c -lpthread + * + * Downloaded from http://people.redhat.com/mingo/cfs-scheduler/tools/hackbench.c + * February 19 2010. + * */ -#if 0 - -Date: Fri, 04 Jan 2008 14:06:26 +0800 -From: "Zhang, Yanmin" -To: LKML -Subject: Improve hackbench -Cc: Ingo Molnar , Arjan van de Ven - -hackbench tests the Linux scheduler. The original program is at -http://devresources.linux-foundation.org/craiger/hackbench/src/hackbench.c -Based on this multi-process version, a nice person created a multi-thread -version. Pls. see -http://www.bullopensource.org/posix/pi-futex/hackbench_pth.c - -When I integrated them into my automation testing system, I found -a couple of issues and did some improvements. - -1) Merge hackbench: I integrated hackbench_pth.c into hackbench and added a -new parameter which can be used to choose process mode or thread mode. The -default mode is process. - -2) It runs too fast and ends in a couple of seconds. Sometimes it's too hard to debug -the issues. On my ia64 Montecito machines, the result looks weird when comparing -process mode and thread mode. -I want a stable result and hope the testing could run for a stable longer time, so I -might use performance tools to debug issues. -I added another new parameter,`loops`, which can be used to change variable loops, -so more messages will be passed from writers to receivers. Parameter 'loops' is equal to -100 by default. - -For example on my 8-core x86_64: -[ymzhang@lkp-st01-x8664 hackbench]$ uname -a -Linux lkp-st01-x8664 2.6.24-rc6 #1 SMP Fri Dec 21 08:32:31 CST 2007 x86_64 x86_64 x86_64 GNU/Linux -[ymzhang@lkp-st01-x8664 hackbench]$ ./hackbench -Usage: hackbench [-pipe] [process|thread] [loops] -[ymzhang@lkp-st01-x8664 hackbench]$ ./hackbench 150 process 1000 -Time: 151.533 -[ymzhang@lkp-st01-x8664 hackbench]$ ./hackbench 150 thread 1000 -Time: 153.666 - - -With the same new parameters, I did captured the SLUB issue discussed on LKML recently. - -3) hackbench_pth.c will fail on ia64 machine because pthread_attr_setstacksize always -fails if the stack size is less than 196*1024. I moved this statement within a __ia64__ check. - - -This new program could be compiled with command line: -#gcc -g -Wall -o hackbench hackbench.c -lpthread - - -Thank Ingo for his great comments! - --yanmin - ---- - -* Nathan Lynch wrote: - -> Here's a fixlet for the hackbench program found at -> -> http://people.redhat.com/mingo/cfs-scheduler/tools/hackbench.c -> -> When redirecting hackbench output I am seeing multiple copies of the -> "Running with %d*40 (== %d) tasks" line. Need to flush the buffered -> output before forking. - -#endif /* Test groups of 20 processes spraying to 20 receivers */ #include @@ -115,6 +50,12 @@ struct receiver_context { }; +typedef union { + pthread_t threadid; + pid_t pid; + long long error; +} childinfo_t; + static void barf(const char *msg) { fprintf(stderr, "%s (error: %s)\n", msg, strerror(errno)); @@ -142,7 +83,7 @@ static void fdpair(int fds[2]) /* Block until we're ready to go */ static void ready(int ready_out, int wakefd) { - char dummy; + char dummy = '*'; struct pollfd pollfd = { .fd = wakefd, .events = POLLIN }; /* Tell them we're ready. */ @@ -161,6 +102,7 @@ static void *sender(struct sender_context *ctx) unsigned int i, j; ready(ctx->ready_out, ctx->wakefd); + memset(&data, '-', DATASIZE); /* Now pump to every receiver. */ for (i = 0; i < loops; i++) { @@ -209,72 +151,110 @@ again: return NULL; } -pthread_t create_worker(void *ctx, void *(*func)(void *)) +childinfo_t create_worker(void *ctx, void *(*func)(void *)) { pthread_attr_t attr; - pthread_t childid; int err; + childinfo_t child; + pid_t childpid; - if (process_mode) { - /* process mode */ - /* Fork the receiver. */ - switch (fork()) { - case -1: barf("fork()"); + switch (process_mode) { + case 1: /* process mode */ + /* Fork the sender/receiver child. */ + switch ((childpid = fork())) { + case -1: + fprintf(stderr, "fork(): %s\n", strerror(errno)); + child.error = -1; + return child; case 0: (*func) (ctx); exit(0); } + child.pid = childpid; + break; - return (pthread_t) 0; - } - - if (pthread_attr_init(&attr) != 0) - barf("pthread_attr_init:"); + case 0: /* threaded mode */ + if (pthread_attr_init(&attr) != 0) { + fprintf(stderr, "pthread_attr_init: %s\n", strerror(errno)); + child.error = -1; + return child; + } #ifndef __ia64__ - if (pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN) != 0) - barf("pthread_attr_setstacksize"); + if (pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN) != 0) { + fprintf(stderr, "pthread_attr_setstacksize: %s\n", strerror(errno)); + child.error = -1; + return child; + } #endif - if ((err=pthread_create(&childid, &attr, func, ctx)) != 0) { - fprintf(stderr, "pthread_create failed: %s (%d)\n", strerror(err), err); - exit(-1); + if ((err=pthread_create(&child.threadid, &attr, func, ctx)) != 0) { + fprintf(stderr, "pthread_create failed: %s (%d)\n", strerror(err), err); + child.error = -1; + return child; + } + break; } - return (childid); + return child; } -void reap_worker(pthread_t id) +unsigned int reap_workers(childinfo_t *child, unsigned int totchld, unsigned int dokill) { - int status; + unsigned int i, rc = 0; + int status, err; + void *thr_status; - if (process_mode) { - /* process mode */ - wait(&status); - if (!WIFEXITED(status)) - exit(1); - } else { - void *status; - - pthread_join(id, &status); + for( i = 0; i < totchld; i++ ) { + switch( process_mode ) { + case 1: /* process mode */ + if( dokill ) { + kill(child[i].pid, SIGTERM); + } + fflush(stdout); + waitpid(child[i].pid, &status, 0); + if (!WIFEXITED(status)) + rc++; + break; + case 0: /* threaded mode */ + if( dokill ) { + pthread_kill(child[i].threadid, SIGTERM); + } + err = pthread_join(child[i].threadid, &thr_status); + if( err != 0 ) { + fprintf(stderr, "pthread_join(): %s\n", strerror(err)); + rc++; + } + break; + } } + return rc; } /* One group of senders and receivers */ -static unsigned int group(pthread_t *pth, - unsigned int num_fds, - int ready_out, - int wakefd) +static unsigned int group(childinfo_t *child, + unsigned int tab_offset, + unsigned int num_fds, + int ready_out, + int wakefd) { unsigned int i; struct sender_context* snd_ctx = malloc (sizeof(struct sender_context) +num_fds*sizeof(int)); + if (!snd_ctx) { + fprintf(stderr, "** malloc() error (sender ctx): %s\n", strerror(errno)); + return 0; + } + + for (i = 0; i < num_fds; i++) { int fds[2]; struct receiver_context* ctx = malloc (sizeof(*ctx)); - if (!ctx) - barf("malloc()"); + if (!ctx) { + fprintf(stderr, "** malloc() error (receiver ctx): %s\n", strerror(errno)); + return (i > 0 ? i-1 : 0); + } /* Create the pipe between client and server */ @@ -286,8 +266,10 @@ static unsigned int group(pthread_t *pth, ctx->ready_out = ready_out; ctx->wakefd = wakefd; - pth[i] = create_worker(ctx, (void *)(void *)receiver); - + child[tab_offset+i] = create_worker(ctx, (void *)(void *)receiver); + if( child[tab_offset+i].error < 0 ) { + return (i > 0 ? i-1 : 0); + } snd_ctx->out_fds[i] = fds[1]; if (process_mode) close(fds[0]); @@ -299,7 +281,10 @@ static unsigned int group(pthread_t *pth, snd_ctx->wakefd = wakefd; snd_ctx->num_fds = num_fds; - pth[num_fds+i] = create_worker(snd_ctx, (void *)(void *)sender); + child[tab_offset+num_fds+i] = create_worker(snd_ctx, (void *)(void *)sender); + if( child[tab_offset+num_fds+i].error < 0 ) { + return (num_fds+i)-1; + } } /* Close the fds we have left */ @@ -318,7 +303,7 @@ int main(int argc, char *argv[]) unsigned int num_fds = 20; int readyfds[2], wakefds[2]; char dummy; - pthread_t *pth_tab; + childinfo_t *child_tab; if (argv[1] && strcmp(argv[1], "-pipe") == 0) { use_pipes = 1; @@ -346,39 +331,52 @@ int main(int argc, char *argv[]) if (argc > 3) loops = atoi(argv[3]); - pth_tab = malloc(num_fds * 2 * num_groups * sizeof(pthread_t)); + child_tab = malloc(num_fds * 2 * num_groups * sizeof(childinfo_t)); - if (!pth_tab) + if (!child_tab) barf("main:malloc()"); fdpair(readyfds); fdpair(wakefds); total_children = 0; - for (i = 0; i < num_groups; i++) - total_children += group(pth_tab+total_children, num_fds, readyfds[1], wakefds[0]); + for (i = 0; i < num_groups; i++) { + int c = group(child_tab, total_children, num_fds, readyfds[1], wakefds[0]); + if( c > (num_fds*2) ) { + reap_workers(child_tab, total_children, 1); + fprintf(stderr, "%i children started?!?!? Expected %i\n", c, num_fds*2); + barf("Creating workers"); + } + if( c < (num_fds*2) ) { + reap_workers(child_tab, total_children + c, 1); + barf("Creating workers"); + } + total_children += c; + } /* Wait for everyone to be ready */ for (i = 0; i < total_children; i++) - if (read(readyfds[0], &dummy, 1) != 1) + if (read(readyfds[0], &dummy, 1) != 1) { + reap_workers(child_tab, total_children, 1); barf("Reading for readyfds"); + } gettimeofday(&start, NULL); /* Kick them off */ - if (write(wakefds[1], &dummy, 1) != 1) + if (write(wakefds[1], &dummy, 1) != 1) { + reap_workers(child_tab, total_children, 1); barf("Writing to start them"); + } /* Reap them all */ - for (i = 0; i < total_children; i++) - reap_worker(pth_tab[i]); + reap_workers(child_tab, total_children, 0); gettimeofday(&stop, NULL); /* Print time... */ timersub(&stop, &start, &diff); printf("Time: %lu.%03lu\n", diff.tv_sec, diff.tv_usec/1000); + free(child_tab); exit(0); } - - From ba166b6415b71090c7b3805ad539cbeea13612ee Mon Sep 17 00:00:00 2001 From: David Sommerseth Date: Tue, 23 Feb 2010 10:39:59 +0100 Subject: [PATCH 3/7] Fixed a memory leak, receiver contexts not getting freed upon exit Signed-off-by: David Sommerseth --- src/hackbench/hackbench.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/hackbench/hackbench.c b/src/hackbench/hackbench.c index 2fccecf..abc21a0 100644 --- a/src/hackbench/hackbench.c +++ b/src/hackbench/hackbench.c @@ -147,7 +147,9 @@ again: if (done < DATASIZE) goto again; } - + if (ctx) { + free(ctx); + } return NULL; } From 5fd7c301eb7574a9fbe25cf1232867003899ee16 Mon Sep 17 00:00:00 2001 From: David Sommerseth Date: Tue, 23 Feb 2010 10:47:13 +0100 Subject: [PATCH 4/7] Simplified and improved error logging, clarified some messages Signed-off-by: David Sommerseth --- src/hackbench/hackbench.c | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/hackbench/hackbench.c b/src/hackbench/hackbench.c index abc21a0..091c765 100644 --- a/src/hackbench/hackbench.c +++ b/src/hackbench/hackbench.c @@ -56,9 +56,13 @@ typedef union { long long error; } childinfo_t; +inline static void sneeze(const char *msg) { + fprintf(stderr, "%s (error: %s)\n", msg, strerror(errno)); +} + static void barf(const char *msg) { - fprintf(stderr, "%s (error: %s)\n", msg, strerror(errno)); + sneeze(msg); exit(1); } @@ -165,7 +169,7 @@ childinfo_t create_worker(void *ctx, void *(*func)(void *)) /* Fork the sender/receiver child. */ switch ((childpid = fork())) { case -1: - fprintf(stderr, "fork(): %s\n", strerror(errno)); + sneeze("fork()"); child.error = -1; return child; case 0: @@ -177,21 +181,21 @@ childinfo_t create_worker(void *ctx, void *(*func)(void *)) case 0: /* threaded mode */ if (pthread_attr_init(&attr) != 0) { - fprintf(stderr, "pthread_attr_init: %s\n", strerror(errno)); + sneeze("pthread_attr_init()"); child.error = -1; return child; } #ifndef __ia64__ if (pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN) != 0) { - fprintf(stderr, "pthread_attr_setstacksize: %s\n", strerror(errno)); + sneeze("pthread_attr_setstacksize()"); child.error = -1; return child; } #endif if ((err=pthread_create(&child.threadid, &attr, func, ctx)) != 0) { - fprintf(stderr, "pthread_create failed: %s (%d)\n", strerror(err), err); + sneeze("pthread_create failed()"); child.error = -1; return child; } @@ -223,7 +227,7 @@ unsigned int reap_workers(childinfo_t *child, unsigned int totchld, unsigned int } err = pthread_join(child[i].threadid, &thr_status); if( err != 0 ) { - fprintf(stderr, "pthread_join(): %s\n", strerror(err)); + sneeze("pthread_join()"); rc++; } break; @@ -244,7 +248,7 @@ static unsigned int group(childinfo_t *child, +num_fds*sizeof(int)); if (!snd_ctx) { - fprintf(stderr, "** malloc() error (sender ctx): %s\n", strerror(errno)); + sneeze("malloc() [sender ctx]"); return 0; } @@ -254,7 +258,7 @@ static unsigned int group(childinfo_t *child, struct receiver_context* ctx = malloc (sizeof(*ctx)); if (!ctx) { - fprintf(stderr, "** malloc() error (receiver ctx): %s\n", strerror(errno)); + sneeze("malloc() [receiver ctx]"); return (i > 0 ? i-1 : 0); } @@ -344,12 +348,8 @@ int main(int argc, char *argv[]) total_children = 0; for (i = 0; i < num_groups; i++) { int c = group(child_tab, total_children, num_fds, readyfds[1], wakefds[0]); - if( c > (num_fds*2) ) { - reap_workers(child_tab, total_children, 1); - fprintf(stderr, "%i children started?!?!? Expected %i\n", c, num_fds*2); - barf("Creating workers"); - } - if( c < (num_fds*2) ) { + if( c != (num_fds*2) ) { + fprintf(stderr, "%i children started. Expected %i\n", c, num_fds*2); reap_workers(child_tab, total_children + c, 1); barf("Creating workers"); } @@ -368,7 +368,7 @@ int main(int argc, char *argv[]) /* Kick them off */ if (write(wakefds[1], &dummy, 1) != 1) { reap_workers(child_tab, total_children, 1); - barf("Writing to start them"); + barf("Writing to start senders"); } /* Reap them all */ From faa1f165c46d6573c62e046c75c59904dd8cdda9 Mon Sep 17 00:00:00 2001 From: David Sommerseth Date: Tue, 23 Feb 2010 10:49:29 +0100 Subject: [PATCH 5/7] (code cleanup) Tabified lines which was not tabbed Signed-off-by: David Sommerseth --- src/hackbench/hackbench.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hackbench/hackbench.c b/src/hackbench/hackbench.c index 091c765..93ecad8 100644 --- a/src/hackbench/hackbench.c +++ b/src/hackbench/hackbench.c @@ -52,7 +52,7 @@ struct receiver_context { typedef union { pthread_t threadid; - pid_t pid; + pid_t pid; long long error; } childinfo_t; @@ -106,7 +106,7 @@ static void *sender(struct sender_context *ctx) unsigned int i, j; ready(ctx->ready_out, ctx->wakefd); - memset(&data, '-', DATASIZE); + memset(&data, '-', DATASIZE); /* Now pump to every receiver. */ for (i = 0; i < loops; i++) { From f273b4bd85fe7d8561bf8c9030b6a9d8bb8b8e9f Mon Sep 17 00:00:00 2001 From: David Sommerseth Date: Tue, 23 Feb 2010 11:34:14 +0100 Subject: [PATCH 6/7] hackbench: Implemented getopt Improved argument/option handling by using getopt_long(). Made more of the parameters tunable as well. Hackbench now accepts the following arguments: -P | --pipe Use pipe -s | --datasize Number of bytes to pass from sender to receiver (default 100 bytes) -l | --loops Number of messages each sender will send (default 100 rounds) -g | --groups Number of groups with sender/receivers (default 10 groups) -f | --fds Number of file descriptors each group will use (default 20*2) -T | --threads Run using pthreads -P | --process Run using fork() Signed-off-by: David Sommerseth --- src/hackbench/hackbench.c | 131 ++++++++++++++++++++++++++++---------- 1 file changed, 98 insertions(+), 33 deletions(-) diff --git a/src/hackbench/hackbench.c b/src/hackbench/hackbench.c index 93ecad8..7a731d3 100644 --- a/src/hackbench/hackbench.c +++ b/src/hackbench/hackbench.c @@ -25,9 +25,13 @@ #include #include #include +#include -#define DATASIZE 100 +static unsigned int datasize = 100; static unsigned int loops = 100; +static unsigned int num_groups = 10; +static unsigned int num_fds = 20; + /* * 0 means thread mode and others mean process (default) */ @@ -68,7 +72,9 @@ static void barf(const char *msg) static void print_usage_exit() { - printf("Usage: hackbench [-pipe] [process|thread] [loops]\n"); + printf("Usage: hackbench [-p|--pipe] [-s|--datasize ] [-l|--loops ]\n" + "\t\t [-g|--groups ]\n" + "\t\t [-T|--threads] [-P|--process] [--help]\n"); exit(1); } @@ -102,11 +108,11 @@ static void ready(int ready_out, int wakefd) /* Sender sprays loops messages down each file descriptor */ static void *sender(struct sender_context *ctx) { - char data[DATASIZE]; + char data[datasize]; unsigned int i, j; ready(ctx->ready_out, ctx->wakefd); - memset(&data, '-', DATASIZE); + memset(&data, '-', datasize); /* Now pump to every receiver. */ for (i = 0; i < loops; i++) { @@ -140,15 +146,15 @@ static void *receiver(struct receiver_context* ctx) /* Receive them all */ for (i = 0; i < ctx->num_packets; i++) { - char data[DATASIZE]; + char data[datasize]; int ret, done = 0; again: - ret = read(ctx->in_fds[0], data + done, DATASIZE - done); + ret = read(ctx->in_fds[0], data + done, datasize - done); if (ret < 0) barf("SERVER: read"); done += ret; - if (done < DATASIZE) + if (done < datasize) goto again; } if (ctx) { @@ -302,43 +308,102 @@ static unsigned int group(childinfo_t *child, return num_fds * 2; } +static void process_options (int argc, char *argv[]) +{ + int error = 0; + + while( 1 ) { + int optind = 0; + + static struct option longopts[] = { + {"pipe", no_argument, NULL, 'p'}, + {"datasize", required_argument, NULL, 's'}, + {"loops", required_argument, NULL, 'l'}, + {"groups", required_argument, NULL, 'g'}, + {"fds", required_argument, NULL, 'f'}, + {"threads", no_argument, NULL, 'T'}, + {"processes", no_argument, NULL, 'P'}, + {"help", no_argument, NULL, 'h'}, + {NULL, 0, NULL, 0} + }; + + int c = getopt_long(argc, argv, "ps:l:g:f:TPh", + longopts, &optind); + if (c == -1) { + break; + } + switch (c) { + case 'p': + use_pipes = 1; + break; + + case 's': + if (!(argv[optind] && (datasize = atoi(optarg)) > 0)) { + fprintf(stderr, "%s: --datasize|-s requires an integer > 0\n", argv[0]); + error = 1; + } + break; + + case 'l': + if (!(argv[optind] && (loops = atoi(optarg)) > 0)) { + fprintf(stderr, "%s: --loops|-l requires an integer > 0\n", argv[0]); + error = 1; + } + break; + + case 'g': + if (!(argv[optind] && (num_groups = atoi(optarg)) > 0)) { + fprintf(stderr, "%s: --groups|-g requires an integer > 0\n", argv[0]); + error = 1; + } + break; + + case 'f': + if (!(argv[optind] && (num_fds = atoi(optarg)) > 0)) { + fprintf(stderr, "%s: --fds|-f requires an integer > 0\n", argv[0]); + error = 1; + } + break; + + case 'T': + process_mode = 0; + break; + case 'P': + process_mode = 1; + break; + + case 'h': + print_usage_exit(); + + default: + error = 1; + } + } + + if( error ) { + exit(1); + } +} + + + int main(int argc, char *argv[]) { - unsigned int i, num_groups = 10, total_children; + unsigned int i, total_children; struct timeval start, stop, diff; - unsigned int num_fds = 20; int readyfds[2], wakefds[2]; char dummy; childinfo_t *child_tab; - if (argv[1] && strcmp(argv[1], "-pipe") == 0) { - use_pipes = 1; - argc--; - argv++; - } - - if (argc >= 2 && (num_groups = atoi(argv[1])) == 0) - print_usage_exit(); - - printf("Running with %d*40 (== %d) tasks.\n", - num_groups, num_groups*40); + process_options (argc, argv); + printf("Running in %s mode with %d groups using %d file descriptors each (== %d tasks)\n", + (process_mode == 0 ? "threaded" : "process"), + num_groups, 2*num_fds, num_groups*(num_fds*2)); + printf("Each sender will pass %d messages of %d bytes\n", loops, datasize); fflush(NULL); - if (argc > 2) { - if ( !strcmp(argv[2], "process") ) - process_mode = 1; - else if ( !strcmp(argv[2], "thread") ) - process_mode = 0; - else - print_usage_exit(); - } - - if (argc > 3) - loops = atoi(argv[3]); - child_tab = malloc(num_fds * 2 * num_groups * sizeof(childinfo_t)); - if (!child_tab) barf("main:malloc()"); From 4e37a2b689fc0d081e41ca656be48aba53ff350b Mon Sep 17 00:00:00 2001 From: David Sommerseth Date: Tue, 23 Feb 2010 15:02:48 +0100 Subject: [PATCH 7/7] Added signal handling in hackbench When receiving SIGINT or SIGTERM, it will now reap all worker threads/processes and properly stop them. --- src/hackbench/hackbench.c | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/src/hackbench/hackbench.c b/src/hackbench/hackbench.c index 7a731d3..c8d12bd 100644 --- a/src/hackbench/hackbench.c +++ b/src/hackbench/hackbench.c @@ -26,6 +26,7 @@ #include #include #include +#include static unsigned int datasize = 100; static unsigned int loops = 100; @@ -60,8 +61,17 @@ typedef union { long long error; } childinfo_t; +childinfo_t *child_tab = NULL; +unsigned int total_children = 0; +unsigned int signal_caught = 0; + inline static void sneeze(const char *msg) { - fprintf(stderr, "%s (error: %s)\n", msg, strerror(errno)); + /* Avoid calling these functions when called from a code path + * which involves sigcatcher(), as they are not reentrant safe. + */ + if( !signal_caught ) { + fprintf(stderr, "%s (error: %s)\n", msg, strerror(errno)); + } } static void barf(const char *msg) @@ -385,15 +395,22 @@ static void process_options (int argc, char *argv[]) } } - +void sigcatcher(int sig) { + /* All caught signals will cause the program to exit */ + signal_caught = 1; + if( child_tab && (total_children > 0) ) { + reap_workers(child_tab, total_children, 1); + } + fprintf(stderr, "** Operation aborted **\n"); + exit(0); +} int main(int argc, char *argv[]) { - unsigned int i, total_children; + unsigned int i; struct timeval start, stop, diff; int readyfds[2], wakefds[2]; char dummy; - childinfo_t *child_tab; process_options (argc, argv); @@ -403,13 +420,18 @@ int main(int argc, char *argv[]) printf("Each sender will pass %d messages of %d bytes\n", loops, datasize); fflush(NULL); - child_tab = malloc(num_fds * 2 * num_groups * sizeof(childinfo_t)); + child_tab = calloc(num_fds * 2 * num_groups, sizeof(childinfo_t)); if (!child_tab) barf("main:malloc()"); fdpair(readyfds); fdpair(wakefds); + /* Catch some signals */ + signal(SIGINT, sigcatcher); + signal(SIGTERM, sigcatcher); + signal(SIGHUP, SIG_IGN); + total_children = 0; for (i = 0; i < num_groups; i++) { int c = group(child_tab, total_children, num_fds, readyfds[1], wakefds[0]);