From 105037e658518c5a4abbe9ba405a58d8b14d0603 Mon Sep 17 00:00:00 2001
From: John Kacur <jkacur@redhat.com>
Date: Fri, 6 Sep 2019 20:59:42 +0200
Subject: [PATCH] rt-tests: Set affinity before applying numa

There are three changes here.

1. If affinity is not specified, but numa is available, then numa
implies AFFINITY_USEALL.

2. Move setting affinity before applying numa
We need to set the affinity before applying numa, so that we don't
apply numa settings to cpus we don't intend to run on.

3. Allow a greater number of threads than cpus to run in a round robin
fashion in the case of numa.

Signed-off-by: John Kacur <jkacur@redhat.com>
---
 src/cyclictest/cyclictest.c | 34 +++++++++++++++++++++-------------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/src/cyclictest/cyclictest.c b/src/cyclictest/cyclictest.c
index d101e3d..52f93da 100644
--- a/src/cyclictest/cyclictest.c
+++ b/src/cyclictest/cyclictest.c
@@ -1349,8 +1349,12 @@ static void process_options (int argc, char *argv[], int max_cpus)
 	/* if smp wasn't requested, test for numa automatically */
 	if (!smp) {
 #ifdef NUMA
-		if (numa_available() != -1)
+		if (numa_available() != -1) {
 			numa = 1;
+			if (setaffinity == AFFINITY_UNSPECIFIED) {
+				setaffinity = AFFINITY_USEALL;
+			}
+		}
 #else
 		warn("cyclictest was not built with the numa option\n");
 		numa = 0;
@@ -1716,6 +1720,7 @@ int main(int argc, char **argv)
 	sigset_t sigset;
 	int signum = SIGALRM;
 	int mode;
+	int cpu;
 	int max_cpus = sysconf(_SC_NPROCESSORS_ONLN);
 	int i, ret = -1;
 	int status;
@@ -1877,6 +1882,16 @@ int main(int argc, char **argv)
 		if (status != 0)
 			fatal("error from pthread_attr_init for thread %d: %s\n", i, strerror(status));
 
+		switch (setaffinity) {
+		case AFFINITY_UNSPECIFIED: cpu = -1; break;
+		case AFFINITY_SPECIFIED:
+			cpu = cpu_for_thread(i, max_cpus);
+			if (verbose)
+				printf("Thread %d using cpu %d.\n", i, cpu);
+			break;
+		case AFFINITY_USEALL: cpu = i % max_cpus; break;
+		}
+
 		node = -1;
 		if (numa) {
 			void *stack;
@@ -1884,7 +1899,7 @@ int main(int argc, char **argv)
 			size_t stksize;
 
 			/* find the memory node associated with the cpu i */
-			node = rt_numa_numa_node_of_cpu(i);
+			node = rt_numa_numa_node_of_cpu(cpu);
 
 			/* get the stack size set for for this thread */
 			if (pthread_attr_getstack(&attr, &currstk, &stksize))
@@ -1895,7 +1910,7 @@ int main(int argc, char **argv)
 				stksize = PTHREAD_STACK_MIN * 2;
 
 			/*  allocate memory for a stack on appropriate node */
-			stack = rt_numa_numa_alloc_onnode(stksize, node, i);
+			stack = rt_numa_numa_alloc_onnode(stksize, node, cpu);
 
 			/* touch the stack pages to pre-fault them in */
 			memset(stack, 0, stksize);
@@ -1965,20 +1980,13 @@ int main(int argc, char **argv)
 			interval += distance;
 		if (verbose)
 			printf("Thread %d Interval: %d\n", i, interval);
+
 		par->max_cycles = max_cycles;
 		par->stats = stat;
 		par->node = node;
 		par->tnum = i;
-		switch (setaffinity) {
-		case AFFINITY_UNSPECIFIED: par->cpu = -1; break;
-		case AFFINITY_SPECIFIED:
-			par->cpu = cpu_for_thread(i, max_cpus);
-			if (verbose)
-				printf("Thread %d using cpu %d.\n", i,
-					par->cpu);
-			break;
-		case AFFINITY_USEALL: par->cpu = i % max_cpus; break;
-		}
+		par->cpu = cpu;
+
 		stat->min = 1000000;
 		stat->max = 0;
 		stat->avg = 0.0;