1 #include <fcntl.h>
2 #include <errno.h>
3 #include <stdio.h>
4 #include <stdlib.h>
5 #include <unistd.h>
6 #include <dirent.h>
7 #include <sys/ioctl.h>
8 #include <sys/mman.h>
9 #include <sys/stat.h>
10 #include <sys/types.h>
11 #include <pthread.h>
12 #include <assert.h>
13 #include <mm/gup_test.h>
14 #include "../kselftest.h"
15 #include "vm_util.h"
16 
17 #define MB (1UL << 20)
18 
19 /* Just the flags we need, copied from mm.h: */
20 #define FOLL_WRITE	0x01	/* check pte is writable */
21 #define FOLL_TOUCH	0x02	/* mark page accessed */
22 
23 #define GUP_TEST_FILE "/sys/kernel/debug/gup_test"
24 
25 static unsigned long cmd = GUP_FAST_BENCHMARK;
26 static int gup_fd, repeats = 1;
27 static unsigned long size = 128 * MB;
28 /* Serialize prints */
29 static pthread_mutex_t print_mutex = PTHREAD_MUTEX_INITIALIZER;
30 
cmd_to_str(unsigned long cmd)31 static char *cmd_to_str(unsigned long cmd)
32 {
33 	switch (cmd) {
34 	case GUP_FAST_BENCHMARK:
35 		return "GUP_FAST_BENCHMARK";
36 	case PIN_FAST_BENCHMARK:
37 		return "PIN_FAST_BENCHMARK";
38 	case PIN_LONGTERM_BENCHMARK:
39 		return "PIN_LONGTERM_BENCHMARK";
40 	case GUP_BASIC_TEST:
41 		return "GUP_BASIC_TEST";
42 	case PIN_BASIC_TEST:
43 		return "PIN_BASIC_TEST";
44 	case DUMP_USER_PAGES_TEST:
45 		return "DUMP_USER_PAGES_TEST";
46 	}
47 	return "Unknown command";
48 }
49 
gup_thread(void * data)50 void *gup_thread(void *data)
51 {
52 	struct gup_test gup = *(struct gup_test *)data;
53 	int i;
54 
55 	/* Only report timing information on the *_BENCHMARK commands: */
56 	if ((cmd == PIN_FAST_BENCHMARK) || (cmd == GUP_FAST_BENCHMARK) ||
57 	     (cmd == PIN_LONGTERM_BENCHMARK)) {
58 		for (i = 0; i < repeats; i++) {
59 			gup.size = size;
60 			if (ioctl(gup_fd, cmd, &gup))
61 				perror("ioctl"), exit(1);
62 
63 			pthread_mutex_lock(&print_mutex);
64 			printf("%s: Time: get:%lld put:%lld us",
65 			       cmd_to_str(cmd), gup.get_delta_usec,
66 			       gup.put_delta_usec);
67 			if (gup.size != size)
68 				printf(", truncated (size: %lld)", gup.size);
69 			printf("\n");
70 			pthread_mutex_unlock(&print_mutex);
71 		}
72 	} else {
73 		gup.size = size;
74 		if (ioctl(gup_fd, cmd, &gup)) {
75 			perror("ioctl");
76 			exit(1);
77 		}
78 
79 		pthread_mutex_lock(&print_mutex);
80 		printf("%s: done\n", cmd_to_str(cmd));
81 		if (gup.size != size)
82 			printf("Truncated (size: %lld)\n", gup.size);
83 		pthread_mutex_unlock(&print_mutex);
84 	}
85 
86 	return NULL;
87 }
88 
main(int argc,char ** argv)89 int main(int argc, char **argv)
90 {
91 	struct gup_test gup = { 0 };
92 	int filed, i, opt, nr_pages = 1, thp = -1, write = 1, nthreads = 1, ret;
93 	int flags = MAP_PRIVATE, touch = 0;
94 	char *file = "/dev/zero";
95 	pthread_t *tid;
96 	char *p;
97 
98 	while ((opt = getopt(argc, argv, "m:r:n:F:f:abcj:tTLUuwWSHpz")) != -1) {
99 		switch (opt) {
100 		case 'a':
101 			cmd = PIN_FAST_BENCHMARK;
102 			break;
103 		case 'b':
104 			cmd = PIN_BASIC_TEST;
105 			break;
106 		case 'L':
107 			cmd = PIN_LONGTERM_BENCHMARK;
108 			break;
109 		case 'c':
110 			cmd = DUMP_USER_PAGES_TEST;
111 			/*
112 			 * Dump page 0 (index 1). May be overridden later, by
113 			 * user's non-option arguments.
114 			 *
115 			 * .which_pages is zero-based, so that zero can mean "do
116 			 * nothing".
117 			 */
118 			gup.which_pages[0] = 1;
119 			break;
120 		case 'p':
121 			/* works only with DUMP_USER_PAGES_TEST */
122 			gup.test_flags |= GUP_TEST_FLAG_DUMP_PAGES_USE_PIN;
123 			break;
124 		case 'F':
125 			/* strtol, so you can pass flags in hex form */
126 			gup.gup_flags = strtol(optarg, 0, 0);
127 			break;
128 		case 'j':
129 			nthreads = atoi(optarg);
130 			break;
131 		case 'm':
132 			size = atoi(optarg) * MB;
133 			break;
134 		case 'r':
135 			repeats = atoi(optarg);
136 			break;
137 		case 'n':
138 			nr_pages = atoi(optarg);
139 			break;
140 		case 't':
141 			thp = 1;
142 			break;
143 		case 'T':
144 			thp = 0;
145 			break;
146 		case 'U':
147 			cmd = GUP_BASIC_TEST;
148 			break;
149 		case 'u':
150 			cmd = GUP_FAST_BENCHMARK;
151 			break;
152 		case 'w':
153 			write = 1;
154 			break;
155 		case 'W':
156 			write = 0;
157 			break;
158 		case 'f':
159 			file = optarg;
160 			break;
161 		case 'S':
162 			flags &= ~MAP_PRIVATE;
163 			flags |= MAP_SHARED;
164 			break;
165 		case 'H':
166 			flags |= (MAP_HUGETLB | MAP_ANONYMOUS);
167 			break;
168 		case 'z':
169 			/* fault pages in gup, do not fault in userland */
170 			touch = 1;
171 			break;
172 		default:
173 			return -1;
174 		}
175 	}
176 
177 	if (optind < argc) {
178 		int extra_arg_count = 0;
179 		/*
180 		 * For example:
181 		 *
182 		 *   ./gup_test -c 0 1 0x1001
183 		 *
184 		 * ...to dump pages 0, 1, and 4097
185 		 */
186 
187 		while ((optind < argc) &&
188 		       (extra_arg_count < GUP_TEST_MAX_PAGES_TO_DUMP)) {
189 			/*
190 			 * Do the 1-based indexing here, so that the user can
191 			 * use normal 0-based indexing on the command line.
192 			 */
193 			long page_index = strtol(argv[optind], 0, 0) + 1;
194 
195 			gup.which_pages[extra_arg_count] = page_index;
196 			extra_arg_count++;
197 			optind++;
198 		}
199 	}
200 
201 	filed = open(file, O_RDWR|O_CREAT);
202 	if (filed < 0) {
203 		perror("open");
204 		exit(filed);
205 	}
206 
207 	gup.nr_pages_per_call = nr_pages;
208 	if (write)
209 		gup.gup_flags |= FOLL_WRITE;
210 
211 	gup_fd = open(GUP_TEST_FILE, O_RDWR);
212 	if (gup_fd == -1) {
213 		switch (errno) {
214 		case EACCES:
215 			if (getuid())
216 				printf("Please run this test as root\n");
217 			break;
218 		case ENOENT:
219 			if (opendir("/sys/kernel/debug") == NULL) {
220 				printf("mount debugfs at /sys/kernel/debug\n");
221 				break;
222 			}
223 			printf("check if CONFIG_GUP_TEST is enabled in kernel config\n");
224 			break;
225 		default:
226 			perror("failed to open " GUP_TEST_FILE);
227 			break;
228 		}
229 		exit(KSFT_SKIP);
230 	}
231 
232 	p = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, filed, 0);
233 	if (p == MAP_FAILED) {
234 		perror("mmap");
235 		exit(1);
236 	}
237 	gup.addr = (unsigned long)p;
238 
239 	if (thp == 1)
240 		madvise(p, size, MADV_HUGEPAGE);
241 	else if (thp == 0)
242 		madvise(p, size, MADV_NOHUGEPAGE);
243 
244 	/*
245 	 * FOLL_TOUCH, in gup_test, is used as an either/or case: either
246 	 * fault pages in from the kernel via FOLL_TOUCH, or fault them
247 	 * in here, from user space. This allows comparison of performance
248 	 * between those two cases.
249 	 */
250 	if (touch) {
251 		gup.gup_flags |= FOLL_TOUCH;
252 	} else {
253 		for (; (unsigned long)p < gup.addr + size; p += psize())
254 			p[0] = 0;
255 	}
256 
257 	tid = malloc(sizeof(pthread_t) * nthreads);
258 	assert(tid);
259 	for (i = 0; i < nthreads; i++) {
260 		ret = pthread_create(&tid[i], NULL, gup_thread, &gup);
261 		assert(ret == 0);
262 	}
263 	for (i = 0; i < nthreads; i++) {
264 		ret = pthread_join(tid[i], NULL);
265 		assert(ret == 0);
266 	}
267 	free(tid);
268 
269 	return 0;
270 }
271