1 /* Repeating a memory blob, with alias mapping optimization.
2    Copyright (C) 2018-2022 Free Software Foundation, Inc.
3    This file is part of the GNU C Library.
4 
5    The GNU C Library is free software; you can redistribute it and/or
6    modify it under the terms of the GNU Lesser General Public
7    License as published by the Free Software Foundation; either
8    version 2.1 of the License, or (at your option) any later version.
9 
10    The GNU C Library is distributed in the hope that it will be useful,
11    but WITHOUT ANY WARRANTY; without even the implied warranty of
12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13    Lesser General Public License for more details.
14 
15    You should have received a copy of the GNU Lesser General Public
16    License along with the GNU C Library; if not, see
17    <https://www.gnu.org/licenses/>.  */
18 
19 #include <errno.h>
20 #include <fcntl.h>
21 #include <stdbool.h>
22 #include <stdlib.h>
23 #include <string.h>
24 #include <support/blob_repeat.h>
25 #include <support/check.h>
26 #include <support/test-driver.h>
27 #include <support/support.h>
28 #include <support/xunistd.h>
29 #include <sys/mman.h>
30 #include <unistd.h>
31 #include <wchar.h>
32 
33 /* Small allocations should use malloc directly instead of the mmap
34    optimization because mappings carry a lot of overhead.  */
35 static const size_t maximum_small_size = 4 * 1024 * 1024;
36 
37 /* Internal helper for fill.  */
38 static void
fill0(char * target,const char * element,size_t element_size,size_t count)39 fill0 (char *target, const char *element, size_t element_size,
40        size_t count)
41 {
42   while (count > 0)
43     {
44       memcpy (target, element, element_size);
45       target += element_size;
46       --count;
47     }
48 }
49 
50 /* Fill the buffer at TARGET with COUNT copies of the ELEMENT_SIZE
51    bytes starting at ELEMENT.  */
52 static void
fill(char * target,const char * element,size_t element_size,size_t count)53 fill (char *target, const char *element, size_t element_size,
54       size_t count)
55 {
56   if (element_size == 0 || count == 0)
57     return;
58   else if (element_size == 1)
59     memset (target, element[0], count);
60   else if (element_size == sizeof (wchar_t))
61     {
62       wchar_t wc;
63       memcpy (&wc, element, sizeof (wc));
64       wmemset ((wchar_t *) target, wc, count);
65     }
66   else if (element_size < 1024 && count > 4096)
67     {
68       /* Use larger copies for really small element sizes.  */
69       char buffer[8192];
70       size_t buffer_count = sizeof (buffer) / element_size;
71       fill0 (buffer, element, element_size, buffer_count);
72       while (count > 0)
73         {
74           size_t copy_count = buffer_count;
75           if (copy_count > count)
76             copy_count = count;
77           size_t copy_bytes = copy_count * element_size;
78           memcpy (target, buffer, copy_bytes);
79           target += copy_bytes;
80           count -= copy_count;
81         }
82     }
83   else
84     fill0 (target, element, element_size, count);
85 }
86 
87 /* Use malloc instead of mmap for small allocations and unusual size
88    combinations.  */
89 static struct support_blob_repeat
allocate_malloc(size_t total_size,const void * element,size_t element_size,size_t count)90 allocate_malloc (size_t total_size, const void *element, size_t element_size,
91                  size_t count)
92 {
93   void *buffer = malloc (total_size);
94   if (buffer == NULL)
95     return (struct support_blob_repeat) { 0 };
96   fill (buffer, element, element_size, count);
97   return (struct support_blob_repeat)
98     {
99       .start = buffer,
100       .size = total_size,
101       .use_malloc = true
102     };
103 }
104 
105 /* Return the least common multiple of PAGE_SIZE and ELEMENT_SIZE,
106    avoiding overflow.  This assumes that PAGE_SIZE is a power of
107    two.  */
108 static size_t
minimum_stride_size(size_t page_size,size_t element_size)109 minimum_stride_size (size_t page_size, size_t element_size)
110 {
111   TEST_VERIFY_EXIT (page_size > 0);
112   TEST_VERIFY_EXIT (element_size > 0);
113 
114   /* Compute the number of trailing zeros common to both sizes.  */
115   unsigned int common_zeros = __builtin_ctzll (page_size | element_size);
116 
117   /* In the product, this power of two appears twice, but in the least
118      common multiple, it appears only once.  Therefore, shift one
119      factor.  */
120   size_t multiple;
121   if (__builtin_mul_overflow (page_size >> common_zeros, element_size,
122 			      &multiple))
123     return 0;
124   return multiple;
125 }
126 
127 /* Allocations larger than maximum_small_size potentially use mmap
128    with alias mappings.  If SHARED, the alias mappings are created
129    using MAP_SHARED instead of MAP_PRIVATE.  */
130 static struct support_blob_repeat
allocate_big(size_t total_size,const void * element,size_t element_size,size_t count,bool shared)131 allocate_big (size_t total_size, const void *element, size_t element_size,
132               size_t count, bool shared)
133 {
134   unsigned long page_size = xsysconf (_SC_PAGESIZE);
135   size_t stride_size = minimum_stride_size (page_size, element_size);
136   if (stride_size == 0)
137     {
138       errno = EOVERFLOW;
139       return (struct support_blob_repeat) { 0 };
140     }
141 
142   /* Ensure that the stride size is at least maximum_small_size.  This
143      is necessary to reduce the number of distinct mappings.  */
144   if (stride_size < maximum_small_size)
145     stride_size
146       = ((maximum_small_size + stride_size - 1) / stride_size) * stride_size;
147 
148   if (stride_size > total_size)
149     /* The mmap optimization would not save anything.  */
150     return allocate_malloc (total_size, element, element_size, count);
151 
152   /* Reserve the memory region.  If we cannot create the mapping,
153      there is no reason to set up the backing file.  */
154   void *target = mmap (NULL, total_size, PROT_NONE,
155                        MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
156   if (target == MAP_FAILED)
157     return (struct support_blob_repeat) { 0 };
158 
159   /* Create the backing file for the repeated mapping.  Call mkstemp
160      directly to remove the resources backing the temporary file
161      immediately, once support_blob_repeat_free is called.  Using
162      create_temp_file would result in a warning during post-test
163      cleanup.  */
164   int fd;
165   {
166     char *temppath = xasprintf ("%s/support_blob_repeat-XXXXXX", test_dir);
167     fd = mkstemp (temppath);
168     if (fd < 0)
169       FAIL_EXIT1 ("mkstemp (\"%s\"): %m", temppath);
170     xunlink (temppath);
171     free (temppath);
172   }
173 
174   /* Make sure that there is backing storage, so that the fill
175      operation will not fault.  */
176   if (posix_fallocate (fd, 0, stride_size) != 0)
177     FAIL_EXIT1 ("posix_fallocate (%zu): %m", stride_size);
178 
179   /* The stride size must still be a multiple of the page size and
180      element size.  */
181   TEST_VERIFY_EXIT ((stride_size % page_size) == 0);
182   TEST_VERIFY_EXIT ((stride_size % element_size) == 0);
183 
184   /* Fill the backing store.  */
185   {
186     void *ptr = mmap (target, stride_size, PROT_READ | PROT_WRITE,
187                       MAP_FIXED | MAP_FILE | MAP_SHARED, fd, 0);
188     if (ptr == MAP_FAILED)
189       {
190         int saved_errno = errno;
191         xmunmap (target, total_size);
192         xclose (fd);
193         errno = saved_errno;
194         return (struct support_blob_repeat) { 0 };
195       }
196     if (ptr != target)
197       FAIL_EXIT1 ("mapping of %zu bytes moved from %p to %p",
198                   stride_size, target, ptr);
199 
200     /* Write the repeating data.  */
201     fill (target, element, element_size, stride_size / element_size);
202 
203     /* Return to a PROT_NONE mapping, just to be on the safe side.  */
204     ptr = mmap (target, stride_size, PROT_NONE,
205                 MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
206     if (ptr == MAP_FAILED)
207       FAIL_EXIT1 ("Failed to reinstate PROT_NONE mapping: %m");
208     if (ptr != target)
209       FAIL_EXIT1 ("PROT_NONE mapping of %zu bytes moved from %p to %p",
210                   stride_size, target, ptr);
211   }
212 
213   /* Create the alias mappings.  */
214   {
215     size_t remaining_size = total_size;
216     char *current = target;
217     int flags = MAP_FIXED | MAP_FILE;
218     if (shared)
219       flags |= MAP_SHARED;
220     else
221       flags |= MAP_PRIVATE;
222 #ifdef MAP_NORESERVE
223     flags |= MAP_NORESERVE;
224 #endif
225     while (remaining_size > 0)
226       {
227         size_t to_map = stride_size;
228         if (to_map > remaining_size)
229           to_map = remaining_size;
230         void *ptr = mmap (current, to_map, PROT_READ | PROT_WRITE,
231                           flags, fd, 0);
232         if (ptr == MAP_FAILED)
233           {
234             int saved_errno = errno;
235             xmunmap (target, total_size);
236             xclose (fd);
237             errno = saved_errno;
238             return (struct support_blob_repeat) { 0 };
239           }
240         if (ptr != current)
241           FAIL_EXIT1 ("MAP_PRIVATE mapping of %zu bytes moved from %p to %p",
242                       to_map, target, ptr);
243         remaining_size -= to_map;
244         current += to_map;
245       }
246   }
247 
248   xclose (fd);
249 
250   return (struct support_blob_repeat)
251     {
252       .start = target,
253       .size = total_size,
254       .use_malloc = false
255     };
256 }
257 
258 struct support_blob_repeat
repeat_allocate(const void * element,size_t element_size,size_t count,bool shared)259 repeat_allocate (const void *element, size_t element_size,
260 		 size_t count, bool shared)
261 {
262   size_t total_size;
263   if (__builtin_mul_overflow (element_size, count, &total_size))
264     {
265       errno = EOVERFLOW;
266       return (struct support_blob_repeat) { 0 };
267     }
268   if (total_size <= maximum_small_size)
269     return allocate_malloc (total_size, element, element_size, count);
270   else
271     return allocate_big (total_size, element, element_size, count, shared);
272 }
273 
274 struct support_blob_repeat
support_blob_repeat_allocate(const void * element,size_t element_size,size_t count)275 support_blob_repeat_allocate (const void *element, size_t element_size,
276                               size_t count)
277 {
278   return repeat_allocate (element, element_size, count, false);
279 }
280 
281 struct support_blob_repeat
support_blob_repeat_allocate_shared(const void * element,size_t element_size,size_t count)282 support_blob_repeat_allocate_shared (const void *element, size_t element_size,
283 				     size_t count)
284 {
285   return repeat_allocate (element, element_size, count, true);
286 }
287 
288 void
support_blob_repeat_free(struct support_blob_repeat * blob)289 support_blob_repeat_free (struct support_blob_repeat *blob)
290 {
291   if (blob->size > 0)
292     {
293       int saved_errno = errno;
294       if (blob->use_malloc)
295         free (blob->start);
296       else
297         xmunmap (blob->start, blob->size);
298       errno = saved_errno;
299     }
300   *blob = (struct support_blob_repeat) { 0 };
301 }
302