1 /*
2  * Copyright 2020 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Christian König
23  */
24 
25 #ifndef _TTM_RESOURCE_H_
26 #define _TTM_RESOURCE_H_
27 
28 #include <linux/types.h>
29 #include <linux/list.h>
30 #include <linux/mutex.h>
31 #include <linux/iosys-map.h>
32 #include <linux/dma-fence.h>
33 
34 #include <drm/drm_print.h>
35 #include <drm/ttm/ttm_caching.h>
36 #include <drm/ttm/ttm_kmap_iter.h>
37 
38 #define TTM_MAX_BO_PRIORITY	4U
39 #define TTM_NUM_MEM_TYPES 8
40 
41 struct ttm_device;
42 struct ttm_resource_manager;
43 struct ttm_resource;
44 struct ttm_place;
45 struct ttm_buffer_object;
46 struct ttm_placement;
47 struct iosys_map;
48 struct io_mapping;
49 struct sg_table;
50 struct scatterlist;
51 
52 struct ttm_resource_manager_func {
53 	/**
54 	 * struct ttm_resource_manager_func member alloc
55 	 *
56 	 * @man: Pointer to a memory type manager.
57 	 * @bo: Pointer to the buffer object we're allocating space for.
58 	 * @place: Placement details.
59 	 * @res: Resulting pointer to the ttm_resource.
60 	 *
61 	 * This function should allocate space in the memory type managed
62 	 * by @man. Placement details if applicable are given by @place. If
63 	 * successful, a filled in ttm_resource object should be returned in
64 	 * @res. @res::start should be set to a value identifying the beginning
65 	 * of the range allocated, and the function should return zero.
66 	 * If the manager can't fulfill the request -ENOSPC should be returned.
67 	 * If a system error occurred, preventing the request to be fulfilled,
68 	 * the function should return a negative error code.
69 	 *
70 	 * This function may not be called from within atomic context and needs
71 	 * to take care of its own locking to protect any data structures
72 	 * managing the space.
73 	 */
74 	int  (*alloc)(struct ttm_resource_manager *man,
75 		      struct ttm_buffer_object *bo,
76 		      const struct ttm_place *place,
77 		      struct ttm_resource **res);
78 
79 	/**
80 	 * struct ttm_resource_manager_func member free
81 	 *
82 	 * @man: Pointer to a memory type manager.
83 	 * @res: Pointer to a struct ttm_resource to be freed.
84 	 *
85 	 * This function frees memory type resources previously allocated.
86 	 * May not be called from within atomic context.
87 	 */
88 	void (*free)(struct ttm_resource_manager *man,
89 		     struct ttm_resource *res);
90 
91 	/**
92 	 * struct ttm_resource_manager_func member intersects
93 	 *
94 	 * @man: Pointer to a memory type manager.
95 	 * @res: Pointer to a struct ttm_resource to be checked.
96 	 * @place: Placement to check against.
97 	 * @size: Size of the check.
98 	 *
99 	 * Test if @res intersects with @place + @size. Used to judge if
100 	 * evictions are valueable or not.
101 	 */
102 	bool (*intersects)(struct ttm_resource_manager *man,
103 			   struct ttm_resource *res,
104 			   const struct ttm_place *place,
105 			   size_t size);
106 
107 	/**
108 	 * struct ttm_resource_manager_func member compatible
109 	 *
110 	 * @man: Pointer to a memory type manager.
111 	 * @res: Pointer to a struct ttm_resource to be checked.
112 	 * @place: Placement to check against.
113 	 * @size: Size of the check.
114 	 *
115 	 * Test if @res compatible with @place + @size. Used to check of
116 	 * the need to move the backing store or not.
117 	 */
118 	bool (*compatible)(struct ttm_resource_manager *man,
119 			   struct ttm_resource *res,
120 			   const struct ttm_place *place,
121 			   size_t size);
122 
123 	/**
124 	 * struct ttm_resource_manager_func member debug
125 	 *
126 	 * @man: Pointer to a memory type manager.
127 	 * @printer: Prefix to be used in printout to identify the caller.
128 	 *
129 	 * This function is called to print out the state of the memory
130 	 * type manager to aid debugging of out-of-memory conditions.
131 	 * It may not be called from within atomic context.
132 	 */
133 	void (*debug)(struct ttm_resource_manager *man,
134 		      struct drm_printer *printer);
135 };
136 
137 /**
138  * struct ttm_resource_manager
139  *
140  * @use_type: The memory type is enabled.
141  * @use_tt: If a TT object should be used for the backing store.
142  * @size: Size of the managed region.
143  * @bdev: ttm device this manager belongs to
144  * @func: structure pointer implementing the range manager. See above
145  * @move_lock: lock for move fence
146  * @move: The fence of the last pipelined move operation.
147  * @lru: The lru list for this memory type.
148  *
149  * This structure is used to identify and manage memory types for a device.
150  */
151 struct ttm_resource_manager {
152 	/*
153 	 * No protection. Constant from start.
154 	 */
155 	bool use_type;
156 	bool use_tt;
157 	struct ttm_device *bdev;
158 	uint64_t size;
159 	const struct ttm_resource_manager_func *func;
160 	spinlock_t move_lock;
161 
162 	/*
163 	 * Protected by @move_lock.
164 	 */
165 	struct dma_fence *move;
166 
167 	/*
168 	 * Protected by the bdev->lru_lock.
169 	 */
170 	struct list_head lru[TTM_MAX_BO_PRIORITY];
171 
172 	/**
173 	 * @usage: How much of the resources are used, protected by the
174 	 * bdev->lru_lock.
175 	 */
176 	uint64_t usage;
177 };
178 
179 /**
180  * struct ttm_bus_placement
181  *
182  * @addr:		mapped virtual address
183  * @offset:		physical addr
184  * @is_iomem:		is this io memory ?
185  * @caching:		See enum ttm_caching
186  *
187  * Structure indicating the bus placement of an object.
188  */
189 struct ttm_bus_placement {
190 	void			*addr;
191 	phys_addr_t		offset;
192 	bool			is_iomem;
193 	enum ttm_caching	caching;
194 };
195 
196 /**
197  * struct ttm_resource
198  *
199  * @start: Start of the allocation.
200  * @num_pages: Actual size of resource in pages.
201  * @mem_type: Resource type of the allocation.
202  * @placement: Placement flags.
203  * @bus: Placement on io bus accessible to the CPU
204  * @bo: weak reference to the BO, protected by ttm_device::lru_lock
205  *
206  * Structure indicating the placement and space resources used by a
207  * buffer object.
208  */
209 struct ttm_resource {
210 	unsigned long start;
211 	unsigned long num_pages;
212 	uint32_t mem_type;
213 	uint32_t placement;
214 	struct ttm_bus_placement bus;
215 	struct ttm_buffer_object *bo;
216 
217 	/**
218 	 * @lru: Least recently used list, see &ttm_resource_manager.lru
219 	 */
220 	struct list_head lru;
221 };
222 
223 /**
224  * struct ttm_resource_cursor
225  *
226  * @priority: the current priority
227  *
228  * Cursor to iterate over the resources in a manager.
229  */
230 struct ttm_resource_cursor {
231 	unsigned int priority;
232 };
233 
234 /**
235  * struct ttm_lru_bulk_move_pos
236  *
237  * @first: first res in the bulk move range
238  * @last: last res in the bulk move range
239  *
240  * Range of resources for a lru bulk move.
241  */
242 struct ttm_lru_bulk_move_pos {
243 	struct ttm_resource *first;
244 	struct ttm_resource *last;
245 };
246 
247 /**
248  * struct ttm_lru_bulk_move
249  *
250  * @pos: first/last lru entry for resources in the each domain/priority
251  *
252  * Container for the current bulk move state. Should be used with
253  * ttm_lru_bulk_move_init() and ttm_bo_set_bulk_move().
254  */
255 struct ttm_lru_bulk_move {
256 	struct ttm_lru_bulk_move_pos pos[TTM_NUM_MEM_TYPES][TTM_MAX_BO_PRIORITY];
257 };
258 
259 /**
260  * struct ttm_kmap_iter_iomap - Specialization for a struct io_mapping +
261  * struct sg_table backed struct ttm_resource.
262  * @base: Embedded struct ttm_kmap_iter providing the usage interface.
263  * @iomap: struct io_mapping representing the underlying linear io_memory.
264  * @st: sg_table into @iomap, representing the memory of the struct ttm_resource.
265  * @start: Offset that needs to be subtracted from @st to make
266  * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
267  * @cache: Scatterlist traversal cache for fast lookups.
268  * @cache.sg: Pointer to the currently cached scatterlist segment.
269  * @cache.i: First index of @sg. PAGE_SIZE granularity.
270  * @cache.end: Last index + 1 of @sg. PAGE_SIZE granularity.
271  * @cache.offs: First offset into @iomap of @sg. PAGE_SIZE granularity.
272  */
273 struct ttm_kmap_iter_iomap {
274 	struct ttm_kmap_iter base;
275 	struct io_mapping *iomap;
276 	struct sg_table *st;
277 	resource_size_t start;
278 	struct {
279 		struct scatterlist *sg;
280 		pgoff_t i;
281 		pgoff_t end;
282 		pgoff_t offs;
283 	} cache;
284 };
285 
286 /**
287  * struct ttm_kmap_iter_linear_io - Iterator specialization for linear io
288  * @base: The base iterator
289  * @dmap: Points to the starting address of the region
290  * @needs_unmap: Whether we need to unmap on fini
291  */
292 struct ttm_kmap_iter_linear_io {
293 	struct ttm_kmap_iter base;
294 	struct iosys_map dmap;
295 	bool needs_unmap;
296 };
297 
298 /**
299  * ttm_resource_manager_set_used
300  *
301  * @man: A memory manager object.
302  * @used: usage state to set.
303  *
304  * Set the manager in use flag. If disabled the manager is no longer
305  * used for object placement.
306  */
307 static inline void
ttm_resource_manager_set_used(struct ttm_resource_manager * man,bool used)308 ttm_resource_manager_set_used(struct ttm_resource_manager *man, bool used)
309 {
310 	int i;
311 
312 	for (i = 0; i < TTM_MAX_BO_PRIORITY; i++)
313 		WARN_ON(!list_empty(&man->lru[i]));
314 	man->use_type = used;
315 }
316 
317 /**
318  * ttm_resource_manager_used
319  *
320  * @man: Manager to get used state for
321  *
322  * Get the in use flag for a manager.
323  * Returns:
324  * true is used, false if not.
325  */
ttm_resource_manager_used(struct ttm_resource_manager * man)326 static inline bool ttm_resource_manager_used(struct ttm_resource_manager *man)
327 {
328 	return man->use_type;
329 }
330 
331 /**
332  * ttm_resource_manager_cleanup
333  *
334  * @man: A memory manager object.
335  *
336  * Cleanup the move fences from the memory manager object.
337  */
338 static inline void
ttm_resource_manager_cleanup(struct ttm_resource_manager * man)339 ttm_resource_manager_cleanup(struct ttm_resource_manager *man)
340 {
341 	dma_fence_put(man->move);
342 	man->move = NULL;
343 }
344 
345 void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk);
346 void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk);
347 
348 void ttm_resource_add_bulk_move(struct ttm_resource *res,
349 				struct ttm_buffer_object *bo);
350 void ttm_resource_del_bulk_move(struct ttm_resource *res,
351 				struct ttm_buffer_object *bo);
352 void ttm_resource_move_to_lru_tail(struct ttm_resource *res);
353 
354 void ttm_resource_init(struct ttm_buffer_object *bo,
355                        const struct ttm_place *place,
356                        struct ttm_resource *res);
357 void ttm_resource_fini(struct ttm_resource_manager *man,
358 		       struct ttm_resource *res);
359 
360 int ttm_resource_alloc(struct ttm_buffer_object *bo,
361 		       const struct ttm_place *place,
362 		       struct ttm_resource **res);
363 void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res);
364 bool ttm_resource_intersects(struct ttm_device *bdev,
365 			     struct ttm_resource *res,
366 			     const struct ttm_place *place,
367 			     size_t size);
368 bool ttm_resource_compatible(struct ttm_device *bdev,
369 			     struct ttm_resource *res,
370 			     const struct ttm_place *place,
371 			     size_t size);
372 bool ttm_resource_compat(struct ttm_resource *res,
373 			 struct ttm_placement *placement);
374 void ttm_resource_set_bo(struct ttm_resource *res,
375 			 struct ttm_buffer_object *bo);
376 
377 void ttm_resource_manager_init(struct ttm_resource_manager *man,
378 			       struct ttm_device *bdev,
379 			       uint64_t size);
380 
381 int ttm_resource_manager_evict_all(struct ttm_device *bdev,
382 				   struct ttm_resource_manager *man);
383 
384 uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man);
385 void ttm_resource_manager_debug(struct ttm_resource_manager *man,
386 				struct drm_printer *p);
387 
388 struct ttm_resource *
389 ttm_resource_manager_first(struct ttm_resource_manager *man,
390 			   struct ttm_resource_cursor *cursor);
391 struct ttm_resource *
392 ttm_resource_manager_next(struct ttm_resource_manager *man,
393 			  struct ttm_resource_cursor *cursor,
394 			  struct ttm_resource *res);
395 
396 /**
397  * ttm_resource_manager_for_each_res - iterate over all resources
398  * @man: the resource manager
399  * @cursor: struct ttm_resource_cursor for the current position
400  * @res: the current resource
401  *
402  * Iterate over all the evictable resources in a resource manager.
403  */
404 #define ttm_resource_manager_for_each_res(man, cursor, res)		\
405 	for (res = ttm_resource_manager_first(man, cursor); res;	\
406 	     res = ttm_resource_manager_next(man, cursor, res))
407 
408 struct ttm_kmap_iter *
409 ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
410 			 struct io_mapping *iomap,
411 			 struct sg_table *st,
412 			 resource_size_t start);
413 
414 struct ttm_kmap_iter_linear_io;
415 
416 struct ttm_kmap_iter *
417 ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
418 			     struct ttm_device *bdev,
419 			     struct ttm_resource *mem);
420 
421 void ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
422 				  struct ttm_device *bdev,
423 				  struct ttm_resource *mem);
424 
425 void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,
426 					 struct dentry * parent,
427 					 const char *name);
428 #endif
429