33 static inline void cachefiles_put_kiocb(struct cachefiles_kiocb *ki)  in cachefiles_put_kiocb()  argument
35 	if (refcount_dec_and_test(&ki->ki_refcnt)) {  in cachefiles_put_kiocb()
36 		cachefiles_put_object(ki->object, cachefiles_obj_put_ioreq);  in cachefiles_put_kiocb()
37 		fput(ki->iocb.ki_filp);  in cachefiles_put_kiocb()
38 		kfree(ki);  in cachefiles_put_kiocb()
47 	struct cachefiles_kiocb *ki = container_of(iocb, struct cachefiles_kiocb, iocb);  in cachefiles_read_complete()  local
48 	struct inode *inode = file_inode(ki->iocb.ki_filp);  in cachefiles_read_complete()
53 		trace_cachefiles_io_error(ki->object, inode, ret,  in cachefiles_read_complete()
56 	if (ki->term_func) {  in cachefiles_read_complete()
58 			if (ki->object->cookie->inval_counter == ki->inval_counter)  in cachefiles_read_complete()
59 				ki->skipped += ret;  in cachefiles_read_complete()
64 		ki->term_func(ki->term_func_priv, ret, ki->was_async);  in cachefiles_read_complete()
67 	cachefiles_put_kiocb(ki);  in cachefiles_read_complete()
81 	struct cachefiles_kiocb *ki;  in cachefiles_read()  local
133 	ki = kzalloc(sizeof(struct cachefiles_kiocb), GFP_KERNEL);  in cachefiles_read()
134 	if (!ki)  in cachefiles_read()
137 	refcount_set(&ki->ki_refcnt, 2);  in cachefiles_read()
138 	ki->iocb.ki_filp	= file;  in cachefiles_read()
139 	ki->iocb.ki_pos		= start_pos + skipped;  in cachefiles_read()
140 	ki->iocb.ki_flags	= IOCB_DIRECT;  in cachefiles_read()
141 	ki->iocb.ki_ioprio	= get_current_ioprio();  in cachefiles_read()
142 	ki->skipped		= skipped;  in cachefiles_read()
143 	ki->object		= object;  in cachefiles_read()
144 	ki->inval_counter	= cres->inval_counter;  in cachefiles_read()
145 	ki->term_func		= term_func;  in cachefiles_read()
146 	ki->term_func_priv	= term_func_priv;  in cachefiles_read()
147 	ki->was_async		= true;  in cachefiles_read()
149 	if (ki->term_func)  in cachefiles_read()
150 		ki->iocb.ki_complete = cachefiles_read_complete;  in cachefiles_read()
152 	get_file(ki->iocb.ki_filp);  in cachefiles_read()
155 	trace_cachefiles_read(object, file_inode(file), ki->iocb.ki_pos, len - skipped);  in cachefiles_read()
159 		ret = vfs_iocb_iter_read(file, &ki->iocb, iter);  in cachefiles_read()
175 		ki->was_async = false;  in cachefiles_read()
176 		cachefiles_read_complete(&ki->iocb, ret);  in cachefiles_read()
183 	cachefiles_put_kiocb(ki);  in cachefiles_read()
256 	struct cachefiles_kiocb *ki = container_of(iocb, struct cachefiles_kiocb, iocb);  in cachefiles_write_complete()  local
257 	struct cachefiles_object *object = ki->object;  in cachefiles_write_complete()
258 	struct inode *inode = file_inode(ki->iocb.ki_filp);  in cachefiles_write_complete()
268 	atomic_long_sub(ki->b_writing, &object->volume->cache->b_writing);  in cachefiles_write_complete()
270 	if (ki->term_func)  in cachefiles_write_complete()
271 		ki->term_func(ki->term_func_priv, ret, ki->was_async);  in cachefiles_write_complete()
272 	cachefiles_put_kiocb(ki);  in cachefiles_write_complete()
286 	struct cachefiles_kiocb *ki;  in __cachefiles_write()  local
298 	ki = kzalloc(sizeof(struct cachefiles_kiocb), GFP_KERNEL);  in __cachefiles_write()
299 	if (!ki) {  in __cachefiles_write()
305 	refcount_set(&ki->ki_refcnt, 2);  in __cachefiles_write()
306 	ki->iocb.ki_filp	= file;  in __cachefiles_write()
307 	ki->iocb.ki_pos		= start_pos;  in __cachefiles_write()
308 	ki->iocb.ki_flags	= IOCB_DIRECT | IOCB_WRITE;  in __cachefiles_write()
309 	ki->iocb.ki_ioprio	= get_current_ioprio();  in __cachefiles_write()
310 	ki->object		= object;  in __cachefiles_write()
311 	ki->start		= start_pos;  in __cachefiles_write()
312 	ki->len			= len;  in __cachefiles_write()
313 	ki->term_func		= term_func;  in __cachefiles_write()
314 	ki->term_func_priv	= term_func_priv;  in __cachefiles_write()
315 	ki->was_async		= true;  in __cachefiles_write()
316 	ki->b_writing		= (len + (1 << cache->bshift) - 1) >> cache->bshift;  in __cachefiles_write()
318 	if (ki->term_func)  in __cachefiles_write()
319 		ki->iocb.ki_complete = cachefiles_write_complete;  in __cachefiles_write()
320 	atomic_long_add(ki->b_writing, &cache->b_writing);  in __cachefiles_write()
322 	kiocb_start_write(&ki->iocb);  in __cachefiles_write()
324 	get_file(ki->iocb.ki_filp);  in __cachefiles_write()
327 	trace_cachefiles_write(object, file_inode(file), ki->iocb.ki_pos, len);  in __cachefiles_write()
331 		ret = vfs_iocb_iter_write(file, &ki->iocb, iter);  in __cachefiles_write()
347 		ki->was_async = false;  in __cachefiles_write()
348 		cachefiles_write_complete(&ki->iocb, ret);  in __cachefiles_write()
355 	cachefiles_put_kiocb(ki);  in __cachefiles_write()