71 static void wake(struct dm_kcopyd_client *kc)  in wake()  argument
73 	queue_work(kc->kcopyd_wq, &kc->kcopyd_work);  in wake()
106 static void kcopyd_put_pages(struct dm_kcopyd_client *kc, struct page_list *pl)  in kcopyd_put_pages()  argument
113 		if (kc->nr_free_pages >= kc->nr_reserved_pages)  in kcopyd_put_pages()
116 			pl->next = kc->pages;  in kcopyd_put_pages()
117 			kc->pages = pl;  in kcopyd_put_pages()
118 			kc->nr_free_pages++;  in kcopyd_put_pages()
125 static int kcopyd_get_pages(struct dm_kcopyd_client *kc,  in kcopyd_get_pages()  argument
136 			pl = kc->pages;  in kcopyd_get_pages()
139 			kc->pages = pl->next;  in kcopyd_get_pages()
140 			kc->nr_free_pages--;  in kcopyd_get_pages()
150 		kcopyd_put_pages(kc, *pages);  in kcopyd_get_pages()
171 static int client_reserve_pages(struct dm_kcopyd_client *kc, unsigned nr_pages)  in client_reserve_pages()  argument
187 	kc->nr_reserved_pages += nr_pages;  in client_reserve_pages()
188 	kcopyd_put_pages(kc, pl);  in client_reserve_pages()
193 static void client_free_pages(struct dm_kcopyd_client *kc)  in client_free_pages()  argument
195 	BUG_ON(kc->nr_free_pages != kc->nr_reserved_pages);  in client_free_pages()
196 	drop_pages(kc->pages);  in client_free_pages()
197 	kc->pages = NULL;  in client_free_pages()
198 	kc->nr_free_pages = kc->nr_reserved_pages = 0;  in client_free_pages()
207 	struct dm_kcopyd_client *kc;  member
276 			      struct dm_kcopyd_client *kc)  in pop()  argument
281 	spin_lock_irqsave(&kc->job_lock, flags);  in pop()
287 	spin_unlock_irqrestore(&kc->job_lock, flags);  in pop()
295 	struct dm_kcopyd_client *kc = job->kc;  in push()  local
297 	spin_lock_irqsave(&kc->job_lock, flags);  in push()
299 	spin_unlock_irqrestore(&kc->job_lock, flags);  in push()
306 	struct dm_kcopyd_client *kc = job->kc;  in push_head()  local
308 	spin_lock_irqsave(&kc->job_lock, flags);  in push_head()
310 	spin_unlock_irqrestore(&kc->job_lock, flags);  in push_head()
328 	struct dm_kcopyd_client *kc = job->kc;  in run_complete_job()  local
331 		kcopyd_put_pages(kc, job->pages);  in run_complete_job()
337 		mempool_free(job, kc->job_pool);  in run_complete_job()
340 	if (atomic_dec_and_test(&kc->nr_jobs))  in run_complete_job()
341 		wake_up(&kc->destroyq);  in run_complete_job()
349 	struct dm_kcopyd_client *kc = job->kc;  in complete_io()  local
358 			push(&kc->complete_jobs, job);  in complete_io()
359 			wake(kc);  in complete_io()
365 		push(&kc->complete_jobs, job);  in complete_io()
369 		push(&kc->io_jobs, job);  in complete_io()
372 	wake(kc);  in complete_io()
389 		.client = job->kc->io_client,  in run_io_job()
405 	r = kcopyd_get_pages(job->kc, nr_pages, &job->pages);  in run_pages_job()
408 		push(&job->kc->io_jobs, job);  in run_pages_job()
423 static int process_jobs(struct list_head *jobs, struct dm_kcopyd_client *kc,  in process_jobs()  argument
429 	while ((job = pop(jobs, kc))) {  in process_jobs()
439 			push(&kc->complete_jobs, job);  in process_jobs()
463 	struct dm_kcopyd_client *kc = container_of(work,  in do_work()  local
475 	process_jobs(&kc->complete_jobs, kc, run_complete_job);  in do_work()
476 	process_jobs(&kc->pages_jobs, kc, run_pages_job);  in do_work()
477 	process_jobs(&kc->io_jobs, kc, run_io_job);  in do_work()
488 	struct dm_kcopyd_client *kc = job->kc;  in dispatch_job()  local
489 	atomic_inc(&kc->nr_jobs);  in dispatch_job()
491 		push(&kc->complete_jobs, job);  in dispatch_job()
493 		push(&kc->io_jobs, job);  in dispatch_job()
495 		push(&kc->pages_jobs, job);  in dispatch_job()
496 	wake(kc);  in dispatch_job()
507 	struct dm_kcopyd_client *kc = job->kc;  in segment_complete()  local
562 		push(&kc->complete_jobs, job);  in segment_complete()
563 		wake(kc);  in segment_complete()
574 	atomic_inc(&master_job->kc->nr_jobs);  in split_job()
583 int dm_kcopyd_copy(struct dm_kcopyd_client *kc, struct dm_io_region *from,  in dm_kcopyd_copy()  argument
593 	job = mempool_alloc(kc->job_pool, GFP_NOIO);  in dm_kcopyd_copy()
598 	job->kc = kc;  in dm_kcopyd_copy()
633 int dm_kcopyd_zero(struct dm_kcopyd_client *kc,  in dm_kcopyd_zero()  argument
637 	return dm_kcopyd_copy(kc, NULL, num_dests, dests, flags, fn, context);  in dm_kcopyd_zero()
641 void *dm_kcopyd_prepare_callback(struct dm_kcopyd_client *kc,  in dm_kcopyd_prepare_callback()  argument
646 	job = mempool_alloc(kc->job_pool, GFP_NOIO);  in dm_kcopyd_prepare_callback()
649 	job->kc = kc;  in dm_kcopyd_prepare_callback()
654 	atomic_inc(&kc->nr_jobs);  in dm_kcopyd_prepare_callback()
663 	struct dm_kcopyd_client *kc = job->kc;  in dm_kcopyd_do_callback()  local
668 	push(&kc->complete_jobs, job);  in dm_kcopyd_do_callback()
669 	wake(kc);  in dm_kcopyd_do_callback()
691 	struct dm_kcopyd_client *kc;  in dm_kcopyd_client_create()  local
693 	kc = kmalloc(sizeof(*kc), GFP_KERNEL);  in dm_kcopyd_client_create()
694 	if (!kc)  in dm_kcopyd_client_create()
697 	spin_lock_init(&kc->job_lock);  in dm_kcopyd_client_create()
698 	INIT_LIST_HEAD(&kc->complete_jobs);  in dm_kcopyd_client_create()
699 	INIT_LIST_HEAD(&kc->io_jobs);  in dm_kcopyd_client_create()
700 	INIT_LIST_HEAD(&kc->pages_jobs);  in dm_kcopyd_client_create()
702 	kc->job_pool = mempool_create_slab_pool(MIN_JOBS, _job_cache);  in dm_kcopyd_client_create()
703 	if (!kc->job_pool)  in dm_kcopyd_client_create()
706 	INIT_WORK(&kc->kcopyd_work, do_work);  in dm_kcopyd_client_create()
707 	kc->kcopyd_wq = alloc_workqueue("kcopyd",  in dm_kcopyd_client_create()
709 	if (!kc->kcopyd_wq)  in dm_kcopyd_client_create()
712 	kc->pages = NULL;  in dm_kcopyd_client_create()
713 	kc->nr_reserved_pages = kc->nr_free_pages = 0;  in dm_kcopyd_client_create()
714 	r = client_reserve_pages(kc, RESERVE_PAGES);  in dm_kcopyd_client_create()
718 	kc->io_client = dm_io_client_create();  in dm_kcopyd_client_create()
719 	if (IS_ERR(kc->io_client)) {  in dm_kcopyd_client_create()
720 		r = PTR_ERR(kc->io_client);  in dm_kcopyd_client_create()
724 	init_waitqueue_head(&kc->destroyq);  in dm_kcopyd_client_create()
725 	atomic_set(&kc->nr_jobs, 0);  in dm_kcopyd_client_create()
727 	return kc;  in dm_kcopyd_client_create()
730 	client_free_pages(kc);  in dm_kcopyd_client_create()
732 	destroy_workqueue(kc->kcopyd_wq);  in dm_kcopyd_client_create()
734 	mempool_destroy(kc->job_pool);  in dm_kcopyd_client_create()
736 	kfree(kc);  in dm_kcopyd_client_create()
742 void dm_kcopyd_client_destroy(struct dm_kcopyd_client *kc)  in dm_kcopyd_client_destroy()  argument
745 	wait_event(kc->destroyq, !atomic_read(&kc->nr_jobs));  in dm_kcopyd_client_destroy()
747 	BUG_ON(!list_empty(&kc->complete_jobs));  in dm_kcopyd_client_destroy()
748 	BUG_ON(!list_empty(&kc->io_jobs));  in dm_kcopyd_client_destroy()
749 	BUG_ON(!list_empty(&kc->pages_jobs));  in dm_kcopyd_client_destroy()
750 	destroy_workqueue(kc->kcopyd_wq);  in dm_kcopyd_client_destroy()
751 	dm_io_client_destroy(kc->io_client);  in dm_kcopyd_client_destroy()
752 	client_free_pages(kc);  in dm_kcopyd_client_destroy()
753 	mempool_destroy(kc->job_pool);  in dm_kcopyd_client_destroy()
754 	kfree(kc);  in dm_kcopyd_client_destroy()