Lines Matching refs:kc
210 static void wake(struct dm_kcopyd_client *kc) in wake() argument
212 queue_work(kc->kcopyd_wq, &kc->kcopyd_work); in wake()
245 static void kcopyd_put_pages(struct dm_kcopyd_client *kc, struct page_list *pl) in kcopyd_put_pages() argument
252 if (kc->nr_free_pages >= kc->nr_reserved_pages) in kcopyd_put_pages()
255 pl->next = kc->pages; in kcopyd_put_pages()
256 kc->pages = pl; in kcopyd_put_pages()
257 kc->nr_free_pages++; in kcopyd_put_pages()
264 static int kcopyd_get_pages(struct dm_kcopyd_client *kc, in kcopyd_get_pages() argument
275 pl = kc->pages; in kcopyd_get_pages()
278 kc->pages = pl->next; in kcopyd_get_pages()
279 kc->nr_free_pages--; in kcopyd_get_pages()
289 kcopyd_put_pages(kc, *pages); in kcopyd_get_pages()
310 static int client_reserve_pages(struct dm_kcopyd_client *kc, unsigned int nr_pages) in client_reserve_pages() argument
326 kc->nr_reserved_pages += nr_pages; in client_reserve_pages()
327 kcopyd_put_pages(kc, pl); in client_reserve_pages()
332 static void client_free_pages(struct dm_kcopyd_client *kc) in client_free_pages() argument
334 BUG_ON(kc->nr_free_pages != kc->nr_reserved_pages); in client_free_pages()
335 drop_pages(kc->pages); in client_free_pages()
336 kc->pages = NULL; in client_free_pages()
337 kc->nr_free_pages = kc->nr_reserved_pages = 0; in client_free_pages()
348 struct dm_kcopyd_client *kc; member
418 struct dm_kcopyd_client *kc) in pop_io_job() argument
444 struct dm_kcopyd_client *kc) in pop() argument
448 spin_lock_irq(&kc->job_lock); in pop()
451 if (jobs == &kc->io_jobs) in pop()
452 job = pop_io_job(jobs, kc); in pop()
458 spin_unlock_irq(&kc->job_lock); in pop()
466 struct dm_kcopyd_client *kc = job->kc; in push() local
468 spin_lock_irqsave(&kc->job_lock, flags); in push()
470 spin_unlock_irqrestore(&kc->job_lock, flags); in push()
476 struct dm_kcopyd_client *kc = job->kc; in push_head() local
478 spin_lock_irq(&kc->job_lock); in push_head()
480 spin_unlock_irq(&kc->job_lock); in push_head()
498 struct dm_kcopyd_client *kc = job->kc; in run_complete_job() local
501 kcopyd_put_pages(kc, job->pages); in run_complete_job()
508 mempool_free(job, &kc->job_pool); in run_complete_job()
512 if (atomic_dec_and_test(&kc->nr_jobs)) in run_complete_job()
513 wake_up(&kc->destroyq); in run_complete_job()
523 struct dm_kcopyd_client *kc = job->kc; in complete_io() local
525 io_job_finish(kc->throttle); in complete_io()
534 push(&kc->complete_jobs, job); in complete_io()
535 wake(kc); in complete_io()
541 push(&kc->complete_jobs, job); in complete_io()
545 push(&kc->io_jobs, job); in complete_io()
548 wake(kc); in complete_io()
565 .client = job->kc->io_client, in run_io_job()
578 io_job_start(job->kc->throttle); in run_io_job()
593 r = kcopyd_get_pages(job->kc, nr_pages, &job->pages); in run_pages_job()
596 push(&job->kc->io_jobs, job); in run_pages_job()
611 static int process_jobs(struct list_head *jobs, struct dm_kcopyd_client *kc, in process_jobs() argument
617 while ((job = pop(jobs, kc))) { in process_jobs()
627 push(&kc->complete_jobs, job); in process_jobs()
628 wake(kc); in process_jobs()
652 struct dm_kcopyd_client *kc = container_of(work, in do_work() local
663 spin_lock_irq(&kc->job_lock); in do_work()
664 list_splice_tail_init(&kc->callback_jobs, &kc->complete_jobs); in do_work()
665 spin_unlock_irq(&kc->job_lock); in do_work()
668 process_jobs(&kc->complete_jobs, kc, run_complete_job); in do_work()
669 process_jobs(&kc->pages_jobs, kc, run_pages_job); in do_work()
670 process_jobs(&kc->io_jobs, kc, run_io_job); in do_work()
681 struct dm_kcopyd_client *kc = job->kc; in dispatch_job() local
683 atomic_inc(&kc->nr_jobs); in dispatch_job()
685 push(&kc->callback_jobs, job); in dispatch_job()
687 push(&kc->io_jobs, job); in dispatch_job()
689 push(&kc->pages_jobs, job); in dispatch_job()
690 wake(kc); in dispatch_job()
701 struct dm_kcopyd_client *kc = job->kc; in segment_complete() local
721 if (count > kc->sub_job_size) in segment_complete()
722 count = kc->sub_job_size; in segment_complete()
757 push(&kc->complete_jobs, job); in segment_complete()
758 wake(kc); in segment_complete()
769 atomic_inc(&master_job->kc->nr_jobs); in split_job()
778 void dm_kcopyd_copy(struct dm_kcopyd_client *kc, struct dm_io_region *from, in dm_kcopyd_copy() argument
789 job = mempool_alloc(&kc->job_pool, GFP_NOIO); in dm_kcopyd_copy()
795 job->kc = kc; in dm_kcopyd_copy()
849 if (job->source.count <= kc->sub_job_size) in dm_kcopyd_copy()
858 void dm_kcopyd_zero(struct dm_kcopyd_client *kc, in dm_kcopyd_zero() argument
862 dm_kcopyd_copy(kc, NULL, num_dests, dests, flags, fn, context); in dm_kcopyd_zero()
866 void *dm_kcopyd_prepare_callback(struct dm_kcopyd_client *kc, in dm_kcopyd_prepare_callback() argument
871 job = mempool_alloc(&kc->job_pool, GFP_NOIO); in dm_kcopyd_prepare_callback()
874 job->kc = kc; in dm_kcopyd_prepare_callback()
879 atomic_inc(&kc->nr_jobs); in dm_kcopyd_prepare_callback()
888 struct dm_kcopyd_client *kc = job->kc; in dm_kcopyd_do_callback() local
893 push(&kc->callback_jobs, job); in dm_kcopyd_do_callback()
894 wake(kc); in dm_kcopyd_do_callback()
919 struct dm_kcopyd_client *kc; in dm_kcopyd_client_create() local
921 kc = kzalloc(sizeof(*kc), GFP_KERNEL); in dm_kcopyd_client_create()
922 if (!kc) in dm_kcopyd_client_create()
925 spin_lock_init(&kc->job_lock); in dm_kcopyd_client_create()
926 INIT_LIST_HEAD(&kc->callback_jobs); in dm_kcopyd_client_create()
927 INIT_LIST_HEAD(&kc->complete_jobs); in dm_kcopyd_client_create()
928 INIT_LIST_HEAD(&kc->io_jobs); in dm_kcopyd_client_create()
929 INIT_LIST_HEAD(&kc->pages_jobs); in dm_kcopyd_client_create()
930 kc->throttle = throttle; in dm_kcopyd_client_create()
932 r = mempool_init_slab_pool(&kc->job_pool, MIN_JOBS, _job_cache); in dm_kcopyd_client_create()
936 INIT_WORK(&kc->kcopyd_work, do_work); in dm_kcopyd_client_create()
937 kc->kcopyd_wq = alloc_workqueue("kcopyd", WQ_MEM_RECLAIM, 0); in dm_kcopyd_client_create()
938 if (!kc->kcopyd_wq) { in dm_kcopyd_client_create()
943 kc->sub_job_size = dm_get_kcopyd_subjob_size(); in dm_kcopyd_client_create()
944 reserve_pages = DIV_ROUND_UP(kc->sub_job_size << SECTOR_SHIFT, PAGE_SIZE); in dm_kcopyd_client_create()
946 kc->pages = NULL; in dm_kcopyd_client_create()
947 kc->nr_reserved_pages = kc->nr_free_pages = 0; in dm_kcopyd_client_create()
948 r = client_reserve_pages(kc, reserve_pages); in dm_kcopyd_client_create()
952 kc->io_client = dm_io_client_create(); in dm_kcopyd_client_create()
953 if (IS_ERR(kc->io_client)) { in dm_kcopyd_client_create()
954 r = PTR_ERR(kc->io_client); in dm_kcopyd_client_create()
958 init_waitqueue_head(&kc->destroyq); in dm_kcopyd_client_create()
959 atomic_set(&kc->nr_jobs, 0); in dm_kcopyd_client_create()
961 return kc; in dm_kcopyd_client_create()
964 client_free_pages(kc); in dm_kcopyd_client_create()
966 destroy_workqueue(kc->kcopyd_wq); in dm_kcopyd_client_create()
968 mempool_exit(&kc->job_pool); in dm_kcopyd_client_create()
970 kfree(kc); in dm_kcopyd_client_create()
976 void dm_kcopyd_client_destroy(struct dm_kcopyd_client *kc) in dm_kcopyd_client_destroy() argument
979 wait_event(kc->destroyq, !atomic_read(&kc->nr_jobs)); in dm_kcopyd_client_destroy()
981 BUG_ON(!list_empty(&kc->callback_jobs)); in dm_kcopyd_client_destroy()
982 BUG_ON(!list_empty(&kc->complete_jobs)); in dm_kcopyd_client_destroy()
983 BUG_ON(!list_empty(&kc->io_jobs)); in dm_kcopyd_client_destroy()
984 BUG_ON(!list_empty(&kc->pages_jobs)); in dm_kcopyd_client_destroy()
985 destroy_workqueue(kc->kcopyd_wq); in dm_kcopyd_client_destroy()
986 dm_io_client_destroy(kc->io_client); in dm_kcopyd_client_destroy()
987 client_free_pages(kc); in dm_kcopyd_client_destroy()
988 mempool_exit(&kc->job_pool); in dm_kcopyd_client_destroy()
989 kfree(kc); in dm_kcopyd_client_destroy()
993 void dm_kcopyd_client_flush(struct dm_kcopyd_client *kc) in dm_kcopyd_client_flush() argument
995 flush_workqueue(kc->kcopyd_wq); in dm_kcopyd_client_flush()