417 static inline void vmballoon_stats_op_inc(struct vmballoon *b, unsigned int op,  in vmballoon_stats_op_inc()  argument
421 		atomic64_inc(&b->stats->ops[op][type]);  in vmballoon_stats_op_inc()
424 static inline void vmballoon_stats_gen_inc(struct vmballoon *b,  in vmballoon_stats_gen_inc()  argument
428 		atomic64_inc(&b->stats->general_stat[stat]);  in vmballoon_stats_gen_inc()
431 static inline void vmballoon_stats_gen_add(struct vmballoon *b,  in vmballoon_stats_gen_add()  argument
436 		atomic64_add(val, &b->stats->general_stat[stat]);  in vmballoon_stats_gen_add()
439 static inline void vmballoon_stats_page_inc(struct vmballoon *b,  in vmballoon_stats_page_inc()  argument
444 		atomic64_inc(&b->stats->page_stat[stat][size]);  in vmballoon_stats_page_inc()
447 static inline void vmballoon_stats_page_add(struct vmballoon *b,  in vmballoon_stats_page_add()  argument
453 		atomic64_add(val, &b->stats->page_stat[stat][size]);  in vmballoon_stats_page_add()
457 __vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1,  in __vmballoon_cmd()  argument
462 	vmballoon_stats_op_inc(b, cmd, VMW_BALLOON_OP_STAT);  in __vmballoon_cmd()
485 		WRITE_ONCE(b->target, local_result);  in __vmballoon_cmd()
489 		vmballoon_stats_op_inc(b, cmd, VMW_BALLOON_OP_FAIL_STAT);  in __vmballoon_cmd()
497 		b->reset_required = true;  in __vmballoon_cmd()
503 vmballoon_cmd(struct vmballoon *b, unsigned long cmd, unsigned long arg1,  in vmballoon_cmd()  argument
508 	return __vmballoon_cmd(b, cmd, arg1, arg2, &dummy);  in vmballoon_cmd()
515 static int vmballoon_send_start(struct vmballoon *b, unsigned long req_caps)  in vmballoon_send_start()  argument
519 	status = __vmballoon_cmd(b, VMW_BALLOON_CMD_START, req_caps, 0,  in vmballoon_send_start()
524 		b->capabilities = capabilities;  in vmballoon_send_start()
527 		b->capabilities = VMW_BALLOON_BASIC_CMDS;  in vmballoon_send_start()
538 	b->max_page_size = VMW_BALLOON_4K_PAGE;  in vmballoon_send_start()
539 	if ((b->capabilities & VMW_BALLOON_BATCHED_2M_CMDS) &&  in vmballoon_send_start()
540 	    (b->capabilities & VMW_BALLOON_BATCHED_CMDS))  in vmballoon_send_start()
541 		b->max_page_size = VMW_BALLOON_2M_PAGE;  in vmballoon_send_start()
559 static int vmballoon_send_guest_id(struct vmballoon *b)  in vmballoon_send_guest_id()  argument
563 	status = vmballoon_cmd(b, VMW_BALLOON_CMD_GUEST_ID,  in vmballoon_send_guest_id()
632 static int vmballoon_send_get_target(struct vmballoon *b)  in vmballoon_send_get_target()  argument
640 	if (!(b->capabilities & VMW_BALLOON_64_BIT_TARGET) &&  in vmballoon_send_get_target()
644 	status = vmballoon_cmd(b, VMW_BALLOON_CMD_GET_TARGET, limit, 0);  in vmballoon_send_get_target()
661 static int vmballoon_alloc_page_list(struct vmballoon *b,  in vmballoon_alloc_page_list()  argument
685 			vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC,  in vmballoon_alloc_page_list()
696 		vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_ALLOC_FAIL,  in vmballoon_alloc_page_list()
714 static int vmballoon_handle_one_result(struct vmballoon *b, struct page *page,  in vmballoon_handle_one_result()  argument
727 	vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_ALLOC,  in vmballoon_handle_one_result()
746 static unsigned long vmballoon_status_page(struct vmballoon *b, int idx,  in vmballoon_status_page()  argument
751 		*p = pfn_to_page(b->batch_page[idx].pfn);  in vmballoon_status_page()
752 		return b->batch_page[idx].status;  in vmballoon_status_page()
756 	*p = b->page;  in vmballoon_status_page()
782 static unsigned long vmballoon_lock_op(struct vmballoon *b,  in vmballoon_lock_op()  argument
789 	lockdep_assert_held(&b->comm_lock);  in vmballoon_lock_op()
801 		pfn = PHYS_PFN(virt_to_phys(b->batch_page));  in vmballoon_lock_op()
805 		pfn = page_to_pfn(b->page);  in vmballoon_lock_op()
812 	return vmballoon_cmd(b, cmd, pfn, num_pages);  in vmballoon_lock_op()
824 static void vmballoon_add_page(struct vmballoon *b, unsigned int idx,  in vmballoon_add_page()  argument
827 	lockdep_assert_held(&b->comm_lock);  in vmballoon_add_page()
830 		b->batch_page[idx] = (struct vmballoon_batch_entry)  in vmballoon_add_page()
833 		b->page = p;  in vmballoon_add_page()
855 static int vmballoon_lock(struct vmballoon *b, struct vmballoon_ctl *ctl)  in vmballoon_lock()  argument
866 	spin_lock(&b->comm_lock);  in vmballoon_lock()
870 		vmballoon_add_page(b, i++, page);  in vmballoon_lock()
872 	batch_status = vmballoon_lock_op(b, ctl->n_pages, ctl->page_size,  in vmballoon_lock()
883 		status = vmballoon_status_page(b, i, &page);  in vmballoon_lock()
893 		if (!vmballoon_handle_one_result(b, page, ctl->page_size,  in vmballoon_lock()
906 	spin_unlock(&b->comm_lock);  in vmballoon_lock()
940 static void vmballoon_release_refused_pages(struct vmballoon *b,  in vmballoon_release_refused_pages()  argument
943 	vmballoon_stats_page_inc(b, VMW_BALLOON_PAGE_STAT_REFUSED_FREE,  in vmballoon_release_refused_pages()
958 static int64_t vmballoon_change(struct vmballoon *b)  in vmballoon_change()  argument
962 	size = atomic64_read(&b->size);  in vmballoon_change()
963 	target = READ_ONCE(b->target);  in vmballoon_change()
970 	if (b->reset_required)  in vmballoon_change()
979 	if (target > size && time_before(jiffies, READ_ONCE(b->shrink_timeout)))  in vmballoon_change()
996 static void vmballoon_enqueue_page_list(struct vmballoon *b,  in vmballoon_enqueue_page_list()  argument
1005 		balloon_page_list_enqueue(&b->b_dev_info, pages);  in vmballoon_enqueue_page_list()
1011 		spin_lock_irqsave(&b->b_dev_info.pages_lock, flags);  in vmballoon_enqueue_page_list()
1017 		list_splice_init(pages, &b->huge_pages);  in vmballoon_enqueue_page_list()
1020 		spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags);  in vmballoon_enqueue_page_list()
1039 static void vmballoon_dequeue_page_list(struct vmballoon *b,  in vmballoon_dequeue_page_list()  argument
1051 		*n_pages = balloon_page_list_dequeue(&b->b_dev_info, pages,  in vmballoon_dequeue_page_list()
1057 	spin_lock_irqsave(&b->b_dev_info.pages_lock, flags);  in vmballoon_dequeue_page_list()
1058 	list_for_each_entry_safe(page, tmp, &b->huge_pages, lru) {  in vmballoon_dequeue_page_list()
1068 	spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags);  in vmballoon_dequeue_page_list()
1103 static void vmballoon_inflate(struct vmballoon *b)  in vmballoon_inflate()  argument
1110 		.page_size = b->max_page_size,  in vmballoon_inflate()
1114 	while ((to_inflate_frames = vmballoon_change(b)) > 0) {  in vmballoon_inflate()
1123 		to_inflate_pages = min_t(unsigned long, b->batch_max_pages,  in vmballoon_inflate()
1128 		alloc_error = vmballoon_alloc_page_list(b, &ctl,  in vmballoon_inflate()
1132 		lock_error = vmballoon_lock(b, &ctl);  in vmballoon_inflate()
1142 		atomic64_add(ctl.n_pages * page_in_frames, &b->size);  in vmballoon_inflate()
1144 		vmballoon_enqueue_page_list(b, &ctl.pages, &ctl.n_pages,  in vmballoon_inflate()
1173 		vmballoon_release_refused_pages(b, &ctl);  in vmballoon_inflate()
1190 static unsigned long vmballoon_deflate(struct vmballoon *b, uint64_t n_frames,  in vmballoon_deflate()  argument
1222 					       -vmballoon_change(b);  in vmballoon_deflate()
1232 		to_deflate_pages = min_t(unsigned long, b->batch_max_pages,  in vmballoon_deflate()
1237 		vmballoon_dequeue_page_list(b, &ctl.pages, &ctl.n_pages,  in vmballoon_deflate()
1254 			vmballoon_lock(b, &ctl);  in vmballoon_deflate()
1266 		atomic64_sub(n_unlocked_frames, &b->size);  in vmballoon_deflate()
1269 		vmballoon_stats_page_add(b, VMW_BALLOON_PAGE_STAT_FREE,  in vmballoon_deflate()
1277 		vmballoon_enqueue_page_list(b, &ctl.refused_pages,  in vmballoon_deflate()
1283 			if (ctl.page_size == b->max_page_size)  in vmballoon_deflate()
1302 static void vmballoon_deinit_batching(struct vmballoon *b)  in vmballoon_deinit_batching()  argument
1304 	free_page((unsigned long)b->batch_page);  in vmballoon_deinit_batching()
1305 	b->batch_page = NULL;  in vmballoon_deinit_batching()
1307 	b->batch_max_pages = 1;  in vmballoon_deinit_batching()
1320 static int vmballoon_init_batching(struct vmballoon *b)  in vmballoon_init_batching()  argument
1328 	b->batch_page = page_address(page);  in vmballoon_init_batching()
1329 	b->batch_max_pages = PAGE_SIZE / sizeof(struct vmballoon_batch_entry);  in vmballoon_init_batching()
1341 	struct vmballoon *b = client_data;  in vmballoon_doorbell()  local
1343 	vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_DOORBELL);  in vmballoon_doorbell()
1345 	mod_delayed_work(system_freezable_wq, &b->dwork, 0);  in vmballoon_doorbell()
1351 static void vmballoon_vmci_cleanup(struct vmballoon *b)  in vmballoon_vmci_cleanup()  argument
1353 	vmballoon_cmd(b, VMW_BALLOON_CMD_VMCI_DOORBELL_SET,  in vmballoon_vmci_cleanup()
1356 	if (!vmci_handle_is_invalid(b->vmci_doorbell)) {  in vmballoon_vmci_cleanup()
1357 		vmci_doorbell_destroy(b->vmci_doorbell);  in vmballoon_vmci_cleanup()
1358 		b->vmci_doorbell = VMCI_INVALID_HANDLE;  in vmballoon_vmci_cleanup()
1372 static int vmballoon_vmci_init(struct vmballoon *b)  in vmballoon_vmci_init()  argument
1376 	if ((b->capabilities & VMW_BALLOON_SIGNALLED_WAKEUP_CMD) == 0)  in vmballoon_vmci_init()
1379 	error = vmci_doorbell_create(&b->vmci_doorbell, VMCI_FLAG_DELAYED_CB,  in vmballoon_vmci_init()
1381 				     vmballoon_doorbell, b);  in vmballoon_vmci_init()
1386 	error =	__vmballoon_cmd(b, VMW_BALLOON_CMD_VMCI_DOORBELL_SET,  in vmballoon_vmci_init()
1387 				b->vmci_doorbell.context,  in vmballoon_vmci_init()
1388 				b->vmci_doorbell.resource, NULL);  in vmballoon_vmci_init()
1395 	vmballoon_vmci_cleanup(b);  in vmballoon_vmci_init()
1408 static void vmballoon_pop(struct vmballoon *b)  in vmballoon_pop()  argument
1412 	while ((size = atomic64_read(&b->size)))  in vmballoon_pop()
1413 		vmballoon_deflate(b, size, false);  in vmballoon_pop()
1421 static void vmballoon_reset(struct vmballoon *b)  in vmballoon_reset()  argument
1425 	down_write(&b->conf_sem);  in vmballoon_reset()
1427 	vmballoon_vmci_cleanup(b);  in vmballoon_reset()
1430 	vmballoon_pop(b);  in vmballoon_reset()
1432 	if (vmballoon_send_start(b, VMW_BALLOON_CAPABILITIES))  in vmballoon_reset()
1435 	if ((b->capabilities & VMW_BALLOON_BATCHED_CMDS) != 0) {  in vmballoon_reset()
1436 		if (vmballoon_init_batching(b)) {  in vmballoon_reset()
1443 			vmballoon_send_start(b, 0);  in vmballoon_reset()
1446 	} else if ((b->capabilities & VMW_BALLOON_BASIC_CMDS) != 0) {  in vmballoon_reset()
1447 		vmballoon_deinit_batching(b);  in vmballoon_reset()
1450 	vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_RESET);  in vmballoon_reset()
1451 	b->reset_required = false;  in vmballoon_reset()
1453 	error = vmballoon_vmci_init(b);  in vmballoon_reset()
1457 	if (vmballoon_send_guest_id(b))  in vmballoon_reset()
1461 	up_write(&b->conf_sem);  in vmballoon_reset()
1475 	struct vmballoon *b = container_of(dwork, struct vmballoon, dwork);  in vmballoon_work()  local
1478 	if (b->reset_required)  in vmballoon_work()
1479 		vmballoon_reset(b);  in vmballoon_work()
1481 	down_read(&b->conf_sem);  in vmballoon_work()
1488 	vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_TIMER);  in vmballoon_work()
1490 	if (!vmballoon_send_get_target(b))  in vmballoon_work()
1491 		change = vmballoon_change(b);  in vmballoon_work()
1495 			 atomic64_read(&b->size), READ_ONCE(b->target));  in vmballoon_work()
1498 			vmballoon_inflate(b);  in vmballoon_work()
1500 			vmballoon_deflate(b, 0, true);  in vmballoon_work()
1503 	up_read(&b->conf_sem);  in vmballoon_work()
1524 	struct vmballoon *b = &balloon;  in vmballoon_shrinker_scan()  local
1527 	pr_debug("%s - size: %llu", __func__, atomic64_read(&b->size));  in vmballoon_shrinker_scan()
1529 	vmballoon_stats_gen_inc(b, VMW_BALLOON_STAT_SHRINK);  in vmballoon_shrinker_scan()
1535 	if (!down_read_trylock(&b->conf_sem))  in vmballoon_shrinker_scan()
1538 	deflated_frames = vmballoon_deflate(b, sc->nr_to_scan, true);  in vmballoon_shrinker_scan()
1540 	vmballoon_stats_gen_add(b, VMW_BALLOON_STAT_SHRINK_FREE,  in vmballoon_shrinker_scan()
1548 	WRITE_ONCE(b->shrink_timeout, jiffies + HZ * VMBALLOON_SHRINK_DELAY);  in vmballoon_shrinker_scan()
1550 	up_read(&b->conf_sem);  in vmballoon_shrinker_scan()
1566 	struct vmballoon *b = &balloon;  in vmballoon_shrinker_count()  local
1568 	return atomic64_read(&b->size);  in vmballoon_shrinker_count()
1571 static void vmballoon_unregister_shrinker(struct vmballoon *b)  in vmballoon_unregister_shrinker()  argument
1573 	if (b->shrinker_registered)  in vmballoon_unregister_shrinker()
1574 		unregister_shrinker(&b->shrinker);  in vmballoon_unregister_shrinker()
1575 	b->shrinker_registered = false;  in vmballoon_unregister_shrinker()
1578 static int vmballoon_register_shrinker(struct vmballoon *b)  in vmballoon_register_shrinker()  argument
1586 	b->shrinker.scan_objects = vmballoon_shrinker_scan;  in vmballoon_register_shrinker()
1587 	b->shrinker.count_objects = vmballoon_shrinker_count;  in vmballoon_register_shrinker()
1588 	b->shrinker.seeks = DEFAULT_SEEKS;  in vmballoon_register_shrinker()
1590 	r = register_shrinker(&b->shrinker);  in vmballoon_register_shrinker()
1593 		b->shrinker_registered = true;  in vmballoon_register_shrinker()
1619 static int vmballoon_enable_stats(struct vmballoon *b)  in vmballoon_enable_stats()  argument
1623 	down_write(&b->conf_sem);  in vmballoon_enable_stats()
1626 	if (b->stats)  in vmballoon_enable_stats()
1629 	b->stats = kzalloc(sizeof(*b->stats), GFP_KERNEL);  in vmballoon_enable_stats()
1631 	if (!b->stats) {  in vmballoon_enable_stats()
1638 	up_write(&b->conf_sem);  in vmballoon_enable_stats()
1655 	struct vmballoon *b = f->private;  in vmballoon_debug_show()  local
1659 	if (!b->stats) {  in vmballoon_debug_show()
1660 		int r = vmballoon_enable_stats(b);  in vmballoon_debug_show()
1669 	seq_printf(f, "%-22s: %#16lx\n", "used capabilities", b->capabilities);  in vmballoon_debug_show()
1671 		   b->reset_required ? "y" : "n");  in vmballoon_debug_show()
1674 	seq_printf(f, "%-22s: %16lu\n", "target", READ_ONCE(b->target));  in vmballoon_debug_show()
1675 	seq_printf(f, "%-22s: %16llu\n", "current", atomic64_read(&b->size));  in vmballoon_debug_show()
1683 			   atomic64_read(&b->stats->ops[i][VMW_BALLOON_OP_STAT]),  in vmballoon_debug_show()
1684 			   atomic64_read(&b->stats->ops[i][VMW_BALLOON_OP_FAIL_STAT]));  in vmballoon_debug_show()
1690 			   atomic64_read(&b->stats->general_stat[i]));  in vmballoon_debug_show()
1697 				   atomic64_read(&b->stats->page_stat[i][j]));  in vmballoon_debug_show()
1705 static void __init vmballoon_debugfs_init(struct vmballoon *b)  in vmballoon_debugfs_init()  argument
1707 	debugfs_create_file("vmmemctl", S_IRUGO, NULL, b,  in vmballoon_debugfs_init()
1711 static void __exit vmballoon_debugfs_exit(struct vmballoon *b)  in vmballoon_debugfs_exit()  argument
1715 	kfree(b->stats);  in vmballoon_debugfs_exit()
1716 	b->stats = NULL;  in vmballoon_debugfs_exit()
1721 static inline void vmballoon_debugfs_init(struct vmballoon *b)  in vmballoon_debugfs_init()  argument
1725 static inline void vmballoon_debugfs_exit(struct vmballoon *b)  in vmballoon_debugfs_exit()  argument
1766 	struct vmballoon *b;  in vmballoon_migratepage()  local
1769 	b = container_of(b_dev_info, struct vmballoon, b_dev_info);  in vmballoon_migratepage()
1775 	if (!down_read_trylock(&b->conf_sem))  in vmballoon_migratepage()
1778 	spin_lock(&b->comm_lock);  in vmballoon_migratepage()
1786 	vmballoon_add_page(b, 0, page);  in vmballoon_migratepage()
1787 	status = vmballoon_lock_op(b, 1, VMW_BALLOON_4K_PAGE,  in vmballoon_migratepage()
1791 		status = vmballoon_status_page(b, 0, &page);  in vmballoon_migratepage()
1798 		spin_unlock(&b->comm_lock);  in vmballoon_migratepage()
1813 	vmballoon_add_page(b, 0, newpage);  in vmballoon_migratepage()
1814 	status = vmballoon_lock_op(b, 1, VMW_BALLOON_4K_PAGE,  in vmballoon_migratepage()
1818 		status = vmballoon_status_page(b, 0, &newpage);  in vmballoon_migratepage()
1820 	spin_unlock(&b->comm_lock);  in vmballoon_migratepage()
1829 		atomic64_dec(&b->size);  in vmballoon_migratepage()
1841 	spin_lock_irqsave(&b->b_dev_info.pages_lock, flags);  in vmballoon_migratepage()
1849 		balloon_page_insert(&b->b_dev_info, newpage);  in vmballoon_migratepage()
1857 	b->b_dev_info.isolated_pages--;  in vmballoon_migratepage()
1858 	spin_unlock_irqrestore(&b->b_dev_info.pages_lock, flags);  in vmballoon_migratepage()
1861 	up_read(&b->conf_sem);  in vmballoon_migratepage()
1870 static void vmballoon_compaction_deinit(struct vmballoon *b)  in vmballoon_compaction_deinit()  argument
1872 	if (!IS_ERR(b->b_dev_info.inode))  in vmballoon_compaction_deinit()
1873 		iput(b->b_dev_info.inode);  in vmballoon_compaction_deinit()
1875 	b->b_dev_info.inode = NULL;  in vmballoon_compaction_deinit()
1891 static __init int vmballoon_compaction_init(struct vmballoon *b)  in vmballoon_compaction_init()  argument
1897 	b->b_dev_info.migratepage = vmballoon_migratepage;  in vmballoon_compaction_init()
1898 	b->b_dev_info.inode = alloc_anon_inode(vmballoon_mnt->mnt_sb);  in vmballoon_compaction_init()
1900 	if (IS_ERR(b->b_dev_info.inode))  in vmballoon_compaction_init()
1901 		return PTR_ERR(b->b_dev_info.inode);  in vmballoon_compaction_init()
1903 	b->b_dev_info.inode->i_mapping->a_ops = &balloon_aops;  in vmballoon_compaction_init()
1909 static void vmballoon_compaction_deinit(struct vmballoon *b)  in vmballoon_compaction_deinit()  argument
1913 static int vmballoon_compaction_init(struct vmballoon *b)  in vmballoon_compaction_init()  argument