53 #define block_start_pfn(pfn, order)	round_down(pfn, 1UL << (order))  argument
54 #define block_end_pfn(pfn, order)	ALIGN((pfn) + 1, 1UL << (order))  argument
87 	unsigned int i, order, nr_pages;  in split_map_pages()  local
94 		order = page_private(page);  in split_map_pages()
95 		nr_pages = 1 << order;  in split_map_pages()
97 		post_alloc_hook(page, order, __GFP_MOVABLE);  in split_map_pages()
98 		if (order)  in split_map_pages()
99 			split_page(page, order);  in split_map_pages()
154 static void defer_compaction(struct zone *zone, int order)  in defer_compaction()  argument
159 	if (order < zone->compact_order_failed)  in defer_compaction()
160 		zone->compact_order_failed = order;  in defer_compaction()
165 	trace_mm_compaction_defer_compaction(zone, order);  in defer_compaction()
169 static bool compaction_deferred(struct zone *zone, int order)  in compaction_deferred()  argument
173 	if (order < zone->compact_order_failed)  in compaction_deferred()
182 	trace_mm_compaction_deferred(zone, order);  in compaction_deferred()
192 void compaction_defer_reset(struct zone *zone, int order,  in compaction_defer_reset()  argument
199 	if (order >= zone->compact_order_failed)  in compaction_defer_reset()
200 		zone->compact_order_failed = order + 1;  in compaction_defer_reset()
202 	trace_mm_compaction_defer_reset(zone, order);  in compaction_defer_reset()
206 static bool compaction_restarting(struct zone *zone, int order)  in compaction_restarting()  argument
208 	if (order < zone->compact_order_failed)  in compaction_restarting()
553 	unsigned int order;  in isolate_freepages_block()  local
585 			const unsigned int order = compound_order(page);  in isolate_freepages_block()  local
587 			if (likely(order < MAX_ORDER)) {  in isolate_freepages_block()
588 				blockpfn += (1UL << order) - 1;  in isolate_freepages_block()
589 				cursor += (1UL << order) - 1;  in isolate_freepages_block()
608 		order = buddy_order(page);  in isolate_freepages_block()
609 		isolated = __isolate_free_page(page, order);  in isolate_freepages_block()
612 		set_page_private(page, order);  in isolate_freepages_block()
828 		next_skip_pfn = block_end_pfn(low_pfn, cc->order);  in isolate_migratepages_block()
853 			next_skip_pfn = block_end_pfn(low_pfn, cc->order);  in isolate_migratepages_block()
956 			const unsigned int order = compound_order(page);  in isolate_migratepages_block()  local
958 			if (likely(order < MAX_ORDER))  in isolate_migratepages_block()
959 				low_pfn += (1UL << order) - 1;  in isolate_migratepages_block()
1150 			next_skip_pfn += 1UL << cc->order;  in isolate_migratepages_block()
1378 static int next_search_order(struct compact_control *cc, int order)  in next_search_order()  argument
1380 	order--;  in next_search_order()
1381 	if (order < 0)  in next_search_order()
1382 		order = cc->order - 1;  in next_search_order()
1385 	if (order == cc->search_order) {  in next_search_order()
1388 			cc->search_order = cc->order - 1;  in next_search_order()
1392 	return order;  in next_search_order()
1405 	int order;  in fast_isolate_freepages()  local
1408 	if (cc->order <= 0)  in fast_isolate_freepages()
1435 	cc->search_order = min_t(unsigned int, cc->order - 1, cc->search_order);  in fast_isolate_freepages()
1437 	for (order = cc->search_order;  in fast_isolate_freepages()
1438 	     !page && order >= 0;  in fast_isolate_freepages()
1439 	     order = next_search_order(cc, order)) {  in fast_isolate_freepages()
1440 		struct free_area *area = &cc->zone->free_area[order];  in fast_isolate_freepages()
1465 				cc->search_order = order;  in fast_isolate_freepages()
1494 			if (__isolate_free_page(page, order)) {  in fast_isolate_freepages()
1495 				set_page_private(page, order);  in fast_isolate_freepages()
1496 				nr_isolated = 1 << order;  in fast_isolate_freepages()
1503 				order = cc->search_order + 1;  in fast_isolate_freepages()
1758 	int order;  in fast_find_migrateblock()  local
1778 	if (cc->order <= PAGE_ALLOC_COSTLY_ORDER)  in fast_find_migrateblock()
1801 	for (order = cc->order - 1;  in fast_find_migrateblock()
1802 	     order >= PAGE_ALLOC_COSTLY_ORDER && !found_block && nr_scanned < limit;  in fast_find_migrateblock()
1803 	     order--) {  in fast_find_migrateblock()
1804 		struct free_area *area = &cc->zone->free_area[order];  in fast_find_migrateblock()
1965 static inline bool is_via_compact_memory(int order)  in is_via_compact_memory()  argument
1967 	return order == -1;  in is_via_compact_memory()
2062 	unsigned int order;  in __compact_finished()  local
2105 	if (is_via_compact_memory(cc->order))  in __compact_finished()
2119 	for (order = cc->order; order < MAX_ORDER; order++) {  in __compact_finished()
2120 		struct free_area *area = &cc->zone->free_area[order];  in __compact_finished()
2137 		if (find_suitable_fallback(area, order, migratetype,  in __compact_finished()
2162 	trace_mm_compaction_finished(cc->zone, cc->order, ret);  in compact_finished()
2169 static enum compact_result __compaction_suitable(struct zone *zone, int order,  in __compaction_suitable()  argument
2176 	if (is_via_compact_memory(order))  in __compaction_suitable()
2184 	if (zone_watermark_ok(zone, order, watermark, highest_zoneidx,  in __compaction_suitable()
2202 	watermark = (order > PAGE_ALLOC_COSTLY_ORDER) ?  in __compaction_suitable()
2204 	watermark += compact_gap(order);  in __compaction_suitable()
2219 enum compact_result compaction_suitable(struct zone *zone, int order,  in compaction_suitable()  argument
2226 	ret = __compaction_suitable(zone, order, alloc_flags, highest_zoneidx,  in compaction_suitable()
2244 	if (ret == COMPACT_CONTINUE && (order > PAGE_ALLOC_COSTLY_ORDER)) {  in compaction_suitable()
2245 		fragindex = fragmentation_index(zone, order);  in compaction_suitable()
2250 	trace_mm_compaction_suitable(zone, order, ret);  in compaction_suitable()
2257 bool compaction_zonelist_suitable(struct alloc_context *ac, int order,  in compaction_zonelist_suitable()  argument
2278 		available = zone_reclaimable_pages(zone) / order;  in compaction_zonelist_suitable()
2280 		compact_result = __compaction_suitable(zone, order, alloc_flags,  in compaction_zonelist_suitable()
2312 	ret = compaction_suitable(cc->zone, cc->order, cc->alloc_flags,  in compact_zone()
2325 	if (compaction_restarting(cc->zone, cc->order))  in compact_zone()
2439 						cc->migrate_pfn - 1, cc->order);  in compact_zone()
2453 		if (cc->order > 0 && last_migrated_pfn) {  in compact_zone()
2455 				block_start_pfn(cc->migrate_pfn, cc->order);  in compact_zone()
2499 static enum compact_result compact_zone_order(struct zone *zone, int order,  in compact_zone_order()  argument
2506 		.order = order,  in compact_zone_order()
2507 		.search_order = order,  in compact_zone_order()
2569 enum compact_result try_to_compact_pages(gfp_t gfp_mask, unsigned int order,  in try_to_compact_pages()  argument
2585 	trace_mm_compaction_try_to_compact_pages(order, gfp_mask, prio);  in try_to_compact_pages()
2593 					&& compaction_deferred(zone, order)) {  in try_to_compact_pages()
2598 		status = compact_zone_order(zone, order, gfp_mask, prio,  in try_to_compact_pages()
2610 			compaction_defer_reset(zone, order, false);  in try_to_compact_pages()
2622 			defer_compaction(zone, order);  in try_to_compact_pages()
2651 		.order = -1,  in proactive_compact_node()
2680 		.order = -1,  in compact_node()
2823 		.order = pgdat->kcompactd_max_order,  in kcompactd_do_work()
2830 	trace_mm_compaction_kcompactd_wake(pgdat->node_id, cc.order,  in kcompactd_do_work()
2841 		if (compaction_deferred(zone, cc.order))  in kcompactd_do_work()
2844 		if (compaction_suitable(zone, cc.order, 0, zoneid) !=  in kcompactd_do_work()
2855 			compaction_defer_reset(zone, cc.order, false);  in kcompactd_do_work()
2869 			defer_compaction(zone, cc.order);  in kcompactd_do_work()
2886 	if (pgdat->kcompactd_max_order <= cc.order)  in kcompactd_do_work()
2892 void wakeup_kcompactd(pg_data_t *pgdat, int order, int highest_zoneidx)  in wakeup_kcompactd()  argument
2894 	if (!order)  in wakeup_kcompactd()
2897 	if (pgdat->kcompactd_max_order < order)  in wakeup_kcompactd()
2898 		pgdat->kcompactd_max_order = order;  in wakeup_kcompactd()
2913 	trace_mm_compaction_wakeup_kcompactd(pgdat->node_id, order,  in wakeup_kcompactd()