203 			   struct hns_roce_hem_table *table, unsigned long *obj,  in hns_roce_calc_hem_mhop()  argument
212 	if (get_hem_table_config(hr_dev, mhop, table->type))  in hns_roce_calc_hem_mhop()
222 	bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);  in hns_roce_calc_hem_mhop()
224 	chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size :  in hns_roce_calc_hem_mhop()
226 	table_idx = *obj / (chunk_size / table->obj_size);  in hns_roce_calc_hem_mhop()
242 			table->type, mhop->hop_num);  in hns_roce_calc_hem_mhop()
336 			   struct hns_roce_hem_table *table, unsigned long obj,  in calc_hem_config()  argument
347 	ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, mhop);  in calc_hem_config()
355 	bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);  in calc_hem_config()
372 			  table->type, mhop->hop_num);  in calc_hem_config()
376 	if (unlikely(index->buf >= table->num_hem)) {  in calc_hem_config()
378 			  table->type, index->buf, table->num_hem);  in calc_hem_config()
386 			  struct hns_roce_hem_table *table,  in free_mhop_hem()  argument
394 		hns_roce_free_hem(hr_dev, table->hem[index->buf]);  in free_mhop_hem()
395 		table->hem[index->buf] = NULL;  in free_mhop_hem()
399 		dma_free_coherent(dev, bt_size, table->bt_l1[index->l1],  in free_mhop_hem()
400 				  table->bt_l1_dma_addr[index->l1]);  in free_mhop_hem()
401 		table->bt_l1[index->l1] = NULL;  in free_mhop_hem()
405 		dma_free_coherent(dev, bt_size, table->bt_l0[index->l0],  in free_mhop_hem()
406 				  table->bt_l0_dma_addr[index->l0]);  in free_mhop_hem()
407 		table->bt_l0[index->l0] = NULL;  in free_mhop_hem()
412 			  struct hns_roce_hem_table *table,  in alloc_mhop_hem()  argument
425 	if ((check_whether_bt_num_3(table->type, mhop->hop_num) ||  in alloc_mhop_hem()
426 	     check_whether_bt_num_2(table->type, mhop->hop_num)) &&  in alloc_mhop_hem()
427 	     !table->bt_l0[index->l0]) {  in alloc_mhop_hem()
428 		table->bt_l0[index->l0] = dma_alloc_coherent(dev, bt_size,  in alloc_mhop_hem()
429 					    &table->bt_l0_dma_addr[index->l0],  in alloc_mhop_hem()
431 		if (!table->bt_l0[index->l0]) {  in alloc_mhop_hem()
439 	if (check_whether_bt_num_3(table->type, mhop->hop_num) &&  in alloc_mhop_hem()
440 	    !table->bt_l1[index->l1])  {  in alloc_mhop_hem()
441 		table->bt_l1[index->l1] = dma_alloc_coherent(dev, bt_size,  in alloc_mhop_hem()
442 					    &table->bt_l1_dma_addr[index->l1],  in alloc_mhop_hem()
444 		if (!table->bt_l1[index->l1]) {  in alloc_mhop_hem()
449 		*(table->bt_l0[index->l0] + mhop->l1_idx) =  in alloc_mhop_hem()
450 					       table->bt_l1_dma_addr[index->l1];  in alloc_mhop_hem()
457 	size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : bt_size;  in alloc_mhop_hem()
459 	table->hem[index->buf] = hns_roce_alloc_hem(hr_dev, size >> PAGE_SHIFT,  in alloc_mhop_hem()
461 	if (!table->hem[index->buf]) {  in alloc_mhop_hem()
467 	hns_roce_hem_first(table->hem[index->buf], &iter);  in alloc_mhop_hem()
469 	if (table->type < HEM_TYPE_MTT) {  in alloc_mhop_hem()
471 			*(table->bt_l1[index->l1] + mhop->l2_idx) = bt_ba;  in alloc_mhop_hem()
473 			*(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba;  in alloc_mhop_hem()
475 		*(table->bt_l0[index->l0] + mhop->l1_idx) = bt_ba;  in alloc_mhop_hem()
480 	free_mhop_hem(hr_dev, table, mhop, index);  in alloc_mhop_hem()
486 			struct hns_roce_hem_table *table, unsigned long obj,  in set_mhop_hem()  argument
495 		ret = hr_dev->hw->set_hem(hr_dev, table, obj, 0);  in set_mhop_hem()
503 		ret = hr_dev->hw->set_hem(hr_dev, table, obj, 1);  in set_mhop_hem()
515 		ret = hr_dev->hw->set_hem(hr_dev, table, obj, step_idx);  in set_mhop_hem()
524 				   struct hns_roce_hem_table *table,  in hns_roce_table_mhop_get()  argument
532 	ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);  in hns_roce_table_mhop_get()
538 	mutex_lock(&table->mutex);  in hns_roce_table_mhop_get()
539 	if (table->hem[index.buf]) {  in hns_roce_table_mhop_get()
540 		refcount_inc(&table->hem[index.buf]->refcount);  in hns_roce_table_mhop_get()
544 	ret = alloc_mhop_hem(hr_dev, table, &mhop, &index);  in hns_roce_table_mhop_get()
551 	if (table->type < HEM_TYPE_MTT) {  in hns_roce_table_mhop_get()
552 		ret = set_mhop_hem(hr_dev, table, obj, &mhop, &index);  in hns_roce_table_mhop_get()
559 	refcount_set(&table->hem[index.buf]->refcount, 1);  in hns_roce_table_mhop_get()
563 	free_mhop_hem(hr_dev, table, &mhop, &index);  in hns_roce_table_mhop_get()
565 	mutex_unlock(&table->mutex);  in hns_roce_table_mhop_get()
570 		       struct hns_roce_hem_table *table, unsigned long obj)  in hns_roce_table_get()  argument
576 	if (hns_roce_check_whether_mhop(hr_dev, table->type))  in hns_roce_table_get()
577 		return hns_roce_table_mhop_get(hr_dev, table, obj);  in hns_roce_table_get()
579 	i = obj / (table->table_chunk_size / table->obj_size);  in hns_roce_table_get()
581 	mutex_lock(&table->mutex);  in hns_roce_table_get()
583 	if (table->hem[i]) {  in hns_roce_table_get()
584 		refcount_inc(&table->hem[i]->refcount);  in hns_roce_table_get()
588 	table->hem[i] = hns_roce_alloc_hem(hr_dev,  in hns_roce_table_get()
589 				       table->table_chunk_size >> PAGE_SHIFT,  in hns_roce_table_get()
590 				       table->table_chunk_size,  in hns_roce_table_get()
592 	if (!table->hem[i]) {  in hns_roce_table_get()
598 	ret = hr_dev->hw->set_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT);  in hns_roce_table_get()
600 		hns_roce_free_hem(hr_dev, table->hem[i]);  in hns_roce_table_get()
601 		table->hem[i] = NULL;  in hns_roce_table_get()
607 	refcount_set(&table->hem[i]->refcount, 1);  in hns_roce_table_get()
609 	mutex_unlock(&table->mutex);  in hns_roce_table_get()
614 			   struct hns_roce_hem_table *table, unsigned long obj,  in clear_mhop_hem()  argument
626 	if (check_whether_bt_num_2(table->type, hop_num)) {  in clear_mhop_hem()
627 		if (hns_roce_check_hem_null(table->hem, index->buf,  in clear_mhop_hem()
628 					    chunk_ba_num, table->num_hem))  in clear_mhop_hem()
630 	} else if (check_whether_bt_num_3(table->type, hop_num)) {  in clear_mhop_hem()
631 		if (hns_roce_check_hem_null(table->hem, index->buf,  in clear_mhop_hem()
632 					    chunk_ba_num, table->num_hem)) {  in clear_mhop_hem()
634 			if (hns_roce_check_bt_null(table->bt_l1, index->l1,  in clear_mhop_hem()
640 	if (table->type < HEM_TYPE_MTT) {  in clear_mhop_hem()
646 		ret = hr_dev->hw->clear_hem(hr_dev, table, obj, step_idx);  in clear_mhop_hem()
652 			ret = hr_dev->hw->clear_hem(hr_dev, table, obj, 1);  in clear_mhop_hem()
659 			ret = hr_dev->hw->clear_hem(hr_dev, table, obj, 0);  in clear_mhop_hem()
668 				    struct hns_roce_hem_table *table,  in hns_roce_table_mhop_put()  argument
677 	ret = calc_hem_config(hr_dev, table, obj, &mhop, &index);  in hns_roce_table_mhop_put()
684 		mutex_lock(&table->mutex);  in hns_roce_table_mhop_put()
685 	else if (!refcount_dec_and_mutex_lock(&table->hem[index.buf]->refcount,  in hns_roce_table_mhop_put()
686 					      &table->mutex))  in hns_roce_table_mhop_put()
689 	clear_mhop_hem(hr_dev, table, obj, &mhop, &index);  in hns_roce_table_mhop_put()
690 	free_mhop_hem(hr_dev, table, &mhop, &index);  in hns_roce_table_mhop_put()
692 	mutex_unlock(&table->mutex);  in hns_roce_table_mhop_put()
696 			struct hns_roce_hem_table *table, unsigned long obj)  in hns_roce_table_put()  argument
702 	if (hns_roce_check_whether_mhop(hr_dev, table->type)) {  in hns_roce_table_put()
703 		hns_roce_table_mhop_put(hr_dev, table, obj, 1);  in hns_roce_table_put()
707 	i = obj / (table->table_chunk_size / table->obj_size);  in hns_roce_table_put()
709 	if (!refcount_dec_and_mutex_lock(&table->hem[i]->refcount,  in hns_roce_table_put()
710 					 &table->mutex))  in hns_roce_table_put()
713 	ret = hr_dev->hw->clear_hem(hr_dev, table, obj, HEM_HOP_STEP_DIRECT);  in hns_roce_table_put()
718 	hns_roce_free_hem(hr_dev, table->hem[i]);  in hns_roce_table_put()
719 	table->hem[i] = NULL;  in hns_roce_table_put()
721 	mutex_unlock(&table->mutex);  in hns_roce_table_put()
725 			  struct hns_roce_hem_table *table,  in hns_roce_table_find()  argument
740 	mutex_lock(&table->mutex);  in hns_roce_table_find()
742 	if (!hns_roce_check_whether_mhop(hr_dev, table->type)) {  in hns_roce_table_find()
743 		obj_per_chunk = table->table_chunk_size / table->obj_size;  in hns_roce_table_find()
744 		hem = table->hem[obj / obj_per_chunk];  in hns_roce_table_find()
746 		dma_offset = offset = idx_offset * table->obj_size;  in hns_roce_table_find()
750 		if (hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop))  in hns_roce_table_find()
761 		hem = table->hem[hem_idx];  in hns_roce_table_find()
789 	mutex_unlock(&table->mutex);  in hns_roce_table_find()
794 			    struct hns_roce_hem_table *table, u32 type,  in hns_roce_init_hem_table()  argument
801 		table->table_chunk_size = hr_dev->caps.chunk_sz;  in hns_roce_init_hem_table()
802 		obj_per_chunk = table->table_chunk_size / obj_size;  in hns_roce_init_hem_table()
805 		table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL);  in hns_roce_init_hem_table()
806 		if (!table->hem)  in hns_roce_init_hem_table()
831 		table->hem = kcalloc(num_hem, sizeof(*table->hem),  in hns_roce_init_hem_table()
833 		if (!table->hem)  in hns_roce_init_hem_table()
840 			table->bt_l1 = kcalloc(num_bt_l1,  in hns_roce_init_hem_table()
841 					       sizeof(*table->bt_l1),  in hns_roce_init_hem_table()
843 			if (!table->bt_l1)  in hns_roce_init_hem_table()
846 			table->bt_l1_dma_addr = kcalloc(num_bt_l1,  in hns_roce_init_hem_table()
847 						 sizeof(*table->bt_l1_dma_addr),  in hns_roce_init_hem_table()
850 			if (!table->bt_l1_dma_addr)  in hns_roce_init_hem_table()
856 			table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0),  in hns_roce_init_hem_table()
858 			if (!table->bt_l0)  in hns_roce_init_hem_table()
861 			table->bt_l0_dma_addr = kcalloc(num_bt_l0,  in hns_roce_init_hem_table()
862 						 sizeof(*table->bt_l0_dma_addr),  in hns_roce_init_hem_table()
864 			if (!table->bt_l0_dma_addr)  in hns_roce_init_hem_table()
869 	table->type = type;  in hns_roce_init_hem_table()
870 	table->num_hem = num_hem;  in hns_roce_init_hem_table()
871 	table->obj_size = obj_size;  in hns_roce_init_hem_table()
872 	mutex_init(&table->mutex);  in hns_roce_init_hem_table()
877 	kfree(table->bt_l0);  in hns_roce_init_hem_table()
878 	table->bt_l0 = NULL;  in hns_roce_init_hem_table()
881 	kfree(table->bt_l1_dma_addr);  in hns_roce_init_hem_table()
882 	table->bt_l1_dma_addr = NULL;  in hns_roce_init_hem_table()
885 	kfree(table->bt_l1);  in hns_roce_init_hem_table()
886 	table->bt_l1 = NULL;  in hns_roce_init_hem_table()
889 	kfree(table->hem);  in hns_roce_init_hem_table()
890 	table->hem = NULL;  in hns_roce_init_hem_table()
897 					    struct hns_roce_hem_table *table)  in hns_roce_cleanup_mhop_hem_table()  argument
904 	if (hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop))  in hns_roce_cleanup_mhop_hem_table()
906 	buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size :  in hns_roce_cleanup_mhop_hem_table()
909 	for (i = 0; i < table->num_hem; ++i) {  in hns_roce_cleanup_mhop_hem_table()
910 		obj = i * buf_chunk_size / table->obj_size;  in hns_roce_cleanup_mhop_hem_table()
911 		if (table->hem[i])  in hns_roce_cleanup_mhop_hem_table()
912 			hns_roce_table_mhop_put(hr_dev, table, obj, 0);  in hns_roce_cleanup_mhop_hem_table()
915 	kfree(table->hem);  in hns_roce_cleanup_mhop_hem_table()
916 	table->hem = NULL;  in hns_roce_cleanup_mhop_hem_table()
917 	kfree(table->bt_l1);  in hns_roce_cleanup_mhop_hem_table()
918 	table->bt_l1 = NULL;  in hns_roce_cleanup_mhop_hem_table()
919 	kfree(table->bt_l1_dma_addr);  in hns_roce_cleanup_mhop_hem_table()
920 	table->bt_l1_dma_addr = NULL;  in hns_roce_cleanup_mhop_hem_table()
921 	kfree(table->bt_l0);  in hns_roce_cleanup_mhop_hem_table()
922 	table->bt_l0 = NULL;  in hns_roce_cleanup_mhop_hem_table()
923 	kfree(table->bt_l0_dma_addr);  in hns_roce_cleanup_mhop_hem_table()
924 	table->bt_l0_dma_addr = NULL;  in hns_roce_cleanup_mhop_hem_table()
928 				struct hns_roce_hem_table *table)  in hns_roce_cleanup_hem_table()  argument
935 	if (hns_roce_check_whether_mhop(hr_dev, table->type)) {  in hns_roce_cleanup_hem_table()
936 		hns_roce_cleanup_mhop_hem_table(hr_dev, table);  in hns_roce_cleanup_hem_table()
940 	for (i = 0; i < table->num_hem; ++i)  in hns_roce_cleanup_hem_table()
941 		if (table->hem[i]) {  in hns_roce_cleanup_hem_table()
942 			obj = i * table->table_chunk_size / table->obj_size;  in hns_roce_cleanup_hem_table()
943 			ret = hr_dev->hw->clear_hem(hr_dev, table, obj, 0);  in hns_roce_cleanup_hem_table()
948 			hns_roce_free_hem(hr_dev, table->hem[i]);  in hns_roce_cleanup_hem_table()
951 	kfree(table->hem);  in hns_roce_cleanup_hem_table()
958 					   &hr_dev->srq_table.table);  in hns_roce_cleanup_hem()
959 	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);  in hns_roce_cleanup_hem()