218 	struct ntb_transport_ctx *nt;  member
272 #define QP_TO_MW(nt, qp)	((qp) % nt->mw_count)  argument
326 static int ntb_bus_init(struct ntb_transport_ctx *nt)  in ntb_bus_init()  argument
328 	list_add_tail(&nt->entry, &ntb_transport_list);  in ntb_bus_init()
332 static void ntb_bus_remove(struct ntb_transport_ctx *nt)  in ntb_bus_remove()  argument
336 	list_for_each_entry_safe(client_dev, cd, &nt->client_devs, entry) {  in ntb_bus_remove()
343 	list_del(&nt->entry);  in ntb_bus_remove()
363 	struct ntb_transport_ctx *nt;  in ntb_transport_unregister_client_dev()  local
365 	list_for_each_entry(nt, &ntb_transport_list, entry)  in ntb_transport_unregister_client_dev()
366 		list_for_each_entry_safe(client, cd, &nt->client_devs, entry)  in ntb_transport_unregister_client_dev()
384 	struct ntb_transport_ctx *nt;  in ntb_transport_register_client_dev()  local
391 	list_for_each_entry(nt, &ntb_transport_list, entry) {  in ntb_transport_register_client_dev()
394 		node = dev_to_node(&nt->ndev->dev);  in ntb_transport_register_client_dev()
409 		dev->parent = &nt->ndev->dev;  in ntb_transport_register_client_dev()
417 		list_add_tail(&client_dev->entry, &nt->client_devs);  in ntb_transport_register_client_dev()
613 static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt,  in ntb_transport_setup_qp_mw()  argument
616 	struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];  in ntb_transport_setup_qp_mw()
618 	struct ntb_dev *ndev = nt->ndev;  in ntb_transport_setup_qp_mw()
625 	mw_count = nt->mw_count;  in ntb_transport_setup_qp_mw()
626 	qp_count = nt->qp_count;  in ntb_transport_setup_qp_mw()
628 	mw_num = QP_TO_MW(nt, qp_num);  in ntb_transport_setup_qp_mw()
629 	mw = &nt->mw_vec[mw_num];  in ntb_transport_setup_qp_mw()
692 static void ntb_transport_setup_qp_peer_msi(struct ntb_transport_ctx *nt,  in ntb_transport_setup_qp_peer_msi()  argument
695 	struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];  in ntb_transport_setup_qp_peer_msi()
696 	int spad = qp_num * 2 + nt->msi_spad_offset;  in ntb_transport_setup_qp_peer_msi()
698 	if (!nt->use_msi)  in ntb_transport_setup_qp_peer_msi()
701 	if (spad >= ntb_spad_count(nt->ndev))  in ntb_transport_setup_qp_peer_msi()
719 static void ntb_transport_setup_qp_msi(struct ntb_transport_ctx *nt,  in ntb_transport_setup_qp_msi()  argument
722 	struct ntb_transport_qp *qp = &nt->qp_vec[qp_num];  in ntb_transport_setup_qp_msi()
723 	int spad = qp_num * 2 + nt->msi_spad_offset;  in ntb_transport_setup_qp_msi()
726 	if (!nt->use_msi)  in ntb_transport_setup_qp_msi()
729 	if (spad >= ntb_spad_count(nt->ndev)) {  in ntb_transport_setup_qp_msi()
765 	devm_free_irq(&nt->ndev->dev, qp->msi_irq, qp);  in ntb_transport_setup_qp_msi()
768 static void ntb_transport_msi_peer_desc_changed(struct ntb_transport_ctx *nt)  in ntb_transport_msi_peer_desc_changed()  argument
772 	dev_dbg(&nt->ndev->pdev->dev, "Peer MSI descriptors changed");  in ntb_transport_msi_peer_desc_changed()
774 	for (i = 0; i < nt->qp_count; i++)  in ntb_transport_msi_peer_desc_changed()
775 		ntb_transport_setup_qp_peer_msi(nt, i);  in ntb_transport_msi_peer_desc_changed()
780 	struct ntb_transport_ctx *nt = data;  in ntb_transport_msi_desc_changed()  local
783 	dev_dbg(&nt->ndev->pdev->dev, "MSI descriptors changed");  in ntb_transport_msi_desc_changed()
785 	for (i = 0; i < nt->qp_count; i++)  in ntb_transport_msi_desc_changed()
786 		ntb_transport_setup_qp_msi(nt, i);  in ntb_transport_msi_desc_changed()
788 	ntb_peer_db_set(nt->ndev, nt->msi_db_mask);  in ntb_transport_msi_desc_changed()
791 static void ntb_free_mw(struct ntb_transport_ctx *nt, int num_mw)  in ntb_free_mw()  argument
793 	struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];  in ntb_free_mw()
794 	struct pci_dev *pdev = nt->ndev->pdev;  in ntb_free_mw()
799 	ntb_mw_clear_trans(nt->ndev, PIDX, num_mw);  in ntb_free_mw()
853 static int ntb_set_mw(struct ntb_transport_ctx *nt, int num_mw,  in ntb_set_mw()  argument
856 	struct ntb_transport_mw *mw = &nt->mw_vec[num_mw];  in ntb_set_mw()
857 	struct pci_dev *pdev = nt->ndev->pdev;  in ntb_set_mw()
866 	rc = ntb_mw_get_align(nt->ndev, PIDX, num_mw, &xlat_align,  in ntb_set_mw()
879 		ntb_free_mw(nt, num_mw);  in ntb_set_mw()
901 	rc = ntb_mw_set_trans(nt->ndev, PIDX, num_mw, mw->dma_addr,  in ntb_set_mw()
905 		ntb_free_mw(nt, num_mw);  in ntb_set_mw()
937 	struct ntb_transport_ctx *nt = qp->transport;  in ntb_qp_link_cleanup()  local
938 	struct pci_dev *pdev = nt->ndev->pdev;  in ntb_qp_link_cleanup()
954 	struct ntb_transport_ctx *nt = qp->transport;  in ntb_qp_link_cleanup_work()  local
958 	if (nt->link_is_up)  in ntb_qp_link_cleanup_work()
968 static void ntb_transport_link_cleanup(struct ntb_transport_ctx *nt)  in ntb_transport_link_cleanup()  argument
974 	qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free;  in ntb_transport_link_cleanup()
977 	for (i = 0; i < nt->qp_count; i++)  in ntb_transport_link_cleanup()
979 			qp = &nt->qp_vec[i];  in ntb_transport_link_cleanup()
985 	if (!nt->link_is_up)  in ntb_transport_link_cleanup()
986 		cancel_delayed_work_sync(&nt->link_work);  in ntb_transport_link_cleanup()
988 	for (i = 0; i < nt->mw_count; i++)  in ntb_transport_link_cleanup()
989 		ntb_free_mw(nt, i);  in ntb_transport_link_cleanup()
995 	count = ntb_spad_count(nt->ndev);  in ntb_transport_link_cleanup()
997 		ntb_spad_write(nt->ndev, i, 0);  in ntb_transport_link_cleanup()
1002 	struct ntb_transport_ctx *nt =  in ntb_transport_link_cleanup_work()  local
1005 	ntb_transport_link_cleanup(nt);  in ntb_transport_link_cleanup_work()
1010 	struct ntb_transport_ctx *nt = data;  in ntb_transport_event_callback()  local
1012 	if (ntb_link_is_up(nt->ndev, NULL, NULL) == 1)  in ntb_transport_event_callback()
1013 		schedule_delayed_work(&nt->link_work, 0);  in ntb_transport_event_callback()
1015 		schedule_work(&nt->link_cleanup);  in ntb_transport_event_callback()
1020 	struct ntb_transport_ctx *nt =  in ntb_transport_link_work()  local
1022 	struct ntb_dev *ndev = nt->ndev;  in ntb_transport_link_work()
1030 	if (nt->use_msi) {  in ntb_transport_link_work()
1036 			nt->use_msi = false;  in ntb_transport_link_work()
1040 	for (i = 0; i < nt->qp_count; i++)  in ntb_transport_link_work()
1041 		ntb_transport_setup_qp_msi(nt, i);  in ntb_transport_link_work()
1043 	for (i = 0; i < nt->mw_count; i++) {  in ntb_transport_link_work()
1044 		size = nt->mw_vec[i].phys_size;  in ntb_transport_link_work()
1056 	ntb_peer_spad_write(ndev, PIDX, NUM_MWS, nt->mw_count);  in ntb_transport_link_work()
1058 	ntb_peer_spad_write(ndev, PIDX, NUM_QPS, nt->qp_count);  in ntb_transport_link_work()
1070 	if (val != nt->qp_count)  in ntb_transport_link_work()
1075 	if (val != nt->mw_count)  in ntb_transport_link_work()
1078 	for (i = 0; i < nt->mw_count; i++) {  in ntb_transport_link_work()
1089 		rc = ntb_set_mw(nt, i, val64);  in ntb_transport_link_work()
1094 	nt->link_is_up = true;  in ntb_transport_link_work()
1096 	for (i = 0; i < nt->qp_count; i++) {  in ntb_transport_link_work()
1097 		struct ntb_transport_qp *qp = &nt->qp_vec[i];  in ntb_transport_link_work()
1099 		ntb_transport_setup_qp_mw(nt, i);  in ntb_transport_link_work()
1100 		ntb_transport_setup_qp_peer_msi(nt, i);  in ntb_transport_link_work()
1109 	for (i = 0; i < nt->mw_count; i++)  in ntb_transport_link_work()
1110 		ntb_free_mw(nt, i);  in ntb_transport_link_work()
1118 		schedule_delayed_work(&nt->link_work,  in ntb_transport_link_work()
1128 	struct ntb_transport_ctx *nt = qp->transport;  in ntb_qp_link_work()  local
1131 	WARN_ON(!nt->link_is_up);  in ntb_qp_link_work()
1133 	val = ntb_spad_read(nt->ndev, QP_LINKS);  in ntb_qp_link_work()
1135 	ntb_peer_spad_write(nt->ndev, PIDX, QP_LINKS, val | BIT(qp->qp_num));  in ntb_qp_link_work()
1151 	} else if (nt->link_is_up)  in ntb_qp_link_work()
1156 static int ntb_transport_init_queue(struct ntb_transport_ctx *nt,  in ntb_transport_init_queue()  argument
1166 	mw_count = nt->mw_count;  in ntb_transport_init_queue()
1167 	qp_count = nt->qp_count;  in ntb_transport_init_queue()
1169 	mw_num = QP_TO_MW(nt, qp_num);  in ntb_transport_init_queue()
1171 	qp = &nt->qp_vec[qp_num];  in ntb_transport_init_queue()
1173 	qp->transport = nt;  in ntb_transport_init_queue()
1174 	qp->ndev = nt->ndev;  in ntb_transport_init_queue()
1184 	mw_base = nt->mw_vec[mw_num].phys_addr;  in ntb_transport_init_queue()
1185 	mw_size = nt->mw_vec[mw_num].phys_size;  in ntb_transport_init_queue()
1194 	qp->tx_mw = nt->mw_vec[mw_num].vbase + qp_offset;  in ntb_transport_init_queue()
1209 	if (nt->debugfs_node_dir) {  in ntb_transport_init_queue()
1214 						     nt->debugfs_node_dir);  in ntb_transport_init_queue()
1243 	struct ntb_transport_ctx *nt;  in ntb_transport_probe()  local
1269 	nt = kzalloc_node(sizeof(*nt), GFP_KERNEL, node);  in ntb_transport_probe()
1270 	if (!nt)  in ntb_transport_probe()
1273 	nt->ndev = ndev;  in ntb_transport_probe()
1283 			nt->use_msi = true;  in ntb_transport_probe()
1292 		nt->mw_count = 0;  in ntb_transport_probe()
1298 	nt->mw_count = min(mw_count, max_mw_count_for_spads);  in ntb_transport_probe()
1300 	nt->msi_spad_offset = nt->mw_count * 2 + MW0_SZ_HIGH;  in ntb_transport_probe()
1302 	nt->mw_vec = kcalloc_node(mw_count, sizeof(*nt->mw_vec),  in ntb_transport_probe()
1304 	if (!nt->mw_vec) {  in ntb_transport_probe()
1310 		mw = &nt->mw_vec[i];  in ntb_transport_probe()
1332 	if (nt->use_msi) {  in ntb_transport_probe()
1334 		nt->msi_db_mask = 1 << qp_count;  in ntb_transport_probe()
1335 		ntb_db_clear_mask(ndev, nt->msi_db_mask);  in ntb_transport_probe()
1340 	else if (nt->mw_count < qp_count)  in ntb_transport_probe()
1341 		qp_count = nt->mw_count;  in ntb_transport_probe()
1345 	nt->qp_count = qp_count;  in ntb_transport_probe()
1346 	nt->qp_bitmap = qp_bitmap;  in ntb_transport_probe()
1347 	nt->qp_bitmap_free = qp_bitmap;  in ntb_transport_probe()
1349 	nt->qp_vec = kcalloc_node(qp_count, sizeof(*nt->qp_vec),  in ntb_transport_probe()
1351 	if (!nt->qp_vec) {  in ntb_transport_probe()
1357 		nt->debugfs_node_dir =  in ntb_transport_probe()
1363 		rc = ntb_transport_init_queue(nt, i);  in ntb_transport_probe()
1368 	INIT_DELAYED_WORK(&nt->link_work, ntb_transport_link_work);  in ntb_transport_probe()
1369 	INIT_WORK(&nt->link_cleanup, ntb_transport_link_cleanup_work);  in ntb_transport_probe()
1371 	rc = ntb_set_ctx(ndev, nt, &ntb_transport_ops);  in ntb_transport_probe()
1375 	INIT_LIST_HEAD(&nt->client_devs);  in ntb_transport_probe()
1376 	rc = ntb_bus_init(nt);  in ntb_transport_probe()
1380 	nt->link_is_up = false;  in ntb_transport_probe()
1389 	kfree(nt->qp_vec);  in ntb_transport_probe()
1392 		mw = &nt->mw_vec[i];  in ntb_transport_probe()
1395 	kfree(nt->mw_vec);  in ntb_transport_probe()
1397 	kfree(nt);  in ntb_transport_probe()
1403 	struct ntb_transport_ctx *nt = ndev->ctx;  in ntb_transport_free()  local
1408 	ntb_transport_link_cleanup(nt);  in ntb_transport_free()
1409 	cancel_work_sync(&nt->link_cleanup);  in ntb_transport_free()
1410 	cancel_delayed_work_sync(&nt->link_work);  in ntb_transport_free()
1412 	qp_bitmap_alloc = nt->qp_bitmap & ~nt->qp_bitmap_free;  in ntb_transport_free()
1415 	for (i = 0; i < nt->qp_count; i++) {  in ntb_transport_free()
1416 		qp = &nt->qp_vec[i];  in ntb_transport_free()
1425 	ntb_bus_remove(nt);  in ntb_transport_free()
1427 	for (i = nt->mw_count; i--; ) {  in ntb_transport_free()
1428 		ntb_free_mw(nt, i);  in ntb_transport_free()
1429 		iounmap(nt->mw_vec[i].vbase);  in ntb_transport_free()
1432 	kfree(nt->qp_vec);  in ntb_transport_free()
1433 	kfree(nt->mw_vec);  in ntb_transport_free()
1434 	kfree(nt);  in ntb_transport_free()
1980 	struct ntb_transport_ctx *nt;  in ntb_transport_create_queue()  local
1991 	nt = ndev->ctx;  in ntb_transport_create_queue()
1995 	free_queue = ffs(nt->qp_bitmap_free);  in ntb_transport_create_queue()
2002 	qp = &nt->qp_vec[free_queue];  in ntb_transport_create_queue()
2005 	nt->qp_bitmap_free &= ~qp_bit;  in ntb_transport_create_queue()
2094 	nt->qp_bitmap_free |= qp_bit;  in ntb_transport_create_queue()
2427 	struct ntb_transport_ctx *nt = data;  in ntb_transport_doorbell_callback()  local
2432 	if (ntb_db_read(nt->ndev) & nt->msi_db_mask) {  in ntb_transport_doorbell_callback()
2433 		ntb_transport_msi_peer_desc_changed(nt);  in ntb_transport_doorbell_callback()
2434 		ntb_db_clear(nt->ndev, nt->msi_db_mask);  in ntb_transport_doorbell_callback()
2437 	db_bits = (nt->qp_bitmap & ~nt->qp_bitmap_free &  in ntb_transport_doorbell_callback()
2438 		   ntb_db_vector_mask(nt->ndev, vector));  in ntb_transport_doorbell_callback()
2442 		qp = &nt->qp_vec[qp_num];  in ntb_transport_doorbell_callback()