315 static int rocker_desc_err(const struct rocker_desc_info *desc_info)  in rocker_desc_err()  argument
317 	int err = desc_info->desc->comp_err & ~ROCKER_DMA_DESC_COMP_ERR_GEN;  in rocker_desc_err()
343 static void rocker_desc_gen_clear(const struct rocker_desc_info *desc_info)  in rocker_desc_gen_clear()  argument
345 	desc_info->desc->comp_err &= ~ROCKER_DMA_DESC_COMP_ERR_GEN;  in rocker_desc_gen_clear()
348 static bool rocker_desc_gen(const struct rocker_desc_info *desc_info)  in rocker_desc_gen()  argument
350 	u32 comp_err = desc_info->desc->comp_err;  in rocker_desc_gen()
356 rocker_desc_cookie_ptr_get(const struct rocker_desc_info *desc_info)  in rocker_desc_cookie_ptr_get()  argument
358 	return (void *)(uintptr_t)desc_info->desc->cookie;  in rocker_desc_cookie_ptr_get()
361 static void rocker_desc_cookie_ptr_set(const struct rocker_desc_info *desc_info,  in rocker_desc_cookie_ptr_set()  argument
364 	desc_info->desc->cookie = (uintptr_t) ptr;  in rocker_desc_cookie_ptr_set()
370 	struct rocker_desc_info *desc_info;  in rocker_desc_head_get()  local
373 	desc_info = &info->desc_info[info->head];  in rocker_desc_head_get()
376 	desc_info->tlv_size = 0;  in rocker_desc_head_get()
377 	return desc_info;  in rocker_desc_head_get()
380 static void rocker_desc_commit(const struct rocker_desc_info *desc_info)  in rocker_desc_commit()  argument
382 	desc_info->desc->buf_size = desc_info->data_size;  in rocker_desc_commit()
383 	desc_info->desc->tlv_size = desc_info->tlv_size;  in rocker_desc_commit()
388 				 const struct rocker_desc_info *desc_info)  in rocker_desc_head_set()  argument
393 	rocker_desc_commit(desc_info);  in rocker_desc_head_set()
401 	struct rocker_desc_info *desc_info;  in rocker_desc_tail_get()  local
405 	desc_info = &info->desc_info[info->tail];  in rocker_desc_tail_get()
406 	if (!rocker_desc_gen(desc_info))  in rocker_desc_tail_get()
409 	desc_info->tlv_size = desc_info->desc->tlv_size;  in rocker_desc_tail_get()
410 	return desc_info;  in rocker_desc_tail_get()
439 	info->desc_info = kcalloc(info->size, sizeof(*info->desc_info),  in rocker_dma_ring_create()
441 	if (!info->desc_info)  in rocker_dma_ring_create()
448 		kfree(info->desc_info);  in rocker_dma_ring_create()
453 		info->desc_info[i].desc = &info->desc[i];  in rocker_dma_ring_create()
471 	kfree(info->desc_info);  in rocker_dma_ring_destroy()
485 		rocker_desc_head_set(rocker, info, &info->desc_info[i]);  in rocker_dma_ring_pass_to_producer()
486 	rocker_desc_commit(&info->desc_info[i]);  in rocker_dma_ring_pass_to_producer()
498 		struct rocker_desc_info *desc_info = &info->desc_info[i];  in rocker_dma_ring_bufs_alloc()  local
517 		desc_info->data = buf;  in rocker_dma_ring_bufs_alloc()
518 		desc_info->data_size = buf_size;  in rocker_dma_ring_bufs_alloc()
519 		dma_unmap_addr_set(desc_info, mapaddr, dma_handle);  in rocker_dma_ring_bufs_alloc()
528 		const struct rocker_desc_info *desc_info = &info->desc_info[i];  in rocker_dma_ring_bufs_alloc()  local
531 				 dma_unmap_addr(desc_info, mapaddr),  in rocker_dma_ring_bufs_alloc()
532 				 desc_info->data_size, direction);  in rocker_dma_ring_bufs_alloc()
533 		kfree(desc_info->data);  in rocker_dma_ring_bufs_alloc()
546 		const struct rocker_desc_info *desc_info = &info->desc_info[i];  in rocker_dma_ring_bufs_free()  local
552 				 dma_unmap_addr(desc_info, mapaddr),  in rocker_dma_ring_bufs_free()
553 				 desc_info->data_size, direction);  in rocker_dma_ring_bufs_free()
554 		kfree(desc_info->data);  in rocker_dma_ring_bufs_free()
558 static int rocker_dma_cmd_ring_wait_alloc(struct rocker_desc_info *desc_info)  in rocker_dma_cmd_ring_wait_alloc()  argument
565 	rocker_desc_cookie_ptr_set(desc_info, wait);  in rocker_dma_cmd_ring_wait_alloc()
570 rocker_dma_cmd_ring_wait_free(const struct rocker_desc_info *desc_info)  in rocker_dma_cmd_ring_wait_free()  argument
572 	struct rocker_wait *wait = rocker_desc_cookie_ptr_get(desc_info);  in rocker_dma_cmd_ring_wait_free()
584 		err = rocker_dma_cmd_ring_wait_alloc(&cmd_ring->desc_info[i]);  in rocker_dma_cmd_ring_waits_alloc()
592 		rocker_dma_cmd_ring_wait_free(&cmd_ring->desc_info[i]);  in rocker_dma_cmd_ring_waits_alloc()
602 		rocker_dma_cmd_ring_wait_free(&cmd_ring->desc_info[i]);  in rocker_dma_cmd_ring_waits_free()
674 				      struct rocker_desc_info *desc_info,  in rocker_dma_rx_ring_skb_map()  argument
685 	if (rocker_tlv_put_u64(desc_info, ROCKER_TLV_RX_FRAG_ADDR, dma_handle))  in rocker_dma_rx_ring_skb_map()
687 	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_RX_FRAG_MAX_LEN, buf_len))  in rocker_dma_rx_ring_skb_map()
693 	desc_info->tlv_size = 0;  in rocker_dma_rx_ring_skb_map()
703 					struct rocker_desc_info *desc_info)  in rocker_dma_rx_ring_skb_alloc()  argument
713 	rocker_desc_cookie_ptr_set(desc_info, NULL);  in rocker_dma_rx_ring_skb_alloc()
714 	desc_info->tlv_size = 0;  in rocker_dma_rx_ring_skb_alloc()
719 	err = rocker_dma_rx_ring_skb_map(rocker_port, desc_info, skb, buf_len);  in rocker_dma_rx_ring_skb_alloc()
724 	rocker_desc_cookie_ptr_set(desc_info, skb);  in rocker_dma_rx_ring_skb_alloc()
744 					const struct rocker_desc_info *desc_info)  in rocker_dma_rx_ring_skb_free()  argument
747 	struct sk_buff *skb = rocker_desc_cookie_ptr_get(desc_info);  in rocker_dma_rx_ring_skb_free()
751 	rocker_tlv_parse_desc(attrs, ROCKER_TLV_RX_MAX, desc_info);  in rocker_dma_rx_ring_skb_free()
765 						   &rx_ring->desc_info[i]);  in rocker_dma_rx_ring_skbs_alloc()
773 		rocker_dma_rx_ring_skb_free(rocker, &rx_ring->desc_info[i]);  in rocker_dma_rx_ring_skbs_alloc()
784 		rocker_dma_rx_ring_skb_free(rocker, &rx_ring->desc_info[i]);  in rocker_dma_rx_ring_skbs_free()
880 	const struct rocker_desc_info *desc_info;  in rocker_cmd_irq_handler()  local
885 	while ((desc_info = rocker_desc_tail_get(&rocker->cmd_ring))) {  in rocker_cmd_irq_handler()
886 		wait = rocker_desc_cookie_ptr_get(desc_info);  in rocker_cmd_irq_handler()
888 			rocker_desc_gen_clear(desc_info);  in rocker_cmd_irq_handler()
973 				const struct rocker_desc_info *desc_info)  in rocker_event_process()  argument
979 	rocker_tlv_parse_desc(attrs, ROCKER_TLV_EVENT_MAX, desc_info);  in rocker_event_process()
1001 	const struct rocker_desc_info *desc_info;  in rocker_event_irq_handler()  local
1005 	while ((desc_info = rocker_desc_tail_get(&rocker->event_ring))) {  in rocker_event_irq_handler()
1006 		err = rocker_desc_err(desc_info);  in rocker_event_irq_handler()
1011 			err = rocker_event_process(rocker, desc_info);  in rocker_event_irq_handler()
1016 		rocker_desc_gen_clear(desc_info);  in rocker_event_irq_handler()
1017 		rocker_desc_head_set(rocker, &rocker->event_ring, desc_info);  in rocker_event_irq_handler()
1050 	struct rocker_desc_info *desc_info;  in rocker_cmd_exec()  local
1057 	desc_info = rocker_desc_head_get(&rocker->cmd_ring);  in rocker_cmd_exec()
1058 	if (!desc_info) {  in rocker_cmd_exec()
1063 	wait = rocker_desc_cookie_ptr_get(desc_info);  in rocker_cmd_exec()
1067 	err = prepare(rocker_port, desc_info, prepare_priv);  in rocker_cmd_exec()
1073 	rocker_desc_head_set(rocker, &rocker->cmd_ring, desc_info);  in rocker_cmd_exec()
1083 	err = rocker_desc_err(desc_info);  in rocker_cmd_exec()
1088 		err = process(rocker_port, desc_info, process_priv);  in rocker_cmd_exec()
1090 	rocker_desc_gen_clear(desc_info);  in rocker_cmd_exec()
1096 				  struct rocker_desc_info *desc_info,  in rocker_cmd_get_port_settings_prep()  argument
1101 	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,  in rocker_cmd_get_port_settings_prep()
1104 	cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);  in rocker_cmd_get_port_settings_prep()
1107 	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_PPORT,  in rocker_cmd_get_port_settings_prep()
1110 	rocker_tlv_nest_end(desc_info, cmd_info);  in rocker_cmd_get_port_settings_prep()
1116 					  const struct rocker_desc_info *desc_info,  in rocker_cmd_get_port_settings_ethtool_proc()  argument
1126 	rocker_tlv_parse_desc(attrs, ROCKER_TLV_CMD_MAX, desc_info);  in rocker_cmd_get_port_settings_ethtool_proc()
1155 					  const struct rocker_desc_info *desc_info,  in rocker_cmd_get_port_settings_macaddr_proc()  argument
1163 	rocker_tlv_parse_desc(attrs, ROCKER_TLV_CMD_MAX, desc_info);  in rocker_cmd_get_port_settings_macaddr_proc()
1182 				       const struct rocker_desc_info *desc_info,  in rocker_cmd_get_port_settings_mode_proc()  argument
1190 	rocker_tlv_parse_desc(attrs, ROCKER_TLV_CMD_MAX, desc_info);  in rocker_cmd_get_port_settings_mode_proc()
1211 					    const struct rocker_desc_info *desc_info,  in rocker_cmd_get_port_settings_phys_name_proc()  argument
1221 	rocker_tlv_parse_desc(attrs, ROCKER_TLV_CMD_MAX, desc_info);  in rocker_cmd_get_port_settings_phys_name_proc()
1252 					  struct rocker_desc_info *desc_info,  in rocker_cmd_set_port_settings_ethtool_prep()  argument
1258 	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,  in rocker_cmd_set_port_settings_ethtool_prep()
1261 	cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);  in rocker_cmd_set_port_settings_ethtool_prep()
1264 	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_PPORT,  in rocker_cmd_set_port_settings_ethtool_prep()
1267 	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_SPEED,  in rocker_cmd_set_port_settings_ethtool_prep()
1270 	if (rocker_tlv_put_u8(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_DUPLEX,  in rocker_cmd_set_port_settings_ethtool_prep()
1273 	if (rocker_tlv_put_u8(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_AUTONEG,  in rocker_cmd_set_port_settings_ethtool_prep()
1276 	rocker_tlv_nest_end(desc_info, cmd_info);  in rocker_cmd_set_port_settings_ethtool_prep()
1282 					  struct rocker_desc_info *desc_info,  in rocker_cmd_set_port_settings_macaddr_prep()  argument
1288 	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,  in rocker_cmd_set_port_settings_macaddr_prep()
1291 	cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);  in rocker_cmd_set_port_settings_macaddr_prep()
1294 	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_PPORT,  in rocker_cmd_set_port_settings_macaddr_prep()
1297 	if (rocker_tlv_put(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_MACADDR,  in rocker_cmd_set_port_settings_macaddr_prep()
1300 	rocker_tlv_nest_end(desc_info, cmd_info);  in rocker_cmd_set_port_settings_macaddr_prep()
1306 				      struct rocker_desc_info *desc_info,  in rocker_cmd_set_port_settings_mtu_prep()  argument
1312 	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,  in rocker_cmd_set_port_settings_mtu_prep()
1315 	cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);  in rocker_cmd_set_port_settings_mtu_prep()
1318 	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_PPORT,  in rocker_cmd_set_port_settings_mtu_prep()
1321 	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_MTU,  in rocker_cmd_set_port_settings_mtu_prep()
1324 	rocker_tlv_nest_end(desc_info, cmd_info);  in rocker_cmd_set_port_settings_mtu_prep()
1330 				  struct rocker_desc_info *desc_info,  in rocker_cmd_set_port_learning_prep()  argument
1336 	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,  in rocker_cmd_set_port_learning_prep()
1339 	cmd_info = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);  in rocker_cmd_set_port_learning_prep()
1342 	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_PPORT,  in rocker_cmd_set_port_learning_prep()
1345 	if (rocker_tlv_put_u8(desc_info, ROCKER_TLV_CMD_PORT_SETTINGS_LEARNING,  in rocker_cmd_set_port_learning_prep()
1348 	rocker_tlv_nest_end(desc_info, cmd_info);  in rocker_cmd_set_port_learning_prep()
1820 				       const struct rocker_desc_info *desc_info)  in rocker_tx_desc_frags_unmap()  argument
1828 	rocker_tlv_parse_desc(attrs, ROCKER_TLV_TX_MAX, desc_info);  in rocker_tx_desc_frags_unmap()
1850 				       struct rocker_desc_info *desc_info,  in rocker_tx_desc_frag_map_put()  argument
1864 	frag = rocker_tlv_nest_start(desc_info, ROCKER_TLV_TX_FRAG);  in rocker_tx_desc_frag_map_put()
1867 	if (rocker_tlv_put_u64(desc_info, ROCKER_TLV_TX_FRAG_ATTR_ADDR,  in rocker_tx_desc_frag_map_put()
1870 	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_TX_FRAG_ATTR_LEN,  in rocker_tx_desc_frag_map_put()
1873 	rocker_tlv_nest_end(desc_info, frag);  in rocker_tx_desc_frag_map_put()
1877 	rocker_tlv_nest_cancel(desc_info, frag);  in rocker_tx_desc_frag_map_put()
1887 	struct rocker_desc_info *desc_info;  in rocker_port_xmit()  local
1892 	desc_info = rocker_desc_head_get(&rocker_port->tx_ring);  in rocker_port_xmit()
1893 	if (unlikely(!desc_info)) {  in rocker_port_xmit()
1899 	rocker_desc_cookie_ptr_set(desc_info, skb);  in rocker_port_xmit()
1901 	frags = rocker_tlv_nest_start(desc_info, ROCKER_TLV_TX_FRAGS);  in rocker_port_xmit()
1904 	err = rocker_tx_desc_frag_map_put(rocker_port, desc_info,  in rocker_port_xmit()
1917 		err = rocker_tx_desc_frag_map_put(rocker_port, desc_info,  in rocker_port_xmit()
1923 	rocker_tlv_nest_end(desc_info, frags);  in rocker_port_xmit()
1925 	rocker_desc_gen_clear(desc_info);  in rocker_port_xmit()
1926 	rocker_desc_head_set(rocker, &rocker_port->tx_ring, desc_info);  in rocker_port_xmit()
1928 	desc_info = rocker_desc_head_get(&rocker_port->tx_ring);  in rocker_port_xmit()
1929 	if (!desc_info)  in rocker_port_xmit()
1935 	rocker_tx_desc_frags_unmap(rocker_port, desc_info);  in rocker_port_xmit()
1937 	rocker_tlv_nest_cancel(desc_info, frags);  in rocker_port_xmit()
2268 			       struct rocker_desc_info *desc_info,  in rocker_cmd_get_port_stats_prep()  argument
2273 	if (rocker_tlv_put_u16(desc_info, ROCKER_TLV_CMD_TYPE,  in rocker_cmd_get_port_stats_prep()
2277 	cmd_stats = rocker_tlv_nest_start(desc_info, ROCKER_TLV_CMD_INFO);  in rocker_cmd_get_port_stats_prep()
2281 	if (rocker_tlv_put_u32(desc_info, ROCKER_TLV_CMD_PORT_STATS_PPORT,  in rocker_cmd_get_port_stats_prep()
2285 	rocker_tlv_nest_end(desc_info, cmd_stats);  in rocker_cmd_get_port_stats_prep()
2292 				       const struct rocker_desc_info *desc_info,  in rocker_cmd_get_port_stats_ethtool_proc()  argument
2302 	rocker_tlv_parse_desc(attrs, ROCKER_TLV_CMD_MAX, desc_info);  in rocker_cmd_get_port_stats_ethtool_proc()
2383 	const struct rocker_desc_info *desc_info;  in rocker_port_poll_tx()  local
2388 	while ((desc_info = rocker_desc_tail_get(&rocker_port->tx_ring))) {  in rocker_port_poll_tx()
2391 		err = rocker_desc_err(desc_info);  in rocker_port_poll_tx()
2395 		rocker_tx_desc_frags_unmap(rocker_port, desc_info);  in rocker_port_poll_tx()
2397 		skb = rocker_desc_cookie_ptr_get(desc_info);  in rocker_port_poll_tx()
2420 			       struct rocker_desc_info *desc_info)  in rocker_port_rx_proc()  argument
2423 	struct sk_buff *skb = rocker_desc_cookie_ptr_get(desc_info);  in rocker_port_rx_proc()
2430 	rocker_tlv_parse_desc(attrs, ROCKER_TLV_RX_MAX, desc_info);  in rocker_port_rx_proc()
2450 	return rocker_dma_rx_ring_skb_alloc(rocker_port, desc_info);  in rocker_port_rx_proc()
2462 	struct rocker_desc_info *desc_info;  in rocker_port_poll_rx()  local
2468 	       (desc_info = rocker_desc_tail_get(&rocker_port->rx_ring))) {  in rocker_port_poll_rx()
2469 		err = rocker_desc_err(desc_info);  in rocker_port_poll_rx()
2476 						  desc_info);  in rocker_port_poll_rx()
2484 		rocker_desc_gen_clear(desc_info);  in rocker_port_poll_rx()
2485 		rocker_desc_head_set(rocker, &rocker_port->rx_ring, desc_info);  in rocker_port_poll_rx()