125 		nr += q->irq_ptr->nr_input_qs;  in qdio_do_eqbs()
127 	ccq = do_eqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count,  in qdio_do_eqbs()
138 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "EQBS part:%02x",  in qdio_do_eqbs()
143 		DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "EQBS again:%2d", ccq);  in qdio_do_eqbs()
149 		q->handler(q->irq_ptr->cdev, QDIO_ERROR_GET_BUF_STATE, q->nr,  in qdio_do_eqbs()
150 			   q->first_to_check, count, q->irq_ptr->int_parm);  in qdio_do_eqbs()
176 		nr += q->irq_ptr->nr_input_qs;  in qdio_do_sqbs()
178 	ccq = do_sqbs(q->irq_ptr->sch_token, state, nr, &tmp_start, &tmp_count);  in qdio_do_sqbs()
188 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "SQBS again:%2d", ccq);  in qdio_do_sqbs()
195 		q->handler(q->irq_ptr->cdev, QDIO_ERROR_SET_BUF_STATE, q->nr,  in qdio_do_sqbs()
196 			   q->first_to_check, count, q->irq_ptr->int_parm);  in qdio_do_sqbs()
271 static void qdio_init_buf_states(struct qdio_irq *irq_ptr)  in qdio_init_buf_states()  argument
276 	for_each_input_queue(irq_ptr, q, i)  in qdio_init_buf_states()
279 	for_each_output_queue(irq_ptr, q, i)  in qdio_init_buf_states()
287 	unsigned long schid = *((u32 *) &q->irq_ptr->schid);  in qdio_siga_sync()
291 	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-s:%1d", q->nr);  in qdio_siga_sync()
295 		schid = q->irq_ptr->sch_token;  in qdio_siga_sync()
326 	unsigned long schid = *((u32 *) &q->irq_ptr->schid);  in qdio_siga_output()
339 		schid = q->irq_ptr->sch_token;  in qdio_siga_output()
357 		DBF_DEV_EVENT(DBF_WARN, q->irq_ptr,  in qdio_siga_output()
359 		DBF_DEV_EVENT(DBF_WARN, q->irq_ptr, "count:%u", retries);  in qdio_siga_output()
366 	unsigned long schid = *((u32 *) &q->irq_ptr->schid);  in qdio_siga_input()
370 	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-r:%1d", q->nr);  in qdio_siga_input()
374 		schid = q->irq_ptr->sch_token;  in qdio_siga_input()
387 	if (qdio_need_siga_sync(q->irq_ptr))  in debug_get_buf_state()
418 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "OUTFULL FTC:%02x", start);  in process_buffer_error()
454 	if (qdio_need_siga_sync(q->irq_ptr))  in get_inbound_buffer_frontier()
463 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in prim:%1d %02x", q->nr,  in get_inbound_buffer_frontier()
469 		if (q->irq_ptr->perf_stat_enabled)  in get_inbound_buffer_frontier()
473 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in err:%1d %02x", q->nr,  in get_inbound_buffer_frontier()
481 		if (q->irq_ptr->perf_stat_enabled)  in get_inbound_buffer_frontier()
485 		if (q->irq_ptr->perf_stat_enabled)  in get_inbound_buffer_frontier()
487 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "in nop:%1d %#02x",  in get_inbound_buffer_frontier()
494 		dev_WARN_ONCE(&q->irq_ptr->cdev->dev, 1,  in get_inbound_buffer_frontier()
533 	if (qdio_need_siga_sync(q->irq_ptr))  in qdio_inbound_q_done()
556 	if (qdio_need_siga_sync(q->irq_ptr))  in get_outbound_buffer_frontier()
569 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr,  in get_outbound_buffer_frontier()
573 		if (q->irq_ptr->perf_stat_enabled)  in get_outbound_buffer_frontier()
577 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out error:%1d %02x",  in get_outbound_buffer_frontier()
583 		if (q->irq_ptr->perf_stat_enabled)  in get_outbound_buffer_frontier()
588 		if (q->irq_ptr->perf_stat_enabled)  in get_outbound_buffer_frontier()
590 		DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "out primed:%1d",  in get_outbound_buffer_frontier()
598 		dev_WARN_ONCE(&q->irq_ptr->cdev->dev, 1,  in get_outbound_buffer_frontier()
636 	if (!qdio_need_siga_out(q->irq_ptr))  in qdio_kick_outbound_q()
639 	DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w:%1d", q->nr);  in qdio_kick_outbound_q()
656 			DBF_DEV_EVENT(DBF_INFO, q->irq_ptr, "siga-w cc2:%1d", q->nr);  in qdio_kick_outbound_q()
673 static inline void qdio_set_state(struct qdio_irq *irq_ptr,  in qdio_set_state()  argument
676 	DBF_DEV_EVENT(DBF_INFO, irq_ptr, "newstate: %1d", state);  in qdio_set_state()
678 	irq_ptr->state = state;  in qdio_set_state()
682 static void qdio_irq_check_sense(struct qdio_irq *irq_ptr, struct irb *irb)  in qdio_irq_check_sense()  argument
685 		DBF_ERROR("%4x sense:", irq_ptr->schid.sch_no);  in qdio_irq_check_sense()
692 static void qdio_int_handler_pci(struct qdio_irq *irq_ptr)  in qdio_int_handler_pci()  argument
694 	if (unlikely(irq_ptr->state != QDIO_IRQ_STATE_ACTIVE))  in qdio_int_handler_pci()
697 	qdio_deliver_irq(irq_ptr);  in qdio_int_handler_pci()
698 	irq_ptr->last_data_irq_time = S390_lowcore.int_clock;  in qdio_int_handler_pci()
701 static void qdio_handle_activate_check(struct qdio_irq *irq_ptr,  in qdio_handle_activate_check()  argument
707 	DBF_ERROR("%4x ACT CHECK", irq_ptr->schid.sch_no);  in qdio_handle_activate_check()
712 	if (irq_ptr->nr_input_qs)  in qdio_handle_activate_check()
713 		first_to_check = irq_ptr->input_qs[0]->first_to_check;  in qdio_handle_activate_check()
715 	irq_ptr->error_handler(irq_ptr->cdev, QDIO_ERROR_ACTIVATE, 0,  in qdio_handle_activate_check()
716 			       first_to_check, 0, irq_ptr->int_parm);  in qdio_handle_activate_check()
717 	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED);  in qdio_handle_activate_check()
725 static void qdio_establish_handle_irq(struct qdio_irq *irq_ptr, int cstat,  in qdio_establish_handle_irq()  argument
728 	DBF_DEV_EVENT(DBF_INFO, irq_ptr, "qest irq");  in qdio_establish_handle_irq()
736 	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ESTABLISHED);  in qdio_establish_handle_irq()
740 	DBF_ERROR("%4x EQ:error", irq_ptr->schid.sch_no);  in qdio_establish_handle_irq()
742 	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR);  in qdio_establish_handle_irq()
749 	struct qdio_irq *irq_ptr = cdev->private->qdio_data;  in qdio_int_handler()  local
753 	if (!intparm || !irq_ptr) {  in qdio_int_handler()
759 	if (irq_ptr->perf_stat_enabled)  in qdio_int_handler()
760 		irq_ptr->perf_stat.qdio_int++;  in qdio_int_handler()
763 		DBF_ERROR("%4x IO error", irq_ptr->schid.sch_no);  in qdio_int_handler()
764 		qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ERR);  in qdio_int_handler()
768 	qdio_irq_check_sense(irq_ptr, irb);  in qdio_int_handler()
772 	switch (irq_ptr->state) {  in qdio_int_handler()
774 		qdio_establish_handle_irq(irq_ptr, cstat, dstat);  in qdio_int_handler()
777 		qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);  in qdio_int_handler()
782 			qdio_int_handler_pci(irq_ptr);  in qdio_int_handler()
786 			qdio_handle_activate_check(irq_ptr, intparm, cstat,  in qdio_int_handler()
856 	struct qdio_irq *irq_ptr = cdev->private->qdio_data;  in qdio_shutdown()  local
860 	if (!irq_ptr)  in qdio_shutdown()
867 	mutex_lock(&irq_ptr->setup_mutex);  in qdio_shutdown()
872 	if (irq_ptr->state == QDIO_IRQ_STATE_INACTIVE) {  in qdio_shutdown()
873 		mutex_unlock(&irq_ptr->setup_mutex);  in qdio_shutdown()
880 	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_STOPPED);  in qdio_shutdown()
882 	qdio_shutdown_debug_entries(irq_ptr);  in qdio_shutdown()
884 	rc = qdio_cancel_ccw(irq_ptr, how);  in qdio_shutdown()
885 	qdio_shutdown_thinint(irq_ptr);  in qdio_shutdown()
886 	qdio_shutdown_irq(irq_ptr);  in qdio_shutdown()
888 	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);  in qdio_shutdown()
889 	mutex_unlock(&irq_ptr->setup_mutex);  in qdio_shutdown()
902 	struct qdio_irq *irq_ptr = cdev->private->qdio_data;  in qdio_free()  local
905 	if (!irq_ptr)  in qdio_free()
910 	DBF_DEV_EVENT(DBF_ERR, irq_ptr, "dbf abandoned");  in qdio_free()
911 	mutex_lock(&irq_ptr->setup_mutex);  in qdio_free()
913 	irq_ptr->debug_area = NULL;  in qdio_free()
915 	mutex_unlock(&irq_ptr->setup_mutex);  in qdio_free()
917 	qdio_free_queues(irq_ptr);  in qdio_free()
918 	free_page((unsigned long) irq_ptr->qdr);  in qdio_free()
919 	free_page(irq_ptr->chsc_page);  in qdio_free()
920 	kfree(irq_ptr->ccw);  in qdio_free()
921 	free_page((unsigned long) irq_ptr);  in qdio_free()
936 	struct qdio_irq *irq_ptr;  in qdio_allocate()  local
946 	irq_ptr = (void *) get_zeroed_page(GFP_KERNEL);  in qdio_allocate()
947 	if (!irq_ptr)  in qdio_allocate()
950 	irq_ptr->ccw = kmalloc(sizeof(*irq_ptr->ccw), GFP_KERNEL | GFP_DMA);  in qdio_allocate()
951 	if (!irq_ptr->ccw)  in qdio_allocate()
955 	kmemleak_not_leak(irq_ptr->ccw);  in qdio_allocate()
957 	irq_ptr->cdev = cdev;  in qdio_allocate()
958 	mutex_init(&irq_ptr->setup_mutex);  in qdio_allocate()
959 	if (qdio_allocate_dbf(irq_ptr))  in qdio_allocate()
962 	DBF_DEV_EVENT(DBF_ERR, irq_ptr, "alloc niq:%1u noq:%1u", no_input_qs,  in qdio_allocate()
971 	irq_ptr->chsc_page = get_zeroed_page(GFP_KERNEL);  in qdio_allocate()
972 	if (!irq_ptr->chsc_page)  in qdio_allocate()
976 	irq_ptr->qdr = (struct qdr *) get_zeroed_page(GFP_KERNEL | GFP_DMA);  in qdio_allocate()
977 	if (!irq_ptr->qdr)  in qdio_allocate()
980 	rc = qdio_allocate_qs(irq_ptr, no_input_qs, no_output_qs);  in qdio_allocate()
984 	cdev->private->qdio_data = irq_ptr;  in qdio_allocate()
985 	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);  in qdio_allocate()
989 	free_page((unsigned long) irq_ptr->qdr);  in qdio_allocate()
991 	free_page(irq_ptr->chsc_page);  in qdio_allocate()
994 	kfree(irq_ptr->ccw);  in qdio_allocate()
996 	free_page((unsigned long) irq_ptr);  in qdio_allocate()
1025 	struct qdio_irq *irq_ptr = cdev->private->qdio_data;  in qdio_establish()  local
1034 	if (!irq_ptr)  in qdio_establish()
1037 	if (init_data->no_input_qs > irq_ptr->max_input_qs ||  in qdio_establish()
1038 	    init_data->no_output_qs > irq_ptr->max_output_qs)  in qdio_establish()
1061 	mutex_lock(&irq_ptr->setup_mutex);  in qdio_establish()
1062 	qdio_trace_init_data(irq_ptr, init_data);  in qdio_establish()
1063 	qdio_setup_irq(irq_ptr, init_data);  in qdio_establish()
1065 	rc = qdio_establish_thinint(irq_ptr);  in qdio_establish()
1070 	irq_ptr->ccw->cmd_code = ciw->cmd;  in qdio_establish()
1071 	irq_ptr->ccw->flags = CCW_FLAG_SLI;  in qdio_establish()
1072 	irq_ptr->ccw->count = ciw->count;  in qdio_establish()
1073 	irq_ptr->ccw->cda = (u32) virt_to_phys(irq_ptr->qdr);  in qdio_establish()
1078 	rc = ccw_device_start(cdev, irq_ptr->ccw, QDIO_DOING_ESTABLISH, 0, 0);  in qdio_establish()
1081 		DBF_ERROR("%4x est IO ERR", irq_ptr->schid.sch_no);  in qdio_establish()
1087 						   irq_ptr->state == QDIO_IRQ_STATE_ESTABLISHED ||  in qdio_establish()
1088 						   irq_ptr->state == QDIO_IRQ_STATE_ERR, HZ);  in qdio_establish()
1094 	if (irq_ptr->state != QDIO_IRQ_STATE_ESTABLISHED) {  in qdio_establish()
1099 	qdio_setup_ssqd_info(irq_ptr);  in qdio_establish()
1102 	qdio_init_buf_states(irq_ptr);  in qdio_establish()
1104 	mutex_unlock(&irq_ptr->setup_mutex);  in qdio_establish()
1105 	qdio_print_subchannel_info(irq_ptr);  in qdio_establish()
1106 	qdio_setup_debug_entries(irq_ptr);  in qdio_establish()
1110 	qdio_cancel_ccw(irq_ptr, QDIO_FLAG_CLEANUP_USING_CLEAR);  in qdio_establish()
1113 	qdio_shutdown_thinint(irq_ptr);  in qdio_establish()
1115 	qdio_shutdown_irq(irq_ptr);  in qdio_establish()
1116 	qdio_set_state(irq_ptr, QDIO_IRQ_STATE_INACTIVE);  in qdio_establish()
1117 	mutex_unlock(&irq_ptr->setup_mutex);  in qdio_establish()
1128 	struct qdio_irq *irq_ptr = cdev->private->qdio_data;  in qdio_activate()  local
1136 	if (!irq_ptr)  in qdio_activate()
1145 	mutex_lock(&irq_ptr->setup_mutex);  in qdio_activate()
1146 	if (irq_ptr->state == QDIO_IRQ_STATE_INACTIVE) {  in qdio_activate()
1151 	irq_ptr->ccw->cmd_code = ciw->cmd;  in qdio_activate()
1152 	irq_ptr->ccw->flags = CCW_FLAG_SLI;  in qdio_activate()
1153 	irq_ptr->ccw->count = ciw->count;  in qdio_activate()
1154 	irq_ptr->ccw->cda = 0;  in qdio_activate()
1159 	rc = ccw_device_start(cdev, irq_ptr->ccw, QDIO_DOING_ACTIVATE,  in qdio_activate()
1163 		DBF_ERROR("%4x act IO ERR", irq_ptr->schid.sch_no);  in qdio_activate()
1171 	switch (irq_ptr->state) {  in qdio_activate()
1177 		qdio_set_state(irq_ptr, QDIO_IRQ_STATE_ACTIVE);  in qdio_activate()
1181 	mutex_unlock(&irq_ptr->setup_mutex);  in qdio_activate()
1209 	if (qdio_need_siga_in(q->irq_ptr))  in handle_inbound()
1225 	struct qdio_irq *irq_ptr = cdev->private->qdio_data;  in qdio_add_bufs_to_input_queue()  local
1230 	if (!irq_ptr)  in qdio_add_bufs_to_input_queue()
1233 	DBF_DEV_EVENT(DBF_INFO, irq_ptr, "addi b:%02x c:%02x", bufnr, count);  in qdio_add_bufs_to_input_queue()
1235 	if (irq_ptr->state != QDIO_IRQ_STATE_ACTIVE)  in qdio_add_bufs_to_input_queue()
1240 	return handle_inbound(irq_ptr->input_qs[q_nr], bufnr, count);  in qdio_add_bufs_to_input_queue()
1270 	} else if (qdio_need_siga_sync(q->irq_ptr)) {  in handle_outbound()
1296 	struct qdio_irq *irq_ptr = cdev->private->qdio_data;  in qdio_add_bufs_to_output_queue()  local
1301 	if (!irq_ptr)  in qdio_add_bufs_to_output_queue()
1304 	DBF_DEV_EVENT(DBF_INFO, irq_ptr, "addo b:%02x c:%02x", bufnr, count);  in qdio_add_bufs_to_output_queue()
1306 	if (irq_ptr->state != QDIO_IRQ_STATE_ACTIVE)  in qdio_add_bufs_to_output_queue()
1311 	return handle_outbound(irq_ptr->output_qs[q_nr], bufnr, count, aob);  in qdio_add_bufs_to_output_queue()
1326 	struct qdio_irq *irq_ptr = cdev->private->qdio_data;  in qdio_start_irq()  local
1329 	if (!irq_ptr)  in qdio_start_irq()
1332 	for_each_input_queue(irq_ptr, q, i)  in qdio_start_irq()
1335 	clear_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state);  in qdio_start_irq()
1341 	if (test_nonshared_ind(irq_ptr))  in qdio_start_irq()
1344 	for_each_input_queue(irq_ptr, q, i) {  in qdio_start_irq()
1352 	if (test_and_set_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state))  in qdio_start_irq()
1370 	struct qdio_irq *irq_ptr = cdev->private->qdio_data;  in qdio_stop_irq()  local
1372 	if (!irq_ptr)  in qdio_stop_irq()
1375 	if (test_and_set_bit(QDIO_IRQ_DISABLED, &irq_ptr->poll_state))  in qdio_stop_irq()