1 /*
2  * zfcp device driver
3  *
4  * Setup and helper functions to access QDIO.
5  *
6  * Copyright IBM Corporation 2002, 2010
7  */
8 
9 #define KMSG_COMPONENT "zfcp"
10 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
11 
12 #include <linux/slab.h>
13 #include <linux/module.h>
14 #include "zfcp_ext.h"
15 #include "zfcp_qdio.h"
16 
17 #define QBUFF_PER_PAGE		(PAGE_SIZE / sizeof(struct qdio_buffer))
18 
19 static bool enable_multibuffer;
20 module_param_named(datarouter, enable_multibuffer, bool, 0400);
21 MODULE_PARM_DESC(datarouter, "Enable hardware data router support");
22 
zfcp_qdio_buffers_enqueue(struct qdio_buffer ** sbal)23 static int zfcp_qdio_buffers_enqueue(struct qdio_buffer **sbal)
24 {
25 	int pos;
26 
27 	for (pos = 0; pos < QDIO_MAX_BUFFERS_PER_Q; pos += QBUFF_PER_PAGE) {
28 		sbal[pos] = (struct qdio_buffer *) get_zeroed_page(GFP_KERNEL);
29 		if (!sbal[pos])
30 			return -ENOMEM;
31 	}
32 	for (pos = 0; pos < QDIO_MAX_BUFFERS_PER_Q; pos++)
33 		if (pos % QBUFF_PER_PAGE)
34 			sbal[pos] = sbal[pos - 1] + 1;
35 	return 0;
36 }
37 
zfcp_qdio_handler_error(struct zfcp_qdio * qdio,char * id,unsigned int qdio_err)38 static void zfcp_qdio_handler_error(struct zfcp_qdio *qdio, char *id,
39 				    unsigned int qdio_err)
40 {
41 	struct zfcp_adapter *adapter = qdio->adapter;
42 
43 	dev_warn(&adapter->ccw_device->dev, "A QDIO problem occurred\n");
44 
45 	if (qdio_err & QDIO_ERROR_SLSB_STATE) {
46 		zfcp_qdio_siosl(adapter);
47 		zfcp_erp_adapter_shutdown(adapter, 0, id);
48 		return;
49 	}
50 	zfcp_erp_adapter_reopen(adapter,
51 				ZFCP_STATUS_ADAPTER_LINK_UNPLUGGED |
52 				ZFCP_STATUS_COMMON_ERP_FAILED, id);
53 }
54 
zfcp_qdio_zero_sbals(struct qdio_buffer * sbal[],int first,int cnt)55 static void zfcp_qdio_zero_sbals(struct qdio_buffer *sbal[], int first, int cnt)
56 {
57 	int i, sbal_idx;
58 
59 	for (i = first; i < first + cnt; i++) {
60 		sbal_idx = i % QDIO_MAX_BUFFERS_PER_Q;
61 		memset(sbal[sbal_idx], 0, sizeof(struct qdio_buffer));
62 	}
63 }
64 
65 /* this needs to be called prior to updating the queue fill level */
zfcp_qdio_account(struct zfcp_qdio * qdio)66 static inline void zfcp_qdio_account(struct zfcp_qdio *qdio)
67 {
68 	unsigned long long now, span;
69 	int used;
70 
71 	now = get_clock_monotonic();
72 	span = (now - qdio->req_q_time) >> 12;
73 	used = QDIO_MAX_BUFFERS_PER_Q - atomic_read(&qdio->req_q_free);
74 	qdio->req_q_util += used * span;
75 	qdio->req_q_time = now;
76 }
77 
zfcp_qdio_int_req(struct ccw_device * cdev,unsigned int qdio_err,int queue_no,int idx,int count,unsigned long parm)78 static void zfcp_qdio_int_req(struct ccw_device *cdev, unsigned int qdio_err,
79 			      int queue_no, int idx, int count,
80 			      unsigned long parm)
81 {
82 	struct zfcp_qdio *qdio = (struct zfcp_qdio *) parm;
83 
84 	if (unlikely(qdio_err)) {
85 		zfcp_qdio_handler_error(qdio, "qdireq1", qdio_err);
86 		return;
87 	}
88 
89 	/* cleanup all SBALs being program-owned now */
90 	zfcp_qdio_zero_sbals(qdio->req_q, idx, count);
91 
92 	spin_lock_irq(&qdio->stat_lock);
93 	zfcp_qdio_account(qdio);
94 	spin_unlock_irq(&qdio->stat_lock);
95 	atomic_add(count, &qdio->req_q_free);
96 	wake_up(&qdio->req_q_wq);
97 }
98 
zfcp_qdio_int_resp(struct ccw_device * cdev,unsigned int qdio_err,int queue_no,int idx,int count,unsigned long parm)99 static void zfcp_qdio_int_resp(struct ccw_device *cdev, unsigned int qdio_err,
100 			       int queue_no, int idx, int count,
101 			       unsigned long parm)
102 {
103 	struct zfcp_qdio *qdio = (struct zfcp_qdio *) parm;
104 	struct zfcp_adapter *adapter = qdio->adapter;
105 	int sbal_no, sbal_idx;
106 
107 	if (unlikely(qdio_err)) {
108 		if (zfcp_adapter_multi_buffer_active(adapter)) {
109 			void *pl[ZFCP_QDIO_MAX_SBALS_PER_REQ + 1];
110 			struct qdio_buffer_element *sbale;
111 			u64 req_id;
112 			u8 scount;
113 
114 			memset(pl, 0,
115 			       ZFCP_QDIO_MAX_SBALS_PER_REQ * sizeof(void *));
116 			sbale = qdio->res_q[idx]->element;
117 			req_id = (u64) sbale->addr;
118 			scount = min(sbale->scount + 1,
119 				     ZFCP_QDIO_MAX_SBALS_PER_REQ + 1);
120 				     /* incl. signaling SBAL */
121 
122 			for (sbal_no = 0; sbal_no < scount; sbal_no++) {
123 				sbal_idx = (idx + sbal_no) %
124 					QDIO_MAX_BUFFERS_PER_Q;
125 				pl[sbal_no] = qdio->res_q[sbal_idx];
126 			}
127 			zfcp_dbf_hba_def_err(adapter, req_id, scount, pl);
128 		}
129 		zfcp_qdio_handler_error(qdio, "qdires1", qdio_err);
130 		return;
131 	}
132 
133 	/*
134 	 * go through all SBALs from input queue currently
135 	 * returned by QDIO layer
136 	 */
137 	for (sbal_no = 0; sbal_no < count; sbal_no++) {
138 		sbal_idx = (idx + sbal_no) % QDIO_MAX_BUFFERS_PER_Q;
139 		/* go through all SBALEs of SBAL */
140 		zfcp_fsf_reqid_check(qdio, sbal_idx);
141 	}
142 
143 	/*
144 	 * put SBALs back to response queue
145 	 */
146 	if (do_QDIO(cdev, QDIO_FLAG_SYNC_INPUT, 0, idx, count))
147 		zfcp_erp_adapter_reopen(qdio->adapter, 0, "qdires2");
148 }
149 
150 static struct qdio_buffer_element *
zfcp_qdio_sbal_chain(struct zfcp_qdio * qdio,struct zfcp_qdio_req * q_req)151 zfcp_qdio_sbal_chain(struct zfcp_qdio *qdio, struct zfcp_qdio_req *q_req)
152 {
153 	struct qdio_buffer_element *sbale;
154 
155 	/* set last entry flag in current SBALE of current SBAL */
156 	sbale = zfcp_qdio_sbale_curr(qdio, q_req);
157 	sbale->eflags |= SBAL_EFLAGS_LAST_ENTRY;
158 
159 	/* don't exceed last allowed SBAL */
160 	if (q_req->sbal_last == q_req->sbal_limit)
161 		return NULL;
162 
163 	/* set chaining flag in first SBALE of current SBAL */
164 	sbale = zfcp_qdio_sbale_req(qdio, q_req);
165 	sbale->sflags |= SBAL_SFLAGS0_MORE_SBALS;
166 
167 	/* calculate index of next SBAL */
168 	q_req->sbal_last++;
169 	q_req->sbal_last %= QDIO_MAX_BUFFERS_PER_Q;
170 
171 	/* keep this requests number of SBALs up-to-date */
172 	q_req->sbal_number++;
173 	BUG_ON(q_req->sbal_number > ZFCP_QDIO_MAX_SBALS_PER_REQ);
174 
175 	/* start at first SBALE of new SBAL */
176 	q_req->sbale_curr = 0;
177 
178 	/* set storage-block type for new SBAL */
179 	sbale = zfcp_qdio_sbale_curr(qdio, q_req);
180 	sbale->sflags |= q_req->sbtype;
181 
182 	return sbale;
183 }
184 
185 static struct qdio_buffer_element *
zfcp_qdio_sbale_next(struct zfcp_qdio * qdio,struct zfcp_qdio_req * q_req)186 zfcp_qdio_sbale_next(struct zfcp_qdio *qdio, struct zfcp_qdio_req *q_req)
187 {
188 	if (q_req->sbale_curr == qdio->max_sbale_per_sbal - 1)
189 		return zfcp_qdio_sbal_chain(qdio, q_req);
190 	q_req->sbale_curr++;
191 	return zfcp_qdio_sbale_curr(qdio, q_req);
192 }
193 
194 /**
195  * zfcp_qdio_sbals_from_sg - fill SBALs from scatter-gather list
196  * @qdio: pointer to struct zfcp_qdio
197  * @q_req: pointer to struct zfcp_qdio_req
198  * @sg: scatter-gather list
199  * @max_sbals: upper bound for number of SBALs to be used
200  * Returns: zero or -EINVAL on error
201  */
zfcp_qdio_sbals_from_sg(struct zfcp_qdio * qdio,struct zfcp_qdio_req * q_req,struct scatterlist * sg)202 int zfcp_qdio_sbals_from_sg(struct zfcp_qdio *qdio, struct zfcp_qdio_req *q_req,
203 			    struct scatterlist *sg)
204 {
205 	struct qdio_buffer_element *sbale;
206 
207 	/* set storage-block type for this request */
208 	sbale = zfcp_qdio_sbale_req(qdio, q_req);
209 	sbale->sflags |= q_req->sbtype;
210 
211 	for (; sg; sg = sg_next(sg)) {
212 		sbale = zfcp_qdio_sbale_next(qdio, q_req);
213 		if (!sbale) {
214 			atomic_inc(&qdio->req_q_full);
215 			zfcp_qdio_zero_sbals(qdio->req_q, q_req->sbal_first,
216 					     q_req->sbal_number);
217 			return -EINVAL;
218 		}
219 		sbale->addr = sg_virt(sg);
220 		sbale->length = sg->length;
221 	}
222 	return 0;
223 }
224 
zfcp_qdio_sbal_check(struct zfcp_qdio * qdio)225 static int zfcp_qdio_sbal_check(struct zfcp_qdio *qdio)
226 {
227 	if (atomic_read(&qdio->req_q_free) ||
228 	    !(atomic_read(&qdio->adapter->status) & ZFCP_STATUS_ADAPTER_QDIOUP))
229 		return 1;
230 	return 0;
231 }
232 
233 /**
234  * zfcp_qdio_sbal_get - get free sbal in request queue, wait if necessary
235  * @qdio: pointer to struct zfcp_qdio
236  *
237  * The req_q_lock must be held by the caller of this function, and
238  * this function may only be called from process context; it will
239  * sleep when waiting for a free sbal.
240  *
241  * Returns: 0 on success, -EIO if there is no free sbal after waiting.
242  */
zfcp_qdio_sbal_get(struct zfcp_qdio * qdio)243 int zfcp_qdio_sbal_get(struct zfcp_qdio *qdio)
244 {
245 	long ret;
246 
247 	ret = wait_event_interruptible_lock_irq_timeout(qdio->req_q_wq,
248 		       zfcp_qdio_sbal_check(qdio), qdio->req_q_lock, 5 * HZ);
249 
250 	if (!(atomic_read(&qdio->adapter->status) & ZFCP_STATUS_ADAPTER_QDIOUP))
251 		return -EIO;
252 
253 	if (ret > 0)
254 		return 0;
255 
256 	if (!ret) {
257 		atomic_inc(&qdio->req_q_full);
258 		/* assume hanging outbound queue, try queue recovery */
259 		zfcp_erp_adapter_reopen(qdio->adapter, 0, "qdsbg_1");
260 	}
261 
262 	return -EIO;
263 }
264 
265 /**
266  * zfcp_qdio_send - set PCI flag in first SBALE and send req to QDIO
267  * @qdio: pointer to struct zfcp_qdio
268  * @q_req: pointer to struct zfcp_qdio_req
269  * Returns: 0 on success, error otherwise
270  */
zfcp_qdio_send(struct zfcp_qdio * qdio,struct zfcp_qdio_req * q_req)271 int zfcp_qdio_send(struct zfcp_qdio *qdio, struct zfcp_qdio_req *q_req)
272 {
273 	int retval;
274 	u8 sbal_number = q_req->sbal_number;
275 
276 	spin_lock(&qdio->stat_lock);
277 	zfcp_qdio_account(qdio);
278 	spin_unlock(&qdio->stat_lock);
279 
280 	retval = do_QDIO(qdio->adapter->ccw_device, QDIO_FLAG_SYNC_OUTPUT, 0,
281 			 q_req->sbal_first, sbal_number);
282 
283 	if (unlikely(retval)) {
284 		zfcp_qdio_zero_sbals(qdio->req_q, q_req->sbal_first,
285 				     sbal_number);
286 		return retval;
287 	}
288 
289 	/* account for transferred buffers */
290 	atomic_sub(sbal_number, &qdio->req_q_free);
291 	qdio->req_q_idx += sbal_number;
292 	qdio->req_q_idx %= QDIO_MAX_BUFFERS_PER_Q;
293 
294 	return 0;
295 }
296 
297 
zfcp_qdio_setup_init_data(struct qdio_initialize * id,struct zfcp_qdio * qdio)298 static void zfcp_qdio_setup_init_data(struct qdio_initialize *id,
299 				      struct zfcp_qdio *qdio)
300 {
301 	memset(id, 0, sizeof(*id));
302 	id->cdev = qdio->adapter->ccw_device;
303 	id->q_format = QDIO_ZFCP_QFMT;
304 	memcpy(id->adapter_name, dev_name(&id->cdev->dev), 8);
305 	ASCEBC(id->adapter_name, 8);
306 	id->qib_rflags = QIB_RFLAGS_ENABLE_DATA_DIV;
307 	if (enable_multibuffer)
308 		id->qdr_ac |= QDR_AC_MULTI_BUFFER_ENABLE;
309 	id->no_input_qs = 1;
310 	id->no_output_qs = 1;
311 	id->input_handler = zfcp_qdio_int_resp;
312 	id->output_handler = zfcp_qdio_int_req;
313 	id->int_parm = (unsigned long) qdio;
314 	id->input_sbal_addr_array = (void **) (qdio->res_q);
315 	id->output_sbal_addr_array = (void **) (qdio->req_q);
316 	id->scan_threshold =
317 		QDIO_MAX_BUFFERS_PER_Q - ZFCP_QDIO_MAX_SBALS_PER_REQ * 2;
318 }
319 
320 /**
321  * zfcp_qdio_allocate - allocate queue memory and initialize QDIO data
322  * @adapter: pointer to struct zfcp_adapter
323  * Returns: -ENOMEM on memory allocation error or return value from
324  *          qdio_allocate
325  */
zfcp_qdio_allocate(struct zfcp_qdio * qdio)326 static int zfcp_qdio_allocate(struct zfcp_qdio *qdio)
327 {
328 	struct qdio_initialize init_data;
329 
330 	if (zfcp_qdio_buffers_enqueue(qdio->req_q) ||
331 	    zfcp_qdio_buffers_enqueue(qdio->res_q))
332 		return -ENOMEM;
333 
334 	zfcp_qdio_setup_init_data(&init_data, qdio);
335 	init_waitqueue_head(&qdio->req_q_wq);
336 
337 	return qdio_allocate(&init_data);
338 }
339 
340 /**
341  * zfcp_close_qdio - close qdio queues for an adapter
342  * @qdio: pointer to structure zfcp_qdio
343  */
zfcp_qdio_close(struct zfcp_qdio * qdio)344 void zfcp_qdio_close(struct zfcp_qdio *qdio)
345 {
346 	struct zfcp_adapter *adapter = qdio->adapter;
347 	int idx, count;
348 
349 	if (!(atomic_read(&adapter->status) & ZFCP_STATUS_ADAPTER_QDIOUP))
350 		return;
351 
352 	/* clear QDIOUP flag, thus do_QDIO is not called during qdio_shutdown */
353 	spin_lock_irq(&qdio->req_q_lock);
354 	atomic_clear_mask(ZFCP_STATUS_ADAPTER_QDIOUP, &adapter->status);
355 	spin_unlock_irq(&qdio->req_q_lock);
356 
357 	wake_up(&qdio->req_q_wq);
358 
359 	qdio_shutdown(adapter->ccw_device, QDIO_FLAG_CLEANUP_USING_CLEAR);
360 
361 	/* cleanup used outbound sbals */
362 	count = atomic_read(&qdio->req_q_free);
363 	if (count < QDIO_MAX_BUFFERS_PER_Q) {
364 		idx = (qdio->req_q_idx + count) % QDIO_MAX_BUFFERS_PER_Q;
365 		count = QDIO_MAX_BUFFERS_PER_Q - count;
366 		zfcp_qdio_zero_sbals(qdio->req_q, idx, count);
367 	}
368 	qdio->req_q_idx = 0;
369 	atomic_set(&qdio->req_q_free, 0);
370 }
371 
372 /**
373  * zfcp_qdio_open - prepare and initialize response queue
374  * @qdio: pointer to struct zfcp_qdio
375  * Returns: 0 on success, otherwise -EIO
376  */
zfcp_qdio_open(struct zfcp_qdio * qdio)377 int zfcp_qdio_open(struct zfcp_qdio *qdio)
378 {
379 	struct qdio_buffer_element *sbale;
380 	struct qdio_initialize init_data;
381 	struct zfcp_adapter *adapter = qdio->adapter;
382 	struct ccw_device *cdev = adapter->ccw_device;
383 	struct qdio_ssqd_desc ssqd;
384 	int cc;
385 
386 	if (atomic_read(&adapter->status) & ZFCP_STATUS_ADAPTER_QDIOUP)
387 		return -EIO;
388 
389 	atomic_clear_mask(ZFCP_STATUS_ADAPTER_SIOSL_ISSUED,
390 			  &qdio->adapter->status);
391 
392 	zfcp_qdio_setup_init_data(&init_data, qdio);
393 
394 	if (qdio_establish(&init_data))
395 		goto failed_establish;
396 
397 	if (qdio_get_ssqd_desc(init_data.cdev, &ssqd))
398 		goto failed_qdio;
399 
400 	if (ssqd.qdioac2 & CHSC_AC2_DATA_DIV_ENABLED)
401 		atomic_set_mask(ZFCP_STATUS_ADAPTER_DATA_DIV_ENABLED,
402 				&qdio->adapter->status);
403 
404 	if (ssqd.qdioac2 & CHSC_AC2_MULTI_BUFFER_ENABLED) {
405 		atomic_set_mask(ZFCP_STATUS_ADAPTER_MB_ACT, &adapter->status);
406 		qdio->max_sbale_per_sbal = QDIO_MAX_ELEMENTS_PER_BUFFER;
407 	} else {
408 		atomic_clear_mask(ZFCP_STATUS_ADAPTER_MB_ACT, &adapter->status);
409 		qdio->max_sbale_per_sbal = QDIO_MAX_ELEMENTS_PER_BUFFER - 1;
410 	}
411 
412 	qdio->max_sbale_per_req =
413 		ZFCP_QDIO_MAX_SBALS_PER_REQ * qdio->max_sbale_per_sbal
414 		- 2;
415 	if (qdio_activate(cdev))
416 		goto failed_qdio;
417 
418 	for (cc = 0; cc < QDIO_MAX_BUFFERS_PER_Q; cc++) {
419 		sbale = &(qdio->res_q[cc]->element[0]);
420 		sbale->length = 0;
421 		sbale->eflags = SBAL_EFLAGS_LAST_ENTRY;
422 		sbale->sflags = 0;
423 		sbale->addr = NULL;
424 	}
425 
426 	if (do_QDIO(cdev, QDIO_FLAG_SYNC_INPUT, 0, 0, QDIO_MAX_BUFFERS_PER_Q))
427 		goto failed_qdio;
428 
429 	/* set index of first available SBALS / number of available SBALS */
430 	qdio->req_q_idx = 0;
431 	atomic_set(&qdio->req_q_free, QDIO_MAX_BUFFERS_PER_Q);
432 	atomic_set_mask(ZFCP_STATUS_ADAPTER_QDIOUP, &qdio->adapter->status);
433 
434 	if (adapter->scsi_host) {
435 		adapter->scsi_host->sg_tablesize = qdio->max_sbale_per_req;
436 		adapter->scsi_host->max_sectors = qdio->max_sbale_per_req * 8;
437 	}
438 
439 	return 0;
440 
441 failed_qdio:
442 	qdio_shutdown(cdev, QDIO_FLAG_CLEANUP_USING_CLEAR);
443 failed_establish:
444 	dev_err(&cdev->dev,
445 		"Setting up the QDIO connection to the FCP adapter failed\n");
446 	return -EIO;
447 }
448 
zfcp_qdio_destroy(struct zfcp_qdio * qdio)449 void zfcp_qdio_destroy(struct zfcp_qdio *qdio)
450 {
451 	int p;
452 
453 	if (!qdio)
454 		return;
455 
456 	if (qdio->adapter->ccw_device)
457 		qdio_free(qdio->adapter->ccw_device);
458 
459 	for (p = 0; p < QDIO_MAX_BUFFERS_PER_Q; p += QBUFF_PER_PAGE) {
460 		free_page((unsigned long) qdio->req_q[p]);
461 		free_page((unsigned long) qdio->res_q[p]);
462 	}
463 
464 	kfree(qdio);
465 }
466 
zfcp_qdio_setup(struct zfcp_adapter * adapter)467 int zfcp_qdio_setup(struct zfcp_adapter *adapter)
468 {
469 	struct zfcp_qdio *qdio;
470 
471 	qdio = kzalloc(sizeof(struct zfcp_qdio), GFP_KERNEL);
472 	if (!qdio)
473 		return -ENOMEM;
474 
475 	qdio->adapter = adapter;
476 
477 	if (zfcp_qdio_allocate(qdio)) {
478 		zfcp_qdio_destroy(qdio);
479 		return -ENOMEM;
480 	}
481 
482 	spin_lock_init(&qdio->req_q_lock);
483 	spin_lock_init(&qdio->stat_lock);
484 
485 	adapter->qdio = qdio;
486 	return 0;
487 }
488 
489 /**
490  * zfcp_qdio_siosl - Trigger logging in FCP channel
491  * @adapter: The zfcp_adapter where to trigger logging
492  *
493  * Call the cio siosl function to trigger hardware logging.  This
494  * wrapper function sets a flag to ensure hardware logging is only
495  * triggered once before going through qdio shutdown.
496  *
497  * The triggers are always run from qdio tasklet context, so no
498  * additional synchronization is necessary.
499  */
zfcp_qdio_siosl(struct zfcp_adapter * adapter)500 void zfcp_qdio_siosl(struct zfcp_adapter *adapter)
501 {
502 	int rc;
503 
504 	if (atomic_read(&adapter->status) & ZFCP_STATUS_ADAPTER_SIOSL_ISSUED)
505 		return;
506 
507 	rc = ccw_device_siosl(adapter->ccw_device);
508 	if (!rc)
509 		atomic_set_mask(ZFCP_STATUS_ADAPTER_SIOSL_ISSUED,
510 				&adapter->status);
511 }
512