1 // SPDX-License-Identifier: GPL-2.0
2 /* ldc.c: Logical Domain Channel link-layer protocol driver.
3  *
4  * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net>
5  */
6 
7 #include <linux/kernel.h>
8 #include <linux/export.h>
9 #include <linux/slab.h>
10 #include <linux/spinlock.h>
11 #include <linux/delay.h>
12 #include <linux/errno.h>
13 #include <linux/string.h>
14 #include <linux/scatterlist.h>
15 #include <linux/interrupt.h>
16 #include <linux/list.h>
17 #include <linux/init.h>
18 #include <linux/bitmap.h>
19 #include <asm/iommu-common.h>
20 
21 #include <asm/hypervisor.h>
22 #include <asm/iommu.h>
23 #include <asm/page.h>
24 #include <asm/ldc.h>
25 #include <asm/mdesc.h>
26 
27 #define DRV_MODULE_NAME		"ldc"
28 #define PFX DRV_MODULE_NAME	": "
29 #define DRV_MODULE_VERSION	"1.1"
30 #define DRV_MODULE_RELDATE	"July 22, 2008"
31 
32 #define COOKIE_PGSZ_CODE	0xf000000000000000ULL
33 #define COOKIE_PGSZ_CODE_SHIFT	60ULL
34 
35 
36 static char version[] =
37 	DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n";
38 
39 /* Packet header layout for unreliable and reliable mode frames.
40  * When in RAW mode, packets are simply straight 64-byte payloads
41  * with no headers.
42  */
43 struct ldc_packet {
44 	u8			type;
45 #define LDC_CTRL		0x01
46 #define LDC_DATA		0x02
47 #define LDC_ERR			0x10
48 
49 	u8			stype;
50 #define LDC_INFO		0x01
51 #define LDC_ACK			0x02
52 #define LDC_NACK		0x04
53 
54 	u8			ctrl;
55 #define LDC_VERS		0x01 /* Link Version		*/
56 #define LDC_RTS			0x02 /* Request To Send		*/
57 #define LDC_RTR			0x03 /* Ready To Receive	*/
58 #define LDC_RDX			0x04 /* Ready for Data eXchange	*/
59 #define LDC_CTRL_MSK		0x0f
60 
61 	u8			env;
62 #define LDC_LEN			0x3f
63 #define LDC_FRAG_MASK		0xc0
64 #define LDC_START		0x40
65 #define LDC_STOP		0x80
66 
67 	u32			seqid;
68 
69 	union {
70 		u8		u_data[LDC_PACKET_SIZE - 8];
71 		struct {
72 			u32	pad;
73 			u32	ackid;
74 			u8	r_data[LDC_PACKET_SIZE - 8 - 8];
75 		} r;
76 	} u;
77 };
78 
79 struct ldc_version {
80 	u16 major;
81 	u16 minor;
82 };
83 
84 /* Ordered from largest major to lowest.  */
85 static struct ldc_version ver_arr[] = {
86 	{ .major = 1, .minor = 0 },
87 };
88 
89 #define LDC_DEFAULT_MTU			(4 * LDC_PACKET_SIZE)
90 #define LDC_DEFAULT_NUM_ENTRIES		(PAGE_SIZE / LDC_PACKET_SIZE)
91 
92 struct ldc_channel;
93 
94 struct ldc_mode_ops {
95 	int (*write)(struct ldc_channel *, const void *, unsigned int);
96 	int (*read)(struct ldc_channel *, void *, unsigned int);
97 };
98 
99 static const struct ldc_mode_ops raw_ops;
100 static const struct ldc_mode_ops nonraw_ops;
101 static const struct ldc_mode_ops stream_ops;
102 
103 int ldom_domaining_enabled;
104 
105 struct ldc_iommu {
106 	/* Protects ldc_unmap.  */
107 	spinlock_t			lock;
108 	struct ldc_mtable_entry		*page_table;
109 	struct iommu_map_table		iommu_map_table;
110 };
111 
112 struct ldc_channel {
113 	/* Protects all operations that depend upon channel state.  */
114 	spinlock_t			lock;
115 
116 	unsigned long			id;
117 
118 	u8				*mssbuf;
119 	u32				mssbuf_len;
120 	u32				mssbuf_off;
121 
122 	struct ldc_packet		*tx_base;
123 	unsigned long			tx_head;
124 	unsigned long			tx_tail;
125 	unsigned long			tx_num_entries;
126 	unsigned long			tx_ra;
127 
128 	unsigned long			tx_acked;
129 
130 	struct ldc_packet		*rx_base;
131 	unsigned long			rx_head;
132 	unsigned long			rx_tail;
133 	unsigned long			rx_num_entries;
134 	unsigned long			rx_ra;
135 
136 	u32				rcv_nxt;
137 	u32				snd_nxt;
138 
139 	unsigned long			chan_state;
140 
141 	struct ldc_channel_config	cfg;
142 	void				*event_arg;
143 
144 	const struct ldc_mode_ops	*mops;
145 
146 	struct ldc_iommu		iommu;
147 
148 	struct ldc_version		ver;
149 
150 	u8				hs_state;
151 #define LDC_HS_CLOSED			0x00
152 #define LDC_HS_OPEN			0x01
153 #define LDC_HS_GOTVERS			0x02
154 #define LDC_HS_SENTRTR			0x03
155 #define LDC_HS_GOTRTR			0x04
156 #define LDC_HS_COMPLETE			0x10
157 
158 	u8				flags;
159 #define LDC_FLAG_ALLOCED_QUEUES		0x01
160 #define LDC_FLAG_REGISTERED_QUEUES	0x02
161 #define LDC_FLAG_REGISTERED_IRQS	0x04
162 #define LDC_FLAG_RESET			0x10
163 
164 	u8				mss;
165 	u8				state;
166 
167 #define LDC_IRQ_NAME_MAX		32
168 	char				rx_irq_name[LDC_IRQ_NAME_MAX];
169 	char				tx_irq_name[LDC_IRQ_NAME_MAX];
170 
171 	struct hlist_head		mh_list;
172 
173 	struct hlist_node		list;
174 };
175 
176 #define ldcdbg(TYPE, f, a...) \
177 do {	if (lp->cfg.debug & LDC_DEBUG_##TYPE) \
178 		printk(KERN_INFO PFX "ID[%lu] " f, lp->id, ## a); \
179 } while (0)
180 
181 #define	LDC_ABORT(lp)	ldc_abort((lp), __func__)
182 
state_to_str(u8 state)183 static const char *state_to_str(u8 state)
184 {
185 	switch (state) {
186 	case LDC_STATE_INVALID:
187 		return "INVALID";
188 	case LDC_STATE_INIT:
189 		return "INIT";
190 	case LDC_STATE_BOUND:
191 		return "BOUND";
192 	case LDC_STATE_READY:
193 		return "READY";
194 	case LDC_STATE_CONNECTED:
195 		return "CONNECTED";
196 	default:
197 		return "<UNKNOWN>";
198 	}
199 }
200 
__advance(unsigned long off,unsigned long num_entries)201 static unsigned long __advance(unsigned long off, unsigned long num_entries)
202 {
203 	off += LDC_PACKET_SIZE;
204 	if (off == (num_entries * LDC_PACKET_SIZE))
205 		off = 0;
206 
207 	return off;
208 }
209 
rx_advance(struct ldc_channel * lp,unsigned long off)210 static unsigned long rx_advance(struct ldc_channel *lp, unsigned long off)
211 {
212 	return __advance(off, lp->rx_num_entries);
213 }
214 
tx_advance(struct ldc_channel * lp,unsigned long off)215 static unsigned long tx_advance(struct ldc_channel *lp, unsigned long off)
216 {
217 	return __advance(off, lp->tx_num_entries);
218 }
219 
handshake_get_tx_packet(struct ldc_channel * lp,unsigned long * new_tail)220 static struct ldc_packet *handshake_get_tx_packet(struct ldc_channel *lp,
221 						  unsigned long *new_tail)
222 {
223 	struct ldc_packet *p;
224 	unsigned long t;
225 
226 	t = tx_advance(lp, lp->tx_tail);
227 	if (t == lp->tx_head)
228 		return NULL;
229 
230 	*new_tail = t;
231 
232 	p = lp->tx_base;
233 	return p + (lp->tx_tail / LDC_PACKET_SIZE);
234 }
235 
236 /* When we are in reliable or stream mode, have to track the next packet
237  * we haven't gotten an ACK for in the TX queue using tx_acked.  We have
238  * to be careful not to stomp over the queue past that point.  During
239  * the handshake, we don't have TX data packets pending in the queue
240  * and that's why handshake_get_tx_packet() need not be mindful of
241  * lp->tx_acked.
242  */
head_for_data(struct ldc_channel * lp)243 static unsigned long head_for_data(struct ldc_channel *lp)
244 {
245 	if (lp->cfg.mode == LDC_MODE_STREAM)
246 		return lp->tx_acked;
247 	return lp->tx_head;
248 }
249 
tx_has_space_for(struct ldc_channel * lp,unsigned int size)250 static int tx_has_space_for(struct ldc_channel *lp, unsigned int size)
251 {
252 	unsigned long limit, tail, new_tail, diff;
253 	unsigned int mss;
254 
255 	limit = head_for_data(lp);
256 	tail = lp->tx_tail;
257 	new_tail = tx_advance(lp, tail);
258 	if (new_tail == limit)
259 		return 0;
260 
261 	if (limit > new_tail)
262 		diff = limit - new_tail;
263 	else
264 		diff = (limit +
265 			((lp->tx_num_entries * LDC_PACKET_SIZE) - new_tail));
266 	diff /= LDC_PACKET_SIZE;
267 	mss = lp->mss;
268 
269 	if (diff * mss < size)
270 		return 0;
271 
272 	return 1;
273 }
274 
data_get_tx_packet(struct ldc_channel * lp,unsigned long * new_tail)275 static struct ldc_packet *data_get_tx_packet(struct ldc_channel *lp,
276 					     unsigned long *new_tail)
277 {
278 	struct ldc_packet *p;
279 	unsigned long h, t;
280 
281 	h = head_for_data(lp);
282 	t = tx_advance(lp, lp->tx_tail);
283 	if (t == h)
284 		return NULL;
285 
286 	*new_tail = t;
287 
288 	p = lp->tx_base;
289 	return p + (lp->tx_tail / LDC_PACKET_SIZE);
290 }
291 
set_tx_tail(struct ldc_channel * lp,unsigned long tail)292 static int set_tx_tail(struct ldc_channel *lp, unsigned long tail)
293 {
294 	unsigned long orig_tail = lp->tx_tail;
295 	int limit = 1000;
296 
297 	lp->tx_tail = tail;
298 	while (limit-- > 0) {
299 		unsigned long err;
300 
301 		err = sun4v_ldc_tx_set_qtail(lp->id, tail);
302 		if (!err)
303 			return 0;
304 
305 		if (err != HV_EWOULDBLOCK) {
306 			lp->tx_tail = orig_tail;
307 			return -EINVAL;
308 		}
309 		udelay(1);
310 	}
311 
312 	lp->tx_tail = orig_tail;
313 	return -EBUSY;
314 }
315 
316 /* This just updates the head value in the hypervisor using
317  * a polling loop with a timeout.  The caller takes care of
318  * upating software state representing the head change, if any.
319  */
__set_rx_head(struct ldc_channel * lp,unsigned long head)320 static int __set_rx_head(struct ldc_channel *lp, unsigned long head)
321 {
322 	int limit = 1000;
323 
324 	while (limit-- > 0) {
325 		unsigned long err;
326 
327 		err = sun4v_ldc_rx_set_qhead(lp->id, head);
328 		if (!err)
329 			return 0;
330 
331 		if (err != HV_EWOULDBLOCK)
332 			return -EINVAL;
333 
334 		udelay(1);
335 	}
336 
337 	return -EBUSY;
338 }
339 
send_tx_packet(struct ldc_channel * lp,struct ldc_packet * p,unsigned long new_tail)340 static int send_tx_packet(struct ldc_channel *lp,
341 			  struct ldc_packet *p,
342 			  unsigned long new_tail)
343 {
344 	BUG_ON(p != (lp->tx_base + (lp->tx_tail / LDC_PACKET_SIZE)));
345 
346 	return set_tx_tail(lp, new_tail);
347 }
348 
handshake_compose_ctrl(struct ldc_channel * lp,u8 stype,u8 ctrl,void * data,int dlen,unsigned long * new_tail)349 static struct ldc_packet *handshake_compose_ctrl(struct ldc_channel *lp,
350 						 u8 stype, u8 ctrl,
351 						 void *data, int dlen,
352 						 unsigned long *new_tail)
353 {
354 	struct ldc_packet *p = handshake_get_tx_packet(lp, new_tail);
355 
356 	if (p) {
357 		memset(p, 0, sizeof(*p));
358 		p->type = LDC_CTRL;
359 		p->stype = stype;
360 		p->ctrl = ctrl;
361 		if (data)
362 			memcpy(p->u.u_data, data, dlen);
363 	}
364 	return p;
365 }
366 
start_handshake(struct ldc_channel * lp)367 static int start_handshake(struct ldc_channel *lp)
368 {
369 	struct ldc_packet *p;
370 	struct ldc_version *ver;
371 	unsigned long new_tail;
372 
373 	ver = &ver_arr[0];
374 
375 	ldcdbg(HS, "SEND VER INFO maj[%u] min[%u]\n",
376 	       ver->major, ver->minor);
377 
378 	p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS,
379 				   ver, sizeof(*ver), &new_tail);
380 	if (p) {
381 		int err = send_tx_packet(lp, p, new_tail);
382 		if (!err)
383 			lp->flags &= ~LDC_FLAG_RESET;
384 		return err;
385 	}
386 	return -EBUSY;
387 }
388 
send_version_nack(struct ldc_channel * lp,u16 major,u16 minor)389 static int send_version_nack(struct ldc_channel *lp,
390 			     u16 major, u16 minor)
391 {
392 	struct ldc_packet *p;
393 	struct ldc_version ver;
394 	unsigned long new_tail;
395 
396 	ver.major = major;
397 	ver.minor = minor;
398 
399 	p = handshake_compose_ctrl(lp, LDC_NACK, LDC_VERS,
400 				   &ver, sizeof(ver), &new_tail);
401 	if (p) {
402 		ldcdbg(HS, "SEND VER NACK maj[%u] min[%u]\n",
403 		       ver.major, ver.minor);
404 
405 		return send_tx_packet(lp, p, new_tail);
406 	}
407 	return -EBUSY;
408 }
409 
send_version_ack(struct ldc_channel * lp,struct ldc_version * vp)410 static int send_version_ack(struct ldc_channel *lp,
411 			    struct ldc_version *vp)
412 {
413 	struct ldc_packet *p;
414 	unsigned long new_tail;
415 
416 	p = handshake_compose_ctrl(lp, LDC_ACK, LDC_VERS,
417 				   vp, sizeof(*vp), &new_tail);
418 	if (p) {
419 		ldcdbg(HS, "SEND VER ACK maj[%u] min[%u]\n",
420 		       vp->major, vp->minor);
421 
422 		return send_tx_packet(lp, p, new_tail);
423 	}
424 	return -EBUSY;
425 }
426 
send_rts(struct ldc_channel * lp)427 static int send_rts(struct ldc_channel *lp)
428 {
429 	struct ldc_packet *p;
430 	unsigned long new_tail;
431 
432 	p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTS, NULL, 0,
433 				   &new_tail);
434 	if (p) {
435 		p->env = lp->cfg.mode;
436 		p->seqid = 0;
437 		lp->rcv_nxt = 0;
438 
439 		ldcdbg(HS, "SEND RTS env[0x%x] seqid[0x%x]\n",
440 		       p->env, p->seqid);
441 
442 		return send_tx_packet(lp, p, new_tail);
443 	}
444 	return -EBUSY;
445 }
446 
send_rtr(struct ldc_channel * lp)447 static int send_rtr(struct ldc_channel *lp)
448 {
449 	struct ldc_packet *p;
450 	unsigned long new_tail;
451 
452 	p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTR, NULL, 0,
453 				   &new_tail);
454 	if (p) {
455 		p->env = lp->cfg.mode;
456 		p->seqid = 0;
457 
458 		ldcdbg(HS, "SEND RTR env[0x%x] seqid[0x%x]\n",
459 		       p->env, p->seqid);
460 
461 		return send_tx_packet(lp, p, new_tail);
462 	}
463 	return -EBUSY;
464 }
465 
send_rdx(struct ldc_channel * lp)466 static int send_rdx(struct ldc_channel *lp)
467 {
468 	struct ldc_packet *p;
469 	unsigned long new_tail;
470 
471 	p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RDX, NULL, 0,
472 				   &new_tail);
473 	if (p) {
474 		p->env = 0;
475 		p->seqid = ++lp->snd_nxt;
476 		p->u.r.ackid = lp->rcv_nxt;
477 
478 		ldcdbg(HS, "SEND RDX env[0x%x] seqid[0x%x] ackid[0x%x]\n",
479 		       p->env, p->seqid, p->u.r.ackid);
480 
481 		return send_tx_packet(lp, p, new_tail);
482 	}
483 	return -EBUSY;
484 }
485 
send_data_nack(struct ldc_channel * lp,struct ldc_packet * data_pkt)486 static int send_data_nack(struct ldc_channel *lp, struct ldc_packet *data_pkt)
487 {
488 	struct ldc_packet *p;
489 	unsigned long new_tail;
490 	int err;
491 
492 	p = data_get_tx_packet(lp, &new_tail);
493 	if (!p)
494 		return -EBUSY;
495 	memset(p, 0, sizeof(*p));
496 	p->type = data_pkt->type;
497 	p->stype = LDC_NACK;
498 	p->ctrl = data_pkt->ctrl & LDC_CTRL_MSK;
499 	p->seqid = lp->snd_nxt + 1;
500 	p->u.r.ackid = lp->rcv_nxt;
501 
502 	ldcdbg(HS, "SEND DATA NACK type[0x%x] ctl[0x%x] seq[0x%x] ack[0x%x]\n",
503 	       p->type, p->ctrl, p->seqid, p->u.r.ackid);
504 
505 	err = send_tx_packet(lp, p, new_tail);
506 	if (!err)
507 		lp->snd_nxt++;
508 
509 	return err;
510 }
511 
ldc_abort(struct ldc_channel * lp,const char * msg)512 static int ldc_abort(struct ldc_channel *lp, const char *msg)
513 {
514 	unsigned long hv_err;
515 
516 	ldcdbg(STATE, "ABORT[%s]\n", msg);
517 	ldc_print(lp);
518 
519 	/* We report but do not act upon the hypervisor errors because
520 	 * there really isn't much we can do if they fail at this point.
521 	 */
522 	hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
523 	if (hv_err)
524 		printk(KERN_ERR PFX "ldc_abort: "
525 		       "sun4v_ldc_tx_qconf(%lx,%lx,%lx) failed, err=%lu\n",
526 		       lp->id, lp->tx_ra, lp->tx_num_entries, hv_err);
527 
528 	hv_err = sun4v_ldc_tx_get_state(lp->id,
529 					&lp->tx_head,
530 					&lp->tx_tail,
531 					&lp->chan_state);
532 	if (hv_err)
533 		printk(KERN_ERR PFX "ldc_abort: "
534 		       "sun4v_ldc_tx_get_state(%lx,...) failed, err=%lu\n",
535 		       lp->id, hv_err);
536 
537 	hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
538 	if (hv_err)
539 		printk(KERN_ERR PFX "ldc_abort: "
540 		       "sun4v_ldc_rx_qconf(%lx,%lx,%lx) failed, err=%lu\n",
541 		       lp->id, lp->rx_ra, lp->rx_num_entries, hv_err);
542 
543 	/* Refetch the RX queue state as well, because we could be invoked
544 	 * here in the queue processing context.
545 	 */
546 	hv_err = sun4v_ldc_rx_get_state(lp->id,
547 					&lp->rx_head,
548 					&lp->rx_tail,
549 					&lp->chan_state);
550 	if (hv_err)
551 		printk(KERN_ERR PFX "ldc_abort: "
552 		       "sun4v_ldc_rx_get_state(%lx,...) failed, err=%lu\n",
553 		       lp->id, hv_err);
554 
555 	return -ECONNRESET;
556 }
557 
find_by_major(u16 major)558 static struct ldc_version *find_by_major(u16 major)
559 {
560 	struct ldc_version *ret = NULL;
561 	int i;
562 
563 	for (i = 0; i < ARRAY_SIZE(ver_arr); i++) {
564 		struct ldc_version *v = &ver_arr[i];
565 		if (v->major <= major) {
566 			ret = v;
567 			break;
568 		}
569 	}
570 	return ret;
571 }
572 
process_ver_info(struct ldc_channel * lp,struct ldc_version * vp)573 static int process_ver_info(struct ldc_channel *lp, struct ldc_version *vp)
574 {
575 	struct ldc_version *vap;
576 	int err;
577 
578 	ldcdbg(HS, "GOT VERSION INFO major[%x] minor[%x]\n",
579 	       vp->major, vp->minor);
580 
581 	if (lp->hs_state == LDC_HS_GOTVERS) {
582 		lp->hs_state = LDC_HS_OPEN;
583 		memset(&lp->ver, 0, sizeof(lp->ver));
584 	}
585 
586 	vap = find_by_major(vp->major);
587 	if (!vap) {
588 		err = send_version_nack(lp, 0, 0);
589 	} else if (vap->major != vp->major) {
590 		err = send_version_nack(lp, vap->major, vap->minor);
591 	} else {
592 		struct ldc_version ver = *vp;
593 		if (ver.minor > vap->minor)
594 			ver.minor = vap->minor;
595 		err = send_version_ack(lp, &ver);
596 		if (!err) {
597 			lp->ver = ver;
598 			lp->hs_state = LDC_HS_GOTVERS;
599 		}
600 	}
601 	if (err)
602 		return LDC_ABORT(lp);
603 
604 	return 0;
605 }
606 
process_ver_ack(struct ldc_channel * lp,struct ldc_version * vp)607 static int process_ver_ack(struct ldc_channel *lp, struct ldc_version *vp)
608 {
609 	ldcdbg(HS, "GOT VERSION ACK major[%x] minor[%x]\n",
610 	       vp->major, vp->minor);
611 
612 	if (lp->hs_state == LDC_HS_GOTVERS) {
613 		if (lp->ver.major != vp->major ||
614 		    lp->ver.minor != vp->minor)
615 			return LDC_ABORT(lp);
616 	} else {
617 		lp->ver = *vp;
618 		lp->hs_state = LDC_HS_GOTVERS;
619 	}
620 	if (send_rts(lp))
621 		return LDC_ABORT(lp);
622 	return 0;
623 }
624 
process_ver_nack(struct ldc_channel * lp,struct ldc_version * vp)625 static int process_ver_nack(struct ldc_channel *lp, struct ldc_version *vp)
626 {
627 	struct ldc_version *vap;
628 	struct ldc_packet *p;
629 	unsigned long new_tail;
630 
631 	if (vp->major == 0 && vp->minor == 0)
632 		return LDC_ABORT(lp);
633 
634 	vap = find_by_major(vp->major);
635 	if (!vap)
636 		return LDC_ABORT(lp);
637 
638 	p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS,
639 					   vap, sizeof(*vap),
640 					   &new_tail);
641 	if (!p)
642 		return LDC_ABORT(lp);
643 
644 	return send_tx_packet(lp, p, new_tail);
645 }
646 
process_version(struct ldc_channel * lp,struct ldc_packet * p)647 static int process_version(struct ldc_channel *lp,
648 			   struct ldc_packet *p)
649 {
650 	struct ldc_version *vp;
651 
652 	vp = (struct ldc_version *) p->u.u_data;
653 
654 	switch (p->stype) {
655 	case LDC_INFO:
656 		return process_ver_info(lp, vp);
657 
658 	case LDC_ACK:
659 		return process_ver_ack(lp, vp);
660 
661 	case LDC_NACK:
662 		return process_ver_nack(lp, vp);
663 
664 	default:
665 		return LDC_ABORT(lp);
666 	}
667 }
668 
process_rts(struct ldc_channel * lp,struct ldc_packet * p)669 static int process_rts(struct ldc_channel *lp,
670 		       struct ldc_packet *p)
671 {
672 	ldcdbg(HS, "GOT RTS stype[%x] seqid[%x] env[%x]\n",
673 	       p->stype, p->seqid, p->env);
674 
675 	if (p->stype     != LDC_INFO	   ||
676 	    lp->hs_state != LDC_HS_GOTVERS ||
677 	    p->env       != lp->cfg.mode)
678 		return LDC_ABORT(lp);
679 
680 	lp->snd_nxt = p->seqid;
681 	lp->rcv_nxt = p->seqid;
682 	lp->hs_state = LDC_HS_SENTRTR;
683 	if (send_rtr(lp))
684 		return LDC_ABORT(lp);
685 
686 	return 0;
687 }
688 
process_rtr(struct ldc_channel * lp,struct ldc_packet * p)689 static int process_rtr(struct ldc_channel *lp,
690 		       struct ldc_packet *p)
691 {
692 	ldcdbg(HS, "GOT RTR stype[%x] seqid[%x] env[%x]\n",
693 	       p->stype, p->seqid, p->env);
694 
695 	if (p->stype     != LDC_INFO ||
696 	    p->env       != lp->cfg.mode)
697 		return LDC_ABORT(lp);
698 
699 	lp->snd_nxt = p->seqid;
700 	lp->hs_state = LDC_HS_COMPLETE;
701 	ldc_set_state(lp, LDC_STATE_CONNECTED);
702 	send_rdx(lp);
703 
704 	return LDC_EVENT_UP;
705 }
706 
rx_seq_ok(struct ldc_channel * lp,u32 seqid)707 static int rx_seq_ok(struct ldc_channel *lp, u32 seqid)
708 {
709 	return lp->rcv_nxt + 1 == seqid;
710 }
711 
process_rdx(struct ldc_channel * lp,struct ldc_packet * p)712 static int process_rdx(struct ldc_channel *lp,
713 		       struct ldc_packet *p)
714 {
715 	ldcdbg(HS, "GOT RDX stype[%x] seqid[%x] env[%x] ackid[%x]\n",
716 	       p->stype, p->seqid, p->env, p->u.r.ackid);
717 
718 	if (p->stype != LDC_INFO ||
719 	    !(rx_seq_ok(lp, p->seqid)))
720 		return LDC_ABORT(lp);
721 
722 	lp->rcv_nxt = p->seqid;
723 
724 	lp->hs_state = LDC_HS_COMPLETE;
725 	ldc_set_state(lp, LDC_STATE_CONNECTED);
726 
727 	return LDC_EVENT_UP;
728 }
729 
process_control_frame(struct ldc_channel * lp,struct ldc_packet * p)730 static int process_control_frame(struct ldc_channel *lp,
731 				 struct ldc_packet *p)
732 {
733 	switch (p->ctrl) {
734 	case LDC_VERS:
735 		return process_version(lp, p);
736 
737 	case LDC_RTS:
738 		return process_rts(lp, p);
739 
740 	case LDC_RTR:
741 		return process_rtr(lp, p);
742 
743 	case LDC_RDX:
744 		return process_rdx(lp, p);
745 
746 	default:
747 		return LDC_ABORT(lp);
748 	}
749 }
750 
process_error_frame(struct ldc_channel * lp,struct ldc_packet * p)751 static int process_error_frame(struct ldc_channel *lp,
752 			       struct ldc_packet *p)
753 {
754 	return LDC_ABORT(lp);
755 }
756 
process_data_ack(struct ldc_channel * lp,struct ldc_packet * ack)757 static int process_data_ack(struct ldc_channel *lp,
758 			    struct ldc_packet *ack)
759 {
760 	unsigned long head = lp->tx_acked;
761 	u32 ackid = ack->u.r.ackid;
762 
763 	while (1) {
764 		struct ldc_packet *p = lp->tx_base + (head / LDC_PACKET_SIZE);
765 
766 		head = tx_advance(lp, head);
767 
768 		if (p->seqid == ackid) {
769 			lp->tx_acked = head;
770 			return 0;
771 		}
772 		if (head == lp->tx_tail)
773 			return LDC_ABORT(lp);
774 	}
775 
776 	return 0;
777 }
778 
send_events(struct ldc_channel * lp,unsigned int event_mask)779 static void send_events(struct ldc_channel *lp, unsigned int event_mask)
780 {
781 	if (event_mask & LDC_EVENT_RESET)
782 		lp->cfg.event(lp->event_arg, LDC_EVENT_RESET);
783 	if (event_mask & LDC_EVENT_UP)
784 		lp->cfg.event(lp->event_arg, LDC_EVENT_UP);
785 	if (event_mask & LDC_EVENT_DATA_READY)
786 		lp->cfg.event(lp->event_arg, LDC_EVENT_DATA_READY);
787 }
788 
ldc_rx(int irq,void * dev_id)789 static irqreturn_t ldc_rx(int irq, void *dev_id)
790 {
791 	struct ldc_channel *lp = dev_id;
792 	unsigned long orig_state, flags;
793 	unsigned int event_mask;
794 
795 	spin_lock_irqsave(&lp->lock, flags);
796 
797 	orig_state = lp->chan_state;
798 
799 	/* We should probably check for hypervisor errors here and
800 	 * reset the LDC channel if we get one.
801 	 */
802 	sun4v_ldc_rx_get_state(lp->id,
803 			       &lp->rx_head,
804 			       &lp->rx_tail,
805 			       &lp->chan_state);
806 
807 	ldcdbg(RX, "RX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n",
808 	       orig_state, lp->chan_state, lp->rx_head, lp->rx_tail);
809 
810 	event_mask = 0;
811 
812 	if (lp->cfg.mode == LDC_MODE_RAW &&
813 	    lp->chan_state == LDC_CHANNEL_UP) {
814 		lp->hs_state = LDC_HS_COMPLETE;
815 		ldc_set_state(lp, LDC_STATE_CONNECTED);
816 
817 		/*
818 		 * Generate an LDC_EVENT_UP event if the channel
819 		 * was not already up.
820 		 */
821 		if (orig_state != LDC_CHANNEL_UP) {
822 			event_mask |= LDC_EVENT_UP;
823 			orig_state = lp->chan_state;
824 		}
825 	}
826 
827 	/* If we are in reset state, flush the RX queue and ignore
828 	 * everything.
829 	 */
830 	if (lp->flags & LDC_FLAG_RESET) {
831 		(void) ldc_rx_reset(lp);
832 		goto out;
833 	}
834 
835 	/* Once we finish the handshake, we let the ldc_read()
836 	 * paths do all of the control frame and state management.
837 	 * Just trigger the callback.
838 	 */
839 	if (lp->hs_state == LDC_HS_COMPLETE) {
840 handshake_complete:
841 		if (lp->chan_state != orig_state) {
842 			unsigned int event = LDC_EVENT_RESET;
843 
844 			if (lp->chan_state == LDC_CHANNEL_UP)
845 				event = LDC_EVENT_UP;
846 
847 			event_mask |= event;
848 		}
849 		if (lp->rx_head != lp->rx_tail)
850 			event_mask |= LDC_EVENT_DATA_READY;
851 
852 		goto out;
853 	}
854 
855 	if (lp->chan_state != orig_state)
856 		goto out;
857 
858 	while (lp->rx_head != lp->rx_tail) {
859 		struct ldc_packet *p;
860 		unsigned long new;
861 		int err;
862 
863 		p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE);
864 
865 		switch (p->type) {
866 		case LDC_CTRL:
867 			err = process_control_frame(lp, p);
868 			if (err > 0)
869 				event_mask |= err;
870 			break;
871 
872 		case LDC_DATA:
873 			event_mask |= LDC_EVENT_DATA_READY;
874 			err = 0;
875 			break;
876 
877 		case LDC_ERR:
878 			err = process_error_frame(lp, p);
879 			break;
880 
881 		default:
882 			err = LDC_ABORT(lp);
883 			break;
884 		}
885 
886 		if (err < 0)
887 			break;
888 
889 		new = lp->rx_head;
890 		new += LDC_PACKET_SIZE;
891 		if (new == (lp->rx_num_entries * LDC_PACKET_SIZE))
892 			new = 0;
893 		lp->rx_head = new;
894 
895 		err = __set_rx_head(lp, new);
896 		if (err < 0) {
897 			(void) LDC_ABORT(lp);
898 			break;
899 		}
900 		if (lp->hs_state == LDC_HS_COMPLETE)
901 			goto handshake_complete;
902 	}
903 
904 out:
905 	spin_unlock_irqrestore(&lp->lock, flags);
906 
907 	send_events(lp, event_mask);
908 
909 	return IRQ_HANDLED;
910 }
911 
ldc_tx(int irq,void * dev_id)912 static irqreturn_t ldc_tx(int irq, void *dev_id)
913 {
914 	struct ldc_channel *lp = dev_id;
915 	unsigned long flags, orig_state;
916 	unsigned int event_mask = 0;
917 
918 	spin_lock_irqsave(&lp->lock, flags);
919 
920 	orig_state = lp->chan_state;
921 
922 	/* We should probably check for hypervisor errors here and
923 	 * reset the LDC channel if we get one.
924 	 */
925 	sun4v_ldc_tx_get_state(lp->id,
926 			       &lp->tx_head,
927 			       &lp->tx_tail,
928 			       &lp->chan_state);
929 
930 	ldcdbg(TX, " TX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n",
931 	       orig_state, lp->chan_state, lp->tx_head, lp->tx_tail);
932 
933 	if (lp->cfg.mode == LDC_MODE_RAW &&
934 	    lp->chan_state == LDC_CHANNEL_UP) {
935 		lp->hs_state = LDC_HS_COMPLETE;
936 		ldc_set_state(lp, LDC_STATE_CONNECTED);
937 
938 		/*
939 		 * Generate an LDC_EVENT_UP event if the channel
940 		 * was not already up.
941 		 */
942 		if (orig_state != LDC_CHANNEL_UP) {
943 			event_mask |= LDC_EVENT_UP;
944 			orig_state = lp->chan_state;
945 		}
946 	}
947 
948 	spin_unlock_irqrestore(&lp->lock, flags);
949 
950 	send_events(lp, event_mask);
951 
952 	return IRQ_HANDLED;
953 }
954 
955 /* XXX ldc_alloc() and ldc_free() needs to run under a mutex so
956  * XXX that addition and removal from the ldc_channel_list has
957  * XXX atomicity, otherwise the __ldc_channel_exists() check is
958  * XXX totally pointless as another thread can slip into ldc_alloc()
959  * XXX and add a channel with the same ID.  There also needs to be
960  * XXX a spinlock for ldc_channel_list.
961  */
962 static HLIST_HEAD(ldc_channel_list);
963 
__ldc_channel_exists(unsigned long id)964 static int __ldc_channel_exists(unsigned long id)
965 {
966 	struct ldc_channel *lp;
967 
968 	hlist_for_each_entry(lp, &ldc_channel_list, list) {
969 		if (lp->id == id)
970 			return 1;
971 	}
972 	return 0;
973 }
974 
alloc_queue(const char * name,unsigned long num_entries,struct ldc_packet ** base,unsigned long * ra)975 static int alloc_queue(const char *name, unsigned long num_entries,
976 		       struct ldc_packet **base, unsigned long *ra)
977 {
978 	unsigned long size, order;
979 	void *q;
980 
981 	size = num_entries * LDC_PACKET_SIZE;
982 	order = get_order(size);
983 
984 	q = (void *) __get_free_pages(GFP_KERNEL, order);
985 	if (!q) {
986 		printk(KERN_ERR PFX "Alloc of %s queue failed with "
987 		       "size=%lu order=%lu\n", name, size, order);
988 		return -ENOMEM;
989 	}
990 
991 	memset(q, 0, PAGE_SIZE << order);
992 
993 	*base = q;
994 	*ra = __pa(q);
995 
996 	return 0;
997 }
998 
free_queue(unsigned long num_entries,struct ldc_packet * q)999 static void free_queue(unsigned long num_entries, struct ldc_packet *q)
1000 {
1001 	unsigned long size, order;
1002 
1003 	if (!q)
1004 		return;
1005 
1006 	size = num_entries * LDC_PACKET_SIZE;
1007 	order = get_order(size);
1008 
1009 	free_pages((unsigned long)q, order);
1010 }
1011 
ldc_cookie_to_index(u64 cookie,void * arg)1012 static unsigned long ldc_cookie_to_index(u64 cookie, void *arg)
1013 {
1014 	u64 szcode = cookie >> COOKIE_PGSZ_CODE_SHIFT;
1015 	/* struct ldc_iommu *ldc_iommu = (struct ldc_iommu *)arg; */
1016 
1017 	cookie &= ~COOKIE_PGSZ_CODE;
1018 
1019 	return (cookie >> (13ULL + (szcode * 3ULL)));
1020 }
1021 
ldc_demap(struct ldc_iommu * iommu,unsigned long id,u64 cookie,unsigned long entry,unsigned long npages)1022 static void ldc_demap(struct ldc_iommu *iommu, unsigned long id, u64 cookie,
1023 		      unsigned long entry, unsigned long npages)
1024 {
1025 	struct ldc_mtable_entry *base;
1026 	unsigned long i, shift;
1027 
1028 	shift = (cookie >> COOKIE_PGSZ_CODE_SHIFT) * 3;
1029 	base = iommu->page_table + entry;
1030 	for (i = 0; i < npages; i++) {
1031 		if (base->cookie)
1032 			sun4v_ldc_revoke(id, cookie + (i << shift),
1033 					 base->cookie);
1034 		base->mte = 0;
1035 	}
1036 }
1037 
1038 /* XXX Make this configurable... XXX */
1039 #define LDC_IOTABLE_SIZE	(8 * 1024)
1040 
ldc_iommu_init(const char * name,struct ldc_channel * lp)1041 static int ldc_iommu_init(const char *name, struct ldc_channel *lp)
1042 {
1043 	unsigned long sz, num_tsb_entries, tsbsize, order;
1044 	struct ldc_iommu *ldc_iommu = &lp->iommu;
1045 	struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table;
1046 	struct ldc_mtable_entry *table;
1047 	unsigned long hv_err;
1048 	int err;
1049 
1050 	num_tsb_entries = LDC_IOTABLE_SIZE;
1051 	tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry);
1052 	spin_lock_init(&ldc_iommu->lock);
1053 
1054 	sz = num_tsb_entries / 8;
1055 	sz = (sz + 7UL) & ~7UL;
1056 	iommu->map = kzalloc(sz, GFP_KERNEL);
1057 	if (!iommu->map) {
1058 		printk(KERN_ERR PFX "Alloc of arena map failed, sz=%lu\n", sz);
1059 		return -ENOMEM;
1060 	}
1061 	iommu_tbl_pool_init(iommu, num_tsb_entries, PAGE_SHIFT,
1062 			    NULL, false /* no large pool */,
1063 			    1 /* npools */,
1064 			    true /* skip span boundary check */);
1065 
1066 	order = get_order(tsbsize);
1067 
1068 	table = (struct ldc_mtable_entry *)
1069 		__get_free_pages(GFP_KERNEL, order);
1070 	err = -ENOMEM;
1071 	if (!table) {
1072 		printk(KERN_ERR PFX "Alloc of MTE table failed, "
1073 		       "size=%lu order=%lu\n", tsbsize, order);
1074 		goto out_free_map;
1075 	}
1076 
1077 	memset(table, 0, PAGE_SIZE << order);
1078 
1079 	ldc_iommu->page_table = table;
1080 
1081 	hv_err = sun4v_ldc_set_map_table(lp->id, __pa(table),
1082 					 num_tsb_entries);
1083 	err = -EINVAL;
1084 	if (hv_err)
1085 		goto out_free_table;
1086 
1087 	return 0;
1088 
1089 out_free_table:
1090 	free_pages((unsigned long) table, order);
1091 	ldc_iommu->page_table = NULL;
1092 
1093 out_free_map:
1094 	kfree(iommu->map);
1095 	iommu->map = NULL;
1096 
1097 	return err;
1098 }
1099 
ldc_iommu_release(struct ldc_channel * lp)1100 static void ldc_iommu_release(struct ldc_channel *lp)
1101 {
1102 	struct ldc_iommu *ldc_iommu = &lp->iommu;
1103 	struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table;
1104 	unsigned long num_tsb_entries, tsbsize, order;
1105 
1106 	(void) sun4v_ldc_set_map_table(lp->id, 0, 0);
1107 
1108 	num_tsb_entries = iommu->poolsize * iommu->nr_pools;
1109 	tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry);
1110 	order = get_order(tsbsize);
1111 
1112 	free_pages((unsigned long) ldc_iommu->page_table, order);
1113 	ldc_iommu->page_table = NULL;
1114 
1115 	kfree(iommu->map);
1116 	iommu->map = NULL;
1117 }
1118 
ldc_alloc(unsigned long id,const struct ldc_channel_config * cfgp,void * event_arg,const char * name)1119 struct ldc_channel *ldc_alloc(unsigned long id,
1120 			      const struct ldc_channel_config *cfgp,
1121 			      void *event_arg,
1122 			      const char *name)
1123 {
1124 	struct ldc_channel *lp;
1125 	const struct ldc_mode_ops *mops;
1126 	unsigned long dummy1, dummy2, hv_err;
1127 	u8 mss, *mssbuf;
1128 	int err;
1129 
1130 	err = -ENODEV;
1131 	if (!ldom_domaining_enabled)
1132 		goto out_err;
1133 
1134 	err = -EINVAL;
1135 	if (!cfgp)
1136 		goto out_err;
1137 	if (!name)
1138 		goto out_err;
1139 
1140 	switch (cfgp->mode) {
1141 	case LDC_MODE_RAW:
1142 		mops = &raw_ops;
1143 		mss = LDC_PACKET_SIZE;
1144 		break;
1145 
1146 	case LDC_MODE_UNRELIABLE:
1147 		mops = &nonraw_ops;
1148 		mss = LDC_PACKET_SIZE - 8;
1149 		break;
1150 
1151 	case LDC_MODE_STREAM:
1152 		mops = &stream_ops;
1153 		mss = LDC_PACKET_SIZE - 8 - 8;
1154 		break;
1155 
1156 	default:
1157 		goto out_err;
1158 	}
1159 
1160 	if (!cfgp->event || !event_arg || !cfgp->rx_irq || !cfgp->tx_irq)
1161 		goto out_err;
1162 
1163 	hv_err = sun4v_ldc_tx_qinfo(id, &dummy1, &dummy2);
1164 	err = -ENODEV;
1165 	if (hv_err == HV_ECHANNEL)
1166 		goto out_err;
1167 
1168 	err = -EEXIST;
1169 	if (__ldc_channel_exists(id))
1170 		goto out_err;
1171 
1172 	mssbuf = NULL;
1173 
1174 	lp = kzalloc(sizeof(*lp), GFP_KERNEL);
1175 	err = -ENOMEM;
1176 	if (!lp)
1177 		goto out_err;
1178 
1179 	spin_lock_init(&lp->lock);
1180 
1181 	lp->id = id;
1182 
1183 	err = ldc_iommu_init(name, lp);
1184 	if (err)
1185 		goto out_free_ldc;
1186 
1187 	lp->mops = mops;
1188 	lp->mss = mss;
1189 
1190 	lp->cfg = *cfgp;
1191 	if (!lp->cfg.mtu)
1192 		lp->cfg.mtu = LDC_DEFAULT_MTU;
1193 
1194 	if (lp->cfg.mode == LDC_MODE_STREAM) {
1195 		mssbuf = kzalloc(lp->cfg.mtu, GFP_KERNEL);
1196 		if (!mssbuf) {
1197 			err = -ENOMEM;
1198 			goto out_free_iommu;
1199 		}
1200 		lp->mssbuf = mssbuf;
1201 	}
1202 
1203 	lp->event_arg = event_arg;
1204 
1205 	/* XXX allow setting via ldc_channel_config to override defaults
1206 	 * XXX or use some formula based upon mtu
1207 	 */
1208 	lp->tx_num_entries = LDC_DEFAULT_NUM_ENTRIES;
1209 	lp->rx_num_entries = LDC_DEFAULT_NUM_ENTRIES;
1210 
1211 	err = alloc_queue("TX", lp->tx_num_entries,
1212 			  &lp->tx_base, &lp->tx_ra);
1213 	if (err)
1214 		goto out_free_mssbuf;
1215 
1216 	err = alloc_queue("RX", lp->rx_num_entries,
1217 			  &lp->rx_base, &lp->rx_ra);
1218 	if (err)
1219 		goto out_free_txq;
1220 
1221 	lp->flags |= LDC_FLAG_ALLOCED_QUEUES;
1222 
1223 	lp->hs_state = LDC_HS_CLOSED;
1224 	ldc_set_state(lp, LDC_STATE_INIT);
1225 
1226 	INIT_HLIST_NODE(&lp->list);
1227 	hlist_add_head(&lp->list, &ldc_channel_list);
1228 
1229 	INIT_HLIST_HEAD(&lp->mh_list);
1230 
1231 	snprintf(lp->rx_irq_name, LDC_IRQ_NAME_MAX, "%s RX", name);
1232 	snprintf(lp->tx_irq_name, LDC_IRQ_NAME_MAX, "%s TX", name);
1233 
1234 	err = request_irq(lp->cfg.rx_irq, ldc_rx, 0,
1235 			  lp->rx_irq_name, lp);
1236 	if (err)
1237 		goto out_free_txq;
1238 
1239 	err = request_irq(lp->cfg.tx_irq, ldc_tx, 0,
1240 			  lp->tx_irq_name, lp);
1241 	if (err) {
1242 		free_irq(lp->cfg.rx_irq, lp);
1243 		goto out_free_txq;
1244 	}
1245 
1246 	return lp;
1247 
1248 out_free_txq:
1249 	free_queue(lp->tx_num_entries, lp->tx_base);
1250 
1251 out_free_mssbuf:
1252 	kfree(mssbuf);
1253 
1254 out_free_iommu:
1255 	ldc_iommu_release(lp);
1256 
1257 out_free_ldc:
1258 	kfree(lp);
1259 
1260 out_err:
1261 	return ERR_PTR(err);
1262 }
1263 EXPORT_SYMBOL(ldc_alloc);
1264 
ldc_unbind(struct ldc_channel * lp)1265 void ldc_unbind(struct ldc_channel *lp)
1266 {
1267 	if (lp->flags & LDC_FLAG_REGISTERED_IRQS) {
1268 		free_irq(lp->cfg.rx_irq, lp);
1269 		free_irq(lp->cfg.tx_irq, lp);
1270 		lp->flags &= ~LDC_FLAG_REGISTERED_IRQS;
1271 	}
1272 
1273 	if (lp->flags & LDC_FLAG_REGISTERED_QUEUES) {
1274 		sun4v_ldc_tx_qconf(lp->id, 0, 0);
1275 		sun4v_ldc_rx_qconf(lp->id, 0, 0);
1276 		lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES;
1277 	}
1278 	if (lp->flags & LDC_FLAG_ALLOCED_QUEUES) {
1279 		free_queue(lp->tx_num_entries, lp->tx_base);
1280 		free_queue(lp->rx_num_entries, lp->rx_base);
1281 		lp->flags &= ~LDC_FLAG_ALLOCED_QUEUES;
1282 	}
1283 
1284 	ldc_set_state(lp, LDC_STATE_INIT);
1285 }
1286 EXPORT_SYMBOL(ldc_unbind);
1287 
ldc_free(struct ldc_channel * lp)1288 void ldc_free(struct ldc_channel *lp)
1289 {
1290 	ldc_unbind(lp);
1291 	hlist_del(&lp->list);
1292 	kfree(lp->mssbuf);
1293 	ldc_iommu_release(lp);
1294 
1295 	kfree(lp);
1296 }
1297 EXPORT_SYMBOL(ldc_free);
1298 
1299 /* Bind the channel.  This registers the LDC queues with
1300  * the hypervisor and puts the channel into a pseudo-listening
1301  * state.  This does not initiate a handshake, ldc_connect() does
1302  * that.
1303  */
ldc_bind(struct ldc_channel * lp)1304 int ldc_bind(struct ldc_channel *lp)
1305 {
1306 	unsigned long hv_err, flags;
1307 	int err = -EINVAL;
1308 
1309 	if (lp->state != LDC_STATE_INIT)
1310 		return -EINVAL;
1311 
1312 	spin_lock_irqsave(&lp->lock, flags);
1313 
1314 	enable_irq(lp->cfg.rx_irq);
1315 	enable_irq(lp->cfg.tx_irq);
1316 
1317 	lp->flags |= LDC_FLAG_REGISTERED_IRQS;
1318 
1319 	err = -ENODEV;
1320 	hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0);
1321 	if (hv_err)
1322 		goto out_free_irqs;
1323 
1324 	hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
1325 	if (hv_err)
1326 		goto out_free_irqs;
1327 
1328 	hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0);
1329 	if (hv_err)
1330 		goto out_unmap_tx;
1331 
1332 	hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
1333 	if (hv_err)
1334 		goto out_unmap_tx;
1335 
1336 	lp->flags |= LDC_FLAG_REGISTERED_QUEUES;
1337 
1338 	hv_err = sun4v_ldc_tx_get_state(lp->id,
1339 					&lp->tx_head,
1340 					&lp->tx_tail,
1341 					&lp->chan_state);
1342 	err = -EBUSY;
1343 	if (hv_err)
1344 		goto out_unmap_rx;
1345 
1346 	lp->tx_acked = lp->tx_head;
1347 
1348 	lp->hs_state = LDC_HS_OPEN;
1349 	ldc_set_state(lp, LDC_STATE_BOUND);
1350 
1351 	if (lp->cfg.mode == LDC_MODE_RAW) {
1352 		/*
1353 		 * There is no handshake in RAW mode, so handshake
1354 		 * is completed.
1355 		 */
1356 		lp->hs_state = LDC_HS_COMPLETE;
1357 	}
1358 
1359 	spin_unlock_irqrestore(&lp->lock, flags);
1360 
1361 	return 0;
1362 
1363 out_unmap_rx:
1364 	lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES;
1365 	sun4v_ldc_rx_qconf(lp->id, 0, 0);
1366 
1367 out_unmap_tx:
1368 	sun4v_ldc_tx_qconf(lp->id, 0, 0);
1369 
1370 out_free_irqs:
1371 	lp->flags &= ~LDC_FLAG_REGISTERED_IRQS;
1372 	free_irq(lp->cfg.tx_irq, lp);
1373 	free_irq(lp->cfg.rx_irq, lp);
1374 
1375 	spin_unlock_irqrestore(&lp->lock, flags);
1376 
1377 	return err;
1378 }
1379 EXPORT_SYMBOL(ldc_bind);
1380 
ldc_connect(struct ldc_channel * lp)1381 int ldc_connect(struct ldc_channel *lp)
1382 {
1383 	unsigned long flags;
1384 	int err;
1385 
1386 	if (lp->cfg.mode == LDC_MODE_RAW)
1387 		return -EINVAL;
1388 
1389 	spin_lock_irqsave(&lp->lock, flags);
1390 
1391 	if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) ||
1392 	    !(lp->flags & LDC_FLAG_REGISTERED_QUEUES) ||
1393 	    lp->hs_state != LDC_HS_OPEN)
1394 		err = ((lp->hs_state > LDC_HS_OPEN) ? 0 : -EINVAL);
1395 	else
1396 		err = start_handshake(lp);
1397 
1398 	spin_unlock_irqrestore(&lp->lock, flags);
1399 
1400 	return err;
1401 }
1402 EXPORT_SYMBOL(ldc_connect);
1403 
ldc_disconnect(struct ldc_channel * lp)1404 int ldc_disconnect(struct ldc_channel *lp)
1405 {
1406 	unsigned long hv_err, flags;
1407 	int err;
1408 
1409 	if (lp->cfg.mode == LDC_MODE_RAW)
1410 		return -EINVAL;
1411 
1412 	if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) ||
1413 	    !(lp->flags & LDC_FLAG_REGISTERED_QUEUES))
1414 		return -EINVAL;
1415 
1416 	spin_lock_irqsave(&lp->lock, flags);
1417 
1418 	err = -ENODEV;
1419 	hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0);
1420 	if (hv_err)
1421 		goto out_err;
1422 
1423 	hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries);
1424 	if (hv_err)
1425 		goto out_err;
1426 
1427 	hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0);
1428 	if (hv_err)
1429 		goto out_err;
1430 
1431 	hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries);
1432 	if (hv_err)
1433 		goto out_err;
1434 
1435 	ldc_set_state(lp, LDC_STATE_BOUND);
1436 	lp->hs_state = LDC_HS_OPEN;
1437 	lp->flags |= LDC_FLAG_RESET;
1438 
1439 	spin_unlock_irqrestore(&lp->lock, flags);
1440 
1441 	return 0;
1442 
1443 out_err:
1444 	sun4v_ldc_tx_qconf(lp->id, 0, 0);
1445 	sun4v_ldc_rx_qconf(lp->id, 0, 0);
1446 	free_irq(lp->cfg.tx_irq, lp);
1447 	free_irq(lp->cfg.rx_irq, lp);
1448 	lp->flags &= ~(LDC_FLAG_REGISTERED_IRQS |
1449 		       LDC_FLAG_REGISTERED_QUEUES);
1450 	ldc_set_state(lp, LDC_STATE_INIT);
1451 
1452 	spin_unlock_irqrestore(&lp->lock, flags);
1453 
1454 	return err;
1455 }
1456 EXPORT_SYMBOL(ldc_disconnect);
1457 
ldc_state(struct ldc_channel * lp)1458 int ldc_state(struct ldc_channel *lp)
1459 {
1460 	return lp->state;
1461 }
1462 EXPORT_SYMBOL(ldc_state);
1463 
ldc_set_state(struct ldc_channel * lp,u8 state)1464 void ldc_set_state(struct ldc_channel *lp, u8 state)
1465 {
1466 	ldcdbg(STATE, "STATE (%s) --> (%s)\n",
1467 	       state_to_str(lp->state),
1468 	       state_to_str(state));
1469 
1470 	lp->state = state;
1471 }
1472 EXPORT_SYMBOL(ldc_set_state);
1473 
ldc_mode(struct ldc_channel * lp)1474 int ldc_mode(struct ldc_channel *lp)
1475 {
1476 	return lp->cfg.mode;
1477 }
1478 EXPORT_SYMBOL(ldc_mode);
1479 
ldc_rx_reset(struct ldc_channel * lp)1480 int ldc_rx_reset(struct ldc_channel *lp)
1481 {
1482 	return __set_rx_head(lp, lp->rx_tail);
1483 }
1484 EXPORT_SYMBOL(ldc_rx_reset);
1485 
__ldc_print(struct ldc_channel * lp,const char * caller)1486 void __ldc_print(struct ldc_channel *lp, const char *caller)
1487 {
1488 	pr_info("%s: id=0x%lx flags=0x%x state=%s cstate=0x%lx hsstate=0x%x\n"
1489 		"\trx_h=0x%lx rx_t=0x%lx rx_n=%ld\n"
1490 		"\ttx_h=0x%lx tx_t=0x%lx tx_n=%ld\n"
1491 		"\trcv_nxt=%u snd_nxt=%u\n",
1492 		caller, lp->id, lp->flags, state_to_str(lp->state),
1493 		lp->chan_state, lp->hs_state,
1494 		lp->rx_head, lp->rx_tail, lp->rx_num_entries,
1495 		lp->tx_head, lp->tx_tail, lp->tx_num_entries,
1496 		lp->rcv_nxt, lp->snd_nxt);
1497 }
1498 EXPORT_SYMBOL(__ldc_print);
1499 
write_raw(struct ldc_channel * lp,const void * buf,unsigned int size)1500 static int write_raw(struct ldc_channel *lp, const void *buf, unsigned int size)
1501 {
1502 	struct ldc_packet *p;
1503 	unsigned long new_tail, hv_err;
1504 	int err;
1505 
1506 	hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail,
1507 					&lp->chan_state);
1508 	if (unlikely(hv_err))
1509 		return -EBUSY;
1510 
1511 	if (unlikely(lp->chan_state != LDC_CHANNEL_UP))
1512 		return LDC_ABORT(lp);
1513 
1514 	if (size > LDC_PACKET_SIZE)
1515 		return -EMSGSIZE;
1516 
1517 	p = data_get_tx_packet(lp, &new_tail);
1518 	if (!p)
1519 		return -EAGAIN;
1520 
1521 	memcpy(p, buf, size);
1522 
1523 	err = send_tx_packet(lp, p, new_tail);
1524 	if (!err)
1525 		err = size;
1526 
1527 	return err;
1528 }
1529 
read_raw(struct ldc_channel * lp,void * buf,unsigned int size)1530 static int read_raw(struct ldc_channel *lp, void *buf, unsigned int size)
1531 {
1532 	struct ldc_packet *p;
1533 	unsigned long hv_err, new;
1534 	int err;
1535 
1536 	if (size < LDC_PACKET_SIZE)
1537 		return -EINVAL;
1538 
1539 	hv_err = sun4v_ldc_rx_get_state(lp->id,
1540 					&lp->rx_head,
1541 					&lp->rx_tail,
1542 					&lp->chan_state);
1543 	if (hv_err)
1544 		return LDC_ABORT(lp);
1545 
1546 	if (lp->chan_state == LDC_CHANNEL_DOWN ||
1547 	    lp->chan_state == LDC_CHANNEL_RESETTING)
1548 		return -ECONNRESET;
1549 
1550 	if (lp->rx_head == lp->rx_tail)
1551 		return 0;
1552 
1553 	p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE);
1554 	memcpy(buf, p, LDC_PACKET_SIZE);
1555 
1556 	new = rx_advance(lp, lp->rx_head);
1557 	lp->rx_head = new;
1558 
1559 	err = __set_rx_head(lp, new);
1560 	if (err < 0)
1561 		err = -ECONNRESET;
1562 	else
1563 		err = LDC_PACKET_SIZE;
1564 
1565 	return err;
1566 }
1567 
1568 static const struct ldc_mode_ops raw_ops = {
1569 	.write		=	write_raw,
1570 	.read		=	read_raw,
1571 };
1572 
write_nonraw(struct ldc_channel * lp,const void * buf,unsigned int size)1573 static int write_nonraw(struct ldc_channel *lp, const void *buf,
1574 			unsigned int size)
1575 {
1576 	unsigned long hv_err, tail;
1577 	unsigned int copied;
1578 	u32 seq;
1579 	int err;
1580 
1581 	hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail,
1582 					&lp->chan_state);
1583 	if (unlikely(hv_err))
1584 		return -EBUSY;
1585 
1586 	if (unlikely(lp->chan_state != LDC_CHANNEL_UP))
1587 		return LDC_ABORT(lp);
1588 
1589 	if (!tx_has_space_for(lp, size))
1590 		return -EAGAIN;
1591 
1592 	seq = lp->snd_nxt;
1593 	copied = 0;
1594 	tail = lp->tx_tail;
1595 	while (copied < size) {
1596 		struct ldc_packet *p = lp->tx_base + (tail / LDC_PACKET_SIZE);
1597 		u8 *data = ((lp->cfg.mode == LDC_MODE_UNRELIABLE) ?
1598 			    p->u.u_data :
1599 			    p->u.r.r_data);
1600 		int data_len;
1601 
1602 		p->type = LDC_DATA;
1603 		p->stype = LDC_INFO;
1604 		p->ctrl = 0;
1605 
1606 		data_len = size - copied;
1607 		if (data_len > lp->mss)
1608 			data_len = lp->mss;
1609 
1610 		BUG_ON(data_len > LDC_LEN);
1611 
1612 		p->env = (data_len |
1613 			  (copied == 0 ? LDC_START : 0) |
1614 			  (data_len == size - copied ? LDC_STOP : 0));
1615 
1616 		p->seqid = ++seq;
1617 
1618 		ldcdbg(DATA, "SENT DATA [%02x:%02x:%02x:%02x:%08x]\n",
1619 		       p->type,
1620 		       p->stype,
1621 		       p->ctrl,
1622 		       p->env,
1623 		       p->seqid);
1624 
1625 		memcpy(data, buf, data_len);
1626 		buf += data_len;
1627 		copied += data_len;
1628 
1629 		tail = tx_advance(lp, tail);
1630 	}
1631 
1632 	err = set_tx_tail(lp, tail);
1633 	if (!err) {
1634 		lp->snd_nxt = seq;
1635 		err = size;
1636 	}
1637 
1638 	return err;
1639 }
1640 
rx_bad_seq(struct ldc_channel * lp,struct ldc_packet * p,struct ldc_packet * first_frag)1641 static int rx_bad_seq(struct ldc_channel *lp, struct ldc_packet *p,
1642 		      struct ldc_packet *first_frag)
1643 {
1644 	int err;
1645 
1646 	if (first_frag)
1647 		lp->rcv_nxt = first_frag->seqid - 1;
1648 
1649 	err = send_data_nack(lp, p);
1650 	if (err)
1651 		return err;
1652 
1653 	err = ldc_rx_reset(lp);
1654 	if (err < 0)
1655 		return LDC_ABORT(lp);
1656 
1657 	return 0;
1658 }
1659 
data_ack_nack(struct ldc_channel * lp,struct ldc_packet * p)1660 static int data_ack_nack(struct ldc_channel *lp, struct ldc_packet *p)
1661 {
1662 	if (p->stype & LDC_ACK) {
1663 		int err = process_data_ack(lp, p);
1664 		if (err)
1665 			return err;
1666 	}
1667 	if (p->stype & LDC_NACK)
1668 		return LDC_ABORT(lp);
1669 
1670 	return 0;
1671 }
1672 
rx_data_wait(struct ldc_channel * lp,unsigned long cur_head)1673 static int rx_data_wait(struct ldc_channel *lp, unsigned long cur_head)
1674 {
1675 	unsigned long dummy;
1676 	int limit = 1000;
1677 
1678 	ldcdbg(DATA, "DATA WAIT cur_head[%lx] rx_head[%lx] rx_tail[%lx]\n",
1679 	       cur_head, lp->rx_head, lp->rx_tail);
1680 	while (limit-- > 0) {
1681 		unsigned long hv_err;
1682 
1683 		hv_err = sun4v_ldc_rx_get_state(lp->id,
1684 						&dummy,
1685 						&lp->rx_tail,
1686 						&lp->chan_state);
1687 		if (hv_err)
1688 			return LDC_ABORT(lp);
1689 
1690 		if (lp->chan_state == LDC_CHANNEL_DOWN ||
1691 		    lp->chan_state == LDC_CHANNEL_RESETTING)
1692 			return -ECONNRESET;
1693 
1694 		if (cur_head != lp->rx_tail) {
1695 			ldcdbg(DATA, "DATA WAIT DONE "
1696 			       "head[%lx] tail[%lx] chan_state[%lx]\n",
1697 			       dummy, lp->rx_tail, lp->chan_state);
1698 			return 0;
1699 		}
1700 
1701 		udelay(1);
1702 	}
1703 	return -EAGAIN;
1704 }
1705 
rx_set_head(struct ldc_channel * lp,unsigned long head)1706 static int rx_set_head(struct ldc_channel *lp, unsigned long head)
1707 {
1708 	int err = __set_rx_head(lp, head);
1709 
1710 	if (err < 0)
1711 		return LDC_ABORT(lp);
1712 
1713 	lp->rx_head = head;
1714 	return 0;
1715 }
1716 
send_data_ack(struct ldc_channel * lp)1717 static void send_data_ack(struct ldc_channel *lp)
1718 {
1719 	unsigned long new_tail;
1720 	struct ldc_packet *p;
1721 
1722 	p = data_get_tx_packet(lp, &new_tail);
1723 	if (likely(p)) {
1724 		int err;
1725 
1726 		memset(p, 0, sizeof(*p));
1727 		p->type = LDC_DATA;
1728 		p->stype = LDC_ACK;
1729 		p->ctrl = 0;
1730 		p->seqid = lp->snd_nxt + 1;
1731 		p->u.r.ackid = lp->rcv_nxt;
1732 
1733 		err = send_tx_packet(lp, p, new_tail);
1734 		if (!err)
1735 			lp->snd_nxt++;
1736 	}
1737 }
1738 
read_nonraw(struct ldc_channel * lp,void * buf,unsigned int size)1739 static int read_nonraw(struct ldc_channel *lp, void *buf, unsigned int size)
1740 {
1741 	struct ldc_packet *first_frag;
1742 	unsigned long hv_err, new;
1743 	int err, copied;
1744 
1745 	hv_err = sun4v_ldc_rx_get_state(lp->id,
1746 					&lp->rx_head,
1747 					&lp->rx_tail,
1748 					&lp->chan_state);
1749 	if (hv_err)
1750 		return LDC_ABORT(lp);
1751 
1752 	if (lp->chan_state == LDC_CHANNEL_DOWN ||
1753 	    lp->chan_state == LDC_CHANNEL_RESETTING)
1754 		return -ECONNRESET;
1755 
1756 	if (lp->rx_head == lp->rx_tail)
1757 		return 0;
1758 
1759 	first_frag = NULL;
1760 	copied = err = 0;
1761 	new = lp->rx_head;
1762 	while (1) {
1763 		struct ldc_packet *p;
1764 		int pkt_len;
1765 
1766 		BUG_ON(new == lp->rx_tail);
1767 		p = lp->rx_base + (new / LDC_PACKET_SIZE);
1768 
1769 		ldcdbg(RX, "RX read pkt[%02x:%02x:%02x:%02x:%08x:%08x] "
1770 		       "rcv_nxt[%08x]\n",
1771 		       p->type,
1772 		       p->stype,
1773 		       p->ctrl,
1774 		       p->env,
1775 		       p->seqid,
1776 		       p->u.r.ackid,
1777 		       lp->rcv_nxt);
1778 
1779 		if (unlikely(!rx_seq_ok(lp, p->seqid))) {
1780 			err = rx_bad_seq(lp, p, first_frag);
1781 			copied = 0;
1782 			break;
1783 		}
1784 
1785 		if (p->type & LDC_CTRL) {
1786 			err = process_control_frame(lp, p);
1787 			if (err < 0)
1788 				break;
1789 			err = 0;
1790 		}
1791 
1792 		lp->rcv_nxt = p->seqid;
1793 
1794 		/*
1795 		 * If this is a control-only packet, there is nothing
1796 		 * else to do but advance the rx queue since the packet
1797 		 * was already processed above.
1798 		 */
1799 		if (!(p->type & LDC_DATA)) {
1800 			new = rx_advance(lp, new);
1801 			break;
1802 		}
1803 		if (p->stype & (LDC_ACK | LDC_NACK)) {
1804 			err = data_ack_nack(lp, p);
1805 			if (err)
1806 				break;
1807 		}
1808 		if (!(p->stype & LDC_INFO)) {
1809 			new = rx_advance(lp, new);
1810 			err = rx_set_head(lp, new);
1811 			if (err)
1812 				break;
1813 			goto no_data;
1814 		}
1815 
1816 		pkt_len = p->env & LDC_LEN;
1817 
1818 		/* Every initial packet starts with the START bit set.
1819 		 *
1820 		 * Singleton packets will have both START+STOP set.
1821 		 *
1822 		 * Fragments will have START set in the first frame, STOP
1823 		 * set in the last frame, and neither bit set in middle
1824 		 * frames of the packet.
1825 		 *
1826 		 * Therefore if we are at the beginning of a packet and
1827 		 * we don't see START, or we are in the middle of a fragmented
1828 		 * packet and do see START, we are unsynchronized and should
1829 		 * flush the RX queue.
1830 		 */
1831 		if ((first_frag == NULL && !(p->env & LDC_START)) ||
1832 		    (first_frag != NULL &&  (p->env & LDC_START))) {
1833 			if (!first_frag)
1834 				new = rx_advance(lp, new);
1835 
1836 			err = rx_set_head(lp, new);
1837 			if (err)
1838 				break;
1839 
1840 			if (!first_frag)
1841 				goto no_data;
1842 		}
1843 		if (!first_frag)
1844 			first_frag = p;
1845 
1846 		if (pkt_len > size - copied) {
1847 			/* User didn't give us a big enough buffer,
1848 			 * what to do?  This is a pretty serious error.
1849 			 *
1850 			 * Since we haven't updated the RX ring head to
1851 			 * consume any of the packets, signal the error
1852 			 * to the user and just leave the RX ring alone.
1853 			 *
1854 			 * This seems the best behavior because this allows
1855 			 * a user of the LDC layer to start with a small
1856 			 * RX buffer for ldc_read() calls and use -EMSGSIZE
1857 			 * as a cue to enlarge it's read buffer.
1858 			 */
1859 			err = -EMSGSIZE;
1860 			break;
1861 		}
1862 
1863 		/* Ok, we are gonna eat this one.  */
1864 		new = rx_advance(lp, new);
1865 
1866 		memcpy(buf,
1867 		       (lp->cfg.mode == LDC_MODE_UNRELIABLE ?
1868 			p->u.u_data : p->u.r.r_data), pkt_len);
1869 		buf += pkt_len;
1870 		copied += pkt_len;
1871 
1872 		if (p->env & LDC_STOP)
1873 			break;
1874 
1875 no_data:
1876 		if (new == lp->rx_tail) {
1877 			err = rx_data_wait(lp, new);
1878 			if (err)
1879 				break;
1880 		}
1881 	}
1882 
1883 	if (!err)
1884 		err = rx_set_head(lp, new);
1885 
1886 	if (err && first_frag)
1887 		lp->rcv_nxt = first_frag->seqid - 1;
1888 
1889 	if (!err) {
1890 		err = copied;
1891 		if (err > 0 && lp->cfg.mode != LDC_MODE_UNRELIABLE)
1892 			send_data_ack(lp);
1893 	}
1894 
1895 	return err;
1896 }
1897 
1898 static const struct ldc_mode_ops nonraw_ops = {
1899 	.write		=	write_nonraw,
1900 	.read		=	read_nonraw,
1901 };
1902 
write_stream(struct ldc_channel * lp,const void * buf,unsigned int size)1903 static int write_stream(struct ldc_channel *lp, const void *buf,
1904 			unsigned int size)
1905 {
1906 	if (size > lp->cfg.mtu)
1907 		size = lp->cfg.mtu;
1908 	return write_nonraw(lp, buf, size);
1909 }
1910 
read_stream(struct ldc_channel * lp,void * buf,unsigned int size)1911 static int read_stream(struct ldc_channel *lp, void *buf, unsigned int size)
1912 {
1913 	if (!lp->mssbuf_len) {
1914 		int err = read_nonraw(lp, lp->mssbuf, lp->cfg.mtu);
1915 		if (err < 0)
1916 			return err;
1917 
1918 		lp->mssbuf_len = err;
1919 		lp->mssbuf_off = 0;
1920 	}
1921 
1922 	if (size > lp->mssbuf_len)
1923 		size = lp->mssbuf_len;
1924 	memcpy(buf, lp->mssbuf + lp->mssbuf_off, size);
1925 
1926 	lp->mssbuf_off += size;
1927 	lp->mssbuf_len -= size;
1928 
1929 	return size;
1930 }
1931 
1932 static const struct ldc_mode_ops stream_ops = {
1933 	.write		=	write_stream,
1934 	.read		=	read_stream,
1935 };
1936 
ldc_write(struct ldc_channel * lp,const void * buf,unsigned int size)1937 int ldc_write(struct ldc_channel *lp, const void *buf, unsigned int size)
1938 {
1939 	unsigned long flags;
1940 	int err;
1941 
1942 	if (!buf)
1943 		return -EINVAL;
1944 
1945 	if (!size)
1946 		return 0;
1947 
1948 	spin_lock_irqsave(&lp->lock, flags);
1949 
1950 	if (lp->hs_state != LDC_HS_COMPLETE)
1951 		err = -ENOTCONN;
1952 	else
1953 		err = lp->mops->write(lp, buf, size);
1954 
1955 	spin_unlock_irqrestore(&lp->lock, flags);
1956 
1957 	return err;
1958 }
1959 EXPORT_SYMBOL(ldc_write);
1960 
ldc_read(struct ldc_channel * lp,void * buf,unsigned int size)1961 int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size)
1962 {
1963 	unsigned long flags;
1964 	int err;
1965 
1966 	ldcdbg(RX, "%s: entered size=%d\n", __func__, size);
1967 
1968 	if (!buf)
1969 		return -EINVAL;
1970 
1971 	if (!size)
1972 		return 0;
1973 
1974 	spin_lock_irqsave(&lp->lock, flags);
1975 
1976 	if (lp->hs_state != LDC_HS_COMPLETE)
1977 		err = -ENOTCONN;
1978 	else
1979 		err = lp->mops->read(lp, buf, size);
1980 
1981 	spin_unlock_irqrestore(&lp->lock, flags);
1982 
1983 	ldcdbg(RX, "%s: mode=%d, head=%lu, tail=%lu rv=%d\n", __func__,
1984 	       lp->cfg.mode, lp->rx_head, lp->rx_tail, err);
1985 
1986 	return err;
1987 }
1988 EXPORT_SYMBOL(ldc_read);
1989 
pagesize_code(void)1990 static u64 pagesize_code(void)
1991 {
1992 	switch (PAGE_SIZE) {
1993 	default:
1994 	case (8ULL * 1024ULL):
1995 		return 0;
1996 	case (64ULL * 1024ULL):
1997 		return 1;
1998 	case (512ULL * 1024ULL):
1999 		return 2;
2000 	case (4ULL * 1024ULL * 1024ULL):
2001 		return 3;
2002 	case (32ULL * 1024ULL * 1024ULL):
2003 		return 4;
2004 	case (256ULL * 1024ULL * 1024ULL):
2005 		return 5;
2006 	}
2007 }
2008 
make_cookie(u64 index,u64 pgsz_code,u64 page_offset)2009 static u64 make_cookie(u64 index, u64 pgsz_code, u64 page_offset)
2010 {
2011 	return ((pgsz_code << COOKIE_PGSZ_CODE_SHIFT) |
2012 		(index << PAGE_SHIFT) |
2013 		page_offset);
2014 }
2015 
2016 
alloc_npages(struct ldc_iommu * iommu,unsigned long npages)2017 static struct ldc_mtable_entry *alloc_npages(struct ldc_iommu *iommu,
2018 					     unsigned long npages)
2019 {
2020 	long entry;
2021 
2022 	entry = iommu_tbl_range_alloc(NULL, &iommu->iommu_map_table,
2023 				      npages, NULL, (unsigned long)-1, 0);
2024 	if (unlikely(entry == IOMMU_ERROR_CODE))
2025 		return NULL;
2026 
2027 	return iommu->page_table + entry;
2028 }
2029 
perm_to_mte(unsigned int map_perm)2030 static u64 perm_to_mte(unsigned int map_perm)
2031 {
2032 	u64 mte_base;
2033 
2034 	mte_base = pagesize_code();
2035 
2036 	if (map_perm & LDC_MAP_SHADOW) {
2037 		if (map_perm & LDC_MAP_R)
2038 			mte_base |= LDC_MTE_COPY_R;
2039 		if (map_perm & LDC_MAP_W)
2040 			mte_base |= LDC_MTE_COPY_W;
2041 	}
2042 	if (map_perm & LDC_MAP_DIRECT) {
2043 		if (map_perm & LDC_MAP_R)
2044 			mte_base |= LDC_MTE_READ;
2045 		if (map_perm & LDC_MAP_W)
2046 			mte_base |= LDC_MTE_WRITE;
2047 		if (map_perm & LDC_MAP_X)
2048 			mte_base |= LDC_MTE_EXEC;
2049 	}
2050 	if (map_perm & LDC_MAP_IO) {
2051 		if (map_perm & LDC_MAP_R)
2052 			mte_base |= LDC_MTE_IOMMU_R;
2053 		if (map_perm & LDC_MAP_W)
2054 			mte_base |= LDC_MTE_IOMMU_W;
2055 	}
2056 
2057 	return mte_base;
2058 }
2059 
pages_in_region(unsigned long base,long len)2060 static int pages_in_region(unsigned long base, long len)
2061 {
2062 	int count = 0;
2063 
2064 	do {
2065 		unsigned long new = (base + PAGE_SIZE) & PAGE_MASK;
2066 
2067 		len -= (new - base);
2068 		base = new;
2069 		count++;
2070 	} while (len > 0);
2071 
2072 	return count;
2073 }
2074 
2075 struct cookie_state {
2076 	struct ldc_mtable_entry		*page_table;
2077 	struct ldc_trans_cookie		*cookies;
2078 	u64				mte_base;
2079 	u64				prev_cookie;
2080 	u32				pte_idx;
2081 	u32				nc;
2082 };
2083 
fill_cookies(struct cookie_state * sp,unsigned long pa,unsigned long off,unsigned long len)2084 static void fill_cookies(struct cookie_state *sp, unsigned long pa,
2085 			 unsigned long off, unsigned long len)
2086 {
2087 	do {
2088 		unsigned long tlen, new = pa + PAGE_SIZE;
2089 		u64 this_cookie;
2090 
2091 		sp->page_table[sp->pte_idx].mte = sp->mte_base | pa;
2092 
2093 		tlen = PAGE_SIZE;
2094 		if (off)
2095 			tlen = PAGE_SIZE - off;
2096 		if (tlen > len)
2097 			tlen = len;
2098 
2099 		this_cookie = make_cookie(sp->pte_idx,
2100 					  pagesize_code(), off);
2101 
2102 		off = 0;
2103 
2104 		if (this_cookie == sp->prev_cookie) {
2105 			sp->cookies[sp->nc - 1].cookie_size += tlen;
2106 		} else {
2107 			sp->cookies[sp->nc].cookie_addr = this_cookie;
2108 			sp->cookies[sp->nc].cookie_size = tlen;
2109 			sp->nc++;
2110 		}
2111 		sp->prev_cookie = this_cookie + tlen;
2112 
2113 		sp->pte_idx++;
2114 
2115 		len -= tlen;
2116 		pa = new;
2117 	} while (len > 0);
2118 }
2119 
sg_count_one(struct scatterlist * sg)2120 static int sg_count_one(struct scatterlist *sg)
2121 {
2122 	unsigned long base = page_to_pfn(sg_page(sg)) << PAGE_SHIFT;
2123 	long len = sg->length;
2124 
2125 	if ((sg->offset | len) & (8UL - 1))
2126 		return -EFAULT;
2127 
2128 	return pages_in_region(base + sg->offset, len);
2129 }
2130 
sg_count_pages(struct scatterlist * sg,int num_sg)2131 static int sg_count_pages(struct scatterlist *sg, int num_sg)
2132 {
2133 	int count;
2134 	int i;
2135 
2136 	count = 0;
2137 	for (i = 0; i < num_sg; i++) {
2138 		int err = sg_count_one(sg + i);
2139 		if (err < 0)
2140 			return err;
2141 		count += err;
2142 	}
2143 
2144 	return count;
2145 }
2146 
ldc_map_sg(struct ldc_channel * lp,struct scatterlist * sg,int num_sg,struct ldc_trans_cookie * cookies,int ncookies,unsigned int map_perm)2147 int ldc_map_sg(struct ldc_channel *lp,
2148 	       struct scatterlist *sg, int num_sg,
2149 	       struct ldc_trans_cookie *cookies, int ncookies,
2150 	       unsigned int map_perm)
2151 {
2152 	unsigned long i, npages;
2153 	struct ldc_mtable_entry *base;
2154 	struct cookie_state state;
2155 	struct ldc_iommu *iommu;
2156 	int err;
2157 	struct scatterlist *s;
2158 
2159 	if (map_perm & ~LDC_MAP_ALL)
2160 		return -EINVAL;
2161 
2162 	err = sg_count_pages(sg, num_sg);
2163 	if (err < 0)
2164 		return err;
2165 
2166 	npages = err;
2167 	if (err > ncookies)
2168 		return -EMSGSIZE;
2169 
2170 	iommu = &lp->iommu;
2171 
2172 	base = alloc_npages(iommu, npages);
2173 
2174 	if (!base)
2175 		return -ENOMEM;
2176 
2177 	state.page_table = iommu->page_table;
2178 	state.cookies = cookies;
2179 	state.mte_base = perm_to_mte(map_perm);
2180 	state.prev_cookie = ~(u64)0;
2181 	state.pte_idx = (base - iommu->page_table);
2182 	state.nc = 0;
2183 
2184 	for_each_sg(sg, s, num_sg, i) {
2185 		fill_cookies(&state, page_to_pfn(sg_page(s)) << PAGE_SHIFT,
2186 			     s->offset, s->length);
2187 	}
2188 
2189 	return state.nc;
2190 }
2191 EXPORT_SYMBOL(ldc_map_sg);
2192 
ldc_map_single(struct ldc_channel * lp,void * buf,unsigned int len,struct ldc_trans_cookie * cookies,int ncookies,unsigned int map_perm)2193 int ldc_map_single(struct ldc_channel *lp,
2194 		   void *buf, unsigned int len,
2195 		   struct ldc_trans_cookie *cookies, int ncookies,
2196 		   unsigned int map_perm)
2197 {
2198 	unsigned long npages, pa;
2199 	struct ldc_mtable_entry *base;
2200 	struct cookie_state state;
2201 	struct ldc_iommu *iommu;
2202 
2203 	if ((map_perm & ~LDC_MAP_ALL) || (ncookies < 1))
2204 		return -EINVAL;
2205 
2206 	pa = __pa(buf);
2207 	if ((pa | len) & (8UL - 1))
2208 		return -EFAULT;
2209 
2210 	npages = pages_in_region(pa, len);
2211 
2212 	iommu = &lp->iommu;
2213 
2214 	base = alloc_npages(iommu, npages);
2215 
2216 	if (!base)
2217 		return -ENOMEM;
2218 
2219 	state.page_table = iommu->page_table;
2220 	state.cookies = cookies;
2221 	state.mte_base = perm_to_mte(map_perm);
2222 	state.prev_cookie = ~(u64)0;
2223 	state.pte_idx = (base - iommu->page_table);
2224 	state.nc = 0;
2225 	fill_cookies(&state, (pa & PAGE_MASK), (pa & ~PAGE_MASK), len);
2226 	BUG_ON(state.nc > ncookies);
2227 
2228 	return state.nc;
2229 }
2230 EXPORT_SYMBOL(ldc_map_single);
2231 
2232 
free_npages(unsigned long id,struct ldc_iommu * iommu,u64 cookie,u64 size)2233 static void free_npages(unsigned long id, struct ldc_iommu *iommu,
2234 			u64 cookie, u64 size)
2235 {
2236 	unsigned long npages, entry;
2237 
2238 	npages = PAGE_ALIGN(((cookie & ~PAGE_MASK) + size)) >> PAGE_SHIFT;
2239 
2240 	entry = ldc_cookie_to_index(cookie, iommu);
2241 	ldc_demap(iommu, id, cookie, entry, npages);
2242 	iommu_tbl_range_free(&iommu->iommu_map_table, cookie, npages, entry);
2243 }
2244 
ldc_unmap(struct ldc_channel * lp,struct ldc_trans_cookie * cookies,int ncookies)2245 void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies,
2246 	       int ncookies)
2247 {
2248 	struct ldc_iommu *iommu = &lp->iommu;
2249 	int i;
2250 	unsigned long flags;
2251 
2252 	spin_lock_irqsave(&iommu->lock, flags);
2253 	for (i = 0; i < ncookies; i++) {
2254 		u64 addr = cookies[i].cookie_addr;
2255 		u64 size = cookies[i].cookie_size;
2256 
2257 		free_npages(lp->id, iommu, addr, size);
2258 	}
2259 	spin_unlock_irqrestore(&iommu->lock, flags);
2260 }
2261 EXPORT_SYMBOL(ldc_unmap);
2262 
ldc_copy(struct ldc_channel * lp,int copy_dir,void * buf,unsigned int len,unsigned long offset,struct ldc_trans_cookie * cookies,int ncookies)2263 int ldc_copy(struct ldc_channel *lp, int copy_dir,
2264 	     void *buf, unsigned int len, unsigned long offset,
2265 	     struct ldc_trans_cookie *cookies, int ncookies)
2266 {
2267 	unsigned int orig_len;
2268 	unsigned long ra;
2269 	int i;
2270 
2271 	if (copy_dir != LDC_COPY_IN && copy_dir != LDC_COPY_OUT) {
2272 		printk(KERN_ERR PFX "ldc_copy: ID[%lu] Bad copy_dir[%d]\n",
2273 		       lp->id, copy_dir);
2274 		return -EINVAL;
2275 	}
2276 
2277 	ra = __pa(buf);
2278 	if ((ra | len | offset) & (8UL - 1)) {
2279 		printk(KERN_ERR PFX "ldc_copy: ID[%lu] Unaligned buffer "
2280 		       "ra[%lx] len[%x] offset[%lx]\n",
2281 		       lp->id, ra, len, offset);
2282 		return -EFAULT;
2283 	}
2284 
2285 	if (lp->hs_state != LDC_HS_COMPLETE ||
2286 	    (lp->flags & LDC_FLAG_RESET)) {
2287 		printk(KERN_ERR PFX "ldc_copy: ID[%lu] Link down hs_state[%x] "
2288 		       "flags[%x]\n", lp->id, lp->hs_state, lp->flags);
2289 		return -ECONNRESET;
2290 	}
2291 
2292 	orig_len = len;
2293 	for (i = 0; i < ncookies; i++) {
2294 		unsigned long cookie_raddr = cookies[i].cookie_addr;
2295 		unsigned long this_len = cookies[i].cookie_size;
2296 		unsigned long actual_len;
2297 
2298 		if (unlikely(offset)) {
2299 			unsigned long this_off = offset;
2300 
2301 			if (this_off > this_len)
2302 				this_off = this_len;
2303 
2304 			offset -= this_off;
2305 			this_len -= this_off;
2306 			if (!this_len)
2307 				continue;
2308 			cookie_raddr += this_off;
2309 		}
2310 
2311 		if (this_len > len)
2312 			this_len = len;
2313 
2314 		while (1) {
2315 			unsigned long hv_err;
2316 
2317 			hv_err = sun4v_ldc_copy(lp->id, copy_dir,
2318 						cookie_raddr, ra,
2319 						this_len, &actual_len);
2320 			if (unlikely(hv_err)) {
2321 				printk(KERN_ERR PFX "ldc_copy: ID[%lu] "
2322 				       "HV error %lu\n",
2323 				       lp->id, hv_err);
2324 				if (lp->hs_state != LDC_HS_COMPLETE ||
2325 				    (lp->flags & LDC_FLAG_RESET))
2326 					return -ECONNRESET;
2327 				else
2328 					return -EFAULT;
2329 			}
2330 
2331 			cookie_raddr += actual_len;
2332 			ra += actual_len;
2333 			len -= actual_len;
2334 			if (actual_len == this_len)
2335 				break;
2336 
2337 			this_len -= actual_len;
2338 		}
2339 
2340 		if (!len)
2341 			break;
2342 	}
2343 
2344 	/* It is caller policy what to do about short copies.
2345 	 * For example, a networking driver can declare the
2346 	 * packet a runt and drop it.
2347 	 */
2348 
2349 	return orig_len - len;
2350 }
2351 EXPORT_SYMBOL(ldc_copy);
2352 
ldc_alloc_exp_dring(struct ldc_channel * lp,unsigned int len,struct ldc_trans_cookie * cookies,int * ncookies,unsigned int map_perm)2353 void *ldc_alloc_exp_dring(struct ldc_channel *lp, unsigned int len,
2354 			  struct ldc_trans_cookie *cookies, int *ncookies,
2355 			  unsigned int map_perm)
2356 {
2357 	void *buf;
2358 	int err;
2359 
2360 	if (len & (8UL - 1))
2361 		return ERR_PTR(-EINVAL);
2362 
2363 	buf = kzalloc(len, GFP_ATOMIC);
2364 	if (!buf)
2365 		return ERR_PTR(-ENOMEM);
2366 
2367 	err = ldc_map_single(lp, buf, len, cookies, *ncookies, map_perm);
2368 	if (err < 0) {
2369 		kfree(buf);
2370 		return ERR_PTR(err);
2371 	}
2372 	*ncookies = err;
2373 
2374 	return buf;
2375 }
2376 EXPORT_SYMBOL(ldc_alloc_exp_dring);
2377 
ldc_free_exp_dring(struct ldc_channel * lp,void * buf,unsigned int len,struct ldc_trans_cookie * cookies,int ncookies)2378 void ldc_free_exp_dring(struct ldc_channel *lp, void *buf, unsigned int len,
2379 			struct ldc_trans_cookie *cookies, int ncookies)
2380 {
2381 	ldc_unmap(lp, cookies, ncookies);
2382 	kfree(buf);
2383 }
2384 EXPORT_SYMBOL(ldc_free_exp_dring);
2385 
ldc_init(void)2386 static int __init ldc_init(void)
2387 {
2388 	unsigned long major, minor;
2389 	struct mdesc_handle *hp;
2390 	const u64 *v;
2391 	int err;
2392 	u64 mp;
2393 
2394 	hp = mdesc_grab();
2395 	if (!hp)
2396 		return -ENODEV;
2397 
2398 	mp = mdesc_node_by_name(hp, MDESC_NODE_NULL, "platform");
2399 	err = -ENODEV;
2400 	if (mp == MDESC_NODE_NULL)
2401 		goto out;
2402 
2403 	v = mdesc_get_property(hp, mp, "domaining-enabled", NULL);
2404 	if (!v)
2405 		goto out;
2406 
2407 	major = 1;
2408 	minor = 0;
2409 	if (sun4v_hvapi_register(HV_GRP_LDOM, major, &minor)) {
2410 		printk(KERN_INFO PFX "Could not register LDOM hvapi.\n");
2411 		goto out;
2412 	}
2413 
2414 	printk(KERN_INFO "%s", version);
2415 
2416 	if (!*v) {
2417 		printk(KERN_INFO PFX "Domaining disabled.\n");
2418 		goto out;
2419 	}
2420 	ldom_domaining_enabled = 1;
2421 	err = 0;
2422 
2423 out:
2424 	mdesc_release(hp);
2425 	return err;
2426 }
2427 
2428 core_initcall(ldc_init);
2429