1 /*
2 *
3 * Copyright (c) 2009, Microsoft Corporation.
4 *
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
8 *
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
12 * more details.
13 *
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
16 * Place - Suite 330, Boston, MA 02111-1307 USA.
17 *
18 * Authors:
19 * Haiyang Zhang <haiyangz@microsoft.com>
20 * Hank Janssen <hjanssen@microsoft.com>
21 *
22 */
23
24 #include <linux/kernel.h>
25 #include <linux/mm.h>
26 #include "logging.h"
27 #include "ring_buffer.h"
28
29
30 /* #defines */
31
32
33 /* Amount of space to write to */
34 #define BYTES_AVAIL_TO_WRITE(r, w, z) ((w) >= (r)) ? ((z) - ((w) - (r))) : ((r) - (w))
35
36
37 /*++
38
39 Name:
40 get_ringbuffer_availbytes()
41
42 Description:
43 Get number of bytes available to read and to write to
44 for the specified ring buffer
45
46 --*/
47 static inline void
get_ringbuffer_availbytes(struct hv_ring_buffer_info * rbi,u32 * read,u32 * write)48 get_ringbuffer_availbytes(struct hv_ring_buffer_info *rbi,
49 u32 *read, u32 *write)
50 {
51 u32 read_loc, write_loc;
52
53 /* Capture the read/write indices before they changed */
54 read_loc = rbi->ring_buffer->read_index;
55 write_loc = rbi->ring_buffer->write_index;
56
57 *write = BYTES_AVAIL_TO_WRITE(read_loc, write_loc, rbi->ring_datasize);
58 *read = rbi->ring_datasize - *write;
59 }
60
61 /*++
62
63 Name:
64 get_next_write_location()
65
66 Description:
67 Get the next write location for the specified ring buffer
68
69 --*/
70 static inline u32
get_next_write_location(struct hv_ring_buffer_info * ring_info)71 get_next_write_location(struct hv_ring_buffer_info *ring_info)
72 {
73 u32 next = ring_info->ring_buffer->write_index;
74
75 /* ASSERT(next < ring_info->RingDataSize); */
76
77 return next;
78 }
79
80 /*++
81
82 Name:
83 set_next_write_location()
84
85 Description:
86 Set the next write location for the specified ring buffer
87
88 --*/
89 static inline void
set_next_write_location(struct hv_ring_buffer_info * ring_info,u32 next_write_location)90 set_next_write_location(struct hv_ring_buffer_info *ring_info,
91 u32 next_write_location)
92 {
93 ring_info->ring_buffer->write_index = next_write_location;
94 }
95
96 /*++
97
98 Name:
99 get_next_read_location()
100
101 Description:
102 Get the next read location for the specified ring buffer
103
104 --*/
105 static inline u32
get_next_read_location(struct hv_ring_buffer_info * ring_info)106 get_next_read_location(struct hv_ring_buffer_info *ring_info)
107 {
108 u32 next = ring_info->ring_buffer->read_index;
109
110 /* ASSERT(next < ring_info->RingDataSize); */
111
112 return next;
113 }
114
115 /*++
116
117 Name:
118 get_next_readlocation_withoffset()
119
120 Description:
121 Get the next read location + offset for the specified ring buffer.
122 This allows the caller to skip
123
124 --*/
125 static inline u32
get_next_readlocation_withoffset(struct hv_ring_buffer_info * ring_info,u32 offset)126 get_next_readlocation_withoffset(struct hv_ring_buffer_info *ring_info,
127 u32 offset)
128 {
129 u32 next = ring_info->ring_buffer->read_index;
130
131 /* ASSERT(next < ring_info->RingDataSize); */
132 next += offset;
133 next %= ring_info->ring_datasize;
134
135 return next;
136 }
137
138 /*++
139
140 Name:
141 set_next_read_location()
142
143 Description:
144 Set the next read location for the specified ring buffer
145
146 --*/
147 static inline void
set_next_read_location(struct hv_ring_buffer_info * ring_info,u32 next_read_location)148 set_next_read_location(struct hv_ring_buffer_info *ring_info,
149 u32 next_read_location)
150 {
151 ring_info->ring_buffer->read_index = next_read_location;
152 }
153
154
155 /*++
156
157 Name:
158 get_ring_buffer()
159
160 Description:
161 Get the start of the ring buffer
162
163 --*/
164 static inline void *
get_ring_buffer(struct hv_ring_buffer_info * ring_info)165 get_ring_buffer(struct hv_ring_buffer_info *ring_info)
166 {
167 return (void *)ring_info->ring_buffer->buffer;
168 }
169
170
171 /*++
172
173 Name:
174 get_ring_buffersize()
175
176 Description:
177 Get the size of the ring buffer
178
179 --*/
180 static inline u32
get_ring_buffersize(struct hv_ring_buffer_info * ring_info)181 get_ring_buffersize(struct hv_ring_buffer_info *ring_info)
182 {
183 return ring_info->ring_datasize;
184 }
185
186 /*++
187
188 Name:
189 get_ring_bufferindices()
190
191 Description:
192 Get the read and write indices as u64 of the specified ring buffer
193
194 --*/
195 static inline u64
get_ring_bufferindices(struct hv_ring_buffer_info * ring_info)196 get_ring_bufferindices(struct hv_ring_buffer_info *ring_info)
197 {
198 return (u64)ring_info->ring_buffer->write_index << 32;
199 }
200
201
202 /*++
203
204 Name:
205 dump_ring_info()
206
207 Description:
208 Dump out to console the ring buffer info
209
210 --*/
dump_ring_info(struct hv_ring_buffer_info * ring_info,char * prefix)211 void dump_ring_info(struct hv_ring_buffer_info *ring_info, char *prefix)
212 {
213 u32 bytes_avail_towrite;
214 u32 bytes_avail_toread;
215
216 get_ringbuffer_availbytes(ring_info,
217 &bytes_avail_toread,
218 &bytes_avail_towrite);
219
220 DPRINT(VMBUS,
221 DEBUG_RING_LVL,
222 "%s <<ringinfo %p buffer %p avail write %u "
223 "avail read %u read idx %u write idx %u>>",
224 prefix,
225 ring_info,
226 ring_info->ring_buffer->buffer,
227 bytes_avail_towrite,
228 bytes_avail_toread,
229 ring_info->ring_buffer->read_index,
230 ring_info->ring_buffer->write_index);
231 }
232
233
234 /* Internal routines */
235
236 static u32
237 copyto_ringbuffer(
238 struct hv_ring_buffer_info *ring_info,
239 u32 start_write_offset,
240 void *src,
241 u32 srclen);
242
243 static u32
244 copyfrom_ringbuffer(
245 struct hv_ring_buffer_info *ring_info,
246 void *dest,
247 u32 destlen,
248 u32 start_read_offset);
249
250
251
252 /*++
253
254 Name:
255 ringbuffer_get_debuginfo()
256
257 Description:
258 Get various debug metrics for the specified ring buffer
259
260 --*/
ringbuffer_get_debuginfo(struct hv_ring_buffer_info * ring_info,struct hv_ring_buffer_debug_info * debug_info)261 void ringbuffer_get_debuginfo(struct hv_ring_buffer_info *ring_info,
262 struct hv_ring_buffer_debug_info *debug_info)
263 {
264 u32 bytes_avail_towrite;
265 u32 bytes_avail_toread;
266
267 if (ring_info->ring_buffer) {
268 get_ringbuffer_availbytes(ring_info,
269 &bytes_avail_toread,
270 &bytes_avail_towrite);
271
272 debug_info->bytes_avail_toread = bytes_avail_toread;
273 debug_info->bytes_avail_towrite = bytes_avail_towrite;
274 debug_info->current_read_index =
275 ring_info->ring_buffer->read_index;
276 debug_info->current_write_index =
277 ring_info->ring_buffer->write_index;
278 debug_info->current_interrupt_mask =
279 ring_info->ring_buffer->interrupt_mask;
280 }
281 }
282
283
284 /*++
285
286 Name:
287 get_ringbuffer_interrupt_mask()
288
289 Description:
290 Get the interrupt mask for the specified ring buffer
291
292 --*/
get_ringbuffer_interrupt_mask(struct hv_ring_buffer_info * rbi)293 u32 get_ringbuffer_interrupt_mask(struct hv_ring_buffer_info *rbi)
294 {
295 return rbi->ring_buffer->interrupt_mask;
296 }
297
298 /*++
299
300 Name:
301 ringbuffer_init()
302
303 Description:
304 Initialize the ring buffer
305
306 --*/
ringbuffer_init(struct hv_ring_buffer_info * ring_info,void * buffer,u32 buflen)307 int ringbuffer_init(struct hv_ring_buffer_info *ring_info,
308 void *buffer, u32 buflen)
309 {
310 if (sizeof(struct hv_ring_buffer) != PAGE_SIZE)
311 return -EINVAL;
312
313 memset(ring_info, 0, sizeof(struct hv_ring_buffer_info));
314
315 ring_info->ring_buffer = (struct hv_ring_buffer *)buffer;
316 ring_info->ring_buffer->read_index =
317 ring_info->ring_buffer->write_index = 0;
318
319 ring_info->ring_size = buflen;
320 ring_info->ring_datasize = buflen - sizeof(struct hv_ring_buffer);
321
322 spin_lock_init(&ring_info->ring_lock);
323
324 return 0;
325 }
326
327 /*++
328
329 Name:
330 ringbuffer_cleanup()
331
332 Description:
333 Cleanup the ring buffer
334
335 --*/
ringbuffer_cleanup(struct hv_ring_buffer_info * ring_info)336 void ringbuffer_cleanup(struct hv_ring_buffer_info *ring_info)
337 {
338 }
339
340 /*++
341
342 Name:
343 ringbuffer_write()
344
345 Description:
346 Write to the ring buffer
347
348 --*/
ringbuffer_write(struct hv_ring_buffer_info * outring_info,struct scatterlist * sglist,u32 sgcount)349 int ringbuffer_write(struct hv_ring_buffer_info *outring_info,
350 struct scatterlist *sglist, u32 sgcount)
351 {
352 int i = 0;
353 u32 bytes_avail_towrite;
354 u32 bytes_avail_toread;
355 u32 totalbytes_towrite = 0;
356
357 struct scatterlist *sg;
358 volatile u32 next_write_location;
359 u64 prev_indices = 0;
360 unsigned long flags;
361
362 for_each_sg(sglist, sg, sgcount, i)
363 {
364 totalbytes_towrite += sg->length;
365 }
366
367 totalbytes_towrite += sizeof(u64);
368
369 spin_lock_irqsave(&outring_info->ring_lock, flags);
370
371 get_ringbuffer_availbytes(outring_info,
372 &bytes_avail_toread,
373 &bytes_avail_towrite);
374
375 DPRINT_DBG(VMBUS, "Writing %u bytes...", totalbytes_towrite);
376
377 /* Dumpring_info(Outring_info, "BEFORE "); */
378
379 /* If there is only room for the packet, assume it is full. */
380 /* Otherwise, the next time around, we think the ring buffer */
381 /* is empty since the read index == write index */
382 if (bytes_avail_towrite <= totalbytes_towrite) {
383 DPRINT_DBG(VMBUS,
384 "No more space left on outbound ring buffer "
385 "(needed %u, avail %u)",
386 totalbytes_towrite,
387 bytes_avail_towrite);
388
389 spin_unlock_irqrestore(&outring_info->ring_lock, flags);
390 return -1;
391 }
392
393 /* Write to the ring buffer */
394 next_write_location = get_next_write_location(outring_info);
395
396 for_each_sg(sglist, sg, sgcount, i)
397 {
398 next_write_location = copyto_ringbuffer(outring_info,
399 next_write_location,
400 sg_virt(sg),
401 sg->length);
402 }
403
404 /* Set previous packet start */
405 prev_indices = get_ring_bufferindices(outring_info);
406
407 next_write_location = copyto_ringbuffer(outring_info,
408 next_write_location,
409 &prev_indices,
410 sizeof(u64));
411
412 /* Make sure we flush all writes before updating the writeIndex */
413 mb();
414
415 /* Now, update the write location */
416 set_next_write_location(outring_info, next_write_location);
417
418 /* Dumpring_info(Outring_info, "AFTER "); */
419
420 spin_unlock_irqrestore(&outring_info->ring_lock, flags);
421 return 0;
422 }
423
424
425 /*++
426
427 Name:
428 ringbuffer_peek()
429
430 Description:
431 Read without advancing the read index
432
433 --*/
ringbuffer_peek(struct hv_ring_buffer_info * Inring_info,void * Buffer,u32 buflen)434 int ringbuffer_peek(struct hv_ring_buffer_info *Inring_info,
435 void *Buffer, u32 buflen)
436 {
437 u32 bytes_avail_towrite;
438 u32 bytes_avail_toread;
439 u32 next_read_location = 0;
440 unsigned long flags;
441
442 spin_lock_irqsave(&Inring_info->ring_lock, flags);
443
444 get_ringbuffer_availbytes(Inring_info,
445 &bytes_avail_toread,
446 &bytes_avail_towrite);
447
448 /* Make sure there is something to read */
449 if (bytes_avail_toread < buflen) {
450 /* DPRINT_DBG(VMBUS,
451 "got callback but not enough to read "
452 "<avail to read %d read size %d>!!",
453 bytes_avail_toread,
454 BufferLen); */
455
456 spin_unlock_irqrestore(&Inring_info->ring_lock, flags);
457
458 return -1;
459 }
460
461 /* Convert to byte offset */
462 next_read_location = get_next_read_location(Inring_info);
463
464 next_read_location = copyfrom_ringbuffer(Inring_info,
465 Buffer,
466 buflen,
467 next_read_location);
468
469 spin_unlock_irqrestore(&Inring_info->ring_lock, flags);
470
471 return 0;
472 }
473
474
475 /*++
476
477 Name:
478 ringbuffer_read()
479
480 Description:
481 Read and advance the read index
482
483 --*/
ringbuffer_read(struct hv_ring_buffer_info * inring_info,void * buffer,u32 buflen,u32 offset)484 int ringbuffer_read(struct hv_ring_buffer_info *inring_info, void *buffer,
485 u32 buflen, u32 offset)
486 {
487 u32 bytes_avail_towrite;
488 u32 bytes_avail_toread;
489 u32 next_read_location = 0;
490 u64 prev_indices = 0;
491 unsigned long flags;
492
493 if (buflen <= 0)
494 return -EINVAL;
495
496 spin_lock_irqsave(&inring_info->ring_lock, flags);
497
498 get_ringbuffer_availbytes(inring_info,
499 &bytes_avail_toread,
500 &bytes_avail_towrite);
501
502 DPRINT_DBG(VMBUS, "Reading %u bytes...", buflen);
503
504 /* Dumpring_info(Inring_info, "BEFORE "); */
505
506 /* Make sure there is something to read */
507 if (bytes_avail_toread < buflen) {
508 DPRINT_DBG(VMBUS,
509 "got callback but not enough to read "
510 "<avail to read %d read size %d>!!",
511 bytes_avail_toread,
512 buflen);
513
514 spin_unlock_irqrestore(&inring_info->ring_lock, flags);
515
516 return -1;
517 }
518
519 next_read_location =
520 get_next_readlocation_withoffset(inring_info, offset);
521
522 next_read_location = copyfrom_ringbuffer(inring_info,
523 buffer,
524 buflen,
525 next_read_location);
526
527 next_read_location = copyfrom_ringbuffer(inring_info,
528 &prev_indices,
529 sizeof(u64),
530 next_read_location);
531
532 /* Make sure all reads are done before we update the read index since */
533 /* the writer may start writing to the read area once the read index */
534 /*is updated */
535 mb();
536
537 /* Update the read index */
538 set_next_read_location(inring_info, next_read_location);
539
540 /* Dumpring_info(Inring_info, "AFTER "); */
541
542 spin_unlock_irqrestore(&inring_info->ring_lock, flags);
543
544 return 0;
545 }
546
547
548 /*++
549
550 Name:
551 copyto_ringbuffer()
552
553 Description:
554 Helper routine to copy from source to ring buffer.
555 Assume there is enough room. Handles wrap-around in dest case only!!
556
557 --*/
558 static u32
copyto_ringbuffer(struct hv_ring_buffer_info * ring_info,u32 start_write_offset,void * src,u32 srclen)559 copyto_ringbuffer(
560 struct hv_ring_buffer_info *ring_info,
561 u32 start_write_offset,
562 void *src,
563 u32 srclen)
564 {
565 void *ring_buffer = get_ring_buffer(ring_info);
566 u32 ring_buffer_size = get_ring_buffersize(ring_info);
567 u32 frag_len;
568
569 /* wrap-around detected! */
570 if (srclen > ring_buffer_size - start_write_offset) {
571 DPRINT_DBG(VMBUS, "wrap-around detected!");
572
573 frag_len = ring_buffer_size - start_write_offset;
574 memcpy(ring_buffer + start_write_offset, src, frag_len);
575 memcpy(ring_buffer, src + frag_len, srclen - frag_len);
576 } else
577 memcpy(ring_buffer + start_write_offset, src, srclen);
578
579 start_write_offset += srclen;
580 start_write_offset %= ring_buffer_size;
581
582 return start_write_offset;
583 }
584
585
586 /*++
587
588 Name:
589 copyfrom_ringbuffer()
590
591 Description:
592 Helper routine to copy to source from ring buffer.
593 Assume there is enough room. Handles wrap-around in src case only!!
594
595 --*/
596 static u32
copyfrom_ringbuffer(struct hv_ring_buffer_info * ring_info,void * dest,u32 destlen,u32 start_read_offset)597 copyfrom_ringbuffer(
598 struct hv_ring_buffer_info *ring_info,
599 void *dest,
600 u32 destlen,
601 u32 start_read_offset)
602 {
603 void *ring_buffer = get_ring_buffer(ring_info);
604 u32 ring_buffer_size = get_ring_buffersize(ring_info);
605
606 u32 frag_len;
607
608 /* wrap-around detected at the src */
609 if (destlen > ring_buffer_size - start_read_offset) {
610 DPRINT_DBG(VMBUS, "src wrap-around detected!");
611
612 frag_len = ring_buffer_size - start_read_offset;
613
614 memcpy(dest, ring_buffer + start_read_offset, frag_len);
615 memcpy(dest + frag_len, ring_buffer, destlen - frag_len);
616 } else
617
618 memcpy(dest, ring_buffer + start_read_offset, destlen);
619
620
621 start_read_offset += destlen;
622 start_read_offset %= ring_buffer_size;
623
624 return start_read_offset;
625 }
626
627
628 /* eof */
629