2 * Copyright (C) 2016 Red Hat, Inc.
3 * Author: Michael S. Tsirkin <mst@redhat.com>
4 * This work is licensed under the terms of the GNU GPL, version 2.
6 * Partial implementation of virtio 0.9. event index is used for signalling,
7 * unconditionally. Design roughly follows linux kernel implementation in order
8 * to be able to judge its performance.
16 #include <linux/virtio_ring.h>
24 /* enabling the below activates experimental ring polling code
25 * (which skips index reads on consumer in favor of looking at
26 * high bits of ring id ^ 0x8000).
28 /* #ifdef RING_POLL */
30 /* how much padding is needed to avoid false cache sharing */
31 #define HOST_GUEST_PADDING 0x80
34 unsigned short avail_idx;
35 unsigned short last_used_idx;
36 unsigned short num_free;
37 unsigned short kicked_avail_idx;
38 unsigned short free_head;
39 unsigned char reserved[HOST_GUEST_PADDING - 10];
43 /* we do not need to track last avail index
44 * unless we have more than one in flight.
46 unsigned short used_idx;
47 unsigned short called_used_idx;
48 unsigned char reserved[HOST_GUEST_PADDING - 4];
51 /* implemented by ring */
58 ret = posix_memalign(&p, 0x1000, vring_size(ring_size, 0x1000));
60 perror("Unable to allocate ring buffer.\n");
63 memset(p, 0, vring_size(ring_size, 0x1000));
64 vring_init(&ring, ring_size, p, 0x1000);
67 guest.kicked_avail_idx = -1;
68 guest.last_used_idx = 0;
69 /* Put everything in free lists. */
71 for (i = 0; i < ring_size - 1; i++)
72 ring.desc[i].next = i + 1;
74 host.called_used_idx = -1;
75 guest.num_free = ring_size;
76 data = malloc(ring_size * sizeof *data);
78 perror("Unable to allocate data buffer.\n");
81 memset(data, 0, ring_size * sizeof *data);
85 int add_inbuf(unsigned len, void *buf, void *datap)
88 struct vring_desc *desc;
93 head = guest.free_head;
97 desc[head].flags = VRING_DESC_F_NEXT;
98 desc[head].addr = (unsigned long)(void *)buf;
100 /* We do it like this to simulate the way
101 * we'd have to flip it if we had multiple
104 desc[head].flags &= ~VRING_DESC_F_NEXT;
105 guest.free_head = desc[head].next;
107 data[head].data = datap;
110 /* Barrier A (for pairing) */
112 avail = guest.avail_idx++;
113 ring.avail->ring[avail & (ring_size - 1)] =
114 (head | (avail & ~(ring_size - 1))) ^ 0x8000;
116 avail = (ring_size - 1) & (guest.avail_idx++);
117 ring.avail->ring[avail] = head;
118 /* Barrier A (for pairing) */
121 ring.avail->idx = guest.avail_idx;
125 void *get_buf(unsigned *lenp, void **bufp)
132 head = (ring_size - 1) & guest.last_used_idx;
133 index = ring.used->ring[head].id;
134 if ((index ^ guest.last_used_idx ^ 0x8000) & ~(ring_size - 1))
136 /* Barrier B (for pairing) */
138 index &= ring_size - 1;
140 if (ring.used->idx == guest.last_used_idx)
142 /* Barrier B (for pairing) */
144 head = (ring_size - 1) & guest.last_used_idx;
145 index = ring.used->ring[head].id;
147 *lenp = ring.used->ring[head].len;
148 datap = data[index].data;
149 *bufp = (void*)(unsigned long)ring.desc[index].addr;
150 data[index].data = NULL;
151 ring.desc[index].next = guest.free_head;
152 guest.free_head = index;
154 guest.last_used_idx++;
161 unsigned head = (ring_size - 1) & guest.last_used_idx;
164 unsigned index = ring.used->ring[head].id;
166 if ((index ^ guest.last_used_idx ^ 0x8000) & ~(ring_size - 1))
172 unsigned head = guest.last_used_idx;
174 while (ring.used->idx == head)
181 /* Doing nothing to disable calls might cause
182 * extra interrupts, but reduces the number of cache misses.
188 unsigned short last_used_idx;
190 vring_used_event(&ring) = (last_used_idx = guest.last_used_idx);
191 /* Flush call index write */
192 /* Barrier D (for pairing) */
196 unsigned short head = last_used_idx & (ring_size - 1);
197 unsigned index = ring.used->ring[head].id;
199 return (index ^ last_used_idx ^ 0x8000) & ~(ring_size - 1);
202 return ring.used->idx == last_used_idx;
206 void kick_available(void)
208 /* Flush in previous flags write */
209 /* Barrier C (for pairing) */
211 if (!vring_need_event(vring_avail_event(&ring),
213 guest.kicked_avail_idx))
216 guest.kicked_avail_idx = guest.avail_idx;
223 /* Doing nothing to disable kicks might cause
224 * extra interrupts, but reduces the number of cache misses.
230 unsigned head = host.used_idx;
232 vring_avail_event(&ring) = head;
233 /* Barrier C (for pairing) */
237 unsigned index = ring.avail->ring[head & (ring_size - 1)];
239 return (index ^ head ^ 0x8000) & ~(ring_size - 1);
242 return head == ring.avail->idx;
246 void poll_avail(void)
248 unsigned head = host.used_idx;
251 unsigned index = ring.avail->ring[head & (ring_size - 1)];
252 if ((index ^ head ^ 0x8000) & ~(ring_size - 1))
258 while (ring.avail->idx == head)
263 bool use_buf(unsigned *lenp, void **bufp)
265 unsigned used_idx = host.used_idx;
266 struct vring_desc *desc;
270 head = ring.avail->ring[used_idx & (ring_size - 1)];
271 if ((used_idx ^ head ^ 0x8000) & ~(ring_size - 1))
273 /* Barrier A (for pairing) */
276 used_idx &= ring_size - 1;
277 desc = &ring.desc[head & (ring_size - 1)];
279 if (used_idx == ring.avail->idx)
282 /* Barrier A (for pairing) */
285 used_idx &= ring_size - 1;
286 head = ring.avail->ring[used_idx];
287 desc = &ring.desc[head];
291 *bufp = (void *)(unsigned long)desc->addr;
293 /* now update used ring */
294 ring.used->ring[used_idx].id = head;
295 ring.used->ring[used_idx].len = desc->len - 1;
296 /* Barrier B (for pairing) */
299 ring.used->idx = host.used_idx;
306 /* Flush in previous flags write */
307 /* Barrier D (for pairing) */
309 if (!vring_need_event(vring_used_event(&ring),
311 host.called_used_idx))
314 host.called_used_idx = host.used_idx;