Line data Source code
1 : /*
2 : * vr_dpdk_virtio.c - implements DPDK forwarding infrastructure for
3 : * virtio interfaces. The virtio data structures are setup by the user
4 : * space vhost server.
5 : *
6 : * Copyright (c) 2014 Juniper Networks, Inc. All rights reserved.
7 : */
8 :
9 : #include "vr_dpdk.h"
10 : #include "vr_dpdk_virtio.h"
11 : #include "vr_uvhost_client.h"
12 :
13 : #include <linux/virtio_net.h>
14 : #include <sys/eventfd.h>
15 :
16 : #include <sys/mman.h>
17 : #include <sys/stat.h>
18 : #include <sys/types.h>
19 : #include <unistd.h>
20 :
21 : #include <rte_malloc.h>
22 : #include <rte_memcpy.h>
23 :
24 : #define VIRTIO_HDR_MRG_RXBUF 1
25 :
26 : void *vr_dpdk_vif_clients[VR_MAX_INTERFACES];
27 : vr_dpdk_virtioq_t vr_dpdk_virtio_rxqs[VR_MAX_INTERFACES][VR_DPDK_VIRTIO_MAX_QUEUES];
28 : vr_dpdk_virtioq_t vr_dpdk_virtio_txqs[VR_MAX_INTERFACES][VR_DPDK_VIRTIO_MAX_QUEUES];
29 :
30 : static int dpdk_virtio_from_vm_rx(void *port, struct rte_mbuf **pkts,
31 : uint32_t max_pkts);
32 : static int dpdk_virtio_to_vm_tx(void *port, struct rte_mbuf *pkt);
33 : static int dpdk_virtio_to_vm_flush(void *port);
34 : static int dpdk_virtio_writer_stats_read(void *port,
35 : struct rte_port_out_stats *stats,
36 : int clear);
37 : static int dpdk_virtio_reader_stats_read(void *port,
38 : struct rte_port_in_stats *stats,
39 : int clear);
40 :
41 : /*
42 : * Virtio writer
43 : */
44 : struct dpdk_virtio_writer {
45 : struct rte_port_out_stats stats;
46 : /* extra statistics */
47 : uint64_t nb_syscalls;
48 : /* last packet TX */
49 : uint64_t last_pkt_tx;
50 : /* last TX flush */
51 : uint64_t last_pkt_tx_flush;
52 :
53 : vr_dpdk_virtioq_t *tx_virtioq;
54 : struct rte_mbuf *tx_buf[VR_DPDK_VIRTIO_TX_BURST_SZ];
55 : /* Total number of mbuf chains
56 : * Say if a mbuf chain contains 10 segments, it is counted as 1
57 : */
58 : uint32_t tx_buf_count;
59 : /* Total number of mbufs in all the chains */
60 : uint32_t tx_mbufs;
61 : };
62 :
63 : struct dpdk_virtio_writer_params {
64 : /* virtio TX queue pointer */
65 : vr_dpdk_virtioq_t *tx_virtioq;
66 : };
67 :
68 : /*
69 : * vr_dpdk_virtio_stop - stop the virtio interface.
70 : *
71 : * Returns 0 on success, -1 otherwise.
72 : */
73 : int
74 554 : vr_dpdk_virtio_stop(unsigned int vif_idx)
75 : {
76 : int i;
77 : vr_dpdk_virtioq_t *vq;
78 :
79 554 : if (vif_idx >= VR_MAX_INTERFACES) {
80 0 : return -1;
81 : }
82 :
83 : /* Disable and reset all the virtio queues. */
84 18282 : for (i = 0; i < VR_DPDK_VIRTIO_MAX_QUEUES*2; i++) {
85 17728 : if (i & 1) {
86 8864 : vq = &vr_dpdk_virtio_rxqs[vif_idx][i/2];
87 : } else {
88 8864 : vq = &vr_dpdk_virtio_txqs[vif_idx][i/2];
89 : }
90 :
91 17728 : if (vq->vdv_ready_state != VQ_NOT_READY) {
92 388 : vr_dpdk_set_virtq_ready(vif_idx, i, VQ_NOT_READY);
93 : rte_wmb();
94 388 : synchronize_rcu();
95 : /*
96 : * TODO: code duplication to minimize the changes.
97 : * See vr_dpdk_virtio_get_vring_base().
98 : */
99 388 : vq->vdv_desc = NULL;
100 388 : if (vq->vdv_callfd) {
101 388 : close(vq->vdv_callfd);
102 388 : vq->vdv_callfd = 0;
103 : }
104 : }
105 : }
106 :
107 554 : return 0;
108 : }
109 :
110 : static void *
111 1328 : dpdk_virtio_writer_create(void *params, int socket_id)
112 : {
113 1328 : struct dpdk_virtio_writer_params *conf =
114 : (struct dpdk_virtio_writer_params *) params;
115 : struct dpdk_virtio_writer *port;
116 :
117 : /* Check input parameters */
118 1328 : if (conf == NULL) {
119 0 : RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", __func__);
120 0 : return NULL;
121 : }
122 :
123 : /* Memory allocation */
124 1328 : port = rte_zmalloc_socket("PORT", sizeof(*port),
125 : RTE_CACHE_LINE_SIZE, socket_id);
126 1328 : if (port == NULL) {
127 0 : RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__);
128 0 : return NULL;
129 : }
130 :
131 : /* Initialization */
132 1328 : port->tx_virtioq = conf->tx_virtioq;
133 :
134 1328 : return port;
135 : }
136 :
137 : static int
138 860 : dpdk_virtio_writer_free(void *port)
139 : {
140 : vr_dpdk_virtioq_t *tx_virtioq;
141 :
142 860 : if (port == NULL) {
143 0 : RTE_LOG(ERR, PORT, "%s: port is NULL\n", __func__);
144 0 : return -EINVAL;
145 : }
146 :
147 860 : tx_virtioq = ((struct dpdk_virtio_writer *)port)->tx_virtioq;
148 :
149 : /* close FDs */
150 860 : if (tx_virtioq->vdv_callfd > 0) {
151 0 : close(tx_virtioq->vdv_callfd);
152 : }
153 860 : if (tx_virtioq->vdv_kickfd > 0) {
154 0 : close(tx_virtioq->vdv_kickfd);
155 : }
156 :
157 : /* reset the virtio */
158 860 : memset(tx_virtioq, 0, sizeof(vr_dpdk_virtioq_t));
159 :
160 860 : rte_free(port);
161 :
162 860 : return 0;
163 : }
164 :
165 : struct rte_port_out_ops vr_dpdk_virtio_writer_ops = {
166 : .f_create = dpdk_virtio_writer_create,
167 : .f_free = dpdk_virtio_writer_free,
168 : .f_tx = dpdk_virtio_to_vm_tx,
169 : .f_tx_bulk = NULL, /* TODO: not implemented */
170 : .f_flush = dpdk_virtio_to_vm_flush,
171 : .f_stats = dpdk_virtio_writer_stats_read
172 : };
173 :
174 : /*
175 : * Virtio reader
176 : */
177 : struct dpdk_virtio_reader {
178 : struct rte_port_in_stats stats;
179 : /* extra statistics */
180 : uint64_t nb_syscalls;
181 : uint64_t nb_nombufs;
182 :
183 : vr_dpdk_virtioq_t *rx_virtioq;
184 : };
185 :
186 : struct dpdk_virtio_reader_params {
187 : /* virtio RX queue pointer */
188 : vr_dpdk_virtioq_t *rx_virtioq;
189 : };
190 :
191 : static void *
192 664 : dpdk_virtio_reader_create(void *params, int socket_id)
193 : {
194 664 : struct dpdk_virtio_reader_params *conf =
195 : (struct dpdk_virtio_reader_params *) params;
196 : struct dpdk_virtio_reader *port;
197 :
198 : /* Check input parameters */
199 664 : if (conf == NULL) {
200 0 : RTE_LOG(ERR, PORT, "%s: Invalid input parameters\n", __func__);
201 0 : return NULL;
202 : }
203 :
204 : /* Memory allocation */
205 664 : port = rte_zmalloc_socket("PORT", sizeof(*port),
206 : RTE_CACHE_LINE_SIZE, socket_id);
207 664 : if (port == NULL) {
208 0 : RTE_LOG(ERR, PORT, "%s: Failed to allocate port\n", __func__);
209 0 : return NULL;
210 : }
211 :
212 : /* Initialization */
213 664 : port->rx_virtioq = conf->rx_virtioq;
214 :
215 664 : return port;
216 : }
217 :
218 : static int
219 430 : dpdk_virtio_reader_free(void *port)
220 : {
221 : vr_dpdk_virtioq_t *rx_virtioq;
222 :
223 430 : if (port == NULL) {
224 0 : RTE_LOG(ERR, PORT, "%s: port is NULL\n", __func__);
225 0 : return -EINVAL;
226 : }
227 :
228 430 : rx_virtioq = ((struct dpdk_virtio_reader *)port)->rx_virtioq;
229 :
230 : /* close FDs */
231 430 : if (rx_virtioq->vdv_callfd > 0) {
232 0 : close(rx_virtioq->vdv_callfd);
233 : }
234 430 : if (rx_virtioq->vdv_kickfd > 0) {
235 0 : close(rx_virtioq->vdv_kickfd);
236 : }
237 :
238 : /* reset the virtio */
239 430 : memset(rx_virtioq, 0, sizeof(vr_dpdk_virtioq_t));
240 :
241 430 : rte_free(port);
242 :
243 430 : return 0;
244 : }
245 :
246 :
247 : struct rte_port_in_ops vr_dpdk_virtio_reader_ops = {
248 : .f_create = dpdk_virtio_reader_create,
249 : .f_free = dpdk_virtio_reader_free,
250 : .f_rx = dpdk_virtio_from_vm_rx,
251 : .f_stats = dpdk_virtio_reader_stats_read
252 : };
253 :
254 : /*
255 : * vr_dpdk_vrtio_uvh_get_blk_size - set the block size of fd.
256 : * On error -1 is returned, otherwise 0.
257 : */
258 : int
259 388 : vr_dpdk_virtio_uvh_get_blk_size(int fd, uint64_t *const blksize)
260 : {
261 : struct stat fd_stat;
262 : int ret;
263 388 : memset(&fd_stat, 0, sizeof(stat));
264 :
265 388 : ret = fstat(fd, &fd_stat);
266 388 : if (!ret){
267 388 : *blksize = (uint64_t)fd_stat.st_blksize;
268 : } else {
269 : RTE_LOG_DP(DEBUG, UVHOST, "Error getting file status for FD %d: %s (%d)\n",
270 : fd, strerror(errno), errno);
271 : }
272 :
273 388 : return ret;
274 : }
275 :
276 : /*
277 : * vr_dpdk_virtio_nrxqs - returns the number of receives queues for a virtio
278 : * interface.
279 : */
280 : uint16_t
281 996 : vr_dpdk_virtio_nrxqs(struct vr_interface *vif)
282 : {
283 996 : return vr_dpdk.nb_fwd_lcores;
284 : }
285 :
286 : /*
287 : * vr_dpdk_virtio_ntxqs - returns the number of transmit queues for a virtio
288 : * interface.
289 : */
290 : uint16_t
291 1328 : vr_dpdk_virtio_ntxqs(struct vr_interface *vif)
292 : {
293 1328 : return vr_dpdk.nb_fwd_lcores;
294 : }
295 :
296 : static unsigned int vif_rx_queue_lcore[VR_MAX_INTERFACES][VR_MAX_INTERFACES];
297 :
298 : /*
299 : * dpdk_virtio_rx_queue_release - releases a virtio RX queue.
300 : *
301 : * Returns nothing.
302 : */
303 : static void
304 430 : dpdk_virtio_rx_queue_release(unsigned lcore_id,
305 : unsigned queue_index __attribute__((unused)),
306 : struct vr_interface *vif)
307 : {
308 430 : struct vr_dpdk_lcore *lcore = vr_dpdk.lcores[lcore_id];
309 430 : struct vr_dpdk_queue *rx_queue = &lcore->lcore_rx_queues[vif->vif_idx];
310 430 : struct vr_dpdk_queue_params *rx_queue_params
311 430 : = &lcore->lcore_rx_queue_params[vif->vif_idx];
312 : /* free the queue */
313 430 : if (rx_queue->rxq_ops.f_free(rx_queue->q_queue_h)) {
314 0 : RTE_LOG(ERR, VROUTER, " error freeing lcore %u virtio device RX queue\n",
315 : lcore_id);
316 : }
317 :
318 : /* reset the queue */
319 430 : vrouter_put_interface(rx_queue->q_vif);
320 430 : memset(rx_queue, 0, sizeof(*rx_queue));
321 430 : memset(rx_queue_params, 0, sizeof(*rx_queue_params));
322 430 : }
323 :
324 : /*
325 : * vr_dpdk_virtio_rx_queue_init - initializes a virtio RX queue.
326 : *
327 : * Returns a pointer to the RX queue on success, NULL otherwise.
328 : */
329 : struct vr_dpdk_queue *
330 664 : vr_dpdk_virtio_rx_queue_init(unsigned int lcore_id, struct vr_interface *vif,
331 : unsigned int queue_or_lcore_id)
332 : {
333 664 : uint16_t queue_id = queue_or_lcore_id;
334 664 : struct vr_dpdk_lcore *lcore = vr_dpdk.lcores[lcore_id];
335 664 : const unsigned int socket_id = rte_lcore_to_socket_id(lcore_id);
336 664 : unsigned int vif_idx = vif->vif_idx;
337 664 : struct vr_dpdk_queue *rx_queue = &lcore->lcore_rx_queues[vif_idx];
338 664 : struct vr_dpdk_queue_params *rx_queue_params =
339 : &lcore->lcore_rx_queue_params[vif_idx];
340 :
341 : /* Check input parameters */
342 664 : if (queue_id >= vr_dpdk_virtio_nrxqs(vif)) {
343 0 : RTE_LOG(ERR, VROUTER, " error creating virtio device %s RX queue %"
344 : PRIu16 "\n", vif->vif_name, queue_id);
345 0 : return NULL;
346 : }
347 :
348 : /* init queue */
349 664 : rx_queue->rxq_ops = vr_dpdk_virtio_reader_ops;
350 664 : rx_queue->q_vif = vrouter_get_interface(vif->vif_rid, vif_idx);
351 :
352 : /* init virtio queue */
353 664 : vr_dpdk_virtio_rxqs[vif_idx][queue_id].vdv_ready_state = VQ_NOT_READY;
354 664 : vr_dpdk_virtio_rxqs[vif_idx][queue_id].vdv_last_used_idx = 0;
355 664 : vr_dpdk_virtio_rxqs[vif_idx][queue_id].vdv_last_used_idx_res = 0;
356 664 : vr_dpdk_virtio_rxqs[vif_idx][queue_id].vdv_vif_idx = vif->vif_idx;
357 :
358 : /* create the queue */
359 664 : struct dpdk_virtio_reader_params reader_params = {
360 664 : .rx_virtioq = &vr_dpdk_virtio_rxqs[vif_idx][queue_id],
361 : };
362 664 : rx_queue->q_queue_h = rx_queue->rxq_ops.f_create(&reader_params, socket_id);
363 664 : if (rx_queue->q_queue_h == NULL) {
364 0 : RTE_LOG(ERR, VROUTER, " error creating virtio device %s RX queue %"
365 : PRIu16 "\n", vif->vif_name, queue_id);
366 0 : return NULL;
367 : }
368 664 : rx_queue->vring_queue_id = queue_id;
369 : /* store queue params */
370 664 : rx_queue_params->qp_release_op = &dpdk_virtio_rx_queue_release;
371 :
372 : /* save the lcore serving the queue for later enabling/disabling */
373 664 : vif_rx_queue_lcore[vif_idx][queue_id] = lcore_id;
374 :
375 664 : return rx_queue;
376 : }
377 :
378 : /*
379 : * dpdk_virtio_tx_queue_release - releases a virtio TX queue.
380 : *
381 : * Returns nothing.
382 : */
383 : static void
384 860 : dpdk_virtio_tx_queue_release(unsigned lcore_id, unsigned queue_index,
385 : struct vr_interface *vif)
386 : {
387 860 : struct vr_dpdk_lcore *lcore = vr_dpdk.lcores[lcore_id];
388 860 : struct vr_dpdk_queue *tx_queue =
389 860 : &lcore->lcore_tx_queues[vif->vif_idx][queue_index];
390 860 : struct vr_dpdk_queue_params *tx_queue_params
391 860 : = &lcore->lcore_tx_queue_params[vif->vif_idx][queue_index];
392 :
393 860 : tx_queue->txq_ops.f_tx = NULL;
394 : rte_wmb();
395 :
396 : /* flush and free the queue */
397 860 : if (tx_queue->txq_ops.f_free(tx_queue->q_queue_h)) {
398 0 : RTE_LOG(ERR, VROUTER, " error freeing lcore %u virtio device TX queue\n",
399 : lcore_id);
400 : }
401 :
402 : /* reset the queue */
403 860 : vrouter_put_interface(tx_queue->q_vif);
404 860 : memset(tx_queue, 0, sizeof(*tx_queue));
405 860 : memset(tx_queue_params, 0, sizeof(*tx_queue_params));
406 860 : }
407 :
408 : /*
409 : * vr_dpdk_virtio_tx_queue_init - initializes a virtio TX queue.
410 : *
411 : * Returns a pointer to the TX queue on success, NULL otherwise.
412 : */
413 : struct vr_dpdk_queue *
414 1328 : vr_dpdk_virtio_tx_queue_init(unsigned int lcore_id, struct vr_interface *vif,
415 : unsigned int queue_or_lcore_id)
416 : {
417 1328 : uint16_t queue_id = queue_or_lcore_id;
418 1328 : struct vr_dpdk_lcore *lcore = vr_dpdk.lcores[lcore_id];
419 1328 : const unsigned int socket_id = rte_lcore_to_socket_id(lcore_id);
420 1328 : unsigned int vif_idx = vif->vif_idx;
421 1328 : struct vr_dpdk_queue *tx_queue = &lcore->lcore_tx_queues[vif_idx][0];
422 1328 : struct vr_dpdk_queue_params *tx_queue_params
423 : = &lcore->lcore_tx_queue_params[vif_idx][0];
424 :
425 : /* Check input parameters */
426 : /* virtio TX is thread safe, so just use one of the rings */
427 1328 : queue_id = queue_id % vr_dpdk_virtio_ntxqs(vif);
428 :
429 : /* init queue */
430 1328 : tx_queue->txq_ops = vr_dpdk_virtio_writer_ops;
431 1328 : tx_queue->q_vif = vrouter_get_interface(vif->vif_rid, vif_idx);
432 :
433 : /* init virtio queue */
434 1328 : vr_dpdk_virtio_txqs[vif_idx][queue_id].vdv_ready_state = VQ_NOT_READY;
435 1328 : vr_dpdk_virtio_txqs[vif_idx][queue_id].vdv_last_used_idx = 0;
436 1328 : vr_dpdk_virtio_txqs[vif_idx][queue_id].vdv_last_used_idx_res = 0;
437 1328 : vr_dpdk_virtio_txqs[vif_idx][queue_id].vdv_vif_idx = vif->vif_idx;
438 :
439 : /* create the queue */
440 1328 : struct dpdk_virtio_writer_params writer_params = {
441 : /*
442 : * Always initialize each lcore's tx_queue with virtio queue number 0.
443 : * If there are more queues, they will be enabled later via
444 : * VHOST_USER_SET_VRING_ENABLE message.
445 : */
446 1328 : .tx_virtioq = &vr_dpdk_virtio_txqs[vif_idx][0],
447 : };
448 1328 : tx_queue->q_queue_h = tx_queue->txq_ops.f_create(&writer_params, socket_id);
449 1328 : if (tx_queue->q_queue_h == NULL) {
450 0 : RTE_LOG(ERR, VROUTER, " error creating virtio device %s TX queue %"
451 : PRIu16 "\n", vif->vif_name, queue_id);
452 0 : return NULL;
453 : }
454 :
455 : /* store queue params */
456 1328 : tx_queue_params->qp_release_op = &dpdk_virtio_tx_queue_release;
457 :
458 1328 : return tx_queue;
459 : }
460 :
461 : struct dpdk_virtio_tx_queue_set_params {
462 : unsigned int vif_id;
463 : unsigned int vif_gen;
464 : unsigned int queue_id;
465 : };
466 :
467 : static unsigned int vif_lcore_tx_queue[VR_MAX_INTERFACES][VR_MAX_CPUS_DPDK];
468 : static unsigned int vif_tx_queues_enabled[VR_MAX_INTERFACES];
469 :
470 : /*
471 : * Enable or disable given queue for a vif.
472 : *
473 : * In current vRouter design, every lcore that can send packets has to have a
474 : * TX queue available for every existing vif. It is because we do not know
475 : * which lcore wil eventually send the packet, and thus each has to have a
476 : * queue to use.
477 : *
478 : * If VM requests more than one virtio queue, then we distribute them among the
479 : * forwarding lcores as evenly as possible.
480 : *
481 : * The entire process (this function, which sends commands to other lcores and
482 : * then vr_dpdk_virtio_tx_queue_set(), which is called from the destination
483 : * lcores' main loop) works fine as long as the QEMU enables/disables each
484 : * queues in ascending order. For example, if the maximal number of queues is
485 : * 4, and inside a VM ethtool -L eth0 combined 2 is issued, the QEMU will send
486 : * the following messages:
487 : * 1. Enable queue 0.
488 : * 2. Enable queue 1.
489 : * 3. Disable queue 2.
490 : * 4. Disable queue 3.
491 : *
492 : * TODO: Remove the above assumption as there is no guarantee that QEMU will
493 : * always work as described.
494 : */
495 : void
496 0 : vr_dpdk_virtio_tx_queue_enable_disable(unsigned int vif_id,
497 : unsigned int vif_gen,
498 : unsigned int queue_id,
499 : bool enable)
500 : {
501 : unsigned int lcore_id;
502 : unsigned int starting_lcore;
503 : struct dpdk_virtio_tx_queue_set_params *arg;
504 : unsigned int qid;
505 : unsigned int queue_num;
506 :
507 : /* If command is 'disable', we enable all lower numbered queues */
508 0 : if (!enable)
509 0 : queue_num = queue_id - 1;
510 : else
511 0 : queue_num = queue_id;
512 :
513 : /*
514 : * Subsequent 'disable' commands are ignored. For example if we enabled
515 : * queues 0 and 1, then all higher queues (2, 3, ..) had already been
516 : * disabled. Thus we ignore the 'disable' request for them
517 : */
518 0 : if (!enable && queue_num > vif_tx_queues_enabled[vif_id])
519 0 : return;
520 :
521 : /*
522 : * Each lcore that does tx has to have a queue assigned for every
523 : * interface. We assign queue 0 for pkt and netlink lcores. All
524 : * other queues (including queue 0) are distributed among forwarding
525 : * lcores.
526 : */
527 0 : if (queue_id == 0)
528 0 : starting_lcore = VR_DPDK_PACKET_LCORE_ID;
529 : else
530 0 : starting_lcore = VR_DPDK_FWD_LCORE_ID;
531 :
532 0 : for (lcore_id = starting_lcore, qid = 0; lcore_id < vr_dpdk.nb_fwd_lcores +
533 0 : VR_DPDK_FWD_LCORE_ID; ++lcore_id) {
534 :
535 : /*
536 : * Send cmd to destination lcore only if it has different queue enabled
537 : * curently.
538 : */
539 0 : if (vif_lcore_tx_queue[vif_id][lcore_id - VR_DPDK_PACKET_LCORE_ID] !=
540 : qid) {
541 0 : vif_lcore_tx_queue[vif_id][lcore_id - VR_DPDK_PACKET_LCORE_ID] =
542 : qid;
543 :
544 0 : arg = rte_malloc("virtio_tx_queue_set", sizeof(*arg), 0);
545 :
546 0 : arg->vif_id = vif_id;
547 0 : arg->queue_id = qid;
548 0 : arg->vif_gen = vif_gen;
549 :
550 0 : vr_dpdk_lcore_cmd_post(lcore_id, VR_DPDK_LCORE_TX_QUEUE_SET_CMD,
551 : (uint64_t)arg);
552 : }
553 :
554 0 : ++qid;
555 0 : qid %= queue_num + 1;
556 : }
557 :
558 : /* Save current number of TX queues enabled for vif */
559 0 : vif_tx_queues_enabled[vif_id] = queue_num;
560 : }
561 :
562 : /*
563 : * Assign given virtio queue to vRouter's dpdk (per lcore) tx queue.
564 : *
565 : * The assignment is done by setting correct virtio queue pointer in the
566 : * lcore's tx queue handler.
567 : *
568 : * This function is called only from the main loops of the lcores that have TX
569 : * queues (packet lcore, netlink lcore, forwarding lcores).
570 : */
571 : void
572 0 : vr_dpdk_virtio_tx_queue_set(void *arg)
573 : {
574 0 : struct dpdk_virtio_tx_queue_set_params *p = arg;
575 : struct vr_dpdk_queue *tx_queue;
576 : struct dpdk_virtio_writer *port;
577 : struct vr_dpdk_lcore *lcore;
578 : struct vr_interface *vif;
579 :
580 : /* Check if vif is still valid */
581 0 : vif = __vrouter_get_interface(vrouter_get(0), p->vif_id);
582 0 : if (!vif || vif->vif_gen != p->vif_gen) {
583 0 : rte_free(arg);
584 0 : return;
585 : }
586 :
587 0 : lcore = vr_dpdk.lcores[rte_lcore_id()];
588 0 : tx_queue = &lcore->lcore_tx_queues[p->vif_id][0];
589 0 : port = (struct dpdk_virtio_writer *)tx_queue->q_queue_h;
590 :
591 : /* Assign new queue to the lcore's tx_queue handler */
592 0 : port->tx_virtioq = &vr_dpdk_virtio_txqs[p->vif_id][p->queue_id];
593 :
594 : /*
595 : * Each tx_queue has to have a f_flush method, but we do not need to crash
596 : * in other case.
597 : */
598 0 : if (tx_queue->txq_ops.f_flush)
599 0 : tx_queue->txq_ops.f_flush(tx_queue->q_queue_h);
600 : else
601 0 : RTE_LOG(ERR, VROUTER, "%s: Flush function for tx_queue(%p) unavailable\n",
602 : __func__, tx_queue);
603 :
604 0 : rte_free(arg);
605 : }
606 :
607 : struct dpdk_virtio_rx_queue_set_params {
608 : bool enable;
609 : unsigned int vif_id;
610 : unsigned int vif_gen;
611 : unsigned int queue_id;
612 : };
613 :
614 :
615 : void
616 : dpdk_lcore_queue_add(unsigned lcore_id, struct vr_dpdk_q_slist *q_head,
617 : struct vr_dpdk_queue *queue);
618 : void
619 : dpdk_lcore_rx_queue_remove(struct vr_dpdk_lcore *lcore,
620 : struct vr_dpdk_queue *rx_queue,
621 : bool clear_f_rx);
622 :
623 : /*
624 : * Called on uvhost lcore only.
625 : */
626 : void
627 0 : vr_dpdk_virtio_rx_queue_enable_disable(unsigned int vif_id,
628 : unsigned int vif_gen,
629 : unsigned int queue_id,
630 : bool enable)
631 : {
632 : struct dpdk_virtio_rx_queue_set_params *arg;
633 :
634 : /*
635 : * Ignore requests for queue number 0. It has already been added to lcore's
636 : * list of queues and can never be disabled (qemu never sends the 'disable'
637 : * command for queue 0). Doing otherwise would result in double adding the
638 : * virtio queue to lcore's list of rx queues.
639 : */
640 0 : if (queue_id == 0)
641 0 : return;
642 :
643 0 : arg = rte_malloc("virtio_rx_queue_set", sizeof(*arg), 0);
644 :
645 0 : arg->vif_id = vif_id;
646 0 : arg->vif_gen = vif_gen;
647 0 : arg->queue_id = queue_id;
648 0 : arg->enable = enable;
649 :
650 0 : vr_dpdk_lcore_cmd_post(VR_DPDK_NETLINK_LCORE_ID,
651 : VR_DPDK_LCORE_RX_QUEUE_SET_CMD, (uint64_t)arg);
652 : }
653 :
654 : /*
655 : * Called only on netlink lcore.
656 : */
657 : void
658 0 : vr_dpdk_virtio_rx_queue_set(void *arg)
659 : {
660 0 : struct dpdk_virtio_rx_queue_set_params *p = arg;
661 : struct vr_interface *vif;
662 : struct vr_dpdk_queue *rx_queue;
663 : struct vr_dpdk_lcore *lcore;
664 : unsigned int lcore_id;
665 : struct vr_dpdk_lcore_rx_queue_remove_arg *rx_rm_arg;
666 :
667 : /* Check if vif is still valid */
668 0 : vif = __vrouter_get_interface(vrouter_get(0), p->vif_id);
669 0 : if (!vif || vif->vif_gen != p->vif_gen) {
670 0 : rte_free(arg);
671 0 : return;
672 : }
673 :
674 0 : if (p->enable) {
675 0 : lcore_id = vif_rx_queue_lcore[p->vif_id][p->queue_id];
676 0 : lcore = vr_dpdk.lcores[lcore_id];
677 0 : rx_queue = &lcore->lcore_rx_queues[p->vif_id];
678 0 : lcore->lcore_rx_queues[p->vif_id].vring_queue_id = p->queue_id;
679 0 : dpdk_lcore_queue_add(lcore_id, &lcore->lcore_rx_head, rx_queue);
680 :
681 : } else {
682 0 : lcore_id = vif_rx_queue_lcore[p->vif_id][p->queue_id];
683 0 : lcore = vr_dpdk.lcores[lcore_id];
684 0 : rx_queue = &lcore->lcore_rx_queues[p->vif_id];
685 0 : if (rx_queue->enabled) {
686 0 : rx_rm_arg = rte_malloc("lcore_rx_queue_rm_cmd", sizeof(*rx_rm_arg),
687 : 0);
688 0 : rx_rm_arg->vif_id = vif->vif_idx;
689 0 : rx_rm_arg->clear_f_rx = false;
690 0 : rx_rm_arg->free_arg = true;
691 0 : vr_dpdk_lcore_cmd_post(lcore_id, VR_DPDK_LCORE_RX_RM_CMD,
692 : (uint64_t)rx_rm_arg);
693 : }
694 : }
695 :
696 0 : rte_free(arg);
697 : }
698 :
699 : /*
700 : * vr_dpdk_guest_phys_to_host_virt - convert a guest physical address
701 : * to a host virtual address. Uses the guest memory map stored in the
702 : * vhost client for the guest interface.
703 : *
704 : * Returns address on success, NULL otherwise.
705 : */
706 : static char *
707 243 : vr_dpdk_guest_phys_to_host_virt(vr_uvh_client_t *vru_cl, uint64_t paddr)
708 : {
709 : int i;
710 : vr_uvh_client_mem_region_t *reg;
711 :
712 403 : for (i = 0; i < vru_cl->vruc_num_mem_regions; i++) {
713 403 : reg = &vru_cl->vruc_mem_regions[i];
714 :
715 403 : if ((paddr >= reg->vrucmr_phys_addr) &&
716 243 : (paddr <= (reg->vrucmr_phys_addr + reg->vrucmr_size))) {
717 243 : return ((char *) reg->vrucmr_mmap_addr) +
718 243 : (paddr - reg->vrucmr_phys_addr);
719 : }
720 : }
721 :
722 0 : return NULL;
723 : }
724 :
725 : #ifdef RTE_PORT_STATS_COLLECT
726 :
727 : #define DPDK_VIRTIO_READER_STATS_PKTS_IN_ADD(port, val) \
728 : port->stats.n_pkts_in += val
729 : #define DPDK_VIRTIO_READER_STATS_PKTS_DROP_ADD(port, val) \
730 : port->stats.n_pkts_drop += val
731 :
732 : #else
733 :
734 : /* keep compiler happy, for unused variables */
735 : #define DPDK_VIRTIO_READER_STATS_PKTS_IN_ADD(port, val) \
736 : (void)(val)
737 : #define DPDK_VIRTIO_READER_STATS_PKTS_DROP_ADD(port, val) \
738 : (void)(val)
739 :
740 : #endif
741 :
742 : static inline uint32_t
743 0 : dpdk_virtio_get_ip_tcp_hdr_len(char *pkt_addr, uint32_t pkt_len)
744 : {
745 0 : struct vr_eth *eth_hdr = (struct vr_eth*)pkt_addr;
746 0 : struct vr_ip6 *ipv6_hdr = NULL;
747 0 : struct vr_tcp *tcp_hdr = NULL;
748 0 : unsigned int pull_len = VR_ETHER_HLEN;
749 : unsigned short eth_proto;
750 :
751 0 : if (unlikely(pkt_len < pull_len))
752 0 : return 0;
753 :
754 0 : eth_proto = eth_hdr->eth_proto;
755 :
756 : /* Skip VLAN tag which may be present if VM sends tagged pkts */
757 0 : while (eth_proto == rte_cpu_to_be_16(VR_ETH_PROTO_VLAN)) {
758 0 : if (unlikely(pkt_len < pull_len + VR_VLAN_HLEN))
759 0 : return 0;
760 0 : eth_proto = ((struct vr_vlan_hdr *)((uintptr_t)eth_hdr + pull_len))->vlan_proto;
761 0 : pull_len += VR_VLAN_HLEN;
762 : }
763 :
764 0 : if (likely(eth_proto == rte_cpu_to_be_16(VR_ETH_PROTO_IP))) {
765 0 : struct vr_ip *ipv4_hdr = NULL;
766 : uint32_t ipv4_hlen;
767 0 : ipv4_hdr = (struct vr_ip *)((uintptr_t)eth_hdr + pull_len);
768 :
769 0 : if (unlikely(pkt_len < pull_len + sizeof(struct vr_ip)))
770 0 : return 0;
771 :
772 0 : ipv4_hlen = ((ipv4_hdr->ip_hl) * RTE_IPV4_IHL_MULTIPLIER);
773 0 : pull_len += ipv4_hlen;
774 0 : tcp_hdr = (struct vr_tcp*)((uint8_t*)ipv4_hdr + ipv4_hlen);
775 0 : } else if (eth_proto == rte_cpu_to_be_16(VR_ETH_PROTO_IP6)) {
776 0 : ipv6_hdr = (struct vr_ip6 *)((uintptr_t)eth_hdr + pull_len);
777 :
778 0 : if (unlikely(pkt_len < pull_len + sizeof(struct vr_ip6)))
779 0 : return 0;
780 :
781 0 : pull_len += sizeof(*ipv6_hdr);
782 0 : tcp_hdr = (struct vr_tcp*)((uint8_t*)ipv6_hdr + sizeof(*ipv6_hdr));
783 : }
784 0 : if (likely(tcp_hdr != NULL)) {
785 0 : pull_len += (VR_TCP_OFFSET(tcp_hdr->tcp_offset_r_flags) << 2);
786 : }
787 :
788 0 : return pull_len;
789 : }
790 :
791 0 : static inline char *dpdk_pktmbuf_append(struct rte_mbuf *m, struct rte_mbuf *last, uint16_t len)
792 : {
793 : void *tail;
794 : struct rte_mbuf *m_last;
795 :
796 : __rte_mbuf_sanity_check(m, 1);
797 : __rte_mbuf_sanity_check(last, 1);
798 :
799 0 : m_last = rte_pktmbuf_lastseg(last);
800 0 : if (unlikely(len > rte_pktmbuf_tailroom(m_last)))
801 0 : return NULL;
802 :
803 0 : tail = (char *)m_last->buf_addr + m_last->data_off + m_last->data_len;
804 0 : m_last->data_len = (uint16_t)(m_last->data_len + len);
805 0 : m->pkt_len = (m->pkt_len + len);
806 0 : return (char*) tail;
807 : }
808 :
809 : /*
810 : * dpdk_virtio_create_mss_sized_mbuf_chain - Create a chained mbuf where each segment
811 : * in the chain is of length 'mss' and copy the data pointed to by pkt_addr
812 : *
813 : * @input -
814 : * mbuf: pointer to the mbuf where the chain needs to be created
815 : * mss: lenght of each segment in the chain
816 : * pkt_addr: pointer to the data which has to be copied to mbuf
817 : * pkt_len: length of the data which has to be copied
818 : * header_len: first segment of the chain will have a length of mss + this value
819 : * to account for the headers
820 : *
821 : * @output -
822 : * 0: success
823 : * -1: failure
824 : */
825 : static int
826 0 : dpdk_virtio_create_mss_sized_mbuf_chain(struct rte_mbuf *mbuf,
827 : uint32_t mss, char* pkt_addr, uint32_t pkt_len, uint32_t header_len)
828 : {
829 0 : char *tail_addr, *append_addr = pkt_addr;
830 0 : uint32_t pktlen_to_copy = pkt_len, copy_len;
831 0 : struct rte_mbuf *new_mbuf, *last_mbuf = rte_pktmbuf_lastseg(mbuf);
832 :
833 : /* header is only applicable for first segment */
834 0 : if (mbuf->nb_segs > 1)
835 0 : header_len = 0;
836 :
837 : /* Cannot compute checksum of odd sized mbufs in chain,
838 : * so make it even sized
839 : */
840 0 : if (mss & 1)
841 0 : mbuf->tso_segsz = mss -=1;
842 :
843 0 : while (pktlen_to_copy > 0) {
844 0 : copy_len = mss + header_len - last_mbuf->data_len;
845 0 : header_len = 0;
846 0 : if (pktlen_to_copy > copy_len) {
847 0 : tail_addr = dpdk_pktmbuf_append(mbuf, last_mbuf, copy_len);
848 0 : if (unlikely(tail_addr == NULL))
849 0 : return -1;
850 0 : rte_memcpy(tail_addr, append_addr, copy_len);
851 0 : pktlen_to_copy -= copy_len;
852 0 : append_addr += copy_len;
853 0 : new_mbuf = rte_pktmbuf_alloc(vr_dpdk.rss_mempool);
854 0 : if (unlikely(new_mbuf == NULL)) {
855 : RTE_LOG_DP(DEBUG, VROUTER, "%s: mbuf alloc failed\n",__func__);
856 0 : return -1;
857 : }
858 0 : last_mbuf->next = new_mbuf;
859 0 : last_mbuf = new_mbuf;
860 0 : mbuf->nb_segs += 1;
861 : } else {
862 : /* for last segment */
863 0 : tail_addr = dpdk_pktmbuf_append(mbuf, last_mbuf, pktlen_to_copy);
864 0 : if (unlikely(tail_addr == NULL))
865 0 : return -1;
866 0 : rte_memcpy(tail_addr, append_addr, pktlen_to_copy);
867 0 : pktlen_to_copy = 0;
868 : }
869 : }
870 0 : return 0;
871 : }
872 :
873 : /*
874 : * dpdk_virtio_create_chained_mbuf - Create a chained mbuf and copy the data pointed
875 : * to by pkt_addr of len pkt_len
876 : */
877 : static int
878 1 : dpdk_virtio_create_chained_mbuf(struct rte_mbuf *mbuf, char* pkt_addr, uint32_t pkt_len)
879 : {
880 1 : char *tail_addr, *append_addr = pkt_addr;
881 1 : uint32_t append_len = pkt_len;
882 : struct rte_mbuf *new_mbuf;
883 :
884 5 : while((tail_addr = rte_pktmbuf_append(mbuf, append_len)) == NULL) {
885 4 : uint32_t pkt_tailroom = rte_pktmbuf_tailroom(rte_pktmbuf_lastseg(mbuf));
886 4 : tail_addr = rte_pktmbuf_append(mbuf, pkt_tailroom);
887 4 : if (unlikely(tail_addr == NULL))
888 0 : return -1;
889 4 : rte_memcpy(tail_addr, append_addr, pkt_tailroom);
890 4 : append_len -= pkt_tailroom;
891 4 : append_addr += pkt_tailroom;
892 4 : new_mbuf = rte_pktmbuf_alloc(vr_dpdk.rss_mempool);
893 4 : if (unlikely(new_mbuf == NULL)) {
894 : RTE_LOG_DP(DEBUG, VROUTER, "%s: mbuf alloc failed\n",__func__);
895 0 : return -1;
896 : }
897 4 : rte_pktmbuf_lastseg(mbuf)->next = new_mbuf;
898 4 : mbuf->nb_segs += 1;
899 : }
900 1 : rte_memcpy(tail_addr, append_addr, append_len);
901 1 : return 0;
902 : }
903 :
904 : /*
905 : * dpdk_virtio_from_vm_rx - receive packets from a virtio client so that
906 : * the packets can be handed to vrouter for forwarding. the virtio client is
907 : * usually a VM.
908 : *
909 : * Returns the number of packets received from the virtio.
910 : */
911 : static int
912 957357717 : dpdk_virtio_from_vm_rx(void *port, struct rte_mbuf **pkts, uint32_t max_pkts)
913 : {
914 957357717 : struct dpdk_virtio_reader *p = (struct dpdk_virtio_reader *)port;
915 957357717 : vr_dpdk_virtioq_t *vq = p->rx_virtioq;
916 : uint16_t vq_hard_avail_idx, i;
917 : uint16_t avail_pkts, next_desc_idx, next_avail_idx;
918 : struct vring_desc *desc;
919 : char *pkt_addr, *tail_addr;
920 : struct rte_mbuf *mbuf;
921 957357717 : uint32_t pkt_len, nb_pkts = 0;
922 : vr_uvh_client_t *vru_cl;
923 :
924 957357717 : if (unlikely(vq->vdv_ready_state == VQ_NOT_READY)) {
925 885091655 : DPDK_UDEBUG(VROUTER, &vq->vdv_hash, "%s: queue %p is not ready\n",
926 : __func__, vq);
927 882469014 : return 0;
928 : }
929 :
930 72266062 : vru_cl = vr_dpdk_virtio_get_vif_client(vq->vdv_vif_idx);
931 99307437 : if (unlikely(vru_cl == NULL))
932 0 : return 0;
933 :
934 99307437 : vq_hard_avail_idx = (*((volatile uint16_t *)&vq->vdv_avail->idx));
935 :
936 : /* Unsigned subtraction gives the right result even with wrap around. */
937 99307437 : avail_pkts = vq_hard_avail_idx - vq->vdv_last_used_idx;
938 99307437 : avail_pkts = RTE_MIN(avail_pkts, max_pkts);
939 99307437 : if (unlikely(avail_pkts == 0)) {
940 99323911 : DPDK_UDEBUG(VROUTER, &vq->vdv_hash, "%s: queue %p has no packets\n",
941 : __func__, vq);
942 99340041 : return 0;
943 : }
944 :
945 0 : DPDK_UDEBUG(VROUTER, &vq->vdv_hash, "%s: queue %p AVAILABLE %u packets\n",
946 : __func__, vq, avail_pkts);
947 320 : for (i = 0; i < avail_pkts; i++) {
948 160 : uint32_t header_len = 0;
949 : /* Allocate a mbuf. */
950 160 : mbuf = rte_pktmbuf_alloc(vr_dpdk.rss_mempool);
951 160 : if (unlikely(mbuf == NULL)) {
952 0 : p->nb_nombufs++;
953 0 : DPDK_UDEBUG(VROUTER, &vq->vdv_hash, "%s: queue %p no_mbufs=%"PRIu64"\n",
954 : __func__, vq, p->nb_nombufs);
955 0 : break;
956 : }
957 :
958 160 : next_avail_idx = (vq->vdv_last_used_idx + i) & (vq->vdv_size - 1);
959 160 : next_desc_idx = vq->vdv_avail->ring[next_avail_idx];
960 : /*
961 : * Move the (chain of) descriptors to the vdv_used list. The used
962 : * index will, however, only be updated at the end of the loop.
963 : */
964 160 : vq->vdv_used->ring[next_avail_idx].id = next_desc_idx;
965 160 : vq->vdv_used->ring[next_avail_idx].len = 0;
966 :
967 160 : desc = &vq->vdv_desc[next_desc_idx];
968 160 : pkt_len = desc->len;
969 160 : pkt_addr = vr_dpdk_guest_phys_to_host_virt(vru_cl, desc->addr);
970 : /* Check the descriptor is sane. */
971 160 : if (unlikely(desc->len < vq->vdv_hlen ||
972 : desc->addr == 0 || pkt_addr == NULL)) {
973 0 : goto free_mbuf;
974 : }
975 160 : mbuf->tso_segsz = 0;
976 : /* Now pkt_addr points to the virtio_net_hdr. */
977 160 : if (((struct virtio_net_hdr *)pkt_addr)->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM)
978 160 : mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD;
979 160 : if (((struct virtio_net_hdr *)pkt_addr)->gso_type == VIRTIO_NET_HDR_GSO_TCPV4) {
980 0 : mbuf->ol_flags |= PKT_RX_GSO_TCP4;
981 0 : mbuf->tso_segsz = ((struct virtio_net_hdr *)pkt_addr)->gso_size;
982 160 : } else if (((struct virtio_net_hdr *)pkt_addr)->gso_type == VIRTIO_NET_HDR_GSO_TCPV6) {
983 0 : mbuf->ol_flags |= PKT_RX_GSO_TCP6;
984 0 : mbuf->tso_segsz = ((struct virtio_net_hdr *)pkt_addr)->gso_size;
985 : }
986 :
987 : /* Skip virtio_net_hdr */
988 160 : if (likely(desc->flags & VRING_DESC_F_NEXT &&
989 : pkt_len == vq->vdv_hlen)) {
990 0 : DPDK_UDEBUG(VROUTER, &vq->vdv_hash, "%s: queue %p pkt %u F_NEXT\n",
991 : __func__, vq, i);
992 0 : desc = &vq->vdv_desc[desc->next];
993 0 : pkt_len = desc->len;
994 0 : pkt_addr = vr_dpdk_guest_phys_to_host_virt(vru_cl, desc->addr);
995 : } else {
996 160 : DPDK_UDEBUG(VROUTER, &vq->vdv_hash, "%s: queue %p pkt %u no F_NEXT\n",
997 : __func__, vq, i);
998 160 : pkt_addr += vq->vdv_hlen;
999 160 : pkt_len -= vq->vdv_hlen;
1000 : }
1001 : /* Now pkt_addr points to the packet data. */
1002 160 : if (mbuf->tso_segsz == 0) {
1003 160 : tail_addr = rte_pktmbuf_append(mbuf, pkt_len);
1004 : /* Check we ready to copy the data. */
1005 160 : if (unlikely(desc->addr == 0 || pkt_addr == NULL)) {
1006 0 : goto free_mbuf;
1007 160 : } else if (unlikely(tail_addr == NULL)) {
1008 : /* If insufficient tailroom, create a chained mbuf and copy the data */
1009 1 : if (unlikely(dpdk_virtio_create_chained_mbuf(mbuf, pkt_addr, pkt_len) < 0)) {
1010 0 : goto free_mbuf;
1011 : }
1012 : } else {
1013 : /* No chaining - Just Copy first descriptor data. */
1014 159 : rte_memcpy(tail_addr, pkt_addr, pkt_len);
1015 : }
1016 : } else {
1017 0 : header_len = dpdk_virtio_get_ip_tcp_hdr_len(pkt_addr, pkt_len);
1018 0 : if (unlikely(dpdk_virtio_create_mss_sized_mbuf_chain(mbuf,
1019 : mbuf->tso_segsz, pkt_addr, pkt_len, header_len) < 0)) {
1020 0 : goto free_mbuf;
1021 : }
1022 : }
1023 :
1024 : /*
1025 : * Gather mbuf from several virtio buffers.
1026 : */
1027 160 : while (unlikely(desc->flags & VRING_DESC_F_NEXT)) {
1028 0 : desc = &vq->vdv_desc[desc->next];
1029 0 : pkt_len = desc->len;
1030 0 : pkt_addr = vr_dpdk_guest_phys_to_host_virt(vru_cl, desc->addr);
1031 0 : if (mbuf->tso_segsz == 0) {
1032 0 : tail_addr = rte_pktmbuf_append(mbuf, pkt_len);
1033 : /* Check we ready to copy the data. */
1034 0 : if (unlikely(desc->addr == 0 || pkt_addr == NULL)) {
1035 0 : goto free_mbuf;
1036 0 : } else if (unlikely(tail_addr == NULL)) {
1037 : /* If insufficient tailroom, create a chained mbuf and copy the data */
1038 0 : if (unlikely(dpdk_virtio_create_chained_mbuf(mbuf, pkt_addr, pkt_len) < 0)) {
1039 0 : goto free_mbuf;
1040 : }
1041 : } else {
1042 : /* No chaining - Just append next descriptor(s) data. */
1043 0 : rte_memcpy(tail_addr, pkt_addr, pkt_len);
1044 : }
1045 : } else {
1046 0 : if (unlikely(dpdk_virtio_create_mss_sized_mbuf_chain(mbuf,
1047 : mbuf->tso_segsz, pkt_addr, pkt_len, header_len) < 0)) {
1048 0 : goto free_mbuf;
1049 : }
1050 :
1051 : }
1052 : }
1053 :
1054 160 : pkts[nb_pkts] = mbuf;
1055 160 : nb_pkts++;
1056 160 : continue;
1057 :
1058 0 : free_mbuf:
1059 0 : DPDK_UDEBUG(VROUTER, &vq->vdv_hash, "%s: queue %p DROP desc->addr %p "
1060 : "pkt_addr %p tail_addr %p len %d\n",
1061 : __func__, vq, desc->addr, pkt_addr, tail_addr, pkt_len);
1062 0 : DPDK_VIRTIO_READER_STATS_PKTS_DROP_ADD(p, 1);
1063 0 : rte_pktmbuf_free(mbuf);
1064 : }
1065 :
1066 : /*
1067 : * Do not call the guest if there are no descriptors processed.
1068 : *
1069 : * If there are no free mbufs on host, the TX queue in guest gets
1070 : * filled up. This makes the guest kernel to switch to interrupt mode
1071 : * and clear the VRING_AVAIL_F_NO_INTERRUPT flag.
1072 : *
1073 : * Meanwhile the host polls the virtio queue, sees the available
1074 : * descriptors and interrupts the guest. Those interrupts get unhandled by
1075 : * the guest virtio driver, so after 100K of the interrupts the IRQ get
1076 : * reported and disabled by the guest kernel.
1077 : */
1078 160 : if (likely(i > 0)) {
1079 160 : vq->vdv_last_used_idx += i;
1080 : rte_wmb();
1081 160 : vq->vdv_used->idx += i;
1082 : RTE_LOG_DP(DEBUG, VROUTER,
1083 : "%s: vif %d vq %p vdv_last_used_idx %d vdv_used->idx %u vdv_avail->idx %u\n",
1084 : __func__, vq->vdv_vif_idx, vq, vq->vdv_last_used_idx,
1085 : vq->vdv_used->idx, vq->vdv_avail->idx);
1086 :
1087 : /* Call guest if required. */
1088 160 : if (unlikely(!(vq->vdv_avail->flags & VRING_AVAIL_F_NO_INTERRUPT))) {
1089 160 : p->nb_syscalls++;
1090 160 : eventfd_write(vq->vdv_callfd, 1);
1091 : }
1092 : }
1093 :
1094 160 : DPDK_UDEBUG(VROUTER, &vq->vdv_hash, "%s: queue %p RETURNS %u pkts\n",
1095 : __func__, vq, nb_pkts);
1096 :
1097 160 : DPDK_VIRTIO_READER_STATS_PKTS_IN_ADD(p, nb_pkts);
1098 :
1099 160 : return nb_pkts;
1100 : }
1101 :
1102 : #ifdef RTE_PORT_STATS_COLLECT
1103 :
1104 : #define DPDK_VIRTIO_WRITER_STATS_PKTS_IN_ADD(port, val) \
1105 : port->stats.n_pkts_in += val
1106 : #define DPDK_VIRTIO_WRITER_STATS_PKTS_DROP_ADD(port, val) \
1107 : port->stats.n_pkts_drop += val
1108 :
1109 : #else
1110 :
1111 : /* keep compiler happy, for unused variables */
1112 : #define DPDK_VIRTIO_WRITER_STATS_PKTS_IN_ADD(port, val) \
1113 : (void)(val)
1114 : #define DPDK_VIRTIO_WRITER_STATS_PKTS_DROP_ADD(port, val) \
1115 : (void)(val)
1116 :
1117 : #endif
1118 :
1119 : static inline int32_t __attribute__((always_inline))
1120 : dpdk_virtio_dev_to_vm_tx_burst_simple(struct dpdk_virtio_writer *p,
1121 : vr_dpdk_virtioq_t *vq, uint16_t res_base_idx, uint16_t res_end_idx,
1122 : struct rte_mbuf **pkts, uint32_t count, uint8_t mrg_hdr)
1123 : {
1124 : struct vring_desc *desc;
1125 : struct rte_mbuf *buff;
1126 : /* The virtio_hdr is initialised to 0. */
1127 83 : struct virtio_net_hdr_mrg_rxbuf virtio_hdr = {{0, 0, 0, 0, 0, 0}, 1};
1128 83 : uint64_t buff_addr = 0;
1129 83 : uint64_t buff_hdr_addr = 0;
1130 : uint32_t head[VR_DPDK_VIRTIO_TX_BURST_SZ];
1131 83 : uint32_t head_idx, packet_success = 0;
1132 : uint16_t res_cur_idx;
1133 : uint8_t virtio_hdr_len;
1134 : vr_uvh_client_t *vru_cl;
1135 :
1136 83 : vru_cl = vr_dpdk_virtio_get_vif_client(vq->vdv_vif_idx);
1137 83 : if (unlikely(vru_cl == NULL))
1138 0 : return 0;
1139 :
1140 83 : res_cur_idx = res_base_idx;
1141 : RTE_LOG_DP(DEBUG, VROUTER, "%s: Current Index %d| End Index %d\n",
1142 : __func__, res_cur_idx, res_end_idx);
1143 :
1144 : /* Prefetch available ring to retrieve indexes. */
1145 83 : rte_prefetch0(&vq->vdv_avail->ring[res_cur_idx & (vq->vdv_size - 1)]);
1146 :
1147 : /* Retrieve all of the head indexes first to avoid caching issues. */
1148 166 : for (head_idx = 0; head_idx < count; head_idx++)
1149 83 : head[head_idx] = vq->vdv_avail->ring[(res_cur_idx + head_idx) &
1150 83 : (vq->vdv_size - 1)];
1151 :
1152 83 : virtio_hdr_len = (mrg_hdr)? sizeof(struct virtio_net_hdr_mrg_rxbuf):
1153 : sizeof(struct virtio_net_hdr);
1154 :
1155 : /* Prefetch descriptor index. */
1156 83 : rte_prefetch0(&vq->vdv_desc[head[packet_success]]);
1157 :
1158 166 : while (res_cur_idx != res_end_idx) {
1159 83 : uint32_t offset = 0, vb_offset = 0;
1160 83 : uint32_t pkt_len, len_to_cpy, data_len, total_copied = 0;
1161 83 : uint8_t hdr = 0, uncompleted_pkt = 0;
1162 :
1163 : /* Get descriptor from available ring */
1164 83 : desc = &vq->vdv_desc[head[packet_success]];
1165 :
1166 83 : buff = pkts[packet_success];
1167 :
1168 : /* Convert from gpa to vva (guest physical addr -> vhost virtual addr) */
1169 83 : buff_addr = (uintptr_t)vr_dpdk_guest_phys_to_host_virt(vru_cl, desc->addr);
1170 :
1171 : /* Copy virtio_hdr to packet and increment buffer address */
1172 83 : buff_hdr_addr = buff_addr;
1173 :
1174 83 : if (unlikely(buff_addr == (uint64_t)NULL)) {
1175 : /* Retry with next descriptor */
1176 0 : uncompleted_pkt = 1;
1177 0 : goto next_descr;
1178 : }
1179 :
1180 : /* Prefetch buffer address. */
1181 83 : rte_prefetch0((void *)(uintptr_t)buff_addr);
1182 :
1183 : /*
1184 : * If the descriptors are chained the header and data are
1185 : * placed in separate buffers.
1186 : */
1187 83 : if (likely(desc->flags & VRING_DESC_F_NEXT)
1188 0 : && !mrg_hdr && (desc->len == sizeof(struct virtio_net_hdr))) {
1189 : /*
1190 : * TODO: verify that desc->next is sane below.
1191 : */
1192 0 : desc = &vq->vdv_desc[desc->next];
1193 : /* Buffer address translation. */
1194 0 : buff_addr = (uintptr_t)vr_dpdk_guest_phys_to_host_virt(vru_cl, desc->addr);
1195 0 : if (unlikely(buff_addr == (uint64_t)NULL)) {
1196 : /* Retry with next descriptor */
1197 0 : uncompleted_pkt = 1;
1198 0 : goto next_descr;
1199 : }
1200 : } else {
1201 83 : vb_offset += virtio_hdr_len;
1202 83 : hdr = 1;
1203 : }
1204 :
1205 83 : pkt_len = rte_pktmbuf_pkt_len(buff);
1206 83 : data_len = rte_pktmbuf_data_len(buff);
1207 83 : len_to_cpy = RTE_MIN(data_len,
1208 : hdr ? desc->len - virtio_hdr_len : desc->len);
1209 87 : while (total_copied < pkt_len) {
1210 : /* Copy mbuf data to buffer */
1211 87 : rte_memcpy((void *)(uintptr_t)(buff_addr + vb_offset),
1212 87 : rte_pktmbuf_mtod_offset(buff, const void *, offset),
1213 : len_to_cpy);
1214 :
1215 87 : offset += len_to_cpy;
1216 87 : vb_offset += len_to_cpy;
1217 87 : total_copied += len_to_cpy;
1218 :
1219 : /* The whole packet completes */
1220 87 : if (likely(total_copied == pkt_len))
1221 83 : break;
1222 :
1223 : /* The current segment completes */
1224 4 : if (offset == data_len) {
1225 4 : buff = buff->next;
1226 4 : offset = 0;
1227 4 : data_len = rte_pktmbuf_data_len(buff);
1228 : }
1229 :
1230 : /* The current vring descriptor done */
1231 4 : if (vb_offset == desc->len) {
1232 0 : if (desc->flags & VRING_DESC_F_NEXT) {
1233 0 : desc = &vq->vdv_desc[desc->next];
1234 0 : buff_addr = (uintptr_t)vr_dpdk_guest_phys_to_host_virt(vru_cl, desc->addr);
1235 0 : if (unlikely(buff_addr == (uint64_t)NULL)) {
1236 : /* Retry with next descriptor */
1237 0 : uncompleted_pkt = 1;
1238 0 : goto next_descr;
1239 : }
1240 0 : vb_offset = 0;
1241 : } else {
1242 : /* Room in vring buffer is not enough */
1243 0 : uncompleted_pkt = 1;
1244 0 : break;
1245 : }
1246 : }
1247 4 : len_to_cpy = RTE_MIN(data_len - offset, desc->len - vb_offset);
1248 : };
1249 :
1250 0 : next_descr:
1251 : /* Update used ring with desc information */
1252 83 : vq->vdv_used->ring[res_cur_idx & (vq->vdv_size - 1)].id =
1253 83 : head[packet_success];
1254 :
1255 : /* Drop the packet if it is uncompleted */
1256 83 : if (unlikely(uncompleted_pkt == 1))
1257 0 : vq->vdv_used->ring[res_cur_idx & (vq->vdv_size - 1)].len =
1258 : virtio_hdr_len;
1259 : else
1260 83 : vq->vdv_used->ring[res_cur_idx & (vq->vdv_size - 1)].len =
1261 83 : pkt_len + virtio_hdr_len;
1262 :
1263 83 : res_cur_idx++;
1264 83 : packet_success++;
1265 :
1266 : /* TODO: in DPDK 2.1 we do not copy the header
1267 : if (unlikely(uncompleted_pkt == 1))
1268 : continue;
1269 : */
1270 83 : if (buff_hdr_addr) {
1271 83 : rte_memcpy((void *)(uintptr_t)buff_hdr_addr,
1272 : (const void *)&virtio_hdr, virtio_hdr_len);
1273 : }
1274 :
1275 83 : if (likely(res_cur_idx < res_end_idx)) {
1276 : /* Prefetch descriptor index. */
1277 0 : rte_prefetch0(&vq->vdv_desc[head[packet_success]]);
1278 : }
1279 : }
1280 :
1281 83 : rte_compiler_barrier();
1282 :
1283 : /* Wait until it's our turn to add our buffer to the used ring. */
1284 83 : while (unlikely(vq->vdv_last_used_idx != res_base_idx))
1285 0 : rte_pause();
1286 :
1287 83 : *(volatile uint16_t *)&vq->vdv_used->idx += count;
1288 83 : vq->vdv_last_used_idx = res_end_idx;
1289 : RTE_LOG_DP(DEBUG, VROUTER, "%s: vif %d vq %p last_used_idx %d used->idx %d\n",
1290 : __func__, vq->vdv_vif_idx, vq, vq->vdv_last_used_idx, vq->vdv_used->idx);
1291 :
1292 : /* flush used->idx update before we read avail->flags. */
1293 : rte_mb();
1294 :
1295 : /* Kick the guest if necessary. */
1296 83 : if (unlikely(!(vq->vdv_avail->flags & VRING_AVAIL_F_NO_INTERRUPT))) {
1297 83 : p->nb_syscalls++;
1298 83 : eventfd_write(vq->vdv_callfd, 1);
1299 : }
1300 83 : return count;
1301 : }
1302 :
1303 : /**
1304 : * This function adds buffers to the virtio devices RX virtqueue. Buffers can
1305 : * be received from the physical port or from another virtio device. A packet
1306 : * count is returned to indicate the number of packets that are succesfully
1307 : * added to the RX queue. This function works when mergeable is disabled.
1308 : *
1309 : * This is an adaptation of DPDK virtio_dev_rx() function.
1310 : * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
1311 : * BSD LICENSE
1312 : */
1313 : static inline uint32_t __attribute__((always_inline))
1314 105 : dpdk_virtio_dev_to_vm_tx_burst(struct dpdk_virtio_writer *p,
1315 : vr_dpdk_virtioq_t *vq, struct rte_mbuf **pkts, uint32_t count)
1316 : {
1317 : uint16_t res_base_idx, res_end_idx, avail_idx, free_entries;
1318 105 : uint8_t success = 0;
1319 :
1320 105 : if (unlikely(vq->vdv_ready_state == VQ_NOT_READY))
1321 22 : return 0;
1322 :
1323 : /*
1324 : * As many data cores may want access to available buffers,
1325 : * they need to be reserved.
1326 : */
1327 : do {
1328 83 : res_base_idx = vq->vdv_last_used_idx_res;
1329 83 : avail_idx = *((volatile uint16_t *)&vq->vdv_avail->idx);
1330 :
1331 83 : free_entries = (avail_idx - res_base_idx);
1332 : /*check that we have enough buffers*/
1333 83 : if (unlikely(count > free_entries))
1334 0 : count = free_entries;
1335 :
1336 83 : if (unlikely(count == 0))
1337 0 : return 0;
1338 :
1339 83 : res_end_idx = res_base_idx + count;
1340 : /* vq->vdv_last_used_idx_res is atomically updated. */
1341 : /* TODO: Allow to disable cmpset if no concurrency in application. */
1342 83 : success = rte_atomic16_cmpset(&vq->vdv_last_used_idx_res,
1343 : res_base_idx, res_end_idx);
1344 83 : } while (unlikely(success == 0));
1345 :
1346 166 : return dpdk_virtio_dev_to_vm_tx_burst_simple(p, vq,
1347 : res_base_idx, res_end_idx,
1348 : pkts, count, !VIRTIO_HDR_MRG_RXBUF);
1349 : }
1350 :
1351 : static inline uint32_t __attribute__((always_inline))
1352 : copy_from_mbuf_to_vring(vr_dpdk_virtioq_t *vq, vr_uvh_client_t *vru_cl, uint16_t res_base_idx,
1353 : uint16_t res_end_idx, struct vq_buf_vector *buf_vec,
1354 : struct virtio_net_hdr_mrg_rxbuf* virtio_hdr,
1355 : struct rte_mbuf *pkt)
1356 : {
1357 0 : uint32_t vec_idx = 0;
1358 0 : uint32_t entry_success = 0;
1359 0 : uint16_t cur_idx = res_base_idx;
1360 0 : uint64_t vb_addr = 0;
1361 0 : uint64_t vb_hdr_addr = 0;
1362 0 : uint32_t seg_offset = 0;
1363 0 : uint32_t vb_offset = 0;
1364 : uint32_t seg_avail;
1365 : uint32_t vb_avail;
1366 : uint32_t cpy_len, entry_len;
1367 :
1368 0 : if (pkt == NULL)
1369 0 : return 0;
1370 :
1371 : RTE_LOG_DP(DEBUG, VROUTER, "%s: Current Index %d| "
1372 : "End Index %d\n",
1373 : __func__, cur_idx, res_end_idx);
1374 :
1375 : /*
1376 : * Convert from gpa to vva
1377 : * (guest physical addr -> vhost virtual addr)
1378 : */
1379 0 : vb_addr = (uintptr_t)vr_dpdk_guest_phys_to_host_virt(vru_cl,
1380 0 : buf_vec[vec_idx].buf_addr);
1381 0 : vb_hdr_addr = vb_addr;
1382 :
1383 : /* Prefetch buffer address. */
1384 0 : rte_prefetch0((void *)(uintptr_t)vb_addr);
1385 :
1386 : RTE_LOG_DP(DEBUG, VROUTER, "%s RX: Num merge buffers %d\n",
1387 : __func__, virtio_hdr->num_buffers);
1388 :
1389 0 : rte_memcpy((void *)(uintptr_t)vb_hdr_addr,
1390 0 : (const void *)virtio_hdr, vq->vdv_hlen);
1391 :
1392 0 : seg_avail = rte_pktmbuf_data_len(pkt);
1393 0 : vb_offset = vq->vdv_hlen;
1394 0 : vb_avail = buf_vec[vec_idx].buf_len - vq->vdv_hlen;
1395 :
1396 0 : entry_len = vq->vdv_hlen;
1397 :
1398 0 : if (vb_avail == 0) {
1399 0 : uint32_t desc_idx =
1400 0 : buf_vec[vec_idx].desc_idx;
1401 :
1402 0 : if ((vq->vdv_desc[desc_idx].flags
1403 0 : & VRING_DESC_F_NEXT) == 0) {
1404 : /* Update vdv_used ring with vdv_desc information */
1405 0 : vq->vdv_used->ring[cur_idx & (vq->vdv_size - 1)].id
1406 0 : = buf_vec[vec_idx].desc_idx;
1407 0 : vq->vdv_used->ring[cur_idx & (vq->vdv_size - 1)].len
1408 0 : = entry_len;
1409 :
1410 0 : entry_len = 0;
1411 0 : cur_idx++;
1412 0 : entry_success++;
1413 : }
1414 :
1415 0 : vec_idx++;
1416 0 : vb_addr = (uintptr_t)vr_dpdk_guest_phys_to_host_virt(vru_cl,
1417 0 : buf_vec[vec_idx].buf_addr);
1418 :
1419 : /* Prefetch buffer address. */
1420 0 : rte_prefetch0((void *)(uintptr_t)vb_addr);
1421 0 : vb_offset = 0;
1422 0 : vb_avail = buf_vec[vec_idx].buf_len;
1423 : }
1424 :
1425 0 : cpy_len = RTE_MIN(vb_avail, seg_avail);
1426 :
1427 0 : while (cpy_len > 0) {
1428 : /* Copy mbuf data to vring buffer */
1429 0 : rte_memcpy((void *)(uintptr_t)(vb_addr + vb_offset),
1430 0 : rte_pktmbuf_mtod_offset(pkt, const void *, seg_offset),
1431 : cpy_len);
1432 :
1433 0 : seg_offset += cpy_len;
1434 0 : vb_offset += cpy_len;
1435 0 : seg_avail -= cpy_len;
1436 0 : vb_avail -= cpy_len;
1437 0 : entry_len += cpy_len;
1438 :
1439 0 : if (seg_avail != 0) {
1440 : /*
1441 : * The virtio buffer in this vring
1442 : * entry reach to its end.
1443 : * But the segment doesn't complete.
1444 : */
1445 0 : if ((vq->vdv_desc[buf_vec[vec_idx].desc_idx].flags &
1446 : VRING_DESC_F_NEXT) == 0) {
1447 : /* Update vdv_used ring with vdv_desc information */
1448 0 : vq->vdv_used->ring[cur_idx & (vq->vdv_size - 1)].id
1449 0 : = buf_vec[vec_idx].desc_idx;
1450 0 : vq->vdv_used->ring[cur_idx & (vq->vdv_size - 1)].len
1451 0 : = entry_len;
1452 0 : entry_len = 0;
1453 0 : cur_idx++;
1454 0 : entry_success++;
1455 : }
1456 :
1457 0 : vec_idx++;
1458 0 : vb_addr = (uintptr_t)vr_dpdk_guest_phys_to_host_virt(vru_cl,
1459 0 : buf_vec[vec_idx].buf_addr);
1460 0 : vb_offset = 0;
1461 0 : vb_avail = buf_vec[vec_idx].buf_len;
1462 0 : cpy_len = RTE_MIN(vb_avail, seg_avail);
1463 : } else {
1464 : /*
1465 : * This current segment complete, need continue to
1466 : * check if the whole packet complete or not.
1467 : */
1468 0 : pkt = pkt->next;
1469 0 : if (pkt != NULL) {
1470 : /*
1471 : * There are more segments.
1472 : */
1473 0 : if (vb_avail == 0) {
1474 : /*
1475 : * This current buffer from vring is
1476 : * vdv_used up, need fetch next buffer
1477 : * from buf_vec.
1478 : */
1479 0 : uint32_t desc_idx =
1480 0 : buf_vec[vec_idx].desc_idx;
1481 :
1482 0 : if ((vq->vdv_desc[desc_idx].flags &
1483 : VRING_DESC_F_NEXT) == 0) {
1484 0 : uint16_t wrapped_idx =
1485 0 : cur_idx & (vq->vdv_size - 1);
1486 : /*
1487 : * Update vdv_used ring with the
1488 : * descriptor information
1489 : */
1490 0 : vq->vdv_used->ring[wrapped_idx].id
1491 0 : = desc_idx;
1492 0 : vq->vdv_used->ring[wrapped_idx].len
1493 0 : = entry_len;
1494 0 : entry_success++;
1495 0 : entry_len = 0;
1496 0 : cur_idx++;
1497 : }
1498 :
1499 : /* Get next buffer from buf_vec. */
1500 0 : vec_idx++;
1501 0 : vb_addr = (uintptr_t)vr_dpdk_guest_phys_to_host_virt(vru_cl,
1502 0 : buf_vec[vec_idx].buf_addr);
1503 0 : vb_avail =
1504 0 : buf_vec[vec_idx].buf_len;
1505 0 : vb_offset = 0;
1506 : }
1507 :
1508 0 : seg_offset = 0;
1509 0 : seg_avail = rte_pktmbuf_data_len(pkt);
1510 0 : cpy_len = RTE_MIN(vb_avail, seg_avail);
1511 : } else {
1512 : /*
1513 : * This whole packet completes.
1514 : */
1515 : /* Update vdv_used ring with vdv_desc information */
1516 0 : vq->vdv_used->ring[cur_idx & (vq->vdv_size - 1)].id
1517 0 : = buf_vec[vec_idx].desc_idx;
1518 0 : vq->vdv_used->ring[cur_idx & (vq->vdv_size - 1)].len
1519 0 : = entry_len;
1520 0 : entry_success++;
1521 0 : break;
1522 : }
1523 : }
1524 : }
1525 :
1526 0 : return entry_success;
1527 : }
1528 :
1529 : static inline void __attribute__((always_inline))
1530 : update_secure_len(vr_dpdk_virtioq_t *vq, uint32_t id,
1531 : uint32_t *secure_len, struct vq_buf_vector *buf_vec, uint32_t *vec_idx)
1532 : {
1533 0 : uint16_t wrapped_idx = id & (vq->vdv_size - 1);
1534 0 : uint32_t idx = vq->vdv_avail->ring[wrapped_idx];
1535 : uint8_t next_desc;
1536 0 : uint32_t len = *secure_len;
1537 0 : uint32_t vec_id = *vec_idx;
1538 :
1539 : do {
1540 0 : if (vec_id >= VR_BUF_VECTOR_MAX)
1541 0 : break;
1542 0 : next_desc = 0;
1543 0 : len += vq->vdv_desc[idx].len;
1544 0 : buf_vec[vec_id].buf_addr = vq->vdv_desc[idx].addr;
1545 0 : buf_vec[vec_id].buf_len = vq->vdv_desc[idx].len;
1546 0 : buf_vec[vec_id].desc_idx = idx;
1547 0 : vec_id++;
1548 :
1549 0 : if (vq->vdv_desc[idx].flags & VRING_DESC_F_NEXT) {
1550 0 : idx = vq->vdv_desc[idx].next;
1551 0 : next_desc = 1;
1552 : }
1553 0 : } while (next_desc);
1554 :
1555 0 : *secure_len = len;
1556 0 : *vec_idx = vec_id;
1557 0 : }
1558 :
1559 : static inline uint32_t __attribute__((always_inline))
1560 0 : dpdk_virtio_dev_to_vm_tx_burst_mergeable(struct dpdk_virtio_writer *p,
1561 : vr_dpdk_virtioq_t *vq, struct rte_mbuf **pkts, uint32_t count)
1562 : {
1563 0 : uint32_t pkt_idx = 0, start_idx = 0, entry_success = 0, simple_count;
1564 : uint16_t avail_idx;
1565 : uint16_t res_base_idx, res_cur_idx;
1566 0 : uint8_t success = 0;
1567 : vr_uvh_client_t *vru_cl;
1568 : struct vq_buf_vector buf_vec[VR_BUF_VECTOR_MAX];
1569 :
1570 0 : if (unlikely(vq->vdv_ready_state == VQ_NOT_READY))
1571 0 : return 0;
1572 :
1573 0 : vru_cl = vr_dpdk_virtio_get_vif_client(vq->vdv_vif_idx);
1574 0 : if (unlikely(vru_cl == NULL))
1575 0 : return 0;
1576 :
1577 0 : count = RTE_MIN((uint32_t)VR_DPDK_VIRTIO_TX_BURST_SZ, count);
1578 :
1579 0 : if (count == 0)
1580 0 : return 0;
1581 :
1582 : /* Check if we can call -
1583 : * dpdk_virtio_dev_to_vm_tx_burst_simple() for some/all pkts
1584 : */
1585 : do {
1586 0 : res_base_idx = vq->vdv_last_used_idx_res;
1587 0 : res_cur_idx = res_base_idx;
1588 0 : avail_idx = *((volatile uint16_t *)&vq->vdv_avail->idx);
1589 0 : for (pkt_idx = 0; pkt_idx < count; pkt_idx++) {
1590 0 : uint32_t pkt_len = pkts[pkt_idx]->pkt_len + vq->vdv_hlen;
1591 0 : if (unlikely(res_cur_idx == avail_idx)) {
1592 : RTE_LOG_DP(DEBUG, VROUTER,
1593 : "Failed "
1594 : "to get enough vdv_desc from "
1595 : "vring\n");
1596 0 : count = pkt_idx;
1597 0 : break;
1598 : } else {
1599 : uint8_t next_desc;
1600 0 : uint16_t wrapped_idx = res_cur_idx & (vq->vdv_size - 1);
1601 0 : uint32_t len = 0, idx = vq->vdv_avail->ring[wrapped_idx];
1602 : do {
1603 0 : next_desc = 0;
1604 0 : len += vq->vdv_desc[idx].len;
1605 0 : if (vq->vdv_desc[idx].flags & VRING_DESC_F_NEXT) {
1606 0 : idx = vq->vdv_desc[idx].next;
1607 0 : next_desc = 1;
1608 : }
1609 0 : if (len > pkt_len)
1610 0 : break;
1611 0 : }while (next_desc);
1612 0 : if (len < pkt_len)
1613 0 : break;
1614 0 : res_cur_idx++;
1615 : }
1616 : }
1617 :
1618 : /* If there are no packets to pass to
1619 : * dpdk_virtio_dev_to_vm_tx_burst_simple() function, break
1620 : */
1621 0 : if (pkt_idx == 0)
1622 0 : break;
1623 :
1624 0 : success = rte_atomic16_cmpset(&vq->vdv_last_used_idx_res,
1625 : res_base_idx, res_cur_idx);
1626 0 : } while (unlikely(success == 0));
1627 :
1628 0 : if (pkt_idx) {
1629 0 : simple_count = dpdk_virtio_dev_to_vm_tx_burst_simple(p, vq,
1630 : res_base_idx, res_cur_idx,
1631 : pkts, pkt_idx, VIRTIO_HDR_MRG_RXBUF);
1632 0 : if (simple_count < pkt_idx)
1633 0 : return simple_count;
1634 : }
1635 :
1636 0 : start_idx = pkt_idx;
1637 0 : for (pkt_idx = start_idx; pkt_idx < count; pkt_idx++) {
1638 0 : struct virtio_net_hdr_mrg_rxbuf virtio_hdr = {
1639 : {0, 0, 0, 0, 0, 0}, 0};
1640 0 : uint32_t pkt_len = pkts[pkt_idx]->pkt_len + vq->vdv_hlen;
1641 :
1642 : do {
1643 : /*
1644 : * As many data cores may want access to available
1645 : * buffers, they need to be reserved.
1646 : */
1647 0 : uint32_t secure_len = 0;
1648 0 : uint32_t vec_idx = 0;
1649 :
1650 0 : res_base_idx = vq->vdv_last_used_idx_res;
1651 0 : res_cur_idx = res_base_idx;
1652 :
1653 : do {
1654 0 : avail_idx = *((volatile uint16_t *)&vq->vdv_avail->idx);
1655 0 : if (unlikely(res_cur_idx == avail_idx)) {
1656 : RTE_LOG_DP(DEBUG, VROUTER,
1657 : "Failed "
1658 : "to get enough vdv_desc from "
1659 : "vring\n");
1660 0 : return pkt_idx;
1661 : } else {
1662 0 : update_secure_len(vq, res_cur_idx, &secure_len, buf_vec, &vec_idx);
1663 0 : res_cur_idx++;
1664 : }
1665 0 : } while (pkt_len > secure_len);
1666 :
1667 : /* vq->vdv_last_used_idx_res is atomically updated. */
1668 0 : success = rte_atomic16_cmpset(&vq->vdv_last_used_idx_res,
1669 : res_base_idx,
1670 : res_cur_idx);
1671 0 : } while (success == 0);
1672 :
1673 : /* Fill the virtio hdr */
1674 0 : virtio_hdr.num_buffers = res_cur_idx - res_base_idx;
1675 0 : if (pkts[pkt_idx]->ol_flags & PKT_RX_GSO_TCP4) {
1676 0 : virtio_hdr.hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
1677 0 : virtio_hdr.hdr.gso_size = pkts[pkt_idx]->tso_segsz;
1678 0 : } else if (pkts[pkt_idx]->ol_flags & PKT_RX_GSO_TCP6) {
1679 0 : virtio_hdr.hdr.gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
1680 0 : virtio_hdr.hdr.gso_size = pkts[pkt_idx]->tso_segsz;
1681 : }
1682 :
1683 0 : entry_success = copy_from_mbuf_to_vring(vq, vru_cl, res_base_idx,
1684 0 : res_cur_idx, buf_vec, &virtio_hdr, pkts[pkt_idx]);
1685 :
1686 0 : rte_compiler_barrier();
1687 :
1688 : /*
1689 : * Wait until it's our turn to add our buffer
1690 : * to the vdv_used ring.
1691 : */
1692 0 : while (unlikely(vq->vdv_last_used_idx != res_base_idx))
1693 0 : rte_pause();
1694 :
1695 0 : *(volatile uint16_t *)&vq->vdv_used->idx += entry_success;
1696 0 : vq->vdv_last_used_idx = res_cur_idx;
1697 :
1698 : /* flush vdv_used->idx update before we read vdv_avail->flags. */
1699 : rte_mb();
1700 :
1701 : /* Kick the guest if necessary. */
1702 0 : if (unlikely(!(vq->vdv_avail->flags & VRING_AVAIL_F_NO_INTERRUPT))) {
1703 0 : p->nb_syscalls++;
1704 0 : eventfd_write(vq->vdv_callfd, 1);
1705 : }
1706 : }
1707 :
1708 0 : return count;
1709 : }
1710 :
1711 : void
1712 0 : vr_dpdk_set_vhost_send_func(unsigned int vif_idx, uint32_t mrg)
1713 : {
1714 : int i;
1715 : vr_dpdk_virtioq_t *vq;
1716 :
1717 0 : if (vif_idx >= VR_MAX_INTERFACES) {
1718 0 : return;
1719 : }
1720 :
1721 0 : for (i = 0; i < VR_DPDK_VIRTIO_MAX_QUEUES*2; i++) {
1722 0 : if (i & 1) {
1723 0 : vq = &vr_dpdk_virtio_rxqs[vif_idx][i/2];
1724 : } else {
1725 0 : vq = &vr_dpdk_virtio_txqs[vif_idx][i/2];
1726 : }
1727 :
1728 0 : if (mrg) {
1729 0 : vq->vdv_send_func = dpdk_virtio_dev_to_vm_tx_burst_mergeable;
1730 0 : vq->vdv_hlen = sizeof(struct virtio_net_hdr_mrg_rxbuf);
1731 : } else {
1732 0 : vq->vdv_send_func = dpdk_virtio_dev_to_vm_tx_burst;
1733 0 : vq->vdv_hlen = sizeof(struct virtio_net_hdr);
1734 : }
1735 : }
1736 : }
1737 :
1738 : static inline void
1739 130 : dpdk_virtio_send_burst(struct dpdk_virtio_writer *p)
1740 : {
1741 130 : uint32_t nb_tx = 0;
1742 : int i;
1743 :
1744 130 : if (likely(p->tx_buf_count)) {
1745 : /*
1746 : * prefetch the tx buffer to be sent
1747 : * This will avoid large cpu cycles in the
1748 : * dpdk_virtio_dev_to_vm_tx_burst_mergeable()
1749 : */
1750 266 : for (i=0;i<p->tx_buf_count;i++)
1751 136 : rte_prefetch0((void *)p->tx_buf[i]);
1752 130 : if (likely(p->tx_virtioq->vdv_send_func != NULL)) {
1753 105 : nb_tx = p->tx_virtioq->vdv_send_func(p, p->tx_virtioq,
1754 105 : p->tx_buf, p->tx_buf_count);
1755 : }
1756 :
1757 130 : DPDK_VIRTIO_WRITER_STATS_PKTS_DROP_ADD(p, p->tx_buf_count - nb_tx);
1758 : /* dpdk_virtio_dev_to_vm_tx_burst() does not free any mbufs */
1759 266 : while (likely(p->tx_buf_count)) {
1760 136 : p->tx_buf_count--;
1761 136 : p->tx_mbufs -= p->tx_buf[p->tx_buf_count]->nb_segs;
1762 136 : rte_pktmbuf_free(p->tx_buf[p->tx_buf_count]);
1763 : }
1764 : }
1765 130 : }
1766 :
1767 : /*
1768 : * dpdk_virtio_to_vm_tx - sends a packet from vrouter to a virtio client. The
1769 : * virtio client is usually a VM.
1770 : *
1771 : * Returns nothing.
1772 : */
1773 : static int
1774 136 : dpdk_virtio_to_vm_tx(void *port, struct rte_mbuf *pkt)
1775 : {
1776 136 : struct dpdk_virtio_writer *p = (struct dpdk_virtio_writer *)port;
1777 136 : const unsigned lcore_id = rte_lcore_id();
1778 136 : struct vr_dpdk_lcore *lcore = NULL;
1779 :
1780 136 : if (lcore_id >= VR_DPDK_FWD_LCORE_ID) {
1781 134 : lcore = vr_dpdk.lcores[lcore_id];
1782 134 : p->last_pkt_tx = lcore->lcore_fwd_loops;
1783 : }
1784 :
1785 136 : p->tx_buf[p->tx_buf_count++] = pkt;
1786 136 : p->tx_mbufs += pkt->nb_segs;
1787 136 : DPDK_VIRTIO_WRITER_STATS_PKTS_IN_ADD(p, 1);
1788 :
1789 136 : if (unlikely(p->tx_mbufs >= VR_DPDK_VIRTIO_TX_BURST_SZ)) {
1790 0 : dpdk_virtio_send_burst(p);
1791 0 : if (lcore) {
1792 0 : p->last_pkt_tx_flush = lcore->lcore_fwd_loops;
1793 : }
1794 : }
1795 :
1796 136 : return 0;
1797 : }
1798 :
1799 : /*
1800 : * dpdk_virtio_to_vm_flush - flushes packets from vrouter to a virtio client.
1801 : * The virtio client is usually a VM.
1802 : *
1803 : * Returns nothing.
1804 : */
1805 : static int
1806 172717844 : dpdk_virtio_to_vm_flush(void *port)
1807 : {
1808 172717844 : struct dpdk_virtio_writer *p = (struct dpdk_virtio_writer *)port;
1809 : unsigned lcore_id;
1810 172717844 : struct vr_dpdk_lcore *lcore = NULL;
1811 :
1812 172717844 : if (p->tx_buf_count == 0) {
1813 172710853 : return 0;
1814 : }
1815 :
1816 6991 : lcore_id = rte_lcore_id();
1817 130 : if (lcore_id >= VR_DPDK_FWD_LCORE_ID) {
1818 128 : lcore = vr_dpdk.lcores[lcore_id];
1819 : }
1820 :
1821 130 : if (lcore) {
1822 : /*
1823 : * Flush the TX queue if it has been a while since it was last done OR
1824 : * if there are packets in the queue and no packets have been enqueued
1825 : * for a short while. The latter condition helps to reduce latency in
1826 : * case there isn't a lot of traffic on the queue.
1827 : */
1828 128 : if ((lcore->lcore_fwd_loops - p->last_pkt_tx_flush) <
1829 : VR_DPDK_TX_FLUSH_LOOPS) {
1830 0 : if ((lcore->lcore_fwd_loops - p->last_pkt_tx) <
1831 : VR_DPDK_TX_IDLE_LOOPS) {
1832 0 : return 0;
1833 : }
1834 : }
1835 : }
1836 :
1837 130 : dpdk_virtio_send_burst(p);
1838 130 : if (lcore) {
1839 128 : p->last_pkt_tx_flush = lcore->lcore_fwd_loops;
1840 : }
1841 :
1842 130 : return 0;
1843 : }
1844 :
1845 : /*
1846 : * vr_dpdk_virtio_set_vring_base - sets the vring base using data sent by
1847 : * vhost client.
1848 : *
1849 : * Returns 0 on success, -1 otherwise.
1850 : */
1851 : int
1852 388 : vr_dpdk_virtio_set_vring_base(unsigned int vif_idx, unsigned int vring_idx,
1853 : unsigned int vring_base)
1854 : {
1855 : vr_dpdk_virtioq_t *vq;
1856 :
1857 388 : if ((vif_idx >= VR_MAX_INTERFACES)
1858 388 : || (vring_idx >= (2 * VR_DPDK_VIRTIO_MAX_QUEUES))) {
1859 0 : return -1;
1860 : }
1861 :
1862 : /*
1863 : * RX rings are even numbered and TX rings are odd numbered from the
1864 : * VM's point of view. From vrouter's point of view, VM's TX ring is
1865 : * vrouter's RX ring and vice versa.
1866 : */
1867 388 : if (vring_idx & 1) {
1868 194 : vq = &vr_dpdk_virtio_rxqs[vif_idx][vring_idx/2];
1869 : } else {
1870 194 : vq = &vr_dpdk_virtio_txqs[vif_idx][vring_idx/2];
1871 : }
1872 :
1873 388 : vq->vdv_last_used_idx = vring_base;
1874 388 : vq->vdv_last_used_idx_res = vring_base;
1875 388 : return 0;
1876 : }
1877 :
1878 : /*
1879 : * vr_dpdk_virtio_get_vring_base - gets the vring base for the specified vring
1880 : * sent by the vhost client.
1881 : *
1882 : * Returns 0 on success, -1 otherwise.
1883 : */
1884 : int
1885 0 : vr_dpdk_virtio_get_vring_base(unsigned int vif_idx, unsigned int vring_idx,
1886 : unsigned int *vring_basep)
1887 : {
1888 : vr_dpdk_virtioq_t *vq;
1889 :
1890 0 : if ((vif_idx >= VR_MAX_INTERFACES)
1891 0 : || (vring_idx >= (2 * VR_DPDK_VIRTIO_MAX_QUEUES))) {
1892 0 : return -1;
1893 : }
1894 :
1895 : /*
1896 : * RX rings are even numbered and TX rings are odd numbered from the
1897 : * VM's point of view. From vrouter's point of view, VM's TX ring is
1898 : * vrouter's RX ring and vice versa.
1899 : */
1900 0 : if (vring_idx & 1) {
1901 0 : vq = &vr_dpdk_virtio_rxqs[vif_idx][vring_idx/2];
1902 : } else {
1903 0 : vq = &vr_dpdk_virtio_txqs[vif_idx][vring_idx/2];
1904 : }
1905 :
1906 0 : *vring_basep = vq->vdv_last_used_idx;
1907 :
1908 : /*
1909 : * This is usually called when qemu shuts down a virtio queue. Set the
1910 : * state to indicate that this queue should not be used any more.
1911 : */
1912 0 : vq->vdv_ready_state = VQ_NOT_READY;
1913 : rte_wmb();
1914 0 : synchronize_rcu();
1915 :
1916 : /* Reset the queue. We reset only those values we analyze in
1917 : * uvhm_check_vring_ready()
1918 : */
1919 0 : vq->vdv_desc = NULL;
1920 0 : if (vq->vdv_callfd) {
1921 0 : close(vq->vdv_callfd);
1922 0 : vq->vdv_callfd = 0;
1923 : }
1924 :
1925 0 : return 0;
1926 : }
1927 :
1928 : /*
1929 : * vr_dpdk_virtio_recover_vring_base - recovers the vring base from the shared
1930 : * memory after vRouter crash.
1931 : *
1932 : * Returns 0 on success, -1 otherwise.
1933 : */
1934 : int
1935 388 : vr_dpdk_virtio_recover_vring_base(unsigned int vif_idx, unsigned int vring_idx)
1936 : {
1937 : vr_dpdk_virtioq_t *vq;
1938 :
1939 388 : if ((vif_idx >= VR_MAX_INTERFACES)
1940 388 : || (vring_idx >= (2 * VR_DPDK_VIRTIO_MAX_QUEUES))) {
1941 0 : return -1;
1942 : }
1943 :
1944 388 : if (vring_idx & 1) {
1945 194 : vq = &vr_dpdk_virtio_rxqs[vif_idx][vring_idx/2];
1946 : } else {
1947 194 : vq = &vr_dpdk_virtio_txqs[vif_idx][vring_idx/2];
1948 : }
1949 :
1950 388 : if (vq->vdv_used) {
1951 : /* Reading base index from the shared memory. */
1952 388 : if (vq->vdv_last_used_idx != vq->vdv_used->idx) {
1953 0 : RTE_LOG(INFO, UVHOST, " recovering vring base %d -> %d\n",
1954 : vq->vdv_last_used_idx, vq->vdv_used->idx);
1955 0 : vr_dpdk_virtio_set_vring_base(vif_idx, vring_idx, vq->vdv_used->idx);
1956 : }
1957 : }
1958 :
1959 388 : return 0;
1960 : }
1961 :
1962 : /*
1963 : * vr_dpdk_set_vring_addr - Sets the address of the virtio descriptor and
1964 : * available/used rings based on messages sent by the vhost client.
1965 : *
1966 : * Returns 0 on success, -1 otherwise.
1967 : */
1968 : int
1969 388 : vr_dpdk_set_vring_addr(unsigned int vif_idx, unsigned int vring_idx,
1970 : struct vring_desc *vrucv_desc,
1971 : struct vring_avail *vrucv_avail,
1972 : struct vring_used *vrucv_used)
1973 : {
1974 : vr_dpdk_virtioq_t *vq;
1975 :
1976 388 : if ((vif_idx >= VR_MAX_INTERFACES)
1977 388 : || (vring_idx >= (2 * VR_DPDK_VIRTIO_MAX_QUEUES))) {
1978 0 : return -1;
1979 : }
1980 :
1981 : /*
1982 : * RX rings are even numbered and TX rings are odd numbered from the
1983 : * VM's point of view. From vrouter's point of view, VM's TX ring is
1984 : * vrouter's RX ring and vice versa.
1985 : */
1986 388 : if (vring_idx & 1) {
1987 194 : vq = &vr_dpdk_virtio_rxqs[vif_idx][vring_idx/2];
1988 : } else {
1989 194 : vq = &vr_dpdk_virtio_txqs[vif_idx][vring_idx/2];
1990 : }
1991 :
1992 388 : vq->vdv_desc = vrucv_desc;
1993 388 : vq->vdv_avail = vrucv_avail;
1994 388 : vq->vdv_used = vrucv_used;
1995 :
1996 : /*
1997 : * Tell the guest that it need not interrupt vrouter when it updates the
1998 : * available ring (as vrouter is polling it).
1999 : */
2000 388 : vq->vdv_used->flags |= VRING_USED_F_NO_NOTIFY;
2001 :
2002 388 : return 0;
2003 : }
2004 :
2005 : /*
2006 : * vr_dpdk_set_ring_num_desc - sets the number of descriptors in a vring
2007 : * based on messages from the vhost client.
2008 : *
2009 : * Returns 0 on success, -1 otherwise.
2010 : */
2011 : int
2012 388 : vr_dpdk_set_ring_num_desc(unsigned int vif_idx, unsigned int vring_idx,
2013 : unsigned int num_desc)
2014 : {
2015 : vr_dpdk_virtioq_t *vq;
2016 :
2017 388 : if ((vif_idx >= VR_MAX_INTERFACES) || (vring_idx > 2 * VR_DPDK_VIRTIO_MAX_QUEUES)) {
2018 0 : return -1;
2019 : }
2020 :
2021 : /*
2022 : * RX rings are even numbered and TX rings are odd numbered from the
2023 : * VM's point of view. From vrouter's point of view, VM's TX ring is
2024 : * vrouter's RX ring and vice versa.
2025 : */
2026 388 : if (vring_idx & 1) {
2027 194 : vq = &vr_dpdk_virtio_rxqs[vif_idx][vring_idx/2];
2028 : } else {
2029 194 : vq = &vr_dpdk_virtio_txqs[vif_idx][vring_idx/2];
2030 : }
2031 :
2032 388 : vq->vdv_size = num_desc;
2033 :
2034 388 : return 0;
2035 : }
2036 :
2037 : /*
2038 : * vr_dpdk_set_ring_callfd - set the eventd used to raise interrupts in
2039 : * the guest (if required). Returns 0 on success, -1 otherwise.
2040 : */
2041 : int
2042 388 : vr_dpdk_set_ring_callfd(unsigned int vif_idx, unsigned int vring_idx,
2043 : int callfd)
2044 : {
2045 : vr_dpdk_virtioq_t *vq;
2046 :
2047 388 : if ((vif_idx >= VR_MAX_INTERFACES)
2048 388 : || (vring_idx >= (2 * VR_DPDK_VIRTIO_MAX_QUEUES))) {
2049 0 : return -1;
2050 : }
2051 :
2052 : /*
2053 : * RX rings are even numbered and TX rings are odd numbered from the
2054 : * VM's point of view. From vrouter's point of view, VM's TX ring is
2055 : * vrouter's RX ring and vice versa.
2056 : */
2057 388 : if (vring_idx & 1) {
2058 194 : vq = &vr_dpdk_virtio_rxqs[vif_idx][vring_idx/2];
2059 : } else {
2060 194 : vq = &vr_dpdk_virtio_txqs[vif_idx][vring_idx/2];
2061 : }
2062 :
2063 388 : if (vq->vdv_callfd > 0) {
2064 0 : close(vq->vdv_callfd);
2065 : }
2066 388 : vq->vdv_callfd = callfd;
2067 :
2068 388 : return 0;
2069 : }
2070 :
2071 : /*
2072 : * vr_dpdk_set_virtq_ready - sets the virtio queue ready state to indicate
2073 : * whether forwarding can start on the virtio queue or not.
2074 : *
2075 : * Returns 0 on success, -1 otherwise.
2076 : */
2077 : int
2078 776 : vr_dpdk_set_virtq_ready(unsigned int vif_idx, unsigned int vring_idx,
2079 : vq_ready_state_t ready)
2080 : {
2081 : vr_dpdk_virtioq_t *vq;
2082 :
2083 776 : if ((vif_idx >= VR_MAX_INTERFACES)
2084 776 : || (vring_idx >= (2 * VR_DPDK_VIRTIO_MAX_QUEUES))) {
2085 0 : return -1;
2086 : }
2087 :
2088 : /*
2089 : * RX rings are even numbered and TX rings are odd numbered from the
2090 : * VM's point of view. From vrouter's point of view, VM's TX ring is
2091 : * vrouter's RX ring and vice versa.
2092 : */
2093 776 : if (vring_idx & 1) {
2094 388 : vq = &vr_dpdk_virtio_rxqs[vif_idx][vring_idx/2];
2095 : } else {
2096 388 : vq = &vr_dpdk_virtio_txqs[vif_idx][vring_idx/2];
2097 : }
2098 :
2099 776 : if (vq->vdv_hlen == 0) {
2100 : struct vr_interface *vif;
2101 266 : vif = __vrouter_get_interface(vrouter_get(0), vq->vdv_vif_idx);
2102 266 : if (vif && (vif->vif_flags & VIF_FLAG_MRG_RXBUF)) {
2103 0 : vq->vdv_send_func = dpdk_virtio_dev_to_vm_tx_burst_mergeable;
2104 0 : vq->vdv_hlen = sizeof(struct virtio_net_hdr_mrg_rxbuf);
2105 : } else {
2106 266 : vq->vdv_send_func = dpdk_virtio_dev_to_vm_tx_burst;
2107 266 : vq->vdv_hlen = sizeof(struct virtio_net_hdr);
2108 : }
2109 : }
2110 :
2111 776 : vq->vdv_ready_state = ready;
2112 :
2113 776 : return 0;
2114 : }
2115 :
2116 : /*
2117 : * vr_dpdk_virtio_set_vif_client - sets a pointer to per vif state. Currently
2118 : * used to store a pointer to the vhost client structure.
2119 : *
2120 : * Returns nothing.
2121 : */
2122 : void
2123 401 : vr_dpdk_virtio_set_vif_client(unsigned int idx, void *client)
2124 : {
2125 401 : if (idx >= VR_MAX_INTERFACES) {
2126 0 : return;
2127 : }
2128 :
2129 401 : vr_dpdk_vif_clients[idx] = client;
2130 :
2131 401 : return;
2132 : }
2133 :
2134 : /*
2135 : * vr_dpdk_virtio_get_vif_client - returns a pointer to per vif state if it
2136 : * exists, NULL otherwise.
2137 : */
2138 : void *
2139 99405963 : vr_dpdk_virtio_get_vif_client(unsigned int idx)
2140 : {
2141 99405963 : if (idx >= VR_MAX_INTERFACES) {
2142 0 : return NULL;
2143 : }
2144 :
2145 99405963 : return vr_dpdk_vif_clients[idx];
2146 : }
2147 :
2148 : static int
2149 336 : dpdk_virtio_reader_stats_read(void *port,
2150 : struct rte_port_in_stats *stats, int clear)
2151 : {
2152 336 : struct dpdk_virtio_reader *p = (struct dpdk_virtio_reader *)port;
2153 :
2154 336 : if (stats != NULL)
2155 336 : memcpy(stats, &p->stats, sizeof(p->stats));
2156 :
2157 336 : if (clear)
2158 20 : memset(&p->stats, 0, sizeof(p->stats));
2159 :
2160 336 : return 0;
2161 : }
2162 :
2163 : static int
2164 672 : dpdk_virtio_writer_stats_read(void *port,
2165 : struct rte_port_out_stats *stats, int clear)
2166 : {
2167 672 : struct dpdk_virtio_reader *p = (struct dpdk_virtio_reader *)port;
2168 :
2169 672 : if (stats != NULL)
2170 672 : memcpy(stats, &p->stats, sizeof(p->stats));
2171 :
2172 672 : if (clear)
2173 40 : memset(&p->stats, 0, sizeof(p->stats));
2174 :
2175 672 : return 0;
2176 : }
2177 :
2178 : /* Update extra statistics for virtio queue */
2179 : void
2180 948 : vr_dpdk_virtio_xstats_update(struct vr_interface_stats *stats,
2181 : struct vr_dpdk_queue *queue)
2182 : {
2183 : struct dpdk_virtio_reader *reader;
2184 : struct dpdk_virtio_writer *writer;
2185 :
2186 948 : if (queue->rxq_ops.f_rx == vr_dpdk_virtio_reader_ops.f_rx) {
2187 316 : reader = (struct dpdk_virtio_reader *)queue->q_queue_h;
2188 316 : stats->vis_port_isyscalls = reader->nb_syscalls;
2189 316 : stats->vis_port_inombufs = reader->nb_nombufs;
2190 632 : } else if (queue->txq_ops.f_tx == vr_dpdk_virtio_writer_ops.f_tx) {
2191 632 : writer = (struct dpdk_virtio_writer *)queue->q_queue_h;
2192 632 : stats->vis_port_osyscalls = writer->nb_syscalls;
2193 : }
2194 948 : }
|
