Line data Source code
1 : /*
2 : * Copyright (C) 2014 Semihalf.
3 : *
4 : * This program is free software; you can redistribute it and/or
5 : * modify it under the terms of the GNU General Public License as
6 : * published by the Free Software Foundation version 2.
7 : *
8 : * This program is distributed "as is" WITHOUT ANY WARRANTY of any
9 : * kind, whether express or implied; without even the implied warranty
10 : * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 : * GNU General Public License for more details.
12 : *
13 : * vr_dpdk.h -- vRouter/DPDK definitions
14 : *
15 : */
16 :
17 : #ifndef _VR_DPDK_H_
18 : #define _VR_DPDK_H_
19 :
20 : #include "vr_os.h"
21 : #include "vr_dpdk_compat.h"
22 : #include "vr_interface.h"
23 : #include "vr_packet.h"
24 : #include "vr_fragment.h"
25 : #include "vr_cpuid.h"
26 :
27 : #include <sys/queue.h>
28 :
29 : #include <urcu-qsbr.h>
30 :
31 : #include <rte_config.h>
32 : #include <rte_port.h>
33 : #include <rte_ip.h>
34 : #include <rte_port_ring.h>
35 : #include <rte_ethdev.h>
36 : #include <rte_spinlock.h>
37 :
38 : #ifdef PKT_RX_VLAN_PKT
39 : #define PKT_RX_VLAN PKT_RX_VLAN_PKT
40 : #endif
41 :
42 : #if (RTE_VERSION < RTE_VERSION_NUM(19, 11, 0, 0))
43 : #error "vRouter needs atleast DPDK version 19.11 to start"
44 : #endif
45 :
46 : extern struct vr_interface_stats *vif_get_stats(struct vr_interface *,
47 : unsigned short);
48 : extern int dpdk_vlan_forwarding_if_add(void);
49 : extern unsigned int vr_flow_hold_limit;
50 : extern int no_huge_set;
51 : extern unsigned vr_packet_sz;
52 :
53 : /*
54 : * Use RTE_LOG_DEBUG to enable debug logs.
55 : * See more debug options below.
56 : */
57 : #undef RTE_LOG_LEVEL
58 : #ifdef COMPILE_DEBUG_LOGS
59 : #define RTE_LOG_LEVEL RTE_LOG_DEBUG
60 : #else
61 : #define RTE_LOG_LEVEL RTE_LOG_INFO
62 : #endif
63 :
64 : /*
65 : * By default all the logtypes are enabled.
66 : * Use VR_DPDK_LOGTYPE_DISABLE option below to disable some of the types.
67 : */
68 : #define RTE_LOGTYPE_VROUTER RTE_LOGTYPE_USER1
69 : #define RTE_LOGTYPE_USOCK RTE_LOGTYPE_USER2
70 : #define RTE_LOGTYPE_UVHOST RTE_LOGTYPE_USER3
71 : #define RTE_LOGTYPE_DPCORE RTE_LOGTYPE_USER4
72 : #define RTE_LOGTYPE_OFFLOAD_PACKET RTE_LOGTYPE_USER5
73 : /* Disable the rest (undefined) logtypes */
74 : #define VR_DPDK_LOGTYPE_DISABLE (RTE_LOGTYPE_USER6 | \
75 : RTE_LOGTYPE_USER7 | RTE_LOGTYPE_USER8)
76 :
77 : /*
78 : * Debug options:
79 : *
80 : #define RTE_LOG_LEVEL RTE_LOG_DEBUG
81 : #define VR_DPDK_LOGTYPE_DISABLE (RTE_LOGTYPE_USOCK | RTE_LOGTYPE_UVHOST)
82 : #define VR_DPDK_NETLINK_DEBUG
83 : #define VR_DPDK_NETLINK_PKT_DUMP
84 : #define VR_DPDK_USOCK_DUMP
85 : #define VR_DPDK_RX_PKT_DUMP
86 : #define VR_DPDK_TX_PKT_DUMP
87 : #define VR_DPDK_PKT_DUMP_VIF_FILTER(vif) (vif->vif_type == VIF_TYPE_AGENT \
88 : || vif->vif_type == VIF_TYPE_VIRTUAL)
89 : */
90 :
91 : #define VR_MAX_CPUS_DPDK 256
92 : /* Default lcore mask. Used only when sched_getaffinity() is failed */
93 : #define VR_DPDK_DEF_LCORE_MASK 0xf
94 : /* Default memory size to allocate at startup (in MBs) */
95 : #define VR_DPDK_DEF_MEM "1024"
96 : /* Number of memory channels to use */
97 : #define VR_DPDK_MAX_MEMCHANNELS "4"
98 : /* Use UDP source port hashing */
99 : #define VR_DPDK_USE_MPLS_UDP_ECMP true
100 : /* Use hardware filtering (Flow Director) */
101 : #define VR_DPDK_USE_HW_FILTERING false
102 : /* KNI generates random MACs for e1000e NICs, so we need this
103 : * option enabled for the development on servers with those NICs */
104 : #define VR_DPDK_ENABLE_PROMISC false
105 : /* Maximum number of hardware RX queues to use for RSS and filtering
106 : * (limited by NIC and number of per queue TX/RX descriptors) */
107 : #define VR_DPDK_MAX_NB_RX_QUEUES 16
108 : /* Maximum number of hardware TX queues to use (limited by the number of lcores) */
109 : #define VR_DPDK_MAX_NB_TX_QUEUES 64
110 : /* bnxt NIC only supports 8 TX queues */
111 : #define VR_DPDK_MAX_NB_TX_Q_BNXT 8
112 : /* bnxt and enic only support 9022 size jumbo frames */
113 : #define VT_DPDK_MAX_RX_PKT_LEN_9022 9022
114 : /*
115 : * Special value for number of queues to indicate that each
116 : * packet forwarding core should be assigned one queue
117 : */
118 : #define VR_DPDK_ONE_QUEUE_PER_CORE ((uint16_t)-1)
119 : /* Maximum number of hardware RX queues to use for RSS (limited by the number of lcores) */
120 : #define VR_DPDK_MAX_NB_RSS_QUEUES 16
121 : /* Maximum number of bond members per ethernet device */
122 : #define VR_DPDK_BOND_MAX_SLAVES 6
123 : /* Maximum RETA table size */
124 : #define VR_DPDK_MAX_RETA_SIZE ETH_RSS_RETA_SIZE_512
125 : #define VR_DPDK_MAX_RETA_ENTRIES (VR_DPDK_MAX_RETA_SIZE/RTE_RETA_GROUP_SIZE)
126 : /* Number of hardware RX ring descriptors per queue */
127 : #define VR_DPDK_NB_RXD 128
128 : /* Number of hardware TX ring descriptors per queue */
129 : #define VR_DPDK_NB_TXD 128
130 : /* Offset to MPLS label for hardware filtering (in 16-bit word units) */
131 : #define VR_DPDK_MPLS_OFFSET ((VR_ETHER_HLEN \
132 : + sizeof(struct vr_ip) \
133 : + sizeof(struct vr_udp))/2)
134 : /* Maximum number of rings per lcore (maximum is VR_MAX_INTERFACES*VR_MAX_CPUS_DPDK) */
135 : #define VR_DPDK_MAX_RINGS (VR_MAX_INTERFACES*2)
136 : /* Maximum number of bond interfaces per lcore */
137 : #define VR_DPDK_MAX_BONDS 2
138 : /* Max size of a single packet used by default */
139 : #define VR_DEF_MAX_PACKET_SZ (9 * 1024)
140 : /* check availability of enough lcores */
141 : #if (RTE_MAX_LCORE < 256)
142 : #error "vrouter needs to have support 256 lcores for AMD processors"
143 : #endif
144 : /* Number of bytes needed for each mbuf header */
145 : #define VR_DPDK_MBUF_HDR_SZ (sizeof(struct rte_mbuf) \
146 : + sizeof(struct vr_packet) \
147 : + RTE_PKTMBUF_HEADROOM)
148 : /* Size of direc mbuf used for fragmentation. It needs a headroom as it holds
149 : * the IP headers of the fragments and we have to prepend an outer (tunnel)
150 : * header. */
151 : #define VR_DPDK_FRAG_DIRECT_MBUF_SZ (sizeof(struct rte_mbuf) \
152 : + 2*RTE_PKTMBUF_HEADROOM)
153 : /* Size of indirect mbufs used for fragmentation. These mbufs holds only a
154 : * pointer to the data in other mbufs, thus they don't need any additional
155 : * buffer size. */
156 : #define VR_DPDK_FRAG_INDIRECT_MBUF_SZ (sizeof(struct rte_mbuf))
157 : /* How many packets to read/write from/to queue in one go */
158 : #define VR_DPDK_RX_BURST_SZ 32
159 : #define VR_DPDK_TX_BURST_SZ 32
160 : /* Default Number of mbufs in RSS mempool */
161 : #define VR_DEF_MEMPOOL_SZ (16 * 1024)
162 : /* How many objects (mbufs) to keep in per-lcore RSS mempool cache */
163 : #define VR_DPDK_RSS_MEMPOOL_CACHE_SZ (VR_DPDK_RX_BURST_SZ*8)
164 : /* Number of mbufs in FRAG_DIRECT mempool */
165 : #define VR_DPDK_FRAG_DIRECT_MEMPOOL_SZ 4096
166 : /* How many objects (mbufs) to keep in per-lcore FRAG_DIRECT mempool cache */
167 : #define VR_DPDK_FRAG_DIRECT_MEMPOOL_CACHE_SZ (VR_DPDK_RX_BURST_SZ*8)
168 : /* Number of mbufs in FRAG_INDIRECT mempool */
169 : #define VR_DPDK_FRAG_INDIRECT_MEMPOOL_SZ 4096
170 : /* How many objects (mbufs) to keep in per-lcore FRAG_INDIRECT mempool cache */
171 : #define VR_DPDK_FRAG_INDIRECT_MEMPOOL_CACHE_SZ (VR_DPDK_RX_BURST_SZ*8)
172 : /* Number of VM mempools */
173 : #define VR_DPDK_MAX_VM_MEMPOOLS (VR_DPDK_MAX_NB_RX_QUEUES*2)
174 : /* Number of mbufs in VM mempool */
175 : #define VR_DPDK_VM_MEMPOOL_SZ 1024
176 : /* How many objects (mbufs) to keep in per-lcore VM mempool cache */
177 : #define VR_DPDK_VM_MEMPOOL_CACHE_SZ (VR_DPDK_RX_BURST_SZ*8)
178 : /* Number of mbufs in TX rings (like ring to push, socket, VLAN rings etc */
179 : #define VR_DPDK_TX_RING_SZ (VR_DPDK_TX_BURST_SZ*32)
180 : /* RX ring minimum number of pointers to transfer (cache line / size of ptr) */
181 : #define VR_DPDK_RX_RING_CHUNK_SZ 1
182 : /* Number of mbufs in lcore RX ring (we retry in case enqueue fails) */
183 : #define VR_DPDK_RX_RING_SZ 1024
184 : /* Number of retries to enqueue packets */
185 : #define VR_DPDK_RETRY_NUM 1
186 : /* Delay between retries */
187 : #define VR_DPDK_RETRY_US 15
188 : /* Use timer to measure flushes (slower, but should improve latency) */
189 : #define VR_DPDK_USE_TIMER false
190 : /* TX flush timeout (in loops or US if USE_TIMER defined) */
191 : #define VR_DPDK_TX_FLUSH_LOOPS 16
192 : /* TX idle timeout - if packets are not sent to a VM for this many
193 : * forwarding loops, its TX queue can be flushed if it is not
194 : * empty (to reduce latency).
195 : */
196 : #define VR_DPDK_TX_IDLE_LOOPS 5
197 : #define VR_DPDK_TX_FLUSH_US 100
198 : /*
199 : * Bond TX timeout (in ms)
200 : * Receive and transmit functions must be invoked on bonded
201 : * interface at least 10 times per second or LACP will not
202 : * work correctly
203 : */
204 : #define VR_DPDK_BOND_TX_MS 100
205 : /* Sleep time in US if there are no queues to poll */
206 : #define VR_DPDK_SLEEP_NO_QUEUES_US 10000
207 : /* Sleep (in US) or yield if no packets received (use 0 to disable) */
208 : #define VR_DPDK_SLEEP_NO_PACKETS_US 0
209 : #define VR_DPDK_YIELD_NO_PACKETS 1
210 : /* Sleep (in US) no packets received on any TAP devs (use 0 to disable) */
211 : #define VR_DPDK_TAPDEV_SLEEP_NO_PACKETS_US 500
212 : /* Timers handling periodicity in US */
213 : #define VR_DPDK_SLEEP_TIMER_US 100
214 : /* KNI handling periodicity in US */
215 : #define VR_DPDK_SLEEP_KNI_US 500
216 : /* Sleep time in US for service lcore */
217 : #define VR_DPDK_SLEEP_SERVICE_US 100
218 : /* Invalid port ID */
219 : #define VR_DPDK_INVALID_PORT_ID 0xFF
220 : /* L3MH supports 3 bond interfaces */
221 : #define VR_DPDK_BOND_PORT_IDS 3
222 : /* Invalid queue ID */
223 : #define VR_DPDK_INVALID_QUEUE_ID 0xFFFF
224 : /* Socket connection retry timeout in seconds (use power of 2) */
225 : #define VR_DPDK_RETRY_CONNECT_SECS 64
226 : /* Maximum number of KNI devices (vhost0 + monitoring) */
227 : #define VR_DPDK_MAX_KNI_INTERFACES 16
228 : /* Maximum number of TAP devices (vhost0 + monitoring) */
229 : #define VR_DPDK_MAX_TAP_INTERFACES 16
230 : /* String buffer size (for logs and EAL arguments) */
231 : #define VR_DPDK_STR_BUF_SZ 512
232 : /* Log timestamp format */
233 : #define VR_DPDK_TIMESTAMP "%F %T"
234 : /* Maximum number of fragments allowed after IP fragmentation. Set to 7 to
235 : * allow for standard jumbo frame size (9000 / 1500 = 6) + 1 additional segment
236 : * for outer headers. */
237 : #define VR_DPDK_FRAG_MAX_IP_FRAGS 7
238 : /* Set to 256 just to ensure we have enough buffers in case mss is set
239 : * to 256, if the packet size is 64K.
240 : */
241 : #define VR_DPDK_FRAG_MAX_IP_SEGS 256
242 : #define VR_DPDK_VLAN_FWD_DEF_NAME "vfw0"
243 : /*
244 : * Use IO lcores:
245 : * true - IO lcores distribute packet among forwarding lcores
246 : * false - forwarding lcores distribute among other forwarding lcores
247 : */
248 : #define VR_DPDK_USE_IO_LCORES false
249 : /*
250 : * Whether IO lcore share CPUs with forwarding lcores
251 : * true - each IO lcore has an affinity mask of all the forwarding
252 : * lcores it distributes packets to
253 : * Example: if core mask is 0xf and FWD_LCORES_PER_IO is 2,
254 : * there are will be 4 forwarding lcores: 7@0,8@1,9@2,10@3
255 : * and 2 shared IO lcores: 3@(0,1),4@(2,3)
256 : * false - IO lcore has a dedicated CPU core
257 : * Example: if core mask is 0x3f and FWD_LCORES_PER_IO is 2,
258 : * there are will be 4 forwarding lcores: 7@1,8@2,9@4,10@5
259 : * and 2 dedicated IO lcores: 3@0,4@3
260 : */
261 : #define VR_DPDK_SHARED_IO_LCORES false
262 : /*
263 : * Create IO lcore for the specified number of forwarding lcores.
264 : * The maximum number of IO lcores is limited by IO lcore IDs below.
265 : */
266 : #define VR_DPDK_FWD_LCORES_PER_IO 3
267 :
268 : /*
269 : * Number of open fds needed for vrouter (other than one uvhost fd per
270 : * interface)
271 : */
272 : #define VR_DPDK_NUM_FDS 512
273 :
274 : /* Maximum number of HOLD entries in flow table */
275 : #define VR_DPDK_MAX_FLOW_TABLE_HOLD_COUNT 1000
276 : /* Maximum number of mbufs in per CPU queues */
277 : #define VR_DPDK_MAX_FRAGMENT_ELEMENTS 1024ULL
278 : /*
279 : * SR-IOV virtual function PMD name suffix.
280 : * Note: only rte_ixgbevf_pmd was tested.
281 : */
282 : #if (RTE_VERSION == RTE_VERSION_NUM(2, 1, 0, 0))
283 : #define VR_DPDK_VF_PMD_SFX "vf_pmd"
284 : #define RTE_LOG_DP RTE_LOG
285 : #else
286 : #define VR_DPDK_VF_PMD_SFX "_vf"
287 : #endif
288 :
289 : #define VR_DPDK_MGMTPATH 0
290 : #define VR_DPDK_DATAPATH 1
291 :
292 : /**
293 : * Macro to iterate over all enabled and ownerless dpdk ethdev ports.
294 : */
295 : #if (RTE_VERSION >= RTE_VERSION_NUM(17, 11, 0, 0))
296 : #define VR_DPDK_RTE_ETH_FOREACH_DEV(p) \
297 : RTE_ETH_FOREACH_DEV(p)
298 : #else
299 : #define VR_DPDK_RTE_ETH_FOREACH_DEV(p) \
300 : for (p = 0; p < rte_eth_dev_count(); p++)
301 : #endif
302 :
303 : /* VHOST Default MTU size*/
304 : #define VR_DPDK_VHOST_DEFAULT_MTU_SIZE 1500
305 :
306 : /* vr_info - DPDK platform dependent Macro functions
307 : * For vr_info, callback functions are registered in vr_info.h,
308 : * those callbacks will be expanded below for function declaration and
309 : * mapping those functions in vr_dpdk_host.c */
310 :
311 : /* Map only DPDK specific callback functions */
312 : #undef VR_INFO_HOST_MAP_DPDK
313 : #define VR_INFO_HOST_MAP_DPDK(MSG, CB) \
314 : .hos_vr_##CB = dpdk_##CB,
315 :
316 : #define VR_INFO_HOST_MAP(MSG, CB, PLTFRM) \
317 : VR_INFO_HOST_MAP_##PLTFRM(MSG, CB)
318 :
319 : #define FOREACH_VR_INFO_MAP() \
320 : VR_INFO_REG(VR_INFO_HOST_MAP)
321 :
322 : #define VR_INFO_DECLARATION(MSG, CB, PLTFRM) \
323 : int dpdk_##CB(VR_INFO_ARGS);
324 :
325 : #define FOREACH_VR_INFO_DECLARATION() \
326 : VR_INFO_REG(VR_INFO_DECLARATION)
327 :
328 : #define VR_DPDK_DDP_NOT_SUPPORTED -1000
329 : #define VR_DPDK_DDP_FAILED -1001
330 :
331 : /*
332 : * DPDK LCore IDs
333 : */
334 : enum {
335 : VR_DPDK_TAPDEV_LCORE_ID = 0,
336 : VR_DPDK_TIMER_LCORE_ID,
337 : VR_DPDK_UVHOST_LCORE_ID,
338 : /*
339 : * The actual number of IO lcores depends on the number of
340 : * forwarding lcores.
341 : */
342 : VR_DPDK_IO_LCORE_ID,
343 : VR_DPDK_IO_LCORE_ID2,
344 : VR_DPDK_IO_LCORE_ID3,
345 : VR_DPDK_IO_LCORE_ID4,
346 : VR_DPDK_LAST_IO_LCORE_ID,
347 : /* [PACKET_ID..FWD_ID) lcores have TX queues, but no RX queues */
348 : VR_DPDK_PACKET_LCORE_ID,
349 : VR_DPDK_NETLINK_LCORE_ID,
350 : /* The actual number of forwarding lcores depends on affinity mask. */
351 : VR_DPDK_FWD_LCORE_ID,
352 : };
353 :
354 : /* Fabric(Master) & Slave Interface Type */
355 : enum {
356 : VR_DPDK_BOND_MASTER = 1,
357 : VR_DPDK_BOND_SLAVE,
358 : };
359 :
360 : struct vr_dpdk_bond_member_info {
361 : char intf_name[VR_INTERFACE_NAME_LEN];
362 : char intf_drv_name[VR_INTERFACE_NAME_LEN];
363 : bool status;
364 : };
365 :
366 : /* structure used to store the list of availbale port_ids
367 : for net_bonding. For L3MH we will have more then 1 bond interface.
368 : */
369 : struct vr_dpdk_bond_port_list {
370 : uint8_t intf_list[VR_DPDK_BOND_PORT_IDS];
371 : uint8_t intf_count;
372 : };
373 :
374 : /* Maximum number of IO lcores */
375 : #define VR_DPDK_MAX_IO_LORES (VR_DPDK_LAST_IO_LCORE_ID - VR_DPDK_IO_LCORE_ID + 1)
376 :
377 : /* needs to have a place holder for RX flags, to allow usuage of dpdk from upstream */
378 : #ifndef PKT_RX_GSO_TCP4
379 : #define PKT_RX_GSO_TCP4 (1ULL << 23) /**< RX packet with TCPv4 segment offload */
380 : #endif /* PKT_RX_GSO_TCP4 */
381 : #ifndef PKT_RX_GSO_TCP6
382 : #define PKT_RX_GSO_TCP6 (1ULL << 24) /**< RX packet with TCPv6 segment offload */
383 : #endif /* PKT_RX_GSO_TCP6 */
384 :
385 : /*
386 : * VRouter/DPDK Data Structures
387 : * ============================
388 : *
389 : * Changes since the initial commit:
390 : * lcore_ctx -> vr_dpdk_lcore
391 : * vif_port -> vr_dpdk_queue
392 : *
393 : * TODO: update the description
394 : */
395 :
396 : struct vr_dpdk_rcu_cb_data {
397 : struct rcu_head rcd_rcu;
398 : vr_defer_cb rcd_user_cb;
399 : struct vrouter *rcd_router;
400 : unsigned char rcd_user_data[];
401 : };
402 :
403 : /* Init queue operation */
404 : typedef struct vr_dpdk_queue *
405 : (*vr_dpdk_queue_init_op)(unsigned lcore_id, struct vr_interface *vif,
406 : unsigned queue_or_lcore_id);
407 : /* Release queue operation */
408 : typedef void
409 : (*vr_dpdk_queue_release_op)(unsigned lcore_id, unsigned queue_index,
410 : struct vr_interface *vif);
411 :
412 : struct vr_dpdk_queue {
413 : SLIST_ENTRY(vr_dpdk_queue) q_next;
414 : union {
415 : /* DPDK TX queue operators */
416 : struct rte_port_out_ops txq_ops;
417 : /* DPDK RX queue operators */
418 : struct rte_port_in_ops rxq_ops;
419 : };
420 : /* Queue handler */
421 : void *q_queue_h;
422 : /* Enabled/disabled (whether polled by lcores) */
423 : bool enabled;
424 : /* Pointer to vRouter interface */
425 : struct vr_interface *q_vif;
426 : /* Incase of multiqueue, store vring queue_id */
427 : uint16_t vring_queue_id;
428 : };
429 :
430 : /* We store the queue params in the separate structure to increase CPU
431 : * cache hit rate
432 : */
433 : struct vr_dpdk_queue_params {
434 : /* Pointer to release function */
435 : vr_dpdk_queue_release_op qp_release_op;
436 : /* Extra queue params */
437 : union {
438 : struct {
439 : struct rte_ring *ring_p;
440 : unsigned host_lcore_id;
441 : } qp_ring;
442 : struct {
443 : uint8_t port_id;
444 : uint16_t queue_id;
445 : } qp_ethdev;
446 : };
447 : };
448 :
449 : struct vr_dpdk_ring_to_push {
450 : /* Ring pointer */
451 : struct rte_ring *rtp_tx_ring;
452 : /* TX queue pointer */
453 : struct vr_dpdk_queue *rtp_tx_queue;
454 : };
455 :
456 : SLIST_HEAD(vr_dpdk_q_slist, vr_dpdk_queue);
457 :
458 : /* Lcore commands */
459 : enum vr_dpdk_lcore_cmd {
460 : /* No command */
461 : VR_DPDK_LCORE_NO_CMD = 0,
462 : /* Command arguments are being published */
463 : VR_DPDK_LCORE_IN_PROGRESS_CMD,
464 : /* Stop and exit the lcore loop */
465 : VR_DPDK_LCORE_STOP_CMD,
466 : /* Remove RX queue */
467 : VR_DPDK_LCORE_RX_RM_CMD,
468 : /* Remove TX queue */
469 : VR_DPDK_LCORE_TX_RM_CMD,
470 : /* Call RCU callback */
471 : VR_DPDK_LCORE_RCU_CMD,
472 : /* TX queue disable/enable command */
473 : VR_DPDK_LCORE_TX_QUEUE_SET_CMD,
474 : /* RX queue disable/enable command */
475 : VR_DPDK_LCORE_RX_QUEUE_SET_CMD,
476 : };
477 :
478 : struct gro_ctrl {
479 : int gro_queued;
480 : int gro_flushed;
481 : int gro_bad_csum;
482 : int gro_cnt;
483 : int gro_flows;
484 : int gro_flush_inactive_flows;
485 :
486 : struct rte_hash *gro_tbl_v4_handle;
487 : struct rte_hash *gro_tbl_v6_handle;
488 : };
489 :
490 : struct vr_dpdk_lcore_rx_queue_remove_arg {
491 : unsigned int vif_id;
492 : bool clear_f_rx;
493 : bool free_arg;
494 : };
495 :
496 : struct vr_dpdk_lcore {
497 : /**********************************************************************/
498 : /* Frequently used fields */
499 : /* RX queues head */
500 : struct vr_dpdk_q_slist lcore_rx_head;
501 : /* TX queues head */
502 : struct vr_dpdk_q_slist lcore_tx_head;
503 : /* Forwarding lcore: number of rings to push for the lcore */
504 : volatile uint16_t lcore_nb_rings_to_push;
505 : /* Forwarding lcore: number of bond queues to TX */
506 : volatile uint16_t lcore_nb_bonds_to_tx;
507 : /* Number of hardware RX queues assigned to the lcore (for the scheduler) */
508 : uint16_t lcore_nb_rx_queues;
509 : /* Lcore command */
510 : volatile uint16_t lcore_cmd;
511 : /* Lcore command arguments */
512 : volatile uint64_t lcore_cmd_arg;
513 : /* RX ring with packets from other lcores (i.e. for MPLSoGRE). */
514 : struct rte_ring *lcore_rx_ring;
515 : /* RX ring with packets from IO lcore. */
516 : struct rte_ring *lcore_io_rx_ring;
517 : /* Number of forwarding loops */
518 : u_int64_t lcore_fwd_loops;
519 : /* Flag controlling the assembler work */
520 : bool do_fragment_assembly;
521 : /* GRO ctrl structure */
522 : struct gro_ctrl gro;
523 :
524 : /**********************************************************************/
525 : /* Big and less frequently used fields */
526 : /* Number of lcores to distribute packets to */
527 : uint16_t lcore_nb_dst_lcores;
528 : /* List of forwarding lcore indexes based on VR_DPDK_FWD_LCORE_ID */
529 : uint16_t lcore_dst_lcore_idxs[VR_MAX_CPUS_DPDK];
530 : /* Table of RX queues */
531 : struct vr_dpdk_queue lcore_rx_queues[VR_MAX_INTERFACES];
532 : /* Table of TX queues */
533 : struct vr_dpdk_queue *lcore_tx_queues[VR_MAX_INTERFACES] __rte_cache_aligned;
534 : /* List of rings to push */
535 : struct vr_dpdk_ring_to_push lcore_rings_to_push[VR_DPDK_MAX_RINGS] __rte_cache_aligned;
536 : /* List of bond queue params to TX LACP packets periodically */
537 : struct vr_dpdk_queue_params *lcore_bonds_to_tx[VR_DPDK_MAX_BONDS] __rte_cache_aligned;
538 : /* Table of RX queue params */
539 : struct vr_dpdk_queue_params lcore_rx_queue_params[VR_MAX_INTERFACES] __rte_cache_aligned;
540 : /* Table of TX queue params */
541 : struct vr_dpdk_queue_params *lcore_tx_queue_params[VR_MAX_INTERFACES] __rte_cache_aligned;
542 : /*
543 : * number of queues/lcore - basically one hardware queue + rings
544 : * to other cores that hosts each hardware queue
545 : *
546 : * If hardware queueing is supported, num_tx_queues_per_lcore will
547 : * be equal to the number of queues that the agent wants to use.
548 : * The number of queues that the agent wants to use is set in the
549 : * vr_interface structure (vif_num_hw_queues). agent will read its
550 : * configuration file and tell vRouter the queue numbers it wants
551 : * to use (and hence the number of queues) and set this information
552 : * in vif_hw_queues
553 : */
554 : uint16_t num_tx_queues_per_lcore[VR_MAX_INTERFACES];
555 : /* for each vif, the first hardware queue that is tied to this lcore */
556 : int16_t lcore_hw_queue[VR_MAX_INTERFACES];
557 : /*
558 : * given a hardware queue, the index to the array of vr_dpdk_queue.
559 : * Enables us to get the queue faster
560 : */
561 : int16_t *lcore_hw_queue_to_dpdk_index[VR_MAX_INTERFACES];
562 : void (*fragment_assembly_func)(void *arg);
563 : void *fragment_assembly_arg;
564 : };
565 :
566 : /* Hardware RX queue state */
567 : enum vr_dpdk_queue_state {
568 : /* No queue available */
569 : VR_DPDK_QUEUE_NONE,
570 : /* The queue is ready to use for RSS or filtering */
571 : VR_DPDK_QUEUE_READY_STATE,
572 : /* The queue is being used for RSS */
573 : VR_DPDK_QUEUE_RSS_STATE,
574 : /* The queue is being used for filtering */
575 : VR_DPDK_QUEUE_FILTERING_STATE
576 : };
577 :
578 : struct vif_queue_dpdk_data {
579 : int16_t vqdd_queue_to_lcore[VR_DPDK_MAX_NB_TX_QUEUES];
580 : };
581 :
582 : /* Ethdev configuration */
583 : struct vr_dpdk_ethdev {
584 : /* Pointer to ethdev or NULL if the device is not used */
585 : struct rte_eth_dev *ethdev_ptr;
586 : /* Number of HW RX queues (limited by NIC hardware) */
587 : uint16_t ethdev_nb_rx_queues;
588 : /* Number of HW TX queues (limited by the nb of lcores) */
589 : uint16_t ethdev_nb_tx_queues;
590 : /* Number of HW RX queues used for RSS (limited by the nb of lcores) */
591 : uint16_t ethdev_nb_rss_queues;
592 : /* Actual size of ethdev RETA */
593 : uint16_t ethdev_reta_size;
594 : /* DPDK port ID */
595 : #if (RTE_VERSION >= RTE_VERSION_NUM(17, 11, 0, 0))
596 : uint16_t ethdev_port_id;
597 : #else
598 : uint8_t ethdev_port_id;
599 : #endif
600 : /* The device is a bond if the number of slaves is > 0 */
601 : int8_t ethdev_nb_slaves;
602 : /* List of slaves port IDs */
603 : #if (RTE_VERSION >= RTE_VERSION_NUM(17, 11, 0, 0))
604 : uint16_t ethdev_slaves[VR_DPDK_BOND_MAX_SLAVES];
605 : #else
606 : uint8_t ethdev_slaves[VR_DPDK_BOND_MAX_SLAVES];
607 : #endif
608 : /* Hardware RX queue states */
609 : uint8_t ethdev_queue_states[VR_DPDK_MAX_NB_RX_QUEUES];
610 : /* Pointers to memory pools */
611 : struct rte_mempool *ethdev_mempools[VR_DPDK_MAX_NB_RX_QUEUES];
612 : /* Vif interface Id */
613 : uint8_t ethdev_vif_idx;
614 : /* Port lock. */
615 : rte_spinlock_t ethdev_lock;
616 : };
617 :
618 : /* Tapdev configuration. */
619 : struct vr_dpdk_tapdev {
620 : /* Tapdev file descriptor. */
621 : volatile int tapdev_fd;
622 : /* Tapdev file descriptor for vhost0. */
623 : int tapdev_vhost_fd;
624 : /* RX ring. */
625 : struct rte_ring *tapdev_rx_ring;
626 : /* TX rings (single-producer single-consumer) */
627 : struct rte_ring *tapdev_tx_rings[RTE_MAX_LCORE];
628 : /* Pointer to vif. */
629 : struct vr_interface *tapdev_vif;
630 : /* Name of the corresponding device on kernel. */
631 : char tapdev_name[VR_INTERFACE_NAME_LEN];
632 : };
633 :
634 : struct vr_dpdk_global {
635 : /**********************************************************************/
636 : /* Frequently used fields */
637 : /* Pointer to main (RSS) memory pool */
638 : struct rte_mempool *rss_mempool;
639 : /* Packet socket ring */
640 : struct rte_ring *packet_ring;
641 : /* Global stop flag */
642 : rte_atomic16_t stop_flag;
643 : /* VLAN tag */
644 : uint16_t vlan_tag;
645 : /* We are use it in unittest application (vtest) for vlan
646 : * adding workaround for virtual interfaces. */
647 : uint16_t vtest_vlan;
648 : /* Number of forwarding lcores */
649 : uint16_t nb_fwd_lcores;
650 : /* Number of IO lcores */
651 : uint16_t nb_io_lcores;
652 : /* Packet lcore event socket
653 : * TODO: refactor to use event FD
654 : */
655 : void *packet_event_sock;
656 : /* NetLink lcore event socket */
657 : void *netlink_event_sock;
658 : /* Event FD to wake up UVHost */
659 : int uvhost_event_fd;
660 :
661 : /* Table of pointers to forwarding lcore
662 : * Must be at the end of the cache line 1 */
663 : struct vr_dpdk_lcore *lcores[VR_MAX_CPUS_DPDK];
664 :
665 : /**********************************************************************/
666 : /* Big and less frequently used fields */
667 : /* Number of free memory pools */
668 : uint16_t nb_free_mempools;
669 : /* NetLink socket handler */
670 : void *netlink_sock;
671 : void *flow_table;
672 : void *bridge_table;
673 : /* Packet socket */
674 : void *packet_transport;
675 : /* Interface configuration mutex
676 : * ATM we use it just to synchronize access between the NetLink interface
677 : * and kernel KNI events. The datapath is not affected. */
678 : pthread_mutex_t if_lock;
679 : /* Pointer to IP fragmentation memory pool (direct) */
680 : struct rte_mempool *frag_direct_mempool;
681 : /* Pointer to IP fragmentation memory pool (indirect) */
682 : struct rte_mempool *frag_indirect_mempool;
683 : /* List of free memory pools */
684 : struct rte_mempool *free_mempools[VR_DPDK_MAX_VM_MEMPOOLS] __rte_cache_aligned;
685 : /* List of KNI interfaces to handle KNI requests */
686 : struct rte_kni *knis[VR_DPDK_MAX_KNI_INTERFACES] __rte_cache_aligned;
687 : /* Table of monitoring redirections (for vifdump) */
688 : uint16_t monitorings[VR_MAX_INTERFACES] __rte_cache_aligned;
689 : /* Table of ethdevs */
690 : struct vr_dpdk_ethdev ethdevs[RTE_MAX_ETHPORTS] __rte_cache_aligned;
691 : /* Table of tapdevs. */
692 : struct vr_dpdk_tapdev tapdevs[VR_DPDK_MAX_TAP_INTERFACES] __rte_cache_aligned;
693 : /* netlink socket to listen to link up/down and mtu change notifications */
694 : volatile int tap_nl_fd;
695 : /* VLAN forwarding interface name */
696 : char vlan_name[VR_INTERFACE_NAME_LEN];
697 : /* VLAN forwarding interface ring */
698 : struct rte_ring *vlan_ring;
699 : /* VLAN forwarding device pointer. */
700 : void *vlan_dev;
701 : /* VLAN forwarding interface vif. */
702 : struct vr_interface *vlan_vif;
703 : /* Dedicated IO lcore for SR-IOV VF. */
704 : unsigned vf_lcore_id;
705 : /*
706 : * KNI global state flag:
707 : * 0 - initial state
708 : * 1 - KNI is enabled
709 : * -1 - KNI is not available, so TAP interfaces are used instead
710 : */
711 : int kni_state;
712 : };
713 :
714 : extern struct vr_dpdk_global vr_dpdk;
715 : extern struct rte_eth_conf ethdev_conf;
716 :
717 : /*
718 : * rte_mbuf <=> vr_packet conversion
719 : *
720 : * The vr_packet structure is right after the rte_mbuf:
721 : * struct rte_mbuf + struct vr_packet + headroom + data + tailroom
722 : *
723 : * rte_mbuf: *buf_addr(buf_len) + headroom + data_off(data_len) + tailroom
724 : *
725 : * rte_mbuf->buf_addr = rte_mbuf + sizeof(rte_mbuf) + sizeof(vr_packet)
726 : * rte_mbuf->buf_len = elt_size - sizeof(rte_mbuf) - sizeof(vr_packet)
727 : * rte_mbuf->data_off = RTE_PKTMBUF_HEADROOM
728 : *
729 : *
730 : * vr_packet: *vp_head + headroom + vp_data(vp_len) + vp_tail + tailroom
731 : * + vp_end
732 : *
733 : * vr_packet->vp_head = rte_mbuf->buf_addr (set in mbuf constructor)
734 : * vr_packet->vp_data = rte_mbuf->data_off
735 : * vr_packet->vp_len = rte_mbuf->data_len
736 : * vr_packet->vp_tail = vr_packet->vp_data + vr_packet->vp_len
737 : * vr_packet->vp_end = rte_mbuf->buf_len (set in mbuf constructor)
738 : */
739 : static inline struct rte_mbuf *
740 1390 : vr_dpdk_pkt_to_mbuf(struct vr_packet *pkt)
741 : {
742 1390 : return (struct rte_mbuf *)((uintptr_t)pkt - sizeof(struct rte_mbuf));
743 : }
744 : static inline struct vr_packet *
745 868534 : vr_dpdk_mbuf_to_pkt(struct rte_mbuf *mbuf)
746 : {
747 868534 : return (struct vr_packet *)((uintptr_t)mbuf + sizeof(struct rte_mbuf));
748 : }
749 :
750 : /*
751 : * vr_dpdk_mbuf_reset - if the mbuf changes, possibley due to
752 : * pskb_may_pull, reset fields of the pkt structure that point at
753 : * the mbuf fields.
754 : * Note: we do not reset pkt->data here
755 : */
756 : static inline void
757 0 : vr_dpdk_mbuf_reset(struct vr_packet *pkt)
758 : {
759 0 : struct rte_mbuf *mbuf = vr_dpdk_pkt_to_mbuf(pkt);
760 :
761 0 : pkt->vp_head = mbuf->buf_addr;
762 0 : pkt->vp_tail = rte_pktmbuf_headroom(mbuf) + mbuf->data_len;
763 0 : pkt->vp_end = mbuf->buf_len;
764 0 : pkt->vp_len = pkt->vp_tail - pkt->vp_data;
765 :
766 0 : return;
767 : }
768 :
769 : /*
770 : * dpdk_vrouter.c
771 : */
772 : /* A packet mbuf pool constructor with vr_packet support */
773 : void vr_dpdk_pktmbuf_pool_init(struct rte_mempool *mp, void *opaque_arg);
774 : /* The packet mbuf constructor with vr_packet support */
775 : void vr_dpdk_pktmbuf_init(struct rte_mempool *mp, void *opaque_arg, void *_m, unsigned i);
776 : /* Check if the stop flag is set */
777 : int vr_dpdk_is_stop_flag_set(void);
778 : /* Called by user space vhost server at exit */
779 : void vr_dpdk_exit_trigger(void);
780 :
781 : /*
782 : * vr_dpdk_ethdev.c
783 : */
784 : /* Init eth RX queue */
785 : struct vr_dpdk_queue *
786 : vr_dpdk_ethdev_rx_queue_init(unsigned lcore_id, struct vr_interface *vif,
787 : unsigned rx_queue_id);
788 : /* Init eth TX queue */
789 : struct vr_dpdk_queue *
790 : vr_dpdk_ethdev_tx_queue_init(unsigned lcore_id, struct vr_interface *vif,
791 : unsigned tx_queue_id);
792 : /* Init ethernet device */
793 : int vr_dpdk_ethdev_init(struct vr_dpdk_ethdev *, struct rte_eth_conf *,
794 : struct rte_eth_txconf *, struct rte_eth_rxconf *);
795 : /* Release ethernet device */
796 : int vr_dpdk_ethdev_release(struct vr_dpdk_ethdev *);
797 : /* Get free queue ID */
798 : uint16_t vr_dpdk_ethdev_ready_queue_id_get(struct vr_interface *vif);
799 :
800 : #if VR_DPDK_USE_HW_FILTERING
801 : /* Add hardware filter */
802 : int vr_dpdk_ethdev_filter_add(struct vr_interface *vif, uint16_t queue_id,
803 : unsigned dst_ip, unsigned mpls_label);
804 : /* Init hardware filtering */
805 : int vr_dpdk_ethdev_filtering_init(struct vr_interface *vif, struct vr_dpdk_ethdev *ethdev);
806 : #endif
807 :
808 : /* Init RSS */
809 : int vr_dpdk_ethdev_rss_init(struct vr_dpdk_ethdev *ethdev);
810 : /*
811 : * vr_dpdk_ethdev_rx_emulate - emulate smart NIC RX:
812 : * - strip VLAN tags for packets received from fabric interface
813 : * - calculate RSS hash if it is not present
814 : * - recalculate RSS hash for MPLSoGRE packets
815 : *
816 : * Returns 0 on no hash changes, otherwise a bitmask of mbufs to distribute.
817 : */
818 : uint64_t vr_dpdk_ethdev_rx_emulate(struct vr_interface *vif,
819 : struct rte_mbuf *pkts[VR_DPDK_RX_BURST_SZ], uint32_t *nb_pkts);
820 : /* Check if port_id is a bond slave. */
821 : bool vr_dpdk_ethdev_bond_port_match(uint8_t port_id, struct vr_dpdk_ethdev *ethdev);
822 : /* Send bond interface state to Agent. */
823 : void vr_dpdk_nl_send_bond_intf_state(struct vr_dpdk_bond_member_info *info,
824 : unsigned type, uint8_t vif_idx);
825 :
826 : int vr_dpdk_table_mem_init(unsigned int, unsigned int, unsigned long,
827 : unsigned int, unsigned long);
828 : int vr_dpdk_flow_init(void);
829 : int vr_dpdk_bridge_init(void);
830 :
831 : extern uint32_t vr_dpdk_master_port_id;
832 :
833 : /*
834 : * vr_dpdk_ddp.c
835 : */
836 : enum {
837 : VR_DPDK_DDP_UNDEFINED = 0,
838 : VR_DPDK_DDP_ADD,
839 : VR_DPDK_DDP_DELETE
840 : };
841 : bool vr_dpdk_get_ddp(void);
842 : void vr_dpdk_set_ddp(void);
843 : void vr_dpdk_reset_ddp(void);
844 : int vr_dpdk_process_ddp_package(unsigned);
845 :
846 : /*
847 : * vr_dpdk_host.c
848 : */
849 : int vr_dpdk_host_init(void);
850 : void vr_dpdk_host_exit(void);
851 : /* Convert internal packet fields */
852 : struct vr_packet * vr_dpdk_packet_get(struct rte_mbuf *m, struct vr_interface *vif);
853 : void vr_dpdk_pfree(struct rte_mbuf *mbuf, struct vr_interface *vif, unsigned short reason);
854 : /* Retry socket connection */
855 : int vr_dpdk_retry_connect(int sockfd, const struct sockaddr *addr,
856 : socklen_t alen);
857 : /* Generates unique log message */
858 : int vr_dpdk_ulog(uint32_t level, uint32_t logtype, uint32_t *last_hash,
859 : const char *format, ...);
860 : #if (RTE_LOG_LEVEL == RTE_LOG_DEBUG)
861 : #define DPDK_DEBUG_VAR(v) v
862 : #define DPDK_UDEBUG(t, h, ...) \
863 : (void)(((RTE_LOG_DEBUG <= RTE_LOG_LEVEL) && \
864 : (RTE_LOG_DEBUG <= rte_logs.level) && \
865 : (RTE_LOGTYPE_ ## t & rte_logs.type)) ? \
866 : vr_dpdk_ulog(RTE_LOG_DEBUG, \
867 : RTE_LOGTYPE_ ## t, h, # t ": " __VA_ARGS__) : 0)
868 : #else
869 : #define DPDK_DEBUG_VAR(v)
870 : #define DPDK_UDEBUG(t, h, ...)
871 : #endif
872 : /* Helper to adjust TCP MSS */
873 : void dpdk_adjust_tcp_mss(struct tcphdr *tcph, unsigned short overlay_len,
874 : unsigned char iph_len);
875 : /* Creates a copy of the given packet mbuf */
876 : struct rte_mbuf *
877 : vr_dpdk_pktmbuf_copy(struct rte_mbuf *md, struct rte_mempool *mp);
878 : struct rte_mbuf *
879 : vr_dpdk_pktmbuf_copy_mon(struct rte_mbuf *md, struct rte_mempool *mp);
880 :
881 : /*
882 : * vr_dpdk_interface.c
883 : */
884 : /* Lock interface operations */
885 1035 : static inline int vr_dpdk_if_lock()
886 1035 : { return pthread_mutex_lock(&vr_dpdk.if_lock); }
887 : /* Unlock interface operations */
888 1035 : static inline int vr_dpdk_if_unlock()
889 1035 : { return pthread_mutex_unlock(&vr_dpdk.if_lock); }
890 : static inline uint16_t
891 126 : dpdk_get_ether_header_len(const void *data)
892 : {
893 126 : struct rte_ether_hdr *eth = (struct rte_ether_hdr *)data;
894 :
895 126 : if (eth->ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_VLAN))
896 0 : return sizeof(struct rte_ether_hdr) + sizeof(struct rte_vlan_hdr);
897 : else
898 126 : return sizeof(struct rte_ether_hdr);
899 : }
900 :
901 :
902 : /*
903 : * vr_dpdk_tapdev.c
904 : */
905 : /* Init TAP device. */
906 : int vr_dpdk_tapdev_init(struct vr_interface *vif);
907 : /* Release TAP device. */
908 : int vr_dpdk_tapdev_release(struct vr_interface *vif);
909 : /* Init TAP RX queue. */
910 : struct vr_dpdk_queue *
911 : vr_dpdk_tapdev_rx_queue_init(unsigned lcore_id, struct vr_interface *vif,
912 : unsigned queue_id);
913 : /* Init TAP TX queue. */
914 : struct vr_dpdk_queue *
915 : vr_dpdk_tapdev_tx_queue_init(unsigned lcore_id, struct vr_interface *vif,
916 : unsigned queue_id);
917 : /* RX/TX to/from all the TAP devices. */
918 : uint64_t vr_dpdk_tapdev_rxtx(void);
919 : /* RX a burst of packets from the TAP device. */
920 : unsigned vr_dpdk_tapdev_rx_burst(struct vr_dpdk_tapdev *, struct rte_mbuf **,
921 : unsigned num, int datapath);
922 : /* Dequeue a burst of packets from the TAP device. */
923 : unsigned vr_dpdk_tapdev_dequeue_burst(struct vr_dpdk_tapdev *, struct rte_mbuf **,
924 : unsigned num);
925 : /* TX a burst of packets to the TAP device. */
926 : unsigned vr_dpdk_tapdev_tx_burst(struct vr_dpdk_tapdev *, struct rte_mbuf **,
927 : unsigned num, int datapath);
928 : /* Enqueue a burst of packets to the TAP device. */
929 : unsigned vr_dpdk_tapdev_enqueue_burst(struct vr_dpdk_tapdev *, struct rte_mbuf **,
930 : unsigned num);
931 : void vr_dpdk_tapdev_handle_notifications(void);
932 :
933 : /*
934 : * vr_dpdk_packet.c
935 : */
936 : void vr_dpdk_packet_wakeup(struct vr_interface *vif);
937 : int dpdk_packet_socket_init(void);
938 : void dpdk_packet_socket_close(void);
939 : int dpdk_packet_io(void);
940 : /* RCU callback called on packet lcore */
941 : void vr_dpdk_packet_rcu_cb(struct rcu_head *rh);
942 : /* Work callback called on packet lcore */
943 : void vr_dpdk_packet_work_cb(void (*fn)(void *), void *arg);
944 :
945 : /*
946 : * vr_dpdk_lcore.c
947 : */
948 : /* Launch lcore main loop */
949 : int vr_dpdk_lcore_launch(void *dummy);
950 : /* Schedule an interface */
951 : int vr_dpdk_lcore_if_schedule(struct vr_interface *vif, unsigned least_used_id,
952 : uint16_t nb_rx_queues, vr_dpdk_queue_init_op rx_queue_init_op,
953 : uint16_t nb_tx_queues, vr_dpdk_queue_init_op tx_queue_init_op);
954 : /* Unschedule an interface */
955 : void vr_dpdk_lcore_if_unschedule(struct vr_interface *vif);
956 : /* Schedule an MPLS label queue */
957 : int vr_dpdk_lcore_mpls_schedule(struct vr_interface *vif, unsigned dst_ip,
958 : unsigned mpls_label);
959 : /* Returns the least used lcore or VR_MAX_CPUS_DPDK */
960 : unsigned vr_dpdk_lcore_least_used_get(void);
961 : size_t vr_dpdk_lcore_free_lcore_get(void);
962 : /* Flush TX queues */
963 : static inline void
964 59280213 : vr_dpdk_lcore_flush(struct vr_dpdk_lcore *lcore)
965 : {
966 : struct vr_dpdk_queue *tx_queue;
967 :
968 231973998 : SLIST_FOREACH(tx_queue, &lcore->lcore_tx_head, q_next) {
969 172741853 : tx_queue->txq_ops.f_flush(tx_queue->q_queue_h);
970 : }
971 59232145 : }
972 : /*
973 : * Distribute mbufs among forwarding lcores using hash.rss.
974 : * The destination lcores are listed in lcore->lcore_dst_lcores.
975 : */
976 : void
977 : vr_dpdk_lcore_distribute(struct vr_dpdk_lcore *lcore, const bool io_lcore,
978 : struct vr_interface *vif, struct rte_mbuf *pkts[VR_DPDK_RX_BURST_SZ],
979 : uint32_t nb_pkts);
980 : /* Pass mbufs to dp-core. */
981 : void
982 : vr_dpdk_lcore_vroute(struct vr_dpdk_lcore *lcore, struct vr_interface *vif,
983 : struct rte_mbuf *pkts[VR_DPDK_RX_BURST_SZ], uint32_t nb_pkts);
984 : /* Handle an IPC command */
985 : int vr_dpdk_lcore_cmd_handle(struct vr_dpdk_lcore *lcore);
986 : /* Busy wait for a command to complete on a specific lcore */
987 : void vr_dpdk_lcore_cmd_wait(unsigned lcore_id);
988 : /* Post an lcore command to a specific lcore */
989 : void
990 : vr_dpdk_lcore_cmd_post(unsigned lcore_id, uint16_t cmd, uint64_t cmd_arg);
991 : /* Post an lcore command to all the lcores */
992 : void vr_dpdk_lcore_cmd_post_all(uint16_t cmd, uint64_t cmd_arg);
993 : /* Schedule an asslembler work on an lcore */
994 : void vr_dpdk_lcore_schedule_assembler_work(struct vr_dpdk_lcore *lcore,
995 : void (*fun)(void *arg), void *arg);
996 : void dpdk_lcore_exit(unsigned lcore_id);
997 : /*
998 : * vr_dpdk_netlink.c
999 : */
1000 : void vr_dpdk_netlink_wakeup(void);
1001 : void dpdk_netlink_exit(void);
1002 : int dpdk_netlink_init(void);
1003 : int dpdk_netlink_receive(void *usockp, char *nl_buf, unsigned int nl_len);
1004 :
1005 : extern unsigned int vr_dpdk_rx_ring_sz, vr_dpdk_tx_ring_sz;
1006 : extern unsigned int vr_dpdk_yield_option;
1007 :
1008 : /*
1009 : * vr_dpdk_ringdev.c
1010 : */
1011 : /* Allocates a new ring */
1012 : struct rte_ring *
1013 : vr_dpdk_ring_allocate(unsigned host_lcore_id, char *ring_name,
1014 : unsigned vr_dpdk_tx_ring_sz, unsigned flags);
1015 : /* Init ring RX queue */
1016 : struct vr_dpdk_queue *
1017 : vr_dpdk_ring_rx_queue_init(unsigned lcore_id, struct vr_interface *vif,
1018 : unsigned host_lcore_id);
1019 : /* Init ring TX queue */
1020 : struct vr_dpdk_queue *
1021 : vr_dpdk_ring_tx_queue_init(unsigned lcore_id, struct vr_interface *vif,
1022 : unsigned int queue_id, unsigned host_lcore_id);
1023 : void dpdk_ring_to_push_add(unsigned lcore_id, struct rte_ring *tx_ring,
1024 : struct vr_dpdk_queue *tx_queue);
1025 :
1026 : void
1027 : dpdk_ring_to_push_remove(unsigned lcore_id, struct rte_ring *tx_ring);
1028 :
1029 : /*
1030 : * vr_dpdk_fragment_assembler.c
1031 : */
1032 : int dpdk_fragment_assembler_init(void);
1033 : void dpdk_fragment_assembler_exit(void);
1034 : int dpdk_fragment_assembler_enqueue(struct vrouter *router,
1035 : struct vr_packet *pkt, struct vr_forwarding_md *fmd);
1036 : void dpdk_fragment_sync_assemble(struct vr_fragment_queue_element *vfqe);
1037 : void dpdk_fragment_assembler_table_scan(void *);
1038 : void dpdk_gro_free_all_flows(struct vr_dpdk_lcore *lcore);
1039 : void dpdk_gro_flush_all_inactive(struct vr_dpdk_lcore *lcore);
1040 : int dpdk_gro_process(struct vr_packet *pkt, struct vr_interface *vif, bool l2_pkt);
1041 : int dpdk_segment_packet(struct vr_packet *pkt, struct rte_mbuf *mbuf_in,
1042 : struct rte_mbuf **mbuf_out, const unsigned short out_num,
1043 : const unsigned short mss_size, bool do_outer_ip_csum);
1044 : uint16_t dpdk_ipv4_udptcp_cksum(struct rte_mbuf *m,
1045 : const struct rte_ipv4_hdr *ipv4_hdr,
1046 : uint8_t *l4_hdr);
1047 : uint16_t dpdk_ipv6_udptcp_cksum(struct rte_mbuf *m,
1048 : const struct rte_ipv6_hdr *ipv6_hdr,
1049 : uint8_t *l4_hdr);
1050 : int dpdk_check_rx_mrgbuf_disable(void);
1051 :
1052 : /*
1053 : * vr_dpdk_cpuflags.c
1054 : */
1055 : void vr_dpdk_init_cpuid(struct vr_cpu_type_t *cpu);
1056 :
1057 : /*
1058 : * Get bond interface port id by drv_name
1059 : */
1060 : uint8_t dpdk_find_port_id_by_vif_name(struct vr_interface *);
1061 :
1062 : /*
1063 : * Get all bond interface port ids
1064 : */
1065 : bool dpdk_find_bond_port_id_list(struct vr_dpdk_bond_port_list *);
1066 :
1067 : /*
1068 : * vr_dpdk_interface.c
1069 : */
1070 : int vr_dpdk_interface_queue_setup(struct vr_interface *vif);
1071 : void vr_dpdk_interface_queue_free(struct vr_interface *vif);
1072 :
1073 : /*
1074 : * Get DPDK info
1075 : */
1076 : /* Below macro would be expanded for declaring the DPDK callback function
1077 : * used for vr_info */
1078 : FOREACH_VR_INFO_DECLARATION()
1079 :
1080 : #endif /*_VR_DPDK_H_ */
|
