Line data Source code
1 : /*
2 : * Copyright (C) 2014 Semihalf.
3 : *
4 : * This program is free software; you can redistribute it and/or
5 : * modify it under the terms of the GNU General Public License as
6 : * published by the Free Software Foundation version 2.
7 : *
8 : * This program is distributed "as is" WITHOUT ANY WARRANTY of any
9 : * kind, whether express or implied; without even the implied warranty
10 : * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 : * GNU General Public License for more details.
12 : *
13 : * vr_dpdk_ethdev.c -- DPDK ethernet device
14 : *
15 : */
16 :
17 : #include "vr_dpdk.h"
18 : #include "vr_dpdk_usocket.h"
19 :
20 : #include <vr_mpls.h>
21 :
22 : #include <rte_eth_bond.h>
23 : #include <rte_errno.h>
24 : #include <rte_ethdev_pci.h>
25 : #include <rte_ethdev.h>
26 : #include <rte_hash_crc.h>
27 : #include <rte_ip.h>
28 : #include <rte_port_ethdev.h>
29 : #include <rte_udp.h>
30 : #include <linux/netlink.h>
31 : #include <linux/rtnetlink.h>
32 :
33 : extern int vr_rxd_sz, vr_txd_sz;
34 : extern unsigned int datapath_offloads;
35 : unsigned int vr_dpdk_master_port_id;
36 : extern bool vr_no_load_balance;
37 :
38 : struct rte_eth_conf ethdev_conf = {
39 : .link_speeds = ETH_LINK_SPEED_AUTONEG,
40 : .rxmode = { /* Port RX configuration. */
41 : /* The multi-queue packet distribution mode to be used, e.g. RSS. */
42 : .mq_mode = ETH_MQ_RX_RSS,
43 : .max_rx_pkt_len = VR_DEF_MAX_PACKET_SZ, /* Only used if jumbo_frame enabled */
44 : .offloads = DEV_RX_OFFLOAD_CHECKSUM | DEV_RX_OFFLOAD_JUMBO_FRAME,
45 : },
46 : .rx_adv_conf = {
47 : .rss_conf = { /* Port RSS configuration */
48 : .rss_key = NULL, /* If not NULL, 40-byte hash key */
49 : .rss_key_len = 0, /* Hash key length in bytes */
50 : /* Hash functions to apply */
51 : .rss_hf = ETH_RSS_IP | ETH_RSS_UDP | ETH_RSS_TCP,
52 : },
53 : },
54 : .txmode = { /* Port TX configuration. */
55 : .mq_mode = ETH_MQ_TX_NONE, /* TX multi-queues mode */
56 : /* For i40e specifically */
57 : .pvid = 0,
58 : .hw_vlan_reject_tagged = 0, /* If set, reject sending out tagged pkts */
59 : .hw_vlan_reject_untagged = 0, /* If set, reject sending out untagged pkts */
60 : .hw_vlan_insert_pvid = 0, /* If set, enable port based VLAN insertion */
61 : },
62 : .fdir_conf = {
63 : #if VR_DPDK_USE_HW_FILTERING
64 : .mode = RTE_FDIR_MODE_PERFECT, /* Flow Director mode. */
65 : .status = RTE_FDIR_REPORT_STATUS, /* How to report FDIR hash. */
66 : #else
67 : .mode = RTE_FDIR_MODE_NONE,
68 : .status = RTE_FDIR_NO_REPORT_STATUS,
69 : #endif
70 : .pballoc = RTE_FDIR_PBALLOC_64K, /* Space for FDIR filters. */
71 : /* RX queue of packets matching a "drop" filter in perfect mode. */
72 : .drop_queue = 0,
73 : .flex_conf = {
74 : },
75 : },
76 : .intr_conf = {
77 : .lsc = 1, /* Enable Link status interrupts */
78 : },
79 : };
80 :
81 : /* RX and TX Prefetch, Host, and Write-back threshold values should be
82 : * carefully set for optimal performance. Consult the network
83 : * controller's datasheet and supporting DPDK documentation for guidance
84 : * on how these parameters should be set.
85 : */
86 : /* RX ring configuration */
87 : static const struct rte_eth_rxconf default_rx_queue_conf = {
88 : .rx_thresh = {
89 : .pthresh = 8, /* Ring prefetch threshold */
90 : .hthresh = 8, /* Ring host threshold */
91 : .wthresh = 4, /* Ring writeback threshold */
92 : },
93 : /* Do not immediately free RX descriptors */
94 : .rx_free_thresh = VR_DPDK_RX_BURST_SZ,
95 : .offloads = DEV_RX_OFFLOAD_CHECKSUM | DEV_RX_OFFLOAD_JUMBO_FRAME,
96 : };
97 :
98 : /*
99 : * These default values are optimized for use with the Intel(R) 82599 10 GbE
100 : * Controller and the DPDK ixgbe PMD. Consider using other values for other
101 : * network controllers and/or network drivers.
102 : */
103 : /* TX ring configuration */
104 : static const struct rte_eth_txconf default_tx_queue_conf = {
105 : .tx_thresh = {
106 : .pthresh = 32, /* Ring prefetch threshold */
107 : .hthresh = 0, /* Ring host threshold */
108 : .wthresh = 0, /* Ring writeback threshold */
109 : },
110 : .offloads = DEV_TX_OFFLOAD_UDP_CKSUM | DEV_TX_OFFLOAD_TCP_CKSUM | DEV_TX_OFFLOAD_IPV4_CKSUM,
111 : .tx_free_thresh = 32,
112 : .tx_rs_thresh = 32, /* Use PMD default values */
113 : };
114 :
115 : #if VR_DPDK_USE_HW_FILTERING
116 : /* Add hardware filter */
117 : int
118 : vr_dpdk_ethdev_filter_add(struct vr_interface *vif, uint16_t queue_id,
119 : unsigned dst_ip, unsigned mpls_label)
120 : {
121 : struct vr_dpdk_ethdev *ethdev = (struct vr_dpdk_ethdev *)vif->vif_os;
122 : uint8_t port_id = ethdev->ethdev_port_id;
123 : struct rte_fdir_filter filter;
124 : int ret;
125 :
126 : /* accept 2-byte labels only */
127 : if (mpls_label > 0xffff)
128 : return -EINVAL;
129 :
130 : if (queue_id > VR_DPDK_MAX_NB_RX_QUEUES)
131 : return -EINVAL;
132 :
133 : memset(&filter, 0, sizeof(filter));
134 : filter.iptype = RTE_FDIR_IPTYPE_IPV4;
135 : filter.l4type = RTE_FDIR_L4TYPE_UDP;
136 : filter.ip_dst.ipv4_addr = dst_ip;
137 : filter.port_dst = rte_cpu_to_be_16((uint16_t)VR_MPLS_OVER_UDP_DST_PORT);
138 : filter.flex_bytes = rte_cpu_to_be_16((uint16_t)mpls_label);
139 :
140 : RTE_LOG_DP(DEBUG, VROUTER, "%s: ip_dst=0x%x port_dst=%d flex_bytes=%d\n", __func__,
141 : (unsigned)dst_ip, (unsigned)VR_MPLS_OVER_UDP_DST_PORT, (unsigned)mpls_label);
142 :
143 : if (queue_id >= 0xFF) {
144 : RTE_LOG(ERR, VROUTER, " error adding perfect filter for eth device %"
145 : PRIu8 ": queue ID %" PRIu16 " is out of range\n",
146 : port_id, queue_id);
147 : return -EINVAL;
148 : }
149 : ret = rte_eth_dev_fdir_add_perfect_filter(port_id, &filter, (uint16_t)mpls_label,
150 : (uint8_t)queue_id, 0);
151 : if (ret == 0)
152 : ethdev->ethdev_queue_states[queue_id] = VR_DPDK_QUEUE_FILTERING_STATE;
153 :
154 : return ret;
155 : }
156 : #endif
157 :
158 : /* Get a ready queue ID */
159 : uint16_t
160 0 : vr_dpdk_ethdev_ready_queue_id_get(struct vr_interface *vif)
161 : {
162 : uint16_t i;
163 0 : struct vr_dpdk_ethdev *ethdev = (struct vr_dpdk_ethdev *)vif->vif_os;
164 :
165 0 : for (i = ethdev->ethdev_nb_rss_queues; i < ethdev->ethdev_nb_rx_queues; i++) {
166 0 : if (ethdev->ethdev_queue_states[i] == VR_DPDK_QUEUE_READY_STATE) {
167 0 : return i;
168 : }
169 : }
170 0 : return VR_DPDK_INVALID_QUEUE_ID;
171 : }
172 :
173 : /* Release ethdev RX queue */
174 : static void
175 0 : dpdk_ethdev_rx_queue_release(unsigned lcore_id,
176 : unsigned queue_index __attribute__((unused)),
177 : struct vr_interface *vif)
178 : {
179 0 : struct vr_dpdk_lcore *lcore = vr_dpdk.lcores[lcore_id];
180 0 : struct vr_dpdk_queue *rx_queue = &lcore->lcore_rx_queues[vif->vif_idx];
181 0 : struct vr_dpdk_queue_params *rx_queue_params
182 0 : = &lcore->lcore_rx_queue_params[vif->vif_idx];
183 :
184 : /* free the queue */
185 0 : if (rx_queue->rxq_ops.f_free(rx_queue->q_queue_h)) {
186 0 : RTE_LOG(ERR, VROUTER, " error freeing lcore %u eth device RX queue\n",
187 : lcore_id);
188 : }
189 :
190 : /* reset the queue */
191 0 : vrouter_put_interface(rx_queue->q_vif);
192 0 : memset(rx_queue, 0, sizeof(*rx_queue));
193 0 : memset(rx_queue_params, 0, sizeof(*rx_queue_params));
194 0 : }
195 :
196 : /* Init eth RX queue */
197 : struct vr_dpdk_queue *
198 0 : vr_dpdk_ethdev_rx_queue_init(unsigned lcore_id, struct vr_interface *vif,
199 : unsigned queue_or_lcore_id)
200 : {
201 0 : uint16_t rx_queue_id = queue_or_lcore_id;
202 : uint8_t port_id;
203 0 : unsigned int vif_idx = vif->vif_idx;
204 0 : const unsigned int socket_id = rte_lcore_to_socket_id(lcore_id);
205 :
206 : struct vr_dpdk_ethdev *ethdev;
207 0 : struct vr_dpdk_lcore *lcore = vr_dpdk.lcores[lcore_id];
208 0 : struct vr_dpdk_queue *rx_queue = &lcore->lcore_rx_queues[vif_idx];
209 0 : struct vr_dpdk_queue_params *rx_queue_params
210 : = &lcore->lcore_rx_queue_params[vif_idx];
211 :
212 0 : ethdev = (struct vr_dpdk_ethdev *)vif->vif_os;
213 0 : port_id = ethdev->ethdev_port_id;
214 :
215 : /* init queue */
216 0 : rx_queue->rxq_ops = rte_port_ethdev_reader_ops;
217 0 : rx_queue->q_queue_h = NULL;
218 0 : rx_queue->q_vif = vrouter_get_interface(vif->vif_rid, vif_idx);
219 :
220 : /* create the queue */
221 0 : struct rte_port_ethdev_reader_params reader_params = {
222 : .port_id = port_id,
223 : .queue_id = rx_queue_id,
224 : };
225 0 : rx_queue->q_queue_h = rx_queue->rxq_ops.f_create(&reader_params, socket_id);
226 0 : if (rx_queue->q_queue_h == NULL) {
227 0 : RTE_LOG(ERR, VROUTER, " error creating eth device %" PRIu8
228 : " RX queue %" PRIu16 "\n", port_id, rx_queue_id);
229 0 : return NULL;
230 : }
231 :
232 0 : rx_queue->vring_queue_id = rx_queue_id;
233 : /* store queue params */
234 0 : rx_queue_params->qp_release_op = &dpdk_ethdev_rx_queue_release;
235 0 : rx_queue_params->qp_ethdev.queue_id = rx_queue_id;
236 0 : rx_queue_params->qp_ethdev.port_id = port_id;
237 :
238 0 : return rx_queue;
239 : }
240 :
241 : /* Release ethdev TX queue */
242 : static void
243 0 : dpdk_ethdev_tx_queue_release(unsigned lcore_id, unsigned queue_index,
244 : struct vr_interface *vif)
245 : {
246 : int i;
247 :
248 0 : struct vr_dpdk_lcore *lcore = vr_dpdk.lcores[lcore_id];
249 : struct vr_dpdk_queue *tx_queue;
250 : struct vr_dpdk_queue_params *tx_queue_params;
251 :
252 0 : tx_queue = &lcore->lcore_tx_queues[vif->vif_idx][queue_index];
253 0 : tx_queue_params =
254 0 : &lcore->lcore_tx_queue_params[vif->vif_idx][queue_index];
255 :
256 : /* remove queue params from the list of bonds to TX */
257 0 : for (i = 0; i < lcore->lcore_nb_bonds_to_tx; i++) {
258 0 : if (likely(lcore->lcore_bonds_to_tx[i] == tx_queue_params)) {
259 0 : lcore->lcore_bonds_to_tx[i] = NULL;
260 0 : lcore->lcore_nb_bonds_to_tx--;
261 0 : RTE_VERIFY(lcore->lcore_nb_bonds_to_tx <= VR_DPDK_MAX_BONDS);
262 : /* copy the last element to the empty spot */
263 0 : lcore->lcore_bonds_to_tx[i] = lcore->lcore_bonds_to_tx[lcore->lcore_nb_bonds_to_tx];
264 0 : break;
265 : }
266 : }
267 :
268 0 : tx_queue->txq_ops.f_tx = NULL;
269 : rte_wmb();
270 :
271 : /* flush and free the queue */
272 0 : if (tx_queue->txq_ops.f_free(tx_queue->q_queue_h)) {
273 0 : RTE_LOG(ERR, VROUTER, " error freeing lcore %u eth device TX queue\n",
274 : lcore_id);
275 : }
276 :
277 : /* reset the queue */
278 0 : vrouter_put_interface(tx_queue->q_vif);
279 0 : memset(tx_queue, 0, sizeof(*tx_queue));
280 0 : memset(tx_queue_params, 0, sizeof(*tx_queue_params));
281 0 : }
282 :
283 : /* Init eth TX queue */
284 : struct vr_dpdk_queue *
285 0 : vr_dpdk_ethdev_tx_queue_init(unsigned lcore_id, struct vr_interface *vif,
286 : unsigned queue_or_lcore_id)
287 : {
288 : uint8_t port_id;
289 0 : uint16_t tx_queue_id = queue_or_lcore_id;
290 0 : unsigned int vif_idx = vif->vif_idx, dpdk_queue_index;
291 0 : const unsigned int socket_id = rte_lcore_to_socket_id(lcore_id);
292 :
293 : struct vr_dpdk_ethdev *ethdev;
294 0 : struct vr_dpdk_lcore *lcore = vr_dpdk.lcores[lcore_id];
295 : struct vr_dpdk_queue *tx_queue;
296 : struct vr_dpdk_queue_params *tx_queue_params;
297 :
298 0 : ethdev = (struct vr_dpdk_ethdev *)vif->vif_os;
299 0 : port_id = ethdev->ethdev_port_id;
300 :
301 0 : if (lcore->lcore_hw_queue_to_dpdk_index[vif->vif_idx]) {
302 0 : dpdk_queue_index =
303 0 : lcore->lcore_hw_queue_to_dpdk_index[vif->vif_idx][tx_queue_id];
304 : } else {
305 0 : dpdk_queue_index = 0;
306 : }
307 :
308 0 : tx_queue = &lcore->lcore_tx_queues[vif_idx][dpdk_queue_index];
309 0 : tx_queue_params = &lcore->lcore_tx_queue_params[vif_idx][dpdk_queue_index];
310 :
311 : /* init queue */
312 0 : tx_queue->txq_ops = rte_port_ethdev_writer_ops;
313 0 : tx_queue->q_queue_h = NULL;
314 0 : tx_queue->q_vif = vrouter_get_interface(vif->vif_rid, vif_idx);
315 :
316 : /* create the queue */
317 0 : struct rte_port_ethdev_writer_params writer_params = {
318 : .port_id = port_id,
319 : .queue_id = tx_queue_id,
320 : .tx_burst_sz = VR_DPDK_TX_BURST_SZ,
321 : };
322 0 : tx_queue->q_queue_h = tx_queue->txq_ops.f_create(&writer_params, socket_id);
323 0 : if (tx_queue->q_queue_h == NULL) {
324 0 : RTE_LOG(ERR, VROUTER, " error creating eth device %" PRIu8
325 : " TX queue %" PRIu16 "\n", port_id, tx_queue_id);
326 0 : return NULL;
327 : }
328 :
329 : /* store queue params */
330 0 : tx_queue_params->qp_release_op = &dpdk_ethdev_tx_queue_release;
331 0 : tx_queue_params->qp_ethdev.queue_id = tx_queue_id;
332 0 : tx_queue_params->qp_ethdev.port_id = port_id;
333 :
334 : /* for the queue 0 add queue params to the list of bonds to TX */
335 0 : if (ethdev->ethdev_nb_slaves > 0 && tx_queue_id == 0) {
336 : /* make sure queue params have been stored */
337 : rte_wmb();
338 0 : lcore->lcore_bonds_to_tx[lcore->lcore_nb_bonds_to_tx++] = tx_queue_params;
339 0 : RTE_VERIFY(lcore->lcore_nb_bonds_to_tx <= VR_DPDK_MAX_BONDS);
340 : }
341 :
342 0 : return tx_queue;
343 : }
344 :
345 : /*
346 : * vr_max_tx_queues_adjust - bond devices always return dev_info indicating
347 : * 512 TX queues are supported. vrouter clips down to a max of VR_DPDK_MAX_NB_TX_QUEUES.
348 : * However, if any of the bond slaves does not support this many TX queues, the
349 : * number needs to be reduced further to a value that the NIC can support. Also,
350 : * bnxt report max_tx_queues of 170, but fails to configure more than 8 TX queues.
351 : */
352 : static void
353 0 : vr_max_tx_queues_adjust(struct vr_dpdk_ethdev *ethdev, uint16_t *nb_tx_q)
354 : {
355 : struct rte_eth_dev_info dev_info;
356 : int i;
357 :
358 0 : VR_DPDK_RTE_ETH_FOREACH_DEV(i)
359 : {
360 0 : rte_eth_dev_info_get(i, &dev_info);
361 0 : if (dev_info.driver_name) {
362 0 : if (strncmp(dev_info.driver_name, "net_bnxt",
363 : strlen("net_bnxt") + 1) == 0) {
364 0 : if (*nb_tx_q > VR_DPDK_MAX_NB_TX_Q_BNXT) {
365 0 : RTE_LOG(INFO, VROUTER, "TX queues changed from %d to %d due to port %d\n",
366 : *nb_tx_q, VR_DPDK_MAX_NB_TX_Q_BNXT, i);
367 0 : *nb_tx_q = VR_DPDK_MAX_NB_TX_Q_BNXT;
368 : }
369 : }
370 : }
371 : }
372 :
373 0 : return;
374 : }
375 :
376 : /* Update device info */
377 : static void
378 0 : dpdk_ethdev_info_update(struct vr_dpdk_ethdev *ethdev)
379 : {
380 : struct rte_eth_dev_info dev_info;
381 :
382 0 : rte_eth_dev_info_get(ethdev->ethdev_port_id, &dev_info);
383 :
384 0 : ethdev->ethdev_nb_rx_queues = RTE_MIN(dev_info.max_rx_queues,
385 : VR_DPDK_MAX_NB_RX_QUEUES);
386 :
387 0 : if (dev_info.max_tx_queues > VR_DPDK_MAX_NB_TX_QUEUES)
388 0 : dev_info.max_tx_queues = VR_DPDK_MAX_NB_TX_QUEUES;
389 :
390 : /*
391 : * If a device advertises max_tx_queues higher than it
392 : * can actually support, reduce the value to a number
393 : * that it can support.
394 : */
395 0 : vr_max_tx_queues_adjust(ethdev, &dev_info.max_tx_queues);
396 :
397 0 : ethdev->ethdev_nb_tx_queues = dev_info.max_tx_queues;
398 :
399 : /* Check if we have dedicated an lcore for SR-IOV VF IO. */
400 0 : if (vr_dpdk.vf_lcore_id) {
401 0 : ethdev->ethdev_nb_rx_queues = ethdev->ethdev_nb_tx_queues = 1;
402 : }
403 :
404 0 : ethdev->ethdev_nb_rss_queues = RTE_MIN(RTE_MIN(ethdev->ethdev_nb_rx_queues,
405 : vr_dpdk.nb_fwd_lcores), VR_DPDK_MAX_NB_RSS_QUEUES);
406 0 : ethdev->ethdev_reta_size = RTE_MIN(dev_info.reta_size,
407 : VR_DPDK_MAX_RETA_SIZE);
408 :
409 : /*
410 : * If the NIC driver sets reta_size to a value that is not a power of
411 : * 2, align it as DPDK expects it to be a power of 2.
412 : */
413 0 : ethdev->ethdev_reta_size = RTE_ALIGN(ethdev->ethdev_reta_size, RTE_RETA_GROUP_SIZE);
414 :
415 : RTE_LOG_DP(DEBUG, VROUTER, "dev_info: driver_name=%s if_index=%u"
416 : " max_rx_queues=%" PRIu16 " max_tx_queues=%" PRIu16
417 : " max_vfs=%" PRIu16 " max_vmdq_pools=%" PRIu16
418 : " rx_offload_capa=%" PRIx64 " tx_offload_capa=%" PRIx64 "\n",
419 : dev_info.driver_name, dev_info.if_index,
420 : dev_info.max_rx_queues, dev_info.max_tx_queues,
421 : dev_info.max_vfs, dev_info.max_vmdq_pools,
422 : dev_info.rx_offload_capa, dev_info.tx_offload_capa);
423 :
424 : #if !VR_DPDK_USE_HW_FILTERING
425 : /* use RSS queues only */
426 0 : ethdev->ethdev_nb_rx_queues = ethdev->ethdev_nb_rss_queues;
427 : #else
428 : /* we use just RSS queues if the device does not support RETA */
429 : if (ethdev->ethdev_reta_size == 0)
430 : ethdev->ethdev_nb_rx_queues = ethdev->ethdev_nb_rss_queues;
431 : #endif
432 :
433 0 : RTE_LOG(INFO, VROUTER, "Using %d TX queues, %d RX queues\n",
434 : ethdev->ethdev_nb_tx_queues,
435 : ethdev->ethdev_nb_rx_queues);
436 :
437 0 : return;
438 : }
439 :
440 : /* Setup ethdev hardware queues */
441 : /* If (tx|rx)_conf is null, default_(tx|rx)_queue_conf will be used instead */
442 : static int
443 0 : dpdk_ethdev_queues_setup(struct vr_dpdk_ethdev *ethdev,
444 : struct rte_eth_txconf *tx_conf, struct rte_eth_rxconf *rx_conf)
445 : {
446 : int ret, i;
447 0 : uint8_t port_id = ethdev->ethdev_port_id;
448 : struct rte_mempool *mempool;
449 :
450 : /* configure RX queues */
451 : RTE_LOG_DP(DEBUG, VROUTER, "%s: nb_rx_queues=%u nb_tx_queues=%u\n",
452 : __func__, (unsigned)ethdev->ethdev_nb_rx_queues,
453 : (unsigned)ethdev->ethdev_nb_tx_queues);
454 :
455 0 : for (i = 0; i < VR_DPDK_MAX_NB_RX_QUEUES; i++) {
456 0 : if (i < ethdev->ethdev_nb_rss_queues) {
457 0 : mempool = vr_dpdk.rss_mempool;
458 0 : ethdev->ethdev_queue_states[i] = VR_DPDK_QUEUE_RSS_STATE;
459 0 : } else if (i < ethdev->ethdev_nb_rx_queues) {
460 0 : if (vr_dpdk.nb_free_mempools == 0) {
461 0 : RTE_LOG(ERR, VROUTER, " error assigning mempool to eth device %"
462 : PRIu8 " RX queue %d\n", port_id, i);
463 0 : return -ENOMEM;
464 : }
465 0 : vr_dpdk.nb_free_mempools--;
466 0 : mempool = vr_dpdk.free_mempools[vr_dpdk.nb_free_mempools];
467 0 : ethdev->ethdev_queue_states[i] = VR_DPDK_QUEUE_READY_STATE;
468 : } else {
469 0 : ethdev->ethdev_queue_states[i] = VR_DPDK_QUEUE_NONE;
470 0 : continue;
471 : }
472 :
473 0 : ret = rte_eth_rx_queue_setup(port_id, i, vr_rxd_sz,
474 : SOCKET_ID_ANY, rx_conf ? rx_conf : &default_rx_queue_conf, mempool);
475 0 : if (ret < 0) {
476 : /* return mempool to the list */
477 0 : if (mempool != vr_dpdk.rss_mempool)
478 0 : vr_dpdk.nb_free_mempools++;
479 0 : RTE_LOG(ERR, VROUTER, " error setting up eth device %" PRIu8 " RX queue %d"
480 : ": %s (%d)\n", port_id, i, rte_strerror(-ret), -ret);
481 0 : return ret;
482 : }
483 : /* map RX queue to stats counter ignoring any errors */
484 0 : rte_eth_dev_set_rx_queue_stats_mapping(port_id, i, i);
485 :
486 : /* save queue mempool pointer */
487 0 : ethdev->ethdev_mempools[i] = mempool;
488 : }
489 0 : i = ethdev->ethdev_nb_rx_queues - ethdev->ethdev_nb_rss_queues;
490 0 : RTE_LOG(INFO, VROUTER, " setup %d RSS queue(s) and %d filtering queue(s)\n",
491 : (int)ethdev->ethdev_nb_rss_queues, i);
492 :
493 : /* configure TX queues */
494 0 : for (i = 0; i < ethdev->ethdev_nb_tx_queues; i++) {
495 0 : ret = rte_eth_tx_queue_setup(port_id, i, vr_txd_sz,
496 : SOCKET_ID_ANY, tx_conf ? tx_conf : &default_tx_queue_conf);
497 0 : if (ret < 0) {
498 0 : RTE_LOG(ERR, VROUTER, " error setting up eth device %" PRIu8 " TX queue %d"
499 : ": %s (%d)\n", port_id, i, rte_strerror(-ret), -ret);
500 0 : return ret;
501 : }
502 : /* map TX queue to stats counter ignoring any errors */
503 0 : rte_eth_dev_set_tx_queue_stats_mapping(port_id, i, i);
504 : }
505 0 : return 0;
506 : }
507 :
508 : static void
509 0 : dpdk_ethdev_reta_show(uint8_t port_id, uint16_t reta_size)
510 0 : {
511 0 : int nb_entries = reta_size/RTE_RETA_GROUP_SIZE;
512 0 : struct rte_eth_rss_reta_entry64 reta_entries[nb_entries];
513 : struct rte_eth_rss_reta_entry64 *reta;
514 : uint16_t i, idx, shift;
515 : int ret, entry;
516 :
517 0 : for (entry = 0; entry < nb_entries; entry++) {
518 0 : reta = &reta_entries[entry];
519 :
520 : /* reset RSS redirection table */
521 0 : memset(reta, 0, sizeof(*reta));
522 0 : reta->mask = 0xffffffffffffffffULL;
523 : }
524 :
525 0 : ret = rte_eth_dev_rss_reta_query(port_id, reta_entries, reta_size);
526 0 : if (ret != 0) {
527 0 : RTE_LOG(ERR, VROUTER, "Error getting RSS RETA info: %s (%d)\n",
528 : rte_strerror(ret), ret);
529 0 : return;
530 : }
531 :
532 0 : for (i = 0; i < reta_size; i++) {
533 0 : idx = i / RTE_RETA_GROUP_SIZE;
534 0 : shift = i % RTE_RETA_GROUP_SIZE;
535 0 : if (!(reta_entries[idx].mask & (1ULL << shift)))
536 0 : continue;
537 : RTE_LOG_DP(DEBUG, VROUTER, " hash index=%u, queue=%u\n",
538 : i, reta_entries[idx].reta[shift]);
539 : }
540 : }
541 :
542 : /* Init RSS */
543 : int
544 0 : vr_dpdk_ethdev_rss_init(struct vr_dpdk_ethdev *ethdev)
545 : {
546 : int ret, i, j, entry;
547 0 : uint8_t port_id = ethdev->ethdev_port_id;
548 0 : int nb_entries = ethdev->ethdev_reta_size/RTE_RETA_GROUP_SIZE;
549 : struct rte_eth_rss_reta_entry64 reta_entries[VR_DPDK_MAX_RETA_ENTRIES];
550 : struct rte_eth_rss_reta_entry64 *reta;
551 :
552 : /* There is nothing to configure if the device does not support RETA.
553 : * If the device reported few RX queues earlier, we assume those
554 : * queues are preconfigured for RSS by default.
555 : */
556 0 : if (ethdev->ethdev_reta_size == 0)
557 0 : return 0;
558 :
559 : RTE_LOG_DP(DEBUG, VROUTER, "%s: RSS RETA BEFORE:\n", __func__);
560 0 : dpdk_ethdev_reta_show(port_id, ethdev->ethdev_reta_size);
561 :
562 0 : for (entry = 0; entry < nb_entries; entry++) {
563 0 : reta = &reta_entries[entry];
564 :
565 : /* create new RSS redirection table */
566 0 : memset(reta, 0, sizeof(*reta));
567 0 : reta->mask = 0xffffffffffffffffULL;
568 0 : for (i = j = 0; i < RTE_RETA_GROUP_SIZE; i++) {
569 0 : reta->reta[i] = j++;
570 0 : if (ethdev->ethdev_queue_states[j] != VR_DPDK_QUEUE_RSS_STATE)
571 0 : j = 0;
572 : }
573 : }
574 :
575 : /* update RSS redirection table */
576 0 : ret = rte_eth_dev_rss_reta_update(port_id, reta_entries,
577 0 : ethdev->ethdev_reta_size);
578 :
579 : /* no error if the device does not support RETA configuration */
580 0 : if (ret == -ENOTSUP)
581 0 : return 0;
582 :
583 0 : if (ret < 0) {
584 0 : RTE_LOG(ERR, VROUTER, " error initializing ethdev %" PRIu8 " RSS: %s (%d)\n",
585 : port_id, rte_strerror(-ret), -ret);
586 : }
587 :
588 : RTE_LOG_DP(DEBUG, VROUTER, "%s: RSS RETA AFTER:\n", __func__);
589 0 : dpdk_ethdev_reta_show(port_id, ethdev->ethdev_reta_size);
590 :
591 0 : return ret;
592 : }
593 :
594 : /* Init hardware filtering */
595 : static void
596 0 : dpdk_ethdev_mempools_free(struct vr_dpdk_ethdev *ethdev)
597 : {
598 : int i;
599 :
600 0 : for (i = ethdev->ethdev_nb_rss_queues; i < ethdev->ethdev_nb_rx_queues; i++) {
601 0 : if (ethdev->ethdev_mempools[i] != NULL
602 0 : && ethdev->ethdev_mempools[i] != vr_dpdk.rss_mempool) {
603 0 : vr_dpdk.free_mempools[vr_dpdk.nb_free_mempools++] =
604 0 : ethdev->ethdev_mempools[i];
605 0 : ethdev->ethdev_mempools[i] = NULL;
606 0 : ethdev->ethdev_queue_states[i] = VR_DPDK_QUEUE_READY_STATE;
607 : }
608 : }
609 0 : }
610 :
611 : #if VR_DPDK_USE_HW_FILTERING
612 : /* Init hardware filtering */
613 : int
614 : vr_dpdk_ethdev_filtering_init(struct vr_interface *vif,
615 : struct vr_dpdk_ethdev *ethdev)
616 : {
617 : int ret;
618 : uint8_t port_id = ethdev->ethdev_port_id;
619 : struct rte_fdir_masks masks;
620 : struct rte_eth_fdir fdir_info;
621 :
622 : /* probe Flow Director */
623 : memset(&fdir_info, 0, sizeof(fdir_info));
624 : ret = rte_eth_dev_fdir_get_infos(port_id, &fdir_info);
625 : if (ret == 0) {
626 : /* enable hardware filtering */
627 : RTE_LOG(INFO, VROUTER, " enable hardware filtering for ethdev %"
628 : PRIu8 "\n", port_id);
629 : vif->vif_flags |= VIF_FLAG_FILTERING_OFFLOAD;
630 : } else {
631 : vif->vif_flags &= ~VIF_FLAG_FILTERING_OFFLOAD;
632 : /* free filtering mempools */
633 : dpdk_ethdev_mempools_free(ethdev);
634 : /* the ethdev does not support hardware filtering - it's not an error */
635 : return 0;
636 : }
637 :
638 : memset(&masks, 0, sizeof(masks));
639 : masks.dst_ipv4_mask = 0xffffffff;
640 : masks.dst_port_mask = 0xffff;
641 : masks.flexbytes = 1;
642 :
643 : ret = rte_eth_dev_fdir_set_masks(port_id, &masks);
644 : if (ret < 0) {
645 : RTE_LOG(ERR, VROUTER, " error setting ethdev %" PRIu8
646 : " Flow Director masks: %s (%d)\n", port_id, rte_strerror(-ret), -ret);
647 : }
648 :
649 : return ret;
650 : }
651 : #endif
652 :
653 : /* Update device bond info */
654 : static void
655 0 : dpdk_ethdev_bond_info_update(struct vr_dpdk_ethdev *ethdev)
656 : {
657 : int i, slave_port_id, ret;
658 0 : int port_id = ethdev->ethdev_port_id;
659 0 : uint16_t mtu = 0;
660 : struct rte_pci_addr *pci_addr;
661 : struct rte_ether_addr bond_mac, mac_addr;
662 0 : struct rte_ether_addr lacp_mac = { .addr_bytes = {0x01, 0x80, 0xc2, 0, 0, 0x02} };
663 :
664 0 : if (rte_eth_bond_mode_get(port_id) == -1) {
665 0 : ethdev->ethdev_nb_slaves = -1;
666 : } else {
667 0 : ethdev->ethdev_nb_slaves = rte_eth_bond_slaves_get(port_id,
668 0 : ethdev->ethdev_slaves, VR_DPDK_BOND_MAX_SLAVES);
669 :
670 0 : memset(&mac_addr, 0, sizeof(bond_mac));
671 0 : rte_eth_macaddr_get(port_id, &bond_mac);
672 0 : RTE_LOG(INFO, VROUTER, " bond eth device %" PRIu8
673 : " configured MAC " MAC_FORMAT "\n",
674 : port_id, MAC_VALUE(bond_mac.addr_bytes));
675 : /* log out and configure bond members */
676 0 : for (i = 0; i < ethdev->ethdev_nb_slaves; i++) {
677 0 : slave_port_id = ethdev->ethdev_slaves[i];
678 0 : if (!rte_eth_devices[port_id].data->mtu) {
679 0 : rte_eth_dev_get_mtu(slave_port_id, &mtu);
680 0 : rte_eth_devices[port_id].data->mtu = mtu;
681 : }
682 0 : memset(&mac_addr, 0, sizeof(mac_addr));
683 0 : rte_eth_macaddr_get(slave_port_id, &mac_addr);
684 0 : pci_addr = &(RTE_DEV_TO_PCI(rte_eth_devices[slave_port_id].device)->addr);
685 0 : RTE_LOG(INFO, VROUTER, " bond member eth device %" PRIu8
686 : " PCI " PCI_PRI_FMT " MAC " MAC_FORMAT "\n",
687 : slave_port_id, pci_addr->domain, pci_addr->bus,
688 : pci_addr->devid, pci_addr->function,
689 : MAC_VALUE(mac_addr.addr_bytes));
690 :
691 : /* We just try to add the mc address. In any case, the bond driver would
692 : * enable the 'all multicast' mode
693 : */
694 0 : if ((ret=rte_eth_dev_set_mc_addr_list(slave_port_id, &lacp_mac, 1)) != 0)
695 0 : RTE_LOG(INFO, VROUTER, " bond member eth device %" PRIu8
696 : ": unable to add multicast addresses %d\n", slave_port_id,ret);
697 :
698 : /*
699 : * Need to set the dev mac to disable the promiscuous mode
700 : */
701 0 : if ((ret=rte_eth_dev_mac_addr_add(slave_port_id, &bond_mac, 0)) != 0)
702 0 : RTE_LOG(INFO, VROUTER, " bond member eth device %" PRIu8
703 : ": unable to add MAC addresses %d\n", slave_port_id,ret);
704 : else {
705 0 : RTE_LOG(INFO, VROUTER, " disabling promiscuous mode for device %d\n", slave_port_id);
706 0 : rte_eth_promiscuous_disable(ethdev->ethdev_port_id);
707 : }
708 :
709 : }
710 0 : rte_eth_allmulticast_enable(ethdev->ethdev_port_id);
711 : }
712 0 : }
713 :
714 : /* Check if port_id is a bond slave. */
715 : bool
716 0 : vr_dpdk_ethdev_bond_port_match(uint8_t port_id, struct vr_dpdk_ethdev *ethdev)
717 : {
718 : int i;
719 :
720 0 : if (ethdev->ethdev_nb_slaves > 0) {
721 0 : for (i = 0; i < ethdev->ethdev_nb_slaves; i++) {
722 0 : if (port_id == ethdev->ethdev_slaves[i])
723 0 : break;
724 : }
725 :
726 0 : if (i < ethdev->ethdev_nb_slaves)
727 0 : return true;
728 : }
729 :
730 0 : return false;
731 : }
732 :
733 :
734 : /*
735 : * Function to send bond interface state to Agent
736 : * info->status 0 - down
737 : * info->status 1 - up
738 : * Called when bond master/slave interface goes down or comes up
739 : */
740 : void
741 0 : vr_dpdk_nl_send_bond_intf_state(struct vr_dpdk_bond_member_info *info,
742 : unsigned type, uint8_t vif_idx)
743 : {
744 : int nl_fd;
745 : char buf[1024];
746 : struct nlmsghdr *nlh;
747 : struct ifinfomsg *ifinfo;
748 : struct nlattr *nla;
749 : int len, n;
750 : char *if_name_buf;
751 : struct sockaddr_nl sa;
752 :
753 0 : if (info == NULL) {
754 0 : RTE_LOG(ERR, VROUTER, "Bond member info is NULL\n");
755 0 : return;
756 : }
757 0 : memset(buf, 0, sizeof(buf));
758 0 : memset(&sa, 0, sizeof(sa));
759 0 : sa.nl_family = AF_NETLINK;
760 0 : sa.nl_pid = 0;
761 0 : sa.nl_groups = RTMGRP_LINK | RTMGRP_IPV4_IFADDR;
762 :
763 0 : nl_fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE);
764 0 : if (nl_fd < 0) {
765 0 : RTE_LOG(ERR, VROUTER, "Error creating netlink socket\n");
766 0 : goto error;
767 : }
768 0 : bind(nl_fd, (struct sockaddr *)&sa, sizeof(sa));
769 0 : nlh = (struct nlmsghdr *)buf;
770 :
771 0 : nlh->nlmsg_len = NLMSG_HDRLEN + sizeof(struct ifinfomsg);
772 0 : nlh->nlmsg_type = RTM_NEWLINK;
773 0 : nlh->nlmsg_flags = NLM_F_REQUEST;
774 0 : nlh->nlmsg_seq = 1;
775 0 : nlh->nlmsg_pid = 0;
776 0 : ifinfo = (struct ifinfomsg *)(buf + NLMSG_HDRLEN);
777 :
778 0 : ifinfo->ifi_family = 0;
779 0 : ifinfo->ifi_type = type;
780 :
781 : /* Set index to -1 as agent needs only the interface name */
782 0 : ifinfo->ifi_index = -1;
783 :
784 0 : if (info->status)
785 0 : ifinfo->ifi_flags = IFF_MULTICAST|IFF_RUNNING|IFF_BROADCAST|IFF_UP;
786 : else
787 0 : ifinfo->ifi_flags = IFF_BROADCAST|IFF_UP;
788 : /* ifi_change needs to be set to 0xFFFFFFFF by default
789 : * as its a reserved field
790 : */
791 0 : ifinfo->ifi_change = 0xFFFFFFFF;
792 :
793 0 : nla = (struct nlattr *)(buf + NLMSG_HDRLEN + sizeof(struct ifinfomsg));
794 :
795 0 : if_name_buf = (char *)nla + NLA_HDRLEN;
796 :
797 0 : sprintf(if_name_buf,"%s %s %x", info->intf_name, info->intf_drv_name,
798 : vif_idx);
799 :
800 0 : len = NLA_HDRLEN + NLA_ALIGN(strlen(if_name_buf) + 1);
801 :
802 0 : nla->nla_len = len;
803 0 : nla->nla_type = IFLA_IFNAME;
804 :
805 :
806 0 : len += (NLMSG_HDRLEN + sizeof(struct ifinfomsg));
807 :
808 0 : nlh->nlmsg_len = len;
809 :
810 0 : n = sendto(nl_fd, buf, len, 0, (struct sockaddr *)&sa, sizeof(sa));
811 0 : if (n != len) {
812 0 : RTE_LOG(ERR, VROUTER, "Error sending netlink bond interface message\n");
813 : }
814 : else
815 0 : RTE_LOG(INFO, VROUTER, "Notifed Link status update to agent for \
816 : interface name %s\n",
817 : info->intf_name);
818 :
819 0 : close(nl_fd);
820 0 : error:
821 0 : return;
822 : }
823 :
824 : static int
825 0 : vr_dpdk_bond_send_port_info(uint16_t port_id, uint8_t vif_idx)
826 : {
827 : int ret;
828 0 : char *str[] = {"UP", "DOWN"};
829 : struct rte_eth_link link;
830 : unsigned bond_type;
831 : uint32_t dev_flags;
832 : struct vr_dpdk_bond_member_info member_info;
833 :
834 0 : memset(&member_info, 0, sizeof(member_info));
835 :
836 0 : rte_eth_link_get_nowait(port_id, &link);
837 0 : member_info.status = link.link_status;
838 :
839 0 : ret = rte_eth_dev_get_name_by_port(port_id, member_info.intf_name);
840 0 : if(ret != 0)
841 0 : RTE_LOG(ERR, VROUTER, "%s: Error getting bond interface name "
842 : "port_id:%d\n", __func__, port_id);
843 :
844 0 : if(rte_eth_devices[port_id].device->driver->name != NULL)
845 0 : snprintf(member_info.intf_drv_name, (VR_INTERFACE_NAME_LEN - 1),
846 0 : "%s", rte_eth_devices[port_id].device->driver->name);
847 :
848 0 : RTE_LOG(INFO, VROUTER, "Port ID: %d Link Status: %s intf_name:%s \
849 : drv_name:%s \n\n", port_id, (link.link_status?str[0]:str[1]),
850 : member_info.intf_name, member_info.intf_drv_name);
851 :
852 0 : dev_flags = rte_eth_devices[port_id].data->dev_flags;
853 0 : bond_type = (dev_flags & RTE_ETH_DEV_BONDED_SLAVE) ? VR_DPDK_BOND_SLAVE :
854 : VR_DPDK_BOND_MASTER;
855 :
856 0 : vr_dpdk_nl_send_bond_intf_state(&member_info, bond_type, vif_idx);
857 :
858 0 : return 0;
859 :
860 : }
861 :
862 : static int
863 0 : vr_dpdk_bond_intf_callback(uint16_t port_id, enum rte_eth_event_type type,
864 : void *param, void *ret_param __rte_unused)
865 : {
866 :
867 0 : struct vr_dpdk_ethdev *ethdev = (struct vr_dpdk_ethdev *)param;
868 :
869 0 : if(NULL == ethdev) {
870 0 : RTE_LOG(ERR, VROUTER, "%s: ethdev is null\n", __func__);
871 0 : return -1;
872 : }
873 :
874 : /* On few platforms, Master lsc(callback) is not called.
875 : * Forcing here to send Master notification before
876 : * sending child interface */
877 0 : if(ethdev->ethdev_port_id != port_id)
878 0 : vr_dpdk_bond_send_port_info(ethdev->ethdev_port_id, ethdev->ethdev_vif_idx);
879 :
880 0 : return vr_dpdk_bond_send_port_info(port_id, ethdev->ethdev_vif_idx);
881 :
882 : }
883 :
884 : static void
885 0 : vr_dpdk_bond_intf_cb_register(struct vr_dpdk_ethdev *ethdev)
886 : {
887 :
888 0 : int i = 0, ret = 0;
889 :
890 : /* Fetching port-id for master bond interface */
891 0 : uint8_t port_id = ethdev->ethdev_port_id;
892 0 : vr_dpdk_master_port_id = port_id;
893 :
894 : /* Registering callback notification for Master bond interface */
895 0 : ret = rte_eth_dev_callback_register(port_id, RTE_ETH_EVENT_INTR_LSC,
896 : vr_dpdk_bond_intf_callback, ethdev);
897 :
898 0 : if(ret)
899 0 : RTE_LOG(ERR, VROUTER, "Failed to setup callback for event \
900 : RTE_ETH_EVENT_INTR_LSC for portid: %d\n", port_id);
901 : else {
902 : /* During system initialization, notify agent about the master
903 : * and number of slaves */
904 0 : vr_dpdk_bond_send_port_info(port_id, ethdev->ethdev_vif_idx);
905 : }
906 :
907 : /* For slave interface*/
908 0 : for (i = 0; i < ethdev->ethdev_nb_slaves; i++) {
909 :
910 0 : port_id = ethdev->ethdev_slaves[i];
911 :
912 : /* Registering callback notification for slave bond interfaces */
913 0 : ret = rte_eth_dev_callback_register(port_id, RTE_ETH_EVENT_INTR_LSC,
914 : vr_dpdk_bond_intf_callback, ethdev);
915 0 : if(ret)
916 0 : RTE_LOG(ERR, VROUTER, "Failed to setup callback for event \
917 : RTE_ETH_EVENT_INTR_LSC for portid: %d\n", port_id);
918 : else
919 0 : vr_dpdk_bond_send_port_info(port_id, ethdev->ethdev_vif_idx);
920 : }
921 0 : }
922 :
923 : /* Init ethernet device */
924 : int
925 0 : vr_dpdk_ethdev_init(struct vr_dpdk_ethdev *ethdev, struct rte_eth_conf *dev_conf,
926 : struct rte_eth_txconf *tx_conf, struct rte_eth_rxconf *rx_conf)
927 : {
928 : uint8_t port_id;
929 : int ret;
930 : struct vr_interface *vif;
931 :
932 0 : port_id = ethdev->ethdev_port_id;
933 0 : ethdev->ethdev_ptr = &rte_eth_devices[port_id];
934 0 : vif = __vrouter_get_interface(vrouter_get(0), ethdev->ethdev_vif_idx);
935 :
936 0 : dpdk_ethdev_info_update(ethdev);
937 :
938 0 : ret = rte_eth_dev_configure(port_id, ethdev->ethdev_nb_rx_queues,
939 0 : ethdev->ethdev_nb_tx_queues, dev_conf);
940 0 : if (ret < 0) {
941 0 : RTE_LOG(ERR, VROUTER, " error configuring eth dev %" PRIu8
942 : ": %s (%d)\n",
943 : port_id, rte_strerror(-ret), -ret);
944 0 : return ret;
945 : }
946 :
947 0 : if (rte_eth_devices[port_id].device->driver != NULL &&
948 0 : (strcmp(rte_eth_devices[port_id].device->driver->name, "net_bonding") == 0)) {
949 0 : dpdk_ethdev_bond_info_update(ethdev);
950 :
951 : }
952 :
953 : /* Register notifications only for fabric device */
954 0 : if (vif_is_fabric(vif))
955 0 : vr_dpdk_bond_intf_cb_register(ethdev);
956 :
957 0 : if(vr_dpdk_get_ddp()) {
958 0 : ret = vr_dpdk_process_ddp_package(VR_DPDK_DDP_ADD);
959 0 : if(ret != 0) {
960 : /* By Default DDP is set, If firmware programming not success on
961 : * any ports, then dont disable software load balancing */
962 0 : vr_dpdk_reset_ddp();
963 : }
964 : }
965 :
966 0 : ret = dpdk_ethdev_queues_setup(ethdev, tx_conf, rx_conf);
967 0 : if (ret < 0)
968 0 : return ret;
969 :
970 : /* Promisc mode
971 : * KNI generates random MACs for e1000e NICs, so we need this
972 : * option enabled for the development on servers with those NICs
973 : */
974 : #if VR_DPDK_ENABLE_PROMISC
975 : rte_eth_promiscuous_enable(port_id);
976 : #endif
977 :
978 0 : rte_spinlock_init(ðdev->ethdev_lock);
979 :
980 0 : return 0;
981 : }
982 :
983 : /* Release ethernet device */
984 : int
985 0 : vr_dpdk_ethdev_release(struct vr_dpdk_ethdev *ethdev)
986 : {
987 0 : ethdev->ethdev_ptr = NULL;
988 :
989 0 : dpdk_ethdev_mempools_free(ethdev);
990 :
991 0 : return 0;
992 : }
993 :
994 : /*
995 : * dpdk_mbuf_rss_hash - emulate RSS hash for the mbuf.
996 : *
997 : * Returns:
998 : * 0 if hash was not calculated
999 : * 1 if hash was calculated.
1000 : */
1001 : static inline int
1002 148 : dpdk_mbuf_rss_hash(struct rte_mbuf *mbuf, struct vr_ip *ipv4_hdr,
1003 : struct vr_ip6 *ipv6_hdr)
1004 : {
1005 : uint64_t *ip_addr_ptr;
1006 148 : uint32_t *l4_ptr = NULL;
1007 148 : uint32_t hash = 0;
1008 : unsigned char ip_proto, i;
1009 :
1010 148 : if (likely(ipv4_hdr != NULL)) {
1011 : /**
1012 : * Both source and destination IPv4 addresses are 4-bytes long,
1013 : * so they can be hashed at once with single 8-byte hashing funcion.
1014 : *
1015 : * We use SSE4.2 CRC hash. No need to match NIC's Toeplitz hash ATM.
1016 : */
1017 125 : ip_addr_ptr = (uint64_t *)((uintptr_t)ipv4_hdr +
1018 : offsetof(struct vr_ip, ip_saddr));
1019 125 : hash = rte_hash_crc_8byte(*ip_addr_ptr, hash);
1020 :
1021 125 : if (likely(!vr_ip_fragment(ipv4_hdr))) {
1022 82 : ip_proto = ipv4_hdr->ip_proto;
1023 82 : l4_ptr = (uint32_t *)((uintptr_t)ipv4_hdr +
1024 82 : (ipv4_hdr->ip_hl) * RTE_IPV4_IHL_MULTIPLIER);
1025 : } else {
1026 43 : ip_proto = 0;
1027 : }
1028 23 : } else if (ipv6_hdr != NULL) {
1029 : /**
1030 : * Both source and destination IPv6 addresses are 16-bytes long,
1031 : * and DPDK hashing functions can calculate 8-bytes at once, so hashing
1032 : * has to be done in four steps, each address hashed from the beginning
1033 : * to the middle, and from the middle to the end. In the header, first
1034 : * comes source address, which is followed by destination address. This
1035 : * lets us to set the pointer to the beginning of source address and
1036 : * move it by 64 bits after hash is calculated.
1037 : */
1038 115 : for (i = 0; i < 4; i++) {
1039 92 : ip_addr_ptr = (uint64_t *)((uintptr_t)ipv6_hdr +
1040 92 : offsetof(struct rte_ipv6_hdr, src_addr) + 8*i);
1041 92 : hash = rte_hash_crc_8byte(*ip_addr_ptr, hash);
1042 : }
1043 :
1044 23 : ip_proto = ipv6_hdr->ip6_nxt;
1045 : /* In case of extended header L4 is not hashed. */
1046 23 : l4_ptr = (uint32_t *)((uintptr_t)ipv6_hdr + sizeof(struct rte_ipv6_hdr));
1047 : } else {
1048 0 : return 0;
1049 : }
1050 :
1051 148 : switch (ip_proto) {
1052 22 : case VR_IP_PROTO_TCP:
1053 22 : hash = rte_hash_crc_4byte(*l4_ptr, hash);
1054 22 : break;
1055 27 : case VR_IP_PROTO_UDP:
1056 27 : hash = rte_hash_crc_4byte(*l4_ptr, hash);
1057 27 : break;
1058 0 : case VR_IP_PROTO_GRE:
1059 0 : if (likely(l4_ptr != NULL)) {
1060 0 : struct vr_gre_key* gre_hdr = (struct vr_gre_key *)l4_ptr;
1061 :
1062 0 : if (likely(gre_hdr->gre_comm_hdr.gre_flags & VR_GRE_FLAG_KEY)) {
1063 0 : hash = rte_hash_crc_4byte(gre_hdr->gre_key , hash);
1064 : }
1065 : }
1066 0 : break;
1067 : }
1068 :
1069 148 : mbuf->ol_flags |= PKT_RX_RSS_HASH;
1070 148 : mbuf->hash.rss = hash;
1071 : RTE_LOG_DP(DEBUG, VROUTER, "%s: RSS hash: 0x%x (emulated)\n",
1072 : __func__, mbuf->hash.rss);
1073 :
1074 148 : return 1;
1075 : }
1076 :
1077 : /* dpdk_mbuf_parse_and_hash_packets
1078 : *
1079 : * Parse incoming packet. Check L2, L3 headers, encapsulation type, perform
1080 : * TCP MSS adjust if needed, then call hashing function to.
1081 : *
1082 : * Return:
1083 : * -1 if packet length is too short to contain valid header
1084 : * 0 if there is no need to perform hashing (ie. unsupported encap type,
1085 : * packet already hashed)
1086 : * dpdk_mbuf_rss_hash() if hashing is needed. dpdk_mbuf_rss_hash() returns 1
1087 : * if hash was calculated, 0 if not.
1088 : *
1089 : * TODO: if we ever need to set L4 lengths or packet type flags, or other info
1090 : * about received packets, this is a good place to do it.
1091 : */
1092 : static int
1093 160 : dpdk_mbuf_parse_and_hash_packets(struct rte_mbuf *mbuf)
1094 : {
1095 160 : struct vr_eth *eth_hdr = rte_pktmbuf_mtod(mbuf, struct vr_eth *);
1096 160 : struct vr_ip *ipv4_hdr = NULL;
1097 160 : struct vr_ip *ipv4_inner_hdr = NULL;
1098 160 : struct vr_ip6 *ipv6_hdr = NULL;
1099 160 : struct vr_ip6 *ipv6_inner_hdr = NULL;
1100 160 : struct vr_udp *udp_hdr = NULL;
1101 160 : struct vr_gre *gre_hdr = NULL;
1102 : struct rte_vlan_hdr *vlan_hdr;
1103 160 : unsigned int pull_len = VR_ETHER_HLEN, ipv4_len;
1104 : int encap_type, helper_ret;
1105 160 : unsigned short gre_udp_encap = 0, gre_hdr_len = VR_GRE_BASIC_HDR_LEN,
1106 : eth_proto, udp_port;
1107 160 : uint16_t mbuf_data_len = rte_pktmbuf_data_len(mbuf);
1108 :
1109 160 : if (unlikely(mbuf_data_len < pull_len))
1110 0 : return -1;
1111 :
1112 160 : eth_proto = eth_hdr->eth_proto;
1113 :
1114 : /* Skip VLAN tag. It may be present if we handle tagged packet from VM. */
1115 160 : while (eth_proto == rte_cpu_to_be_16(VR_ETH_PROTO_VLAN)) {
1116 0 : if (unlikely(mbuf_data_len < pull_len + VR_VLAN_HLEN))
1117 0 : return -1;
1118 :
1119 : /* Store the first VLAN TCI for further use. */
1120 0 : if (likely((mbuf->ol_flags & PKT_RX_VLAN) == 0)) {
1121 0 : vlan_hdr = (struct rte_vlan_hdr *)(eth_hdr + 1);
1122 0 : mbuf->ol_flags |= PKT_RX_VLAN;
1123 0 : mbuf->vlan_tci = rte_be_to_cpu_16(vlan_hdr->vlan_tci);
1124 : }
1125 :
1126 0 : eth_proto = ((struct vr_vlan_hdr *)((uintptr_t)eth_hdr + pull_len))->vlan_proto;
1127 0 : pull_len += VR_VLAN_HLEN;
1128 : }
1129 :
1130 160 : if (likely(eth_proto == rte_cpu_to_be_16(VR_ETH_PROTO_IP))) {
1131 125 : ipv4_hdr = (struct vr_ip *)((uintptr_t)eth_hdr + pull_len);
1132 :
1133 125 : if (unlikely(mbuf_data_len < pull_len + sizeof(struct vr_ip)))
1134 0 : return -1;
1135 :
1136 125 : ipv4_len = (ipv4_hdr->ip_hl) * RTE_IPV4_IHL_MULTIPLIER;
1137 125 : pull_len += ipv4_len;
1138 :
1139 125 : if (ipv4_hdr->ip_proto == VR_IP_PROTO_GRE) {
1140 0 : gre_hdr = (struct vr_gre *)((uintptr_t)ipv4_hdr + ipv4_len);
1141 :
1142 : /* Don't do hashing for GRE IP fragments since it may not have
1143 : * the actual GRE header, do regular routing instead */
1144 0 : if (unlikely(vr_ip_fragment(ipv4_hdr))) {
1145 0 : return 0;
1146 : }
1147 :
1148 0 : if (unlikely(mbuf_data_len < pull_len + VR_GRE_BASIC_HDR_LEN))
1149 0 : return -1;
1150 :
1151 0 : if (likely(gre_hdr->gre_proto == VR_GRE_PROTO_MPLS_NO)) {
1152 : /* We are not RFC 1701 compliant receiver. */
1153 0 : if (unlikely(gre_hdr->gre_flags & (~(VR_GRE_FLAG_CSUM |
1154 : VR_GRE_FLAG_KEY))))
1155 0 : return 0;
1156 :
1157 0 : if (gre_hdr->gre_flags & VR_GRE_FLAG_CSUM) {
1158 0 : gre_hdr_len += (VR_GRE_CKSUM_HDR_LEN -
1159 : VR_GRE_BASIC_HDR_LEN);
1160 : }
1161 0 : if (gre_hdr->gre_flags & VR_GRE_FLAG_KEY) {
1162 0 : gre_hdr_len += (VR_GRE_KEY_HDR_LEN -
1163 : VR_GRE_BASIC_HDR_LEN);
1164 : }
1165 :
1166 0 : pull_len += gre_hdr_len;
1167 0 : gre_udp_encap = gre_hdr->gre_proto;
1168 :
1169 : /* If ddp enabled, hashing not required */
1170 0 : if (vr_dpdk_get_ddp()) {
1171 0 : mbuf->ol_flags |= PKT_RX_RSS_HASH;
1172 : } else {
1173 : /*
1174 : * mbuf->ol_flags & PKT_RX_RSS_HASH is mistakenly set
1175 : * by the NIC driver for MPLS over GRE packets. It is
1176 : * removed here and will be set after we perform hashing.
1177 : */
1178 0 : mbuf->ol_flags &= ~PKT_RX_RSS_HASH;
1179 : }
1180 : /* Go to parsing. */
1181 : } else {
1182 0 : return dpdk_mbuf_rss_hash(mbuf, ipv4_hdr, NULL); /* Looks like GRE, but no MPLS. */
1183 : }
1184 125 : } else if (ipv4_hdr->ip_proto == VR_IP_PROTO_UDP) {
1185 : /* At this point the packet may be:
1186 : * IP with inner packet carried in MPLS-over-UDP, or
1187 : * IP with inner packet carried in VXLAN, or
1188 : * just regular UDP inside IP.
1189 : */
1190 24 : udp_hdr = (struct vr_udp *)((uintptr_t)ipv4_hdr + ipv4_len);
1191 :
1192 24 : if (likely(vr_ip_transport_header_valid(ipv4_hdr))) {
1193 24 : if (unlikely(mbuf_data_len < pull_len + sizeof(struct vr_udp)))
1194 0 : return -1;
1195 : }
1196 :
1197 : /*
1198 : * If it is a packet from VM, it for sure will not be MPLS-over-UDP,
1199 : * so go directly to hashing procedure.
1200 : */
1201 : /*
1202 : * TODO: we can't rely on RSS_HASH flag here, since there might be
1203 : * NICs which does not set the flag yet carry MPLSoUDP packets.
1204 : *
1205 : * Instead we have to check the vif type to make sure the packet is
1206 : * from a VM.
1207 : */
1208 24 : if (unlikely((mbuf->ol_flags & PKT_RX_RSS_HASH) == 0))
1209 24 : return dpdk_mbuf_rss_hash(mbuf, ipv4_hdr, ipv6_hdr);
1210 :
1211 0 : if (likely(!vr_ip_fragment(ipv4_hdr))) {
1212 0 : udp_port = rte_be_to_cpu_16(udp_hdr->udp_dport);
1213 0 : if (likely(vr_mpls_udp_port(udp_port) || vr_vxlan_udp_port(udp_port))) {
1214 0 : pull_len += sizeof(struct vr_udp);
1215 0 : gre_udp_encap = udp_hdr->udp_dport;
1216 : /* Go to parsing. */
1217 : } else {
1218 : /* UDP from the wire, but not MPLS-over-UDP nor VXLAN. */
1219 0 : return 0;
1220 : }
1221 : }
1222 101 : } else if ((mbuf->ol_flags & PKT_RX_RSS_HASH) == 0) {
1223 : /* Looks like no tunneling, perhaps a packet from a VM. */
1224 101 : return dpdk_mbuf_rss_hash(mbuf, ipv4_hdr, ipv6_hdr);
1225 : } else {
1226 : RTE_LOG_DP(DEBUG, VROUTER, "%s: RSS hash: 0x%x (from NIC)\n",
1227 : __func__, mbuf->hash.rss);
1228 0 : return 0; /* Not MPLS-over-GRE, not MPLS-over-UDP, not anything from VM. */
1229 : }
1230 :
1231 0 : helper_ret = vr_inner_pkt_parse(rte_pktmbuf_mtod(mbuf, unsigned char *),
1232 : vr_mpls_tunnel_type, &encap_type,
1233 0 : NULL, &pull_len, mbuf->buf_len,
1234 : &ipv4_inner_hdr, &ipv6_inner_hdr,
1235 0 : gre_udp_encap, ipv4_hdr->ip_proto);
1236 0 : if (helper_ret == PKT_RET_SLOW_PATH)
1237 0 : return -1;
1238 0 : else if (helper_ret == PKT_RET_UNHANDLED)
1239 0 : return 0;
1240 :
1241 : /* If not inner IPv4 nor IPv6 - nothing to do. */
1242 0 : if (unlikely(ipv4_inner_hdr == NULL))
1243 0 : return 0; /* Inner IPv6 packets have ipv4_inner_hdr != NULL */
1244 :
1245 0 : helper_ret = vr_ip_transport_parse(ipv4_inner_hdr, ipv6_inner_hdr,
1246 0 : NULL, mbuf->buf_len,
1247 : dpdk_adjust_tcp_mss, NULL, NULL,
1248 : NULL, &pull_len);
1249 0 : if (unlikely(helper_ret == PKT_RET_SLOW_PATH))
1250 0 : return -1;
1251 :
1252 : /* Packet may already be hashed by the NIC */
1253 0 : if (mbuf->ol_flags & PKT_RX_RSS_HASH) {
1254 : RTE_LOG_DP(DEBUG, VROUTER, "%s: RSS hash: 0x%x (from NIC)\n",
1255 : __func__, mbuf->hash.rss);
1256 0 : return 0;
1257 : } else {
1258 : /* For GRE packets we need to hash inner packet */
1259 0 : if (gre_hdr) {
1260 0 : if (ipv6_inner_hdr) {
1261 0 : ipv6_hdr = ipv6_inner_hdr;
1262 0 : ipv4_hdr = NULL;
1263 0 : } else if (ipv4_inner_hdr) {
1264 0 : ipv4_hdr = ipv4_inner_hdr;
1265 : }
1266 : }
1267 : /* Go to hashing */
1268 : }
1269 35 : } else if (eth_proto == rte_cpu_to_be_16(VR_ETH_PROTO_IP6)) {
1270 23 : ipv6_hdr = (struct vr_ip6 *)((uintptr_t)eth_hdr + pull_len);
1271 :
1272 23 : if (unlikely(mbuf_data_len < pull_len + sizeof(struct vr_ip6)))
1273 0 : return -1;
1274 :
1275 : /**
1276 : * There's no IPv6 as a tunnel between nodes, so it is either a packet
1277 : * from VM, or some non-related IPv6 packet from the wire.
1278 : */
1279 23 : if ((mbuf->ol_flags & PKT_RX_RSS_HASH) == 0) {
1280 23 : return dpdk_mbuf_rss_hash(mbuf, ipv4_hdr, ipv6_hdr);
1281 : } else {
1282 : RTE_LOG_DP(DEBUG, VROUTER, "%s: RSS hash: 0x%x (from NIC)\n",
1283 : __func__, mbuf->hash.rss);
1284 0 : return 0;
1285 : }
1286 : } else {
1287 12 : return 0;
1288 : }
1289 :
1290 0 : return dpdk_mbuf_rss_hash(mbuf, ipv4_hdr, ipv6_hdr);
1291 : }
1292 :
1293 : /*
1294 : * vr_dpdk_ethdev_rx_emulate - emulate smart NIC RX:
1295 : * - strip VLAN tags for packets received from fabric interface
1296 : * - calculate RSS hash if not present
1297 : * - recalculate RSS hash for MPLSoGRE packets
1298 : *
1299 : * Returns 0 on no hash changes, otherwise a bitmask of mbufs to distribute.
1300 : */
1301 : uint64_t
1302 160 : vr_dpdk_ethdev_rx_emulate(struct vr_interface *vif,
1303 : struct rte_mbuf *pkts[VR_DPDK_RX_BURST_SZ], uint32_t *nb_pkts)
1304 : {
1305 160 : uint64_t mask_to_distribute = 0, mask_to_distribute_ret = 0,
1306 160 : mask_to_drop = 0;
1307 160 : unsigned i, offload_en, nb_pkts_ret = 0;
1308 : int ret;
1309 :
1310 : /* prefetch the mbufs */
1311 320 : for (i = 0; i < *nb_pkts; i++) {
1312 160 : rte_prefetch0(rte_pktmbuf_mtod(pkts[i], uint8_t *));
1313 160 : rte_prefetch0(rte_pktmbuf_mtod_offset(pkts[i], uint8_t *, RTE_CACHE_LINE_SIZE));
1314 : }
1315 :
1316 : /* emulate VLAN stripping if needed */
1317 320 : for (i = 0; i < *nb_pkts; i++) {
1318 160 : rte_vlan_strip(pkts[i]);
1319 : }
1320 :
1321 : /* indicate packets from namespace don't have checksum calculated */
1322 160 : if (unlikely(vif_is_namespace(vif)))
1323 0 : for (i = 0; i < *nb_pkts; i++)
1324 0 : pkts[i]->ol_flags |= PKT_RX_IP_CKSUM_BAD;
1325 :
1326 160 : offload_en = vif_is_fabric(vif) && datapath_offloads;
1327 : /* parse packet headers and emulate RSS hash */
1328 320 : for (i = 0; i < *nb_pkts; i++) {
1329 :
1330 : /* datapath offloads calculated the RSS for flow tagged packets */
1331 160 : if (!(offload_en && (pkts[i]->ol_flags & PKT_RX_FDIR_ID))) {
1332 160 : ret = dpdk_mbuf_parse_and_hash_packets(pkts[i]);
1333 :
1334 : /* If software load-balancing is not required, continue */
1335 160 : if (vr_no_load_balance)
1336 0 : continue;
1337 :
1338 : /**
1339 : * ret:
1340 : * -1 -> packet is invalid and needs to be dropped
1341 : * 1 -> packet to be distributed (bit in mask_to_distribute set)
1342 : * 0 -> packet to be routed (bit in mask_to_distribute not set)
1343 : */
1344 160 : if (ret == 1) {
1345 148 : mask_to_distribute |= 1ULL << i;
1346 12 : } else if (unlikely(ret == -1)) {
1347 0 : mask_to_drop |= 1ULL << i;
1348 : }
1349 : }
1350 : }
1351 :
1352 : /**
1353 : * Drop invalid packets masked with mask_to_drop and remove them from the
1354 : * array. Bits in mask_to_distribute need to be rewritten in order to get
1355 : * rid of bits refering to dropped packets.
1356 : */
1357 160 : if (unlikely(mask_to_drop != 0)) {
1358 0 : for (i = 0; i < *nb_pkts; i++) {
1359 0 : if (mask_to_drop & (1ULL << i)) {
1360 0 : vr_dpdk_pfree(pkts[i], vif, VP_DROP_PULL);
1361 : } else {
1362 0 : pkts[nb_pkts_ret] = pkts[i];
1363 0 : if (mask_to_distribute & (1ULL << i)) {
1364 0 : mask_to_distribute_ret |= 1ULL << nb_pkts_ret;
1365 : }
1366 0 : nb_pkts_ret++;
1367 : }
1368 : }
1369 :
1370 : /* Return number of valid packets and update mask_to_distribute */
1371 0 : *nb_pkts = nb_pkts_ret;
1372 0 : mask_to_distribute = mask_to_distribute_ret;
1373 : }
1374 :
1375 160 : return mask_to_distribute;
1376 : }
|
