Line data Source code
1 : /* SPDX-License-Identifier: BSD-2-Clause
2 : * Copyright(c) HCL TECHNOLOGIES LTD
3 : * Submitted on behalf of a third-party: Intel Corporation, a
4 : * Delaware corporation, having its principal place of business
5 : * at 2200 Mission College Boulevard,
6 : * Santa Clara, California 95052, USA
7 : */
8 : #include "vr_dpdk_n3k_flow.h"
9 :
10 : #include <rte_eth_bond.h>
11 : #include <rte_hash.h>
12 : #include <rte_log.h>
13 : #include <rte_malloc.h>
14 :
15 : #include "offload_entry/vr_dpdk_n3k_offload_converter.h"
16 : #include "offload_entry/vr_dpdk_n3k_offload_entry.h"
17 : #include "vr_dpdk_n3k_packet_metadata.h"
18 : #include "vr_dpdk_n3k_interface.h"
19 : #include "vr_dpdk_n3k_nexthop.h"
20 : #include "vr_dpdk_n3k_offload_hold.h"
21 : #include "vr_dpdk_n3k_config.h"
22 : #include "vr_dpdk.h"
23 : #include "vr_flow.h"
24 :
25 : static uint32_t flows_count;
26 : static struct vr_n3k_offload_flow **flows = NULL;
27 :
28 : int
29 56 : vr_dpdk_n3k_offload_flow_table_add_unlocked(
30 : struct vr_n3k_offload_flowtable_key *key,
31 : struct vr_n3k_offload_flow *flow)
32 : {
33 56 : if (!flows || key->fe_index >= flows_count)
34 0 : return -EINVAL;
35 :
36 56 : flows[key->fe_index] = flow;
37 56 : return 0;
38 : }
39 :
40 : static int
41 0 : vr_dpdk_n3k_offload_flow_table_del(struct vr_n3k_offload_flowtable_key *key)
42 : {
43 0 : if (!flows || key->fe_index >= flows_count)
44 0 : return -EINVAL;
45 :
46 0 : rte_free(flows[key->fe_index]);
47 0 : flows[key->fe_index] = NULL;
48 0 : return 0;
49 : }
50 :
51 : static inline void
52 : vr_dpdk_n3k_offload_flow_del_internal(struct vr_n3k_offload_flow *fe);
53 :
54 : static inline uint16_t
55 0 : vr_dpdk_n3k_offload_flow_get_underlying_hw_port_id(uint16_t port_id)
56 : {
57 : int ret;
58 :
59 0 : if ((ret = rte_eth_bond_primary_get(port_id)) < 0) {
60 0 : return port_id;
61 : }
62 :
63 0 : return ret;
64 : }
65 :
66 :
67 : static int
68 0 : vr_dpdk_n3k_allocate_flow_handles(const struct vr_nexthop *snh, struct vr_n3k_offload_flow *flow)
69 : {
70 0 : if (snh->nh_type == NH_COMPOSITE) {
71 : /* Note: We assume composite has non-zero components, this is checked
72 : * in vr_dpdk_n3k_offload_nexthop_get */
73 0 : flow->handle_count = snh->nh_component_cnt;
74 : } else {
75 0 : flow->handle_count = 1;
76 : }
77 :
78 0 : flow->handles = rte_zmalloc("n3k_flow_handles",
79 0 : sizeof(*flow->handles) * flow->handle_count, 0);
80 0 : if (!flow->handles) {
81 0 : RTE_LOG(ERR, VROUTER, "%s(): Failed to allocate flow handles array\n", __func__);
82 0 : return -ENOMEM;
83 : }
84 :
85 0 : return 0;
86 : }
87 :
88 : static int
89 0 : vr_dpdk_n3k_offload_correct_missing_rfe_index(struct vr_flow_entry *fe,
90 : struct vr_n3k_offload_flow *oflow)
91 : {
92 : int ret;
93 : struct vr_n3k_offload_flow *old_rfe;
94 : struct vr_n3k_offload_flow new_rfe;
95 0 : uint32_t old_rfe_id = oflow->reverse_id;
96 0 : struct vr_n3k_offload_flowtable_key old_key = {
97 0 : .fe_index = oflow->reverse_id
98 : };
99 0 : struct vr_n3k_offload_flowtable_key new_key = {
100 0 : .fe_index = fe->fe_rflow
101 : };
102 :
103 0 : oflow->reverse_id = fe->fe_rflow;
104 :
105 0 : if (old_rfe_id == 0 || old_rfe_id == fe->fe_rflow)
106 0 : return 0;
107 :
108 0 : RTE_LOG(WARNING, VROUTER, "%s(): rfe_index changed from %d to %d\n",
109 : __func__, old_rfe_id, fe->fe_rflow);
110 :
111 0 : old_rfe = vr_dpdk_n3k_offload_flow_get(&old_key);
112 0 : if (!old_rfe) {
113 0 : RTE_LOG(ERR, VROUTER,
114 : "%s(): Failed to get old rfe; fe_index: %d\n", __func__,
115 : old_key.fe_index);
116 0 : return -ENOENT;
117 : }
118 :
119 0 : memcpy(&new_rfe, old_rfe, sizeof(struct vr_n3k_offload_flow));
120 0 : new_rfe.id = new_key.fe_index;
121 :
122 : /* Updating flow table */
123 0 : ret = vr_dpdk_n3k_offload_flow_table_del(&old_key);
124 0 : if (ret < 0) {
125 0 : RTE_LOG(ERR, VROUTER,
126 : "%s(): Failed to remove old flow; fe_index: %d\n", __func__,
127 : old_key.fe_index);
128 0 : return ret;
129 : }
130 :
131 0 : struct vr_n3k_offload_flow *flow = vr_dpdk_n3k_offload_flow_get(&new_key);
132 0 : if (flow == NULL) {
133 0 : flow = rte_zmalloc("n3k_flow", sizeof(*flow), 0);
134 0 : if (flow == NULL) {
135 0 : RTE_LOG(ERR, VROUTER,
136 : "%s(): Insufficient memory for flow allocation.\n", __func__);
137 0 : ret = -ENOMEM;
138 0 : goto flow_del;
139 : }
140 : }
141 :
142 0 : memcpy(flow, &new_rfe, sizeof(struct vr_n3k_offload_flow));
143 :
144 0 : ret = vr_dpdk_n3k_offload_flow_table_add_unlocked(&new_key, flow);
145 0 : if (ret < 0) {
146 0 : RTE_LOG(ERR, VROUTER,
147 : "%s(): vr_dpdk_n3k_offload_flow_table_add failed: ret = %d;\n",
148 : __func__, ret);
149 0 : rte_free(flow);
150 0 : goto flow_del;
151 : }
152 0 : return 0;
153 :
154 0 : flow_del:
155 0 : vr_dpdk_n3k_offload_flow_del_internal(&new_rfe);
156 0 : return ret;
157 : }
158 :
159 : static inline void
160 0 : vr_dpdk_n3k_offload_flow_construct(
161 : struct vr_flow_entry *fe,
162 : struct vr_flow_entry *rfe,
163 : struct vr_n3k_offload_flow *oflow)
164 : {
165 0 : oflow->id = fe->fe_hentry.hentry_index;
166 0 : oflow->action = fe->fe_action;
167 0 : oflow->flags = fe->fe_flags;
168 0 : oflow->tcp_flags = fe->fe_tcp_flags;
169 :
170 0 : memset(&oflow->ip, 0, sizeof(oflow->ip));
171 :
172 0 : if (fe->fe_key.flow_key_len == VR_FLOW_IPV4_HASH_SIZE) {
173 0 : oflow->ip.type = VR_N3K_IP_TYPE_IPV4;
174 0 : oflow->ip.src.ipv4 = fe->fe_key.flow4_sip;
175 0 : oflow->ip.dst.ipv4 = fe->fe_key.flow4_dip;
176 : } else {
177 0 : oflow->ip.type = VR_N3K_IP_TYPE_IPV6;
178 0 : memcpy(oflow->ip.src.ipv6, fe->fe_key.flow6_sip,
179 : VR_IP6_ADDRESS_LEN);
180 0 : memcpy(oflow->ip.dst.ipv6, fe->fe_key.flow6_dip,
181 : VR_IP6_ADDRESS_LEN);
182 : }
183 :
184 0 : oflow->proto = fe->fe_key.flow4_proto;
185 0 : oflow->src_port = fe->fe_key.flow4_sport;
186 0 : oflow->dst_port = fe->fe_key.flow4_dport;
187 0 : oflow->tunnel_udp_src_port = RTE_BE16(fe->fe_udp_src_port);
188 0 : oflow->nh_id = fe->fe_key.flow_nh_id;
189 0 : oflow->src_vrf_id = fe->fe_vrf;
190 0 : oflow->ecmp_nh_idx = fe->fe_ecmp_nh_index;
191 0 : oflow->underlay_ecmp_index = fe->fe_underlay_ecmp_index;
192 :
193 : // There is a flag in vr_flow_entry to determine if mirroring is enabled.
194 : // At the same time VR_MAX_MIRROR_INDICES is an invalid value that
195 : // fits into uint8_t, so the flag seems to be redundant.
196 : // VR_MAX_MIRROR_INDICES is used in vrouter to initialize mirroring id
197 : // when mirroring is disabled.
198 : // To keep consistency with vouter code, the flag is checked here;
199 : // further in the processing VR_MAX_MIRROR_INDICES is used
200 : // to mark that mirroring is disabled.
201 : // Only one vif is currently supported in offloading procedure, so
202 : // secondary mirroring id (present in vr_flow_entry) is currently ignored.
203 : // TODO: decide if fe_sec_mirror_id requires further attention.
204 0 : if (fe->fe_flags & VR_FLOW_FLAG_MIRROR) {
205 0 : oflow->mirror_id = fe->fe_mirror_id;
206 : } else {
207 0 : oflow->mirror_id = VR_MAX_MIRROR_INDICES;
208 : }
209 :
210 0 : if (rfe != NULL)
211 0 : oflow->reverse_id = rfe->fe_hentry.hentry_index;
212 0 : }
213 :
214 : static inline struct vr_n3k_offload_flowtable_key
215 0 : vr_dpdk_n3k_offload_flow_key_construct(struct vr_flow_entry *fe)
216 : {
217 0 : struct vr_n3k_offload_flowtable_key key = {
218 0 : .fe_index = fe->fe_hentry.hentry_index
219 : };
220 :
221 0 : return key;
222 : }
223 :
224 : static void
225 28 : vr_dpdk_n3k_flow_discard_all_flows(void)
226 : {
227 28 : uint32_t i = 0;
228 :
229 2828 : for (i = 0; i < flows_count; ++i) {
230 2800 : rte_free(flows[i]);
231 2800 : flows[i] = NULL;
232 : }
233 28 : }
234 :
235 : struct vr_n3k_offload_flow *
236 28 : vr_dpdk_n3k_offload_flow_get(struct vr_n3k_offload_flowtable_key *key)
237 : {
238 28 : if (!flows || key->fe_index >= flows_count)
239 0 : return NULL;
240 :
241 28 : return flows[key->fe_index];
242 : }
243 :
244 : int
245 28 : vr_dpdk_n3k_offload_flow_init(size_t flowtable_sz)
246 : {
247 28 : if (flows) {
248 0 : vr_dpdk_n3k_flow_discard_all_flows();
249 0 : rte_free(flows);
250 0 : flows = NULL;
251 : }
252 :
253 28 : flows = rte_zmalloc("n3k_offload_flow", flowtable_sz * sizeof(*flows), 0);
254 28 : if (flows == NULL)
255 0 : return -ENOMEM;
256 :
257 28 : flows_count = flowtable_sz;
258 :
259 28 : return 0;
260 : }
261 :
262 : void
263 28 : vr_dpdk_n3k_offload_flow_exit(void)
264 : {
265 28 : vr_dpdk_n3k_flow_discard_all_flows();
266 28 : rte_free(flows);
267 28 : flows = NULL;
268 28 : }
269 :
270 : static struct vr_n3k_offload_flow *
271 0 : vr_dpdk_n3k_offload_flow_save_copy(
272 : struct vr_flow_entry *fe, struct vr_flow_entry *rfe)
273 : {
274 : struct vr_n3k_offload_flowtable_key key =
275 0 : vr_dpdk_n3k_offload_flow_key_construct(fe);
276 :
277 0 : struct vr_n3k_offload_flow *oflow = vr_dpdk_n3k_offload_flow_get(&key);
278 0 : if (oflow == NULL) {
279 0 : oflow = rte_zmalloc("n3k_flow", sizeof(*oflow), 0);
280 0 : if (oflow == NULL) {
281 0 : RTE_LOG(ERR, VROUTER,
282 : "%s(): Insufficient memory for flow allocation.\n", __func__);
283 0 : return NULL;
284 : }
285 : }
286 :
287 0 : vr_dpdk_n3k_offload_flow_construct(fe, rfe, oflow);
288 :
289 0 : if (vr_dpdk_n3k_offload_flow_table_add_unlocked(&key, oflow)) {
290 0 : rte_free(oflow);
291 0 : return NULL;
292 : }
293 :
294 0 : return oflow;
295 : }
296 :
297 : static inline void
298 0 : log_flow_details(struct vr_flow_entry *fe, const char *prefix)
299 : {
300 : char saddr[40], daddr[40];
301 0 : struct vr_n3k_ips ip = { };
302 :
303 0 : if (fe->fe_key.flow_key_len == VR_FLOW_IPV4_HASH_SIZE) {
304 0 : ip.type = VR_N3K_IP_TYPE_IPV4;
305 0 : ip.src.ipv4 = fe->fe_key.flow4_sip;
306 0 : ip.dst.ipv4 = fe->fe_key.flow4_dip;
307 : } else {
308 0 : ip.type = VR_N3K_IP_TYPE_IPV6;
309 0 : memcpy(ip.src.ipv6, fe->fe_key.flow6_sip,
310 : VR_IP6_ADDRESS_LEN);
311 0 : memcpy(ip.dst.ipv6, fe->fe_key.flow6_dip,
312 : VR_IP6_ADDRESS_LEN);
313 : }
314 :
315 0 : RTE_LOG(DEBUG, VROUTER,
316 : "%10s: id=%d; action=%d; flags=%#x; nh=%d; vrf=%d; rflow=%d;\n",
317 : prefix, fe->fe_hentry.hentry_index, fe->fe_action, fe->fe_flags,
318 : fe->fe_key.flow_nh_id, fe->fe_vrf, fe->fe_rflow
319 : );
320 :
321 0 : RTE_LOG(DEBUG, VROUTER,
322 : " sip=%s; dip=%s; "
323 : "proto=%d; sport=%d; dport=%d;\n",
324 : vr_dpdk_n3k_convert_ip_to_str(saddr, &ip.src, ip.type),
325 : vr_dpdk_n3k_convert_ip_to_str(daddr, &ip.dst, ip.type),
326 : fe->fe_key.flow4_proto,
327 : rte_be_to_cpu_16(fe->fe_key.flow4_sport),
328 : rte_be_to_cpu_16(fe->fe_key.flow4_dport)
329 : );
330 0 : }
331 :
332 : static int
333 0 : vr_dpdk_n3k_offload_destroy_hw_flow(
334 : struct rte_flow *handle, uint16_t port_id, uint32_t flow_idx)
335 : {
336 0 : struct rte_flow_error error = { 0 };
337 0 : int ret = rte_flow_destroy(port_id, handle, &error);
338 0 : if (ret) {
339 0 : RTE_LOG(ERR, VROUTER,
340 : "%s(): Failed to destroy hardware flow %d: ret = %d; [%d]: %s\n",
341 : __func__, flow_idx, ret, error.type, error.message ?
342 : error.message : "No error message");
343 : }
344 :
345 0 : return ret;
346 : }
347 :
348 : static void
349 0 : vr_dpdk_n3k_offload_destroy_hw_flows(
350 : struct vr_n3k_offload_flow *flow)
351 : {
352 : size_t i;
353 0 : if (flow->handles != NULL) {
354 0 : for (i = 0; i < flow->handle_count; ++i) {
355 0 : if (!flow->handles[i])
356 0 : continue;
357 :
358 0 : vr_dpdk_n3k_offload_destroy_hw_flow(
359 0 : flow->handles[i], flow->hw_port_id, flow->id);
360 0 : flow->handles[i] = NULL;
361 : }
362 0 : rte_free(flow->handles);
363 0 : flow->handles = NULL;
364 : }
365 0 : }
366 :
367 : static inline void
368 0 : vr_dpdk_n3k_offload_flow_del_internal(struct vr_n3k_offload_flow *fe) {
369 0 : vr_dpdk_n3k_offload_destroy_hw_flows(fe);
370 0 : vr_dpdk_n3k_packet_metadata_schedule_delete(fe);
371 0 : }
372 :
373 : static inline bool
374 0 : vr_dpdk_n3k_offload_flow_is_tcp_not_established(const struct vr_n3k_offload_flow *flow)
375 : {
376 0 : if (flow->proto != VR_IP_PROTO_TCP)
377 0 : return false;
378 :
379 0 : if (flow->tcp_flags & VR_FLOW_TCP_ESTABLISHED)
380 0 : return false;
381 :
382 0 : if (flow->tcp_flags & VR_FLOW_TCP_ESTABLISHED_R)
383 0 : return false;
384 :
385 0 : return true;
386 : }
387 :
388 : static inline bool
389 0 : vr_dpdk_n3k_offload_flow_is_offloadable_proto(const struct vr_n3k_offload_flow *flow)
390 : {
391 0 : if (flow->proto == VR_IP_PROTO_TCP) {
392 0 : if (flow->action == VR_FLOW_ACTION_DROP)
393 0 : return true;
394 :
395 0 : return !vr_dpdk_n3k_offload_flow_is_tcp_not_established(flow);
396 0 : } else if (flow->proto== VR_IP_PROTO_UDP) {
397 0 : return true;
398 : } else {
399 0 : return false;
400 : }
401 : }
402 :
403 : static int
404 0 : vr_dpdk_n3k_offload_flow_set_internal(
405 : struct vr_n3k_offload_flow *flow,
406 : uint32_t fe_index,
407 : struct vr_n3k_offload_flow *reverse_flow)
408 : {
409 : size_t i;
410 0 : vr_dpdk_n3k_offload_destroy_hw_flows(flow);
411 :
412 0 : int ret = vr_dpdk_n3k_packet_metadata_ensure_entry_for_flow_exists(
413 : flow, reverse_flow);
414 0 : if (ret) {
415 0 : RTE_LOG(ERR, VROUTER,
416 : "%s(): Cannot rebuild packet metadata for flow %d: %d\n",
417 : __func__, flow->id, ret);
418 0 : return ret;
419 : }
420 :
421 0 : if (!vr_dpdk_n3k_config_is_drop_offload_enabled() && flow->action == VR_FLOW_ACTION_DROP) {
422 0 : RTE_LOG(DEBUG, VROUTER,
423 : "%s(): Skipping offload: DROP packets go to slowpath\n", __func__);
424 0 : return 0;
425 : }
426 :
427 0 : if (!vr_dpdk_n3k_offload_flow_is_offloadable_proto(flow)) {
428 : /* We do not offload tcp handshakes, only data. Skip. */
429 0 : RTE_LOG(DEBUG, VROUTER,
430 : "%s(): Skipping offload: proto not offloadable\n", __func__);
431 0 : return 0;
432 : }
433 :
434 : struct vr_n3k_offload_entry pre_offload_entry;
435 0 : ret = vr_dpdk_n3k_fill_offload_entry_partial_start(flow, &pre_offload_entry);
436 0 : if (ret) {
437 0 : RTE_LOG(ERR, VROUTER, "%s(): vr_dpdk_n3k_fill_offload_entry_partial_start failed: %d\n",
438 : __func__, ret);
439 0 : return ret;
440 : }
441 :
442 0 : const struct vr_nexthop *snh = pre_offload_entry.src_nh;
443 0 : ret = vr_dpdk_n3k_allocate_flow_handles(snh, flow);
444 0 : if (ret) {
445 0 : RTE_LOG(ERR, VROUTER, "%s(): vr_dpdk_n3k_allocate_flow_handles failed: %d\n",
446 : __func__, ret);
447 0 : return ret;
448 : }
449 :
450 0 : for (i = 0; i < flow->handle_count; ++i) {
451 0 : struct vr_n3k_offload_entry offload_entry = pre_offload_entry;
452 :
453 0 : if (snh->nh_type == NH_COMPOSITE) {
454 : uint32_t component_nh_idx;
455 : /* We've checked all the preconditions already so
456 : * vr_dpdk_n3k_offload_nexthop_get_cnh_idx can either succeed or
457 : * return -ENOENT */
458 0 : ret = vr_dpdk_n3k_offload_nexthop_get_cnh_idx(snh, i, &component_nh_idx);
459 0 : if (ret == -ENOENT) {
460 0 : RTE_LOG(ERR, VROUTER, "%s(): Failed to get component_nh[%zu], nh is NULL\n",
461 : __func__, i);
462 0 : continue;
463 : }
464 0 : offload_entry.src_nh = vr_dpdk_n3k_offload_nexthop_get(component_nh_idx);
465 0 : if (offload_entry.src_nh == NULL) {
466 0 : RTE_LOG(ERR, VROUTER, "%s(): Failed to get component_nh[%zu], nh_idx=%d\n",
467 : __func__, i, component_nh_idx);
468 0 : continue;
469 : }
470 :
471 0 : ret = vr_dpdk_n3k_offload_nexthop_get_cnh_label(snh, i, &offload_entry.tunnel_label);
472 0 : if (ret) {
473 0 : RTE_LOG(ERR, VROUTER, "%s(): getting composite nexthop's label failed: %d\n",
474 : __func__, ret);
475 0 : return ret;
476 : }
477 : }
478 :
479 0 : ret = vr_dpdk_n3k_fill_offload_entry_vifs(&offload_entry);
480 0 : if (ret) {
481 0 : RTE_LOG(ERR, VROUTER, "%s(): filling offload entry vifs failed: %d\n",
482 : __func__, ret);
483 0 : return ret;
484 : }
485 :
486 0 : ret = vr_dpdk_n3k_fill_offload_entry_partial_end(flow, &offload_entry);
487 0 : if (ret) {
488 0 : RTE_LOG(ERR, VROUTER, "%s(): filling offload entry failed: %d\n",
489 : __func__, ret);
490 0 : return ret;
491 : }
492 :
493 : struct vr_n3k_rte_flow_package flow_package =
494 0 : vr_dpdk_n3k_offload_entry_to_rte_flow(&offload_entry);
495 0 : if (flow_package.error) {
496 0 : RTE_LOG(ERR, VROUTER,
497 : "%s(): vr_dpdk_n3k_offload_entry_to_rte_flow failed: %d\n",
498 : __func__, flow_package.error);
499 0 : return flow_package.error;
500 : }
501 :
502 0 : flow->hw_port_id = vr_dpdk_n3k_offload_flow_get_underlying_hw_port_id(
503 0 : vif_port_id(offload_entry.src_vif));
504 :
505 : struct rte_flow_error error;
506 : static struct rte_flow_attr attr = {
507 : .ingress = 1,
508 : };
509 :
510 0 : flow->handles[i] = rte_flow_create(
511 0 : flow->hw_port_id,
512 : &attr,
513 0 : flow_package.pattern,
514 0 : flow_package.actions,
515 : &error
516 : );
517 :
518 0 : if (flow->handles[i] == NULL) {
519 0 : RTE_LOG(ERR, VROUTER, "%s(): Failed to create flow %d: [%d]: %s\n",
520 : __func__, fe_index, error.type, error.message ?
521 : error.message : "No error message");
522 0 : return -EINVAL;
523 : }
524 : }
525 :
526 0 : vr_dpdk_n3k_packet_metadata_mark_used(flow);
527 :
528 0 : return 0;
529 : }
530 :
531 : int
532 0 : vr_dpdk_n3k_offload_flow_set_unlocked(struct vr_flow_entry *fe, uint32_t fe_index,
533 : struct vr_flow_entry *rfe)
534 : {
535 : int ret;
536 0 : RTE_LOG(DEBUG, VROUTER, "%s() called; fe=%p; fe_index=%u; rfe=%p\n",
537 : __func__, fe, fe_index, rfe);
538 0 : log_flow_details(fe, "fe");
539 0 : if (rfe != NULL)
540 0 : log_flow_details(rfe, "rfe");
541 :
542 0 : struct vr_n3k_offload_flow *reverse_flow = NULL;
543 : struct vr_n3k_offload_flow *flow =
544 0 : vr_dpdk_n3k_offload_flow_save_copy(fe, rfe);
545 0 : if (flow == NULL)
546 0 : return -ENOMEM;
547 :
548 0 : if (rfe != NULL)
549 0 : reverse_flow = vr_dpdk_n3k_offload_flow_save_copy(rfe, fe);
550 : else {
551 0 : ret = vr_dpdk_n3k_offload_correct_missing_rfe_index(fe, flow);
552 0 : if (ret < 0)
553 0 : RTE_LOG(ERR, VROUTER,
554 : "%s() vr_dpdk_n3k_offload_correct_missing_rfe_index failed "
555 : "with error ret=%d; fe=%p; fe_index=%u; rfe=%p\n",
556 : __func__, ret, fe, fe_index, rfe);
557 : }
558 :
559 0 : bool hold_entry_exist = vr_dpdk_n3k_offload_hold_entry_exist(flow);
560 0 : if (!hold_entry_exist && vr_dpdk_n3k_offload_hold_should_wait(flow, reverse_flow)) {
561 0 : ret = vr_dpdk_n3k_offload_hold_save_flow(flow);
562 0 : if (ret < 0) {
563 0 : RTE_LOG(ERR, VROUTER,
564 : "%s() vr_dpdk_n3k_offload_hold_save_flow failed for fe_index=%d; ret=%d\n",
565 : __func__,
566 : flow->reverse_id, ret);
567 : }
568 0 : return 0;
569 : }
570 0 : else if(hold_entry_exist && vr_dpdk_n3k_offload_hold_get_held(flow, &reverse_flow)) {
571 0 : RTE_LOG(WARNING, VROUTER,
572 : "%s() Offloading held flows fe_index=%d; rfe_index=%d\n",
573 : __func__, flow->reverse_id, flow->id);
574 :
575 0 : ret = vr_dpdk_n3k_offload_flow_set_internal(reverse_flow, flow->reverse_id, flow);
576 0 : if (ret < 0) {
577 0 : RTE_LOG(ERR,
578 : VROUTER,
579 : "%s() vr_dpdk_n3k_offload_flow_set_internal(fe=%d, "
580 : "rfe=%d) failed with ret=%d\n",
581 : __func__,
582 : reverse_flow->id, flow->id, ret);
583 0 : return ret;
584 : }
585 : }
586 :
587 0 : return vr_dpdk_n3k_offload_flow_set_internal(flow, fe_index, reverse_flow);
588 : }
589 :
590 : int
591 0 : vr_dpdk_n3k_offload_flow_del_unlocked(struct vr_flow_entry *fe)
592 : {
593 : struct vr_n3k_offload_flowtable_key ftable_key;
594 : struct vr_n3k_offload_flow* flow;
595 :
596 0 : RTE_LOG(DEBUG, VROUTER, "%s() called; fe=%p\n", __func__, fe);
597 0 : log_flow_details(fe, "fe");
598 :
599 0 : ftable_key = vr_dpdk_n3k_offload_flow_key_construct(fe);
600 0 : flow = vr_dpdk_n3k_offload_flow_get(&ftable_key);
601 0 : if (flow == NULL)
602 0 : return -ENOENT;
603 :
604 0 : vr_dpdk_n3k_offload_hold_del_flow(flow);
605 :
606 0 : vr_dpdk_n3k_offload_flow_del_internal(flow);
607 :
608 0 : vr_dpdk_n3k_offload_flow_table_del(&ftable_key);
609 :
610 0 : return 0;
611 : }
612 :
613 : int
614 0 : vr_dpdk_n3k_offload_flow_stats_update_unlocked(struct vr_flow_entry *fe)
615 : {
616 : struct vr_n3k_offload_flowtable_key ftable_key;
617 : struct vr_n3k_offload_flow* flow;
618 : size_t i;
619 :
620 0 : ftable_key = vr_dpdk_n3k_offload_flow_key_construct(fe);
621 0 : flow = vr_dpdk_n3k_offload_flow_get(&ftable_key);
622 :
623 0 : if (flow == NULL || flow->handles == NULL)
624 0 : return -ENOENT;
625 :
626 0 : struct rte_flow_query_count query = {0};
627 0 : query.hits_set = 1;
628 0 : query.bytes_set = 1;
629 :
630 0 : struct rte_flow_error error = {0};
631 0 : struct rte_flow_action actions[] = {
632 : [0] = {
633 : .type = RTE_FLOW_ACTION_TYPE_COUNT,
634 : .conf = NULL,
635 : },
636 : [1] = {
637 : .type = RTE_FLOW_ACTION_TYPE_END,
638 : .conf = NULL,
639 : },
640 : };
641 :
642 0 : uint64_t hits = 0;
643 0 : uint64_t bytes = 0;
644 0 : for (i = 0; i < flow->handle_count; ++i) {
645 0 : if (!flow->handles[i])
646 0 : continue;
647 :
648 0 : int ret = rte_flow_query(flow->hw_port_id, flow->handles[i],
649 : actions, &query, &error);
650 :
651 0 : if (ret != 0) {
652 0 : RTE_LOG(ERR, VROUTER,
653 : "Failed to get statistics for flow id %u - %s\n",
654 : flow->id,
655 : error.message ? error.message : "no error message");
656 0 : return ret;
657 : }
658 :
659 0 : bytes += query.bytes;
660 0 : hits += query.hits;
661 : }
662 :
663 0 : int stats_bytes_bits_nb = sizeof(fe->fe_stats.flow_bytes) * 8;
664 0 : int stats_packets_bits_nb = sizeof(fe->fe_stats.flow_packets) * 8;
665 :
666 0 : uint64_t delta_dev_stats_bytes = bytes - flow->stats.bytes;
667 0 : uint64_t delta_dev_stats_packets = hits - flow->stats.packets;
668 :
669 0 : uint64_t new_flow_bytes = delta_dev_stats_bytes + fe->fe_stats.flow_bytes;
670 0 : uint64_t new_flow_packets = delta_dev_stats_packets + fe->fe_stats.flow_packets;
671 :
672 0 : fe->fe_stats.flow_bytes = new_flow_bytes;
673 0 : fe->fe_stats.flow_bytes_oflow += new_flow_bytes >> stats_bytes_bits_nb;
674 0 : fe->fe_stats.flow_packets = new_flow_packets;
675 0 : fe->fe_stats.flow_packets_oflow += new_flow_packets >> stats_packets_bits_nb;
676 :
677 0 : flow->stats.bytes = bytes;
678 0 : flow->stats.packets = hits;
679 :
680 0 : return 0;
681 : }
682 :
683 : static int
684 0 : vr_dpdk_n3k_offload_flow_iter(struct vr_n3k_offload_flow **flow, uint32_t *next)
685 : {
686 0 : if (!flows)
687 0 : return -EINVAL;
688 :
689 0 : while (*next < flows_count && flows[*next] == NULL)
690 0 : (*next)++;
691 :
692 0 : if (*next >= flows_count)
693 0 : return -ENOENT;
694 :
695 0 : *flow = flows[*next];
696 0 : (*next)++;
697 0 : return 0;
698 : }
699 :
700 : static int
701 0 : vr_dpdk_n3k_offload_flow_update_with_offload_entry(
702 : struct vr_n3k_offload_flow *flow,
703 : struct vr_n3k_offload_entry *offload_entry,
704 : size_t flow_handle_id)
705 : {
706 : struct vr_n3k_rte_flow_package flow_package =
707 0 : vr_dpdk_n3k_offload_entry_to_rte_flow(offload_entry);
708 0 : if (flow_package.error) {
709 0 : RTE_LOG(ERR, VROUTER,
710 : "%s(): vr_dpdk_n3k_offload_entry_to_rte_flow "
711 : "failed: %d\n",
712 : __func__, flow_package.error);
713 0 : return -1;
714 : }
715 :
716 0 : if (!flow->handles) {
717 0 : RTE_LOG(ERR, VROUTER,
718 : "%s(): flow->handles is NULL\n",
719 : __func__);
720 0 : return -ENOENT;
721 : }
722 :
723 0 : if (flow->handles[flow_handle_id]) {
724 0 : vr_dpdk_n3k_offload_destroy_hw_flow(
725 0 : flow->handles[flow_handle_id], flow->hw_port_id, flow->id);
726 0 : flow->handles[flow_handle_id] = NULL;
727 : }
728 :
729 : /* Offload updated flow */
730 : struct rte_flow_error error;
731 : static struct rte_flow_attr attr = {
732 : .ingress = 1,
733 : };
734 :
735 0 : flow->hw_port_id = vr_dpdk_n3k_offload_flow_get_underlying_hw_port_id(
736 0 : vif_port_id(offload_entry->src_vif));
737 :
738 0 : flow->handles[flow_handle_id] = rte_flow_create(flow->hw_port_id, &attr,
739 0 : flow_package.pattern,
740 0 : flow_package.actions, &error);
741 :
742 0 : if (flow->handles[flow_handle_id] == NULL) {
743 0 : RTE_LOG(ERR, VROUTER,
744 : "%s(): Failed to create flow %d: [%d]: %s\n",
745 : __func__, flow->id, error.type,
746 : error.message ? error.message
747 : : "No error message");
748 0 : return -1;
749 : }
750 :
751 0 : vr_dpdk_n3k_packet_metadata_mark_used(flow);
752 :
753 0 : return 0;
754 : }
755 :
756 : int
757 0 : vr_dpdk_n3k_offload_flow_update_unlocked(
758 : struct vr_n3k_offload_flow *flow)
759 : {
760 0 : size_t i = 0;
761 : int ret;
762 : struct vr_n3k_offload_entry pre_offload_entry;
763 0 : ret = vr_dpdk_n3k_fill_offload_entry_partial_start(flow, &pre_offload_entry);
764 0 : if (ret)
765 0 : return ret;
766 :
767 0 : const struct vr_nexthop *snh = pre_offload_entry.src_nh;
768 :
769 0 : if (snh->nh_type == NH_COMPOSITE && snh->nh_component_cnt != flow->handle_count) {
770 0 : RTE_LOG(WARNING, VROUTER, "%s(): Source nexthop %d changed # of components, cannot update\n",
771 : __func__, snh->nh_id);
772 : /* TODO in this we should just reallocate flow->handles array. This
773 : * array most likely contains just NULL handles (it was initialized in
774 : * the original flow_set, but actual flow handle creation failed
775 : * because of missing mirror) */
776 0 : return -EINVAL;
777 : }
778 :
779 0 : for (i = 0; i < flow->handle_count; ++i) {
780 0 : struct vr_n3k_offload_entry offload_entry = pre_offload_entry;
781 :
782 0 : if (snh->nh_type == NH_COMPOSITE) {
783 : uint32_t component_nh_idx;
784 0 : ret = vr_dpdk_n3k_offload_nexthop_get_cnh_idx(snh, i, &component_nh_idx);
785 0 : if (ret == -ENOENT) {
786 0 : RTE_LOG(ERR, VROUTER, "%s(): Failed to get component_nh[%zu], nh is NULL\n",
787 : __func__, i);
788 0 : continue;
789 : }
790 0 : offload_entry.src_nh = vr_dpdk_n3k_offload_nexthop_get(component_nh_idx);
791 0 : if (offload_entry.src_nh == NULL) {
792 0 : RTE_LOG(ERR, VROUTER, "%s(): Failed to get component_nh[%zu], nh_idx=%d\n",
793 : __func__, i, component_nh_idx);
794 0 : continue;
795 : }
796 :
797 0 : ret = vr_dpdk_n3k_offload_nexthop_get_cnh_label(snh, i, &offload_entry.tunnel_label);
798 0 : if (ret) {
799 0 : RTE_LOG(ERR, VROUTER, "%s(): getting composite nexthop's label failed: %d\n",
800 : __func__, ret);
801 0 : return ret;
802 : }
803 : }
804 :
805 0 : if (vr_dpdk_n3k_fill_offload_entry_vifs(&offload_entry) != 0)
806 0 : continue;
807 :
808 0 : if (vr_dpdk_n3k_fill_offload_entry_partial_end(flow, &offload_entry) != 0)
809 0 : continue;
810 :
811 0 : vr_dpdk_n3k_offload_flow_update_with_offload_entry(flow, &offload_entry, i);
812 :
813 : }
814 0 : return 0;
815 : }
816 :
817 : int
818 0 : vr_dpdk_n3k_offload_flow_vif_update_unlocked(const struct vr_interface *vif)
819 : {
820 0 : struct vr_n3k_offload_flow *flow = NULL;
821 0 : uint32_t i = 0;
822 0 : size_t j = 0;
823 0 : int ret = -1;
824 :
825 0 : while (vr_dpdk_n3k_offload_flow_iter(&flow, &i) >= 0) {
826 : struct vr_n3k_offload_entry pre_offload_entry;
827 0 : if (vr_dpdk_n3k_fill_offload_entry_partial_start(flow, &pre_offload_entry) != 0)
828 0 : continue;
829 :
830 0 : const struct vr_nexthop *snh = pre_offload_entry.src_nh;
831 :
832 0 : if (snh->nh_type == NH_COMPOSITE && snh->nh_component_cnt != flow->handle_count) {
833 0 : RTE_LOG(WARNING, VROUTER, "%s(): Source nexthop %d changed # of components, cannot update\n",
834 : __func__, snh->nh_id);
835 0 : continue;
836 : }
837 :
838 0 : for (j = 0; j < flow->handle_count; ++j) {
839 0 : struct vr_n3k_offload_entry offload_entry = pre_offload_entry;
840 :
841 0 : if (snh->nh_type == NH_COMPOSITE) {
842 : uint32_t component_nh_idx;
843 0 : ret = vr_dpdk_n3k_offload_nexthop_get_cnh_idx(snh, i, &component_nh_idx);
844 0 : if (ret == -ENOENT) {
845 0 : RTE_LOG(ERR, VROUTER, "%s(): Failed to get component_nh[%u], nh is NULL\n",
846 : __func__, i);
847 0 : continue;
848 : }
849 0 : offload_entry.src_nh = vr_dpdk_n3k_offload_nexthop_get(component_nh_idx);
850 0 : if (offload_entry.src_nh == NULL) {
851 0 : RTE_LOG(ERR, VROUTER, "%s(): Failed to get component_nh[%u], nh_idx=%d\n",
852 : __func__, i, component_nh_idx);
853 0 : continue;
854 : }
855 :
856 0 : ret = vr_dpdk_n3k_offload_nexthop_get_cnh_label(snh, i, &offload_entry.tunnel_label);
857 0 : if (ret) {
858 0 : RTE_LOG(ERR, VROUTER, "%s(): getting composite nexthop's label failed: %d\n",
859 : __func__, ret);
860 0 : return ret;
861 : }
862 : }
863 :
864 0 : if (vr_dpdk_n3k_fill_offload_entry_vifs(&offload_entry) != 0)
865 0 : continue;
866 :
867 0 : if (offload_entry.src_vif != vif &&
868 0 : offload_entry.dst_vif != vif)
869 0 : continue;
870 :
871 0 : if (vr_dpdk_n3k_fill_offload_entry_partial_end(flow, &offload_entry) != 0)
872 0 : continue;
873 :
874 0 : ret = vr_dpdk_n3k_offload_flow_update_with_offload_entry(flow, &offload_entry, j);
875 0 : if (ret) {
876 0 : RTE_LOG(ERR, VROUTER,
877 : "%s(): Failed to update flow %d\n",
878 : __func__, flow->id);
879 0 : return ret;
880 : }
881 : }
882 : }
883 :
884 0 : return 0;
885 : }
886 :
887 : void
888 0 : vr_dpdk_n3k_offload_flow_reset_unlocked()
889 : {
890 0 : RTE_LOG(DEBUG, VROUTER, "%s() called\n", __func__);
891 : int i;
892 : struct vr_n3k_offload_flow* flow;
893 0 : for (i = 0; i < flows_count; ++i) {
894 0 : flow = flows[i];
895 0 : if (flow == NULL)
896 0 : continue;
897 :
898 0 : vr_dpdk_n3k_offload_hold_del_flow(flow);
899 0 : vr_dpdk_n3k_offload_flow_del_internal(flow);
900 :
901 0 : rte_free(flows[i]);
902 0 : flows[i] = NULL;
903 : }
904 0 : }
|
