Line data Source code
1 : /*
2 : * vr_offloads.c -- datapath flow offloads management
3 : *
4 : * Copyright 2018 Mellanox Technologies, Ltd
5 : */
6 : #include <vrouter.h>
7 : #include <vr_offloads_dp.h>
8 : #include <vr_btable.h>
9 : #include <vr_packet.h>
10 :
11 : extern unsigned int vr_interfaces;
12 : extern unsigned int vr_flow_entries;
13 : extern unsigned int vr_oflow_entries;
14 :
15 : struct vr_interface *pvif;
16 : unsigned int host_ip;
17 : struct vr_btable *offload_flows;
18 : struct vr_btable *offload_tags;
19 : unsigned int datapath_offloads;
20 :
21 : static int
22 0 : vr_offloads_interface_add(struct vr_interface *vif)
23 : {
24 0 : if (!vif)
25 0 : return -EINVAL;
26 0 : if (vif->vif_type == VIF_TYPE_PHYSICAL) {
27 0 : if (pvif && pvif != vif)
28 0 : vr_printf("offload: More than one physical interface\n");
29 : else
30 0 : pvif = vif;
31 0 : } else if (vif->vif_type == VIF_TYPE_HOST) {
32 0 : if (host_ip && vif->vif_ip != host_ip)
33 0 : vr_printf("offload: More than one host interface\n");
34 : else
35 0 : host_ip = vif->vif_ip;
36 : }
37 0 : return 0;
38 : }
39 :
40 : static int
41 0 : vr_offloads_interface_del(struct vr_interface *vif)
42 : {
43 0 : if (!vif)
44 0 : return -EINVAL;
45 0 : if (vif->vif_type == VIF_TYPE_PHYSICAL) {
46 0 : if(vif != pvif)
47 0 : vr_printf("offload: More than one physical interface\n");
48 : else
49 0 : pvif = NULL;
50 0 : } else if (vif->vif_type == VIF_TYPE_HOST) {
51 0 : if (vif->vif_ip != host_ip)
52 0 : vr_printf("offload: More than one host interface\n");
53 : else
54 0 : host_ip = 0;
55 : }
56 :
57 0 : return 0;
58 : }
59 :
60 : static bool
61 0 : vr_offloads_is_offloaded_nexthop(struct vr_nexthop *nh)
62 : {
63 0 : return ((nh->nh_flags & NH_FLAG_VALID) && !(nh->nh_flags & NH_FLAG_MCAST) &&
64 0 : (nh->nh_flags & NH_FLAG_POLICY_ENABLED) && (nh->nh_type == NH_ENCAP) &&
65 0 : nh->nh_dev);
66 : }
67 :
68 : static int
69 0 : vr_offloads_mpls_add(struct vr_nexthop *nh, int label)
70 : {
71 : struct vr_offload_tag *otag;
72 :
73 0 : if (!nh)
74 0 : return -EINVAL;
75 0 : if (!vr_offloads_is_offloaded_nexthop(nh))
76 0 : return 0;
77 :
78 0 : otag = (struct vr_offload_tag *)vr_btable_get(offload_tags, nh->nh_dev->vif_idx);
79 0 : if (!otag) {
80 0 : vr_printf("offload: Invalid tag for nexthop ID %u\n",nh->nh_id);
81 0 : return 0;
82 : }
83 :
84 0 : if (nh->nh_family == AF_BRIDGE && (nh->nh_flags & NH_FLAG_L2_CONTROL_DATA))
85 0 : otag += VR_OFFLOADS_TAG_TYPE_MPLS_L2;
86 0 : else if (nh->nh_family != AF_BRIDGE)
87 0 : otag += VR_OFFLOADS_TAG_TYPE_MPLS_L3;
88 : else
89 0 : return 0;
90 :
91 0 : if (!otag->nh) {
92 0 : otag->tag = label;
93 0 : otag->nh = nh;
94 0 : } else if (otag->tag != label) {
95 0 : vr_printf("offload: 2 different MPLS labels (%u,%u) point to the same"
96 : " tag type of vif %u nexthop %u\n", otag->tag, label,
97 0 : nh->nh_dev->vif_idx, nh->nh_id);
98 : }
99 :
100 0 : return 0;
101 : }
102 :
103 : static int
104 0 : vr_offloads_vxlan_add(struct vr_nexthop * nh, int vnid)
105 : {
106 : struct vr_offload_tag *otag;
107 :
108 0 : if (!nh)
109 0 : return -EINVAL;
110 0 : if (!vr_offloads_is_offloaded_nexthop(nh))
111 0 : return 0;
112 :
113 0 : otag = (struct vr_offload_tag *)vr_btable_get(offload_tags, nh->nh_dev->vif_idx);
114 0 : if (!otag) {
115 0 : vr_printf("offload: Invalid tag for nexthop ID %u\n",nh->nh_id);
116 0 : return 0;
117 : }
118 :
119 0 : otag += VR_OFFLOADS_TAG_TYPE_VXLAN;
120 :
121 0 : if (!otag->nh) {
122 0 : otag->tag = vnid;
123 0 : otag->nh = nh;
124 0 : } else if (otag->tag != vnid) {
125 0 : vr_printf("offload: 2 different VXLAN VNIs (%u,%u) point to the same"
126 : " tag type of vif %u nexthop %u\n", otag->tag, vnid,
127 0 : nh->nh_dev->vif_idx, nh->nh_id);
128 : }
129 :
130 0 : return 0;
131 : }
132 :
133 : static int
134 0 : vr_offloads_mpls_del(int label)
135 : {
136 : struct vr_offload_tag *otag;
137 0 : struct vrouter *router = vrouter_get(0);
138 0 : struct vr_nexthop *nh = __vrouter_get_label(router, label);
139 :
140 0 : if (!nh)
141 0 : return -EINVAL;
142 :
143 0 : if (!vr_offloads_is_offloaded_nexthop(nh))
144 0 : return 0;
145 :
146 0 : otag = (struct vr_offload_tag *)vr_btable_get(offload_tags, nh->nh_dev->vif_idx);
147 0 : if (!otag) {
148 0 : vr_printf("offload: Invalid tag for nexthop ID %u\n",nh->nh_id);
149 0 : return 0;
150 : }
151 :
152 0 : if (nh->nh_family == AF_BRIDGE && (nh->nh_flags & NH_FLAG_L2_CONTROL_DATA))
153 0 : otag += VR_OFFLOADS_TAG_TYPE_MPLS_L2;
154 0 : else if (nh->nh_family != AF_BRIDGE)
155 0 : otag += VR_OFFLOADS_TAG_TYPE_MPLS_L3;
156 : else
157 0 : return 0;
158 :
159 0 : if (otag->nh && otag->tag == label)
160 0 : memset(otag, 0, sizeof(*otag));
161 : else
162 0 : vr_printf("offload: Unexpected tag %u for MPLS label %u, nexthop %u\n",
163 : otag->tag, label, nh->nh_id);
164 :
165 0 : return 0;
166 : }
167 :
168 : static int
169 0 : vr_offloads_vxlan_del(int vnid)
170 : {
171 : struct vr_offload_tag *otag;
172 0 : struct vrouter *router = vrouter_get(0);
173 0 : struct vr_nexthop *nh = (struct vr_nexthop *)vr_itable_get(router->vr_vxlan_table, vnid);
174 :
175 0 : if (!nh)
176 0 : return -EINVAL;
177 :
178 0 : if (!vr_offloads_is_offloaded_nexthop(nh))
179 0 : return 0;
180 :
181 0 : otag = (struct vr_offload_tag *)vr_btable_get(offload_tags, nh->nh_dev->vif_idx);
182 0 : if (!otag) {
183 0 : vr_printf("offload: Invalid tag for nexthop ID %u\n",nh->nh_id);
184 0 : return 0;
185 : }
186 :
187 0 : otag += VR_OFFLOADS_TAG_TYPE_VXLAN;
188 :
189 0 : if (otag->nh && otag->tag == vnid)
190 0 : memset(otag, 0, sizeof(*otag));
191 : else
192 0 : vr_printf("offload: Unexpected tag %u for VXLAN vni %u, nexthop %u\n",
193 : otag->tag, vnid, nh->nh_id);
194 :
195 0 : return 0;
196 : }
197 :
198 : static int
199 0 : vr_offloads_flow_del(struct vr_flow_entry * fe)
200 : {
201 : struct vr_offload_flow *oflow;
202 0 : int ret = 0;
203 :
204 0 : if (!fe)
205 0 : return -EINVAL;
206 :
207 0 : if (!offload_flows)
208 0 : return 0;
209 :
210 0 : oflow = (struct vr_offload_flow *)vr_btable_get(offload_flows,
211 : fe->fe_hentry.hentry_index);
212 0 : if (!oflow) {
213 0 : vr_printf("offload: Invalid tag for flow ID %u\n",
214 : fe->fe_hentry.hentry_index);
215 0 : return 0;
216 : }
217 :
218 0 : if(oflow->flow_handle) {
219 0 : ret = vr_offload_flow_destroy(oflow);
220 0 : if (!ret)
221 0 : memset(oflow, 0, sizeof(*oflow));
222 : else
223 0 : vr_printf("offload: Failed to destroy flow ID %u\n", oflow->fe_index);
224 : }
225 :
226 0 : return 0;
227 : }
228 :
229 : static int
230 0 : vr_offloads_flow_set(struct vr_flow_entry * fe, unsigned int fe_index,
231 : struct vr_flow_entry * rfe)
232 : {
233 : struct vr_nexthop *nh, *snh;
234 : struct vr_offload_flow *oflow;
235 : struct vr_offload_tag *otag;
236 0 : struct vrouter *router = vrouter_get(0);
237 0 : int ret = 0;
238 :
239 0 : if (!fe)
240 0 : return -EINVAL;
241 :
242 0 : if (!pvif || !host_ip) {
243 0 : vr_printf("offload: Missing physical/host interface\n");
244 0 : return 0;
245 : }
246 :
247 0 : oflow = (struct vr_offload_flow *)vr_btable_get(offload_flows, fe_index);
248 0 : if (!oflow) {
249 0 : vr_printf("offload: Invalid tag for flow ID %u\n", fe_index);
250 0 : return 0;
251 : }
252 :
253 : /* For flow change do destroy and create */
254 0 : if (oflow->flow_handle) {
255 0 : ret = vr_offload_flow_destroy(oflow);
256 0 : if (!ret) {
257 0 : memset(oflow, 0, sizeof(*oflow));
258 : } else {
259 0 : vr_printf("offload: Failed to change flow ID %u\n", oflow->fe_index);
260 0 : return 0;
261 : }
262 : }
263 :
264 0 : nh = __vrouter_get_nexthop(router, fe->fe_key.flow_nh_id);
265 0 : if (!nh)
266 0 : return -EINVAL;
267 :
268 0 : if (!(fe->fe_flags & VR_FLOW_FLAG_ACTIVE) || fe->fe_action != VR_FLOW_ACTION_FORWARD ||
269 0 : !vr_offloads_is_offloaded_nexthop(nh))
270 : /* Not a valid flow to be offloaded */
271 0 : return 0;
272 :
273 0 : snh = __vrouter_get_nexthop(router, fe->fe_src_nh_index);
274 0 : if (!snh)
275 0 : return -EINVAL;
276 :
277 0 : if (snh->nh_type != NH_TUNNEL || !(snh->nh_flags & NH_FLAG_VALID))
278 : /* Not a valid flow to be offloaded */
279 0 : return 0;
280 :
281 0 : otag = (struct vr_offload_tag *)vr_btable_get(offload_tags, nh->nh_dev->vif_idx);
282 0 : if (!otag) {
283 0 : vr_printf("offload: Invalid tag for nexthop ID %u\n", nh->nh_id);
284 0 : return 0;
285 : }
286 :
287 0 : if (snh->nh_flags & NH_FLAG_TUNNEL_GRE) {
288 0 : if (otag[VR_OFFLOADS_TAG_TYPE_MPLS_L2].nh && (nh->nh_dev->vif_flags &
289 : VIF_FLAG_L2_ENABLED)) {
290 0 : oflow->tunnel_type = NH_FLAG_TUNNEL_GRE;
291 0 : oflow->tunnel_tag =otag[VR_OFFLOADS_TAG_TYPE_MPLS_L2].tag;
292 0 : oflow->nh = otag[VR_OFFLOADS_TAG_TYPE_MPLS_L2].nh;
293 0 : oflow->is_mpls_l2 = true;
294 0 : } else if (otag[VR_OFFLOADS_TAG_TYPE_MPLS_L3].nh && (nh->nh_dev->vif_flags &
295 : VIF_FLAG_L3_ENABLED)) {
296 0 : oflow->tunnel_type = NH_FLAG_TUNNEL_GRE;
297 0 : oflow->tunnel_tag = otag[VR_OFFLOADS_TAG_TYPE_MPLS_L3].tag;
298 0 : oflow->nh = otag[VR_OFFLOADS_TAG_TYPE_MPLS_L3].nh;
299 0 : oflow->is_mpls_l2 = false;
300 : }
301 0 : } else if (snh->nh_flags & NH_FLAG_TUNNEL_UDP_MPLS) {
302 0 : if (otag[VR_OFFLOADS_TAG_TYPE_MPLS_L2].nh && (nh->nh_dev->vif_flags &
303 : VIF_FLAG_L2_ENABLED)) {
304 0 : oflow->tunnel_type = NH_FLAG_TUNNEL_UDP_MPLS;
305 0 : oflow->tunnel_tag =otag[VR_OFFLOADS_TAG_TYPE_MPLS_L2].tag;
306 0 : oflow->nh = otag[VR_OFFLOADS_TAG_TYPE_MPLS_L2].nh;
307 0 : oflow->is_mpls_l2 = true;
308 0 : } else if (otag[VR_OFFLOADS_TAG_TYPE_MPLS_L3].nh && (nh->nh_dev->vif_flags &
309 : VIF_FLAG_L3_ENABLED)) {
310 0 : oflow->tunnel_type = NH_FLAG_TUNNEL_UDP_MPLS;
311 0 : oflow->tunnel_tag = otag[VR_OFFLOADS_TAG_TYPE_MPLS_L3].tag;
312 0 : oflow->nh = otag[VR_OFFLOADS_TAG_TYPE_MPLS_L3].nh;
313 0 : oflow->is_mpls_l2 = false;
314 : }
315 0 : } else if (snh->nh_flags & NH_FLAG_TUNNEL_VXLAN) {
316 0 : if (otag[VR_OFFLOADS_TAG_TYPE_VXLAN].nh) {
317 0 : oflow->tunnel_type = NH_FLAG_TUNNEL_VXLAN;
318 0 : oflow->tunnel_tag = otag[VR_OFFLOADS_TAG_TYPE_VXLAN].tag;
319 0 : oflow->nh = otag[VR_OFFLOADS_TAG_TYPE_VXLAN].nh;
320 : }
321 : } else {
322 : /* Not a valid flow to be offloaded */
323 0 : return 0;
324 : }
325 :
326 0 : if (!oflow->nh) {
327 0 : vr_printf("offload: Invalid tag type for flow create\n");
328 0 : return 0;
329 : }
330 :
331 0 : oflow->pvif = pvif;
332 0 : oflow->fe = fe;
333 0 : oflow->ip = host_ip;
334 0 : oflow->fe_index = fe_index;
335 :
336 0 : ret = vr_offload_flow_create(oflow);
337 0 : if (ret) {
338 0 : vr_printf("offload: Failed to create flow ID %u\n", fe_index);
339 0 : memset(oflow, 0, sizeof(*oflow));
340 : }
341 :
342 0 : return 0;
343 : }
344 :
345 : struct vr_offload_ops vr_offload_ops = {
346 : .voo_flow_set = vr_offloads_flow_set,
347 : .voo_flow_del = vr_offloads_flow_del,
348 : .voo_interface_add = vr_offloads_interface_add,
349 : .voo_interface_del = vr_offloads_interface_del,
350 : .voo_mpls_add = vr_offloads_mpls_add,
351 : .voo_mpls_del = vr_offloads_mpls_del,
352 : .voo_vxlan_add = vr_offloads_vxlan_add,
353 : .voo_vxlan_del = vr_offloads_vxlan_del,
354 : };
355 :
356 : int
357 53 : vr_offloads_init(struct vrouter *router)
358 : {
359 : unsigned int entry_size;
360 : struct vr_offload_ops *offload;
361 :
362 53 : if (!datapath_offloads)
363 53 : return 0;
364 :
365 : /* Do not initialize twice. E.g. a soft reset would not have unregistered
366 : * the offloads. */
367 0 : offload = vr_rcu_dereference(offload_ops);
368 0 : if (offload)
369 0 : return 0;
370 :
371 0 : if (!vr_offload_flow_destroy || !vr_offload_flow_create ||
372 0 : !vr_offload_prepare) {
373 : /* Not an error necessarily. Offloads are not implemented for this host
374 : * type, so don't register anything. External implementation can still
375 : * be registered after initialization. */
376 0 : vr_printf("offload: no built-in offload implementation for current context\n");
377 0 : return 0;
378 : }
379 :
380 0 : if (!offload_tags) {
381 : /* Round up to the next divisor */
382 0 : for (entry_size = sizeof(struct vr_offload_tag) * VR_OFFLOADS_TAG_TYPE_MAX;
383 0 : VR_SINGLE_ALLOC_LIMIT % entry_size; entry_size++);
384 0 : offload_tags = vr_btable_alloc(vr_interfaces, entry_size);
385 0 : if (!offload_tags) {
386 0 : return -ENOMEM;
387 : }
388 : }
389 :
390 0 : if (!offload_flows) {
391 : /* Round up to the next divisor */
392 0 : for (entry_size = sizeof(struct vr_offload_flow);
393 0 : VR_SINGLE_ALLOC_LIMIT % entry_size; entry_size++);
394 0 : offload_flows = vr_btable_alloc(vr_flow_entries + vr_oflow_entries, entry_size);
395 0 : if (!offload_flows) {
396 0 : vr_btable_free(offload_tags);
397 0 : offload_tags = NULL;
398 0 : return -ENOMEM;
399 : }
400 : }
401 :
402 0 : vr_offload_register(&vr_offload_ops);
403 :
404 0 : return 0;
405 :
406 : }
407 :
408 : static void
409 53 : _vr_offloads_exit(struct vrouter *router, bool soft_reset)
410 : {
411 : struct vr_offload_flow *oflow;
412 : struct vr_offload_tag *otag;
413 : unsigned int i;
414 : unsigned int entry_num, entry_size;
415 :
416 53 : if (!datapath_offloads)
417 53 : return;
418 :
419 0 : if (!vr_offload_flow_destroy || !vr_offload_flow_create ||
420 0 : !vr_offload_prepare)
421 0 : return;
422 :
423 0 : if (offload_flows) {
424 0 : entry_num = vr_btable_entries(offload_flows);
425 0 : entry_size = vr_btable_size(offload_flows) / entry_num;
426 0 : for (i = 0; i < entry_num; i++) {
427 0 : oflow = (struct vr_offload_flow *)vr_btable_get(offload_flows, i);
428 0 : if (!oflow)
429 0 : continue;
430 0 : if(oflow->flow_handle)
431 0 : vr_offload_flow_destroy(oflow);
432 0 : memset(oflow, 0, entry_size);
433 : }
434 :
435 0 : if (!soft_reset) {
436 0 : vr_btable_free(offload_flows);
437 0 : offload_flows = NULL;
438 : }
439 : }
440 :
441 0 : if (offload_tags) {
442 0 : entry_num = vr_btable_entries(offload_tags);
443 0 : entry_size = vr_btable_size(offload_tags) / entry_num;
444 0 : for (i = 0; i < entry_num; i++) {
445 0 : otag = (struct vr_offload_tag *)vr_btable_get(offload_tags, i);
446 0 : if (otag)
447 0 : memset(otag, 0, entry_size);
448 : }
449 :
450 0 : if (!soft_reset) {
451 0 : vr_btable_free(offload_tags);
452 0 : offload_tags = NULL;
453 : }
454 : }
455 :
456 0 : pvif = NULL;
457 0 : host_ip = 0;
458 : }
459 :
460 : inline struct vr_offload_flow *
461 0 : vr_offloads_flow_get(unsigned int index)
462 : {
463 : struct vr_offload_flow *oflow = (struct vr_offload_flow *)
464 0 : vr_btable_get(offload_flows, index);
465 :
466 0 : if (!oflow || !oflow->flow_handle)
467 : /* An invalid packet flow */
468 0 : return NULL;
469 :
470 0 : return oflow;
471 : }
472 :
473 : /*
474 : * Called by an external offload module to register itself with vrouter.
475 : */
476 : int
477 0 : vr_offload_register(const struct vr_offload_ops *new_handler)
478 : {
479 : struct vr_offload_ops *offload;
480 :
481 0 : if (!datapath_offloads || !new_handler)
482 0 : return -EINVAL;
483 :
484 0 : offload = vr_rcu_dereference(offload_ops);
485 0 : if (offload)
486 0 : return -EBUSY;
487 :
488 0 : offload = vr_malloc(sizeof(*offload), VR_MALLOC_OBJECT);
489 0 : if (!offload)
490 0 : return -ENOMEM;
491 0 : *offload = *new_handler;
492 0 : vr_rcu_assign_pointer(offload_ops, offload);
493 0 : vr_synchronize_rcu();
494 :
495 0 : return 0;
496 : }
497 : #if defined(__linux__) && defined(__KERNEL__)
498 : EXPORT_SYMBOL(vr_offload_register);
499 : #endif
500 :
501 : /*
502 : * Called by an external offload module to unregister itself with vrouter.
503 : */
504 : static void
505 53 : _vr_offload_unregister(void)
506 : {
507 53 : struct vr_offload_ops *offload = vr_rcu_dereference(offload_ops);
508 :
509 53 : if (offload) {
510 0 : vr_rcu_assign_pointer(offload_ops, NULL);
511 0 : vr_synchronize_rcu();
512 0 : vr_free(offload, VR_MALLOC_OBJECT);
513 : }
514 53 : }
515 :
516 : int
517 0 : vr_offload_unregister(void)
518 : {
519 0 : struct vrouter *router = vrouter_get(0);
520 :
521 0 : _vr_offloads_exit(router, false);
522 0 : _vr_offload_unregister();
523 :
524 0 : return 0;
525 : }
526 : #if defined(__linux__) && defined(__KERNEL__)
527 : EXPORT_SYMBOL(vr_offload_unregister);
528 : #endif
529 :
530 : void
531 53 : vr_offloads_exit(struct vrouter *router, bool soft_reset)
532 : {
533 53 : _vr_offloads_exit(router, soft_reset);
534 53 : if (!soft_reset)
535 53 : _vr_offload_unregister();
536 53 : }
537 :
538 : /* Statistics update functions used by offload module */
539 : #if defined(__linux__) && defined(__KERNEL__)
540 : EXPORT_SYMBOL(vr_flow_incr_stats);
541 : EXPORT_SYMBOL(vr_nexthop_update_offload_vrfstats);
542 : #endif
|
