Line data Source code
1 : /*
2 : * Copyright (c) 2015 Juniper Networks, Inc. All rights reserved.
3 : */
4 : #ifndef __AGENT_FLOW_TABLE_MGMT_H__
5 : #define __AGENT_FLOW_TABLE_MGMT_H__
6 :
7 : #include <boost/scoped_ptr.hpp>
8 : #include "pkt/flow_table.h"
9 : #include <pkt/flow_mgmt/flow_mgmt_dbclient.h>
10 : #include <pkt/flow_mgmt/flow_mgmt_tree.h>
11 : #include "pkt/flow_event.h"
12 :
13 : ////////////////////////////////////////////////////////////////////////////
14 : // Flow Management module is responsible to keep flow action in-sync with
15 : // changes to operational-db changes.
16 : //
17 : // Flow Management module tracks following,
18 : // 1. Changes to fields of interest in the operational-db
19 : // 2. Flows dependenent on operational-db entries
20 : //
21 : // On any change to an operational-db, it will ensure the dependent flows are
22 : // revaluated to keep the flow action in-sync with latest operational state
23 : //
24 : // A flow can depend on following operational-entires,
25 : // 1. Interface : Flow depends on the SG-Id of interface
26 : // 2. VN : Flow depends on policy of VN
27 : // 3. ACL : Flow depends on ACL Entries in ACL
28 : // 4. Route : Flow depends on the SG-Id list in the route for both source
29 : // and destination addresses
30 : // 5. NextHop : Flow depends on NH for RPF checks and for ECMP Component index
31 : //
32 : // An operational entry can potentially have many flows dependent on it. On
33 : // change to an operational entry if flows are processed inline, it can lead to
34 : // significant latency. The flow management module runs from a work-queue
35 : // (flow-manager queue) to avoid the latency.
36 : //
37 : // The Flow Management module is responsible only to track the dependency
38 : // between flows and operational entries. It does not *own* the flow entries.
39 : // When flow management module identifies that a flow has to be revaluated
40 : // it will invoke flow-table APIs.
41 : //
42 : // There are two sub-modules,
43 : // - Flow Management DBClient
44 : // This module registers to DB notification for all tables of interest. It
45 : // enqueues events for flow revaluation in response to DBTable changes
46 : // - Flow Management Tree module
47 : // This module maintains different data-structures to track flows affected
48 : // by change to a DBEntry. Its also responsible to trigger revaluation of
49 : // flows on change to a DBEntry
50 : //
51 : // There are 2 work-queues defined for flow-management,
52 : // - Flow Management Request
53 : // All request events to flow-management module are enqueued thru this.
54 : // The Flow Management Tree module acts on events on this queue.
55 : // * Add of FLow
56 : // * Delete of Flow
57 : // * Add of DBEntry
58 : // * Change of DBEntry
59 : // * Delete of DBEntry
60 : // * Export of Flow
61 : //
62 : // - Flow Events
63 : // Flow Management Tree module may generate events in response to
64 : // requests. The response events are enqueued here. Example events are,
65 : // * Flow revaluation in response to DBEntry change
66 : // * Flow revaluation in response to DBEntry Add/Delete
67 : // * Flow deletion in response to DBEntry delete
68 : //
69 : // Workflow for flow manager module is given below,
70 : // 1. Flow Table module will enqueue message to Flow Management queue on
71 : // add/delete/change of a flow. On Flow delete event, Flow Table module will
72 : // will also enqueue export of the flow.
73 : // 2. Flow stats collection module will enqueue message to Flow Management
74 : // queue on export of flow
75 : // 3. Flow Management module builds the following tracking information
76 : // - Operational entry to list of dependent flows
77 : // - Flow entry to list of operational-entries it is dependent on
78 : // 4. DBClient module registers to DBTables of interest and tracks changes to
79 : // operational-db entries
80 : // 5. Flow Table module will enqueue a message to Flow Management queue on
81 : // add/delete/change of operational entries
82 : // 6. The action in flow-management module for operational entry events will
83 : // depend on the operational entry type
84 : //
85 : // VN Add/Change : Revaluate flows for change in policy
86 : // VN Delete : Delete dependent flows
87 : // VMI Add/Change : Revaluate flows for change in SG entry in VMI
88 : // Interface Delete : Delete dependent flows
89 : // NH Add/Change : Revaluate flows for change in RPF-NH
90 : // NH Delete : Delete dependent flows
91 : // ACL Add/Change : Revaluate flows for change in ACL rules
92 : // ACL Delete : Delete dependent flows
93 : // Bridge Add/Change : Flow entries are revaluated for change in SG
94 : // Bridge Delete : Delete dependent flows
95 : // Inet Route Add : Add of an inet route can potentially alter the route
96 : // used for a route. It can also potentially alter
97 : // other attributes such as floating-ip used etc...
98 : // So, all flows dependent on the "covering route" are
99 : // enqueued for revluation by PktFlowInfo module
100 : // Inet Route Change : Revluate for for change in RPF-NH, VN and SG
101 : // Inet Route Delete : Flows depending on this route will start using the
102 : // covering route. It can also potentially alter
103 : // other attributes such as floating-ip used etc...
104 : // So, all flows dependent on the "covering route" are
105 : // enqueued for revluation by PktFlowInfo module
106 : //
107 : // Concurrency and references
108 : // The Flow Management module can often lead to compute spikes. To ensure this
109 : // doesnt affect flow setup rates, it is preferable to run flow-management
110 : // module in parallel to both DBTable and Flow setup/teardown task.
111 : //
112 : // Since flow-management module runs in parallel to DBTable/Flow tasks, we must
113 : // ensure that flows and the operational dbentries are not deleted till
114 : // flow-management module references are gone. This is achieved by following,
115 : //
116 : // Flow reference
117 : // --------------
118 : // 1. All message between flow-management and flow-table/flow-stats module will
119 : // hold object references. This will ensure ref-count for object dont drop
120 : // till messages are processed.
121 : // 2. Each flow seen by flow-management will hold a reference to FlowEntryInfo
122 : // in FlowEntry as flow_mgmt_info_
123 : //
124 : // Per FlowEntry mutex is used to synchronize access to same Flow between
125 : // FlowTable and Flow Management module
126 : //
127 : // DBEntry reference
128 : // -----------------
129 : // Flow Management module will rely on DBState set by DBClient module to
130 : // ensure a DBEntry is not deleted till all its flows are deleted.
131 : //
132 : // Flow Management DBClient Flow Management module
133 : //
134 : // 1. On DBEntry deletion, enqueue a
135 : // message to Flow Management module
136 : // 2. Find all dependent flows and
137 : // enqueue flow-delete request for
138 : // all flows. Mark the entry as
139 : // deleted
140 : // 3. Delete the flow and enqueue a
141 : // flow-deleted message to
142 : // flow-management module
143 : // 4. Find all operational entries the
144 : // flow is dependent on and remove
145 : // the flow from their dependent tree.
146 : // If an operational entry is deleted
147 : // and there are no more dependent
148 : // flows, enqueue a message to delete
149 : // the operational entry
150 : // 5. Remove DBState for the DBEntry
151 : //
152 : // RouteTable reference
153 : // -----------------
154 : // The Inet and Bridge route tables should not be deleted till all flows using
155 : // them are deleted. Flow Management module holds Lifetime reference to ensure
156 : // DBTables are deleted only after all its flows are deleted
157 : //
158 : // VRF Reference
159 : // -------------
160 : // When a VRF add is seen, the DBClient sub-module will register for Inet and
161 : // Bridge route tables for the VRF. The unregister for these tables should
162 : // happen only after all flows on them are deleted. The Flow Management module
163 : // will enqueue delete of VRF only after flow entries in all Inet and Bridge
164 : // routes are deleted
165 : //
166 : // Additional functionality:
167 : // -------------------------
168 : // A few more additional functionality is pushed to flow-management module
169 : // 1. Logging of flows
170 : // 2. Flow UVE
171 : // 3. Tracking per VN flows
172 : //
173 : // Duplicate Deletes:
174 : // -----------------
175 : // Consider following sequence of events,
176 : // 1. Add ACL
177 : // 2. Delete ACL
178 : // 3. Delete ACL
179 : //
180 : // Correspondingly, 3 events are enqueued to Flow Management Request queue.
181 : // When event (2) is processed, if ACL does not have any more flows, it will
182 : // result in removing DBState for the ACL and eventual free of ACL before
183 : // event (3) is processed. When event (3) is being processed, it may refer to
184 : // a free'd DBEntry
185 : //
186 : // We use a gen-id to handle this scenario.The gen-id is incremented on every
187 : // Delete notification received for DBEntry. When DELETE event is enqueued, it
188 : // will also carry the gen-id field. The gen-id is also carried in the
189 : // response events generated.
190 : //
191 : // When Flow Management Dbclient module receives FREE_DBENTRY event, it will
192 : // ignore the message if gen-id does not match with latest value.
193 : ////////////////////////////////////////////////////////////////////////////
194 :
195 : ////////////////////////////////////////////////////////////////////////////
196 : // Flow Management module maintains following data structures
197 : //
198 : // - FlowEntryTree : Tree of all flow entries
199 : // FlowEntryPtr : Key for the tree. Holds reference to FlowEntry
200 : // FlowEntryInfo : Data for the tree. Contains tree of DBEntries the flow is
201 : // dependent on.
202 : //
203 : // - FlowMgmtTree : Per operational entry tree. Tracks flow entries dependent
204 : // on the operational entry
205 : //
206 : // An entry into the tree can be added when,
207 : // 1. Flow is added/changed and it refers to the DBEntry
208 : // or
209 : // 2. DBEntry add/change message is got from DBClient
210 : //
211 : // Entry from the tree is removed,
212 : // 1. All flows dependent on the entry are deleted
213 : // AND
214 : // 2. DBEntry delete message is got from DBClient
215 : //
216 : // FlowEntryKey : Key for the tree. Contains DBEntry pointer as key
217 : // FlowMgmtEntry : Data fot the tree. Maintains intrusive-list of
218 : // FlowMgmtKeyNodes(which contains references to Flow entries)
219 : // dependent on the DBEntry
220 : //
221 : // - AclFlowMgmtTree : FlowMgmtTree for ACL
222 : // - InterfaceFlowMgmtTree : FlowMgmtTree for VM-Interfaces
223 : // - VnFlowMgmtTree : FlowMgmtTree for VN
224 : // - InetRouteFlowMgmtTree : FlowMgmtTree for IPv4 routes
225 : // - InetRouteFlowMgmtTree : FlowMgmtTree for IPv6 routes
226 : // - BridgeRouteFlowMgmtTree: FlowMgmtTree for Bridge routes
227 : // - NhFlowMgmtTree : FlowMgmtTree for NH
228 : // - VrfFlowMgmtTree : FlowMgmtTree for VRF. It doesnt track the flows
229 : // for the VRF. But is used to ensure VRF entry is
230 : // not deleted till all route-entries are freed and
231 : // DELETE event for VRF is processed
232 : // - BgpAsAServiceFlowMgmtTree : FlowMgmtTree per control-node. This is
233 : // maintained per control node because VMI can
234 : // establish a bgp peer session with each control
235 : // node.
236 : ////////////////////////////////////////////////////////////////////////////
237 :
238 : class FlowMgmtManager {
239 : public:
240 : typedef boost::shared_ptr<FlowMgmtRequest> FlowMgmtRequestPtr;
241 : typedef WorkQueue<FlowMgmtRequestPtr> FlowMgmtQueue;
242 :
243 : // Comparator for FlowEntryPtr
244 : struct FlowEntryRefCmp {
245 : bool operator()(const FlowEntryPtr &l, const FlowEntryPtr &r) {
246 : FlowEntry *lhs = l.get();
247 : FlowEntry *rhs = r.get();
248 :
249 : return (lhs < rhs);
250 : }
251 : };
252 :
253 : // We want flow to be valid till Flow Management task is complete. So,
254 : // use FlowEntryPtr as key and hold reference to flow till we are done
255 : typedef std::map<FlowEntryPtr, FlowEntryInfo, FlowEntryRefCmp>
256 : FlowEntryTree;
257 :
258 : FlowMgmtManager(Agent *agent, uint16_t table_index);
259 8 : virtual ~FlowMgmtManager() { }
260 :
261 : void Init();
262 : void Shutdown();
263 : static void InitLogQueue(Agent *agent);
264 : static void ShutdownLogQueue();
265 : static void LogFlowUnlocked(FlowEntry *flow, const std::string &op);
266 : static bool LogHandler(FlowMgmtRequestPtr req);
267 :
268 : bool RequestHandler(FlowMgmtRequestPtr req);
269 : bool DBRequestHandler(FlowMgmtRequestPtr req);
270 : static bool ProcessEvent(FlowMgmtRequest *req, FlowMgmtKey *key,
271 : FlowMgmtTree *tree);
272 :
273 : bool DBRequestHandler(FlowMgmtRequest *req, const DBEntry *entry);
274 : bool BgpAsAServiceRequestHandler(FlowMgmtRequest *req);
275 : bool DbClientHandler(const DBEntry *entry);
276 : void EnqueueFlowEvent(FlowEvent *event);
277 : void NonOperEntryEvent(FlowEvent::Event event, FlowEntry *flow);
278 : void DBEntryEvent(FlowEvent::Event event, FlowMgmtKey *key,
279 : FlowEntry *flow);
280 : void FreeDBEntryEvent(FlowEvent::Event event, FlowMgmtKey *key,
281 : uint32_t gen_id);
282 :
283 298 : Agent *agent() const { return agent_; }
284 : uint16_t table_index() const { return table_index_; }
285 : void AddEvent(FlowEntry *low);
286 : void DeleteEvent(FlowEntry *flow, const RevFlowDepParams ¶ms);
287 : void FlowStatsUpdateEvent(FlowEntry *flow, uint32_t bytes, uint32_t packets,
288 : uint32_t oflow_bytes,
289 : const boost::uuids::uuid &u);
290 : void AddDBEntryEvent(const DBEntry *entry, uint32_t gen_id);
291 : void ChangeDBEntryEvent(const DBEntry *entry, uint32_t gen_id);
292 : void DeleteDBEntryEvent(const DBEntry *entry, uint32_t gen_id);
293 : void RouteNHChangeEvent(const DBEntry *entry, uint32_t gen_id);
294 : void RetryVrfDeleteEvent(const VrfEntry *vrf);
295 : void RetryVrfDelete(uint32_t vrf_id);
296 : // Dummy event used for testing
297 : void DummyEvent();
298 :
299 : void VnFlowCounters(const VnEntry *vn,
300 : uint32_t *ingress_flow_count,
301 : uint32_t *egress_flow_count);
302 : void InterfaceFlowCount(const Interface *itf, uint64_t *created,
303 : uint64_t *aged, uint32_t *active_flows);
304 : bool HasVrfFlows(uint32_t vrf);
305 :
306 313 : FlowMgmtDbClient *flow_mgmt_dbclient() const {
307 313 : return flow_mgmt_dbclient_.get();
308 : }
309 :
310 0 : const FlowMgmtQueue *request_queue() const { return &request_queue_; }
311 : const FlowMgmtQueue *log_queue() const { return log_queue_; }
312 1 : void DisableWorkQueue(bool disable) { request_queue_.set_disable(disable); }
313 : void BgpAsAServiceNotify(const boost::uuids::uuid &vm_uuid,
314 : uint32_t source_port);
315 : void BgpAsAServiceHealthCheckNotify(const boost::uuids::uuid &vm_uuid,
316 : uint32_t source_port,
317 : const boost::uuids::uuid &hc_uuid,
318 : bool add);
319 : void EnqueueUveAddEvent(const FlowEntry *flow) const;
320 : void EnqueueUveDeleteEvent(const FlowEntry *flow) const;
321 :
322 : void FlowUpdateQueueDisable(bool val);
323 : size_t FlowUpdateQueueLength();
324 : size_t FlowDBQueueLength();
325 16 : InetRouteFlowMgmtTree* ip4_route_flow_mgmt_tree() {
326 16 : return &ip4_route_flow_mgmt_tree_;
327 : }
328 :
329 : BgpAsAServiceFlowMgmtKey *FindBgpAsAServiceInfo(
330 : FlowEntry *flow,
331 : BgpAsAServiceFlowMgmtKey &key);
332 :
333 : private:
334 : // Handle Add/Change of a flow. Builds FlowMgmtKeyTree for all objects
335 : void AddFlow(FlowEntryPtr &flow);
336 : // Handle Delete of a flow. Updates FlowMgmtKeyTree for all objects
337 : void DeleteFlow(FlowEntryPtr &flow, const RevFlowDepParams &p);
338 : void UpdateFlowStats(FlowEntryPtr &flow, uint32_t bytes, uint32_t packets,
339 : uint32_t oflow_bytes, const boost::uuids::uuid &u);
340 :
341 : // Add a FlowMgmtKey into the FlowMgmtKeyTree for an object
342 : // The FlowMgmtKeyTree for object is passed as argument
343 : void AddFlowMgmtKey(FlowEntry *flow, FlowEntryInfo *info,
344 : FlowMgmtKey *key, FlowMgmtKey *old_key);
345 : // Delete a FlowMgmtKey from FlowMgmtKeyTree for an object
346 : // The FlowMgmtKeyTree for object is passed as argument
347 : void DeleteFlowMgmtKey(FlowEntry *flow, FlowEntryInfo *info,
348 : FlowMgmtKey *key, FlowMgmtKeyNode *node);
349 : FlowEntryInfo *FindFlowEntryInfo(const FlowEntryPtr &flow);
350 : FlowEntryInfo *LocateFlowEntryInfo(FlowEntryPtr &flow);
351 : void DeleteFlowEntryInfo(FlowEntryPtr &flow);
352 : void MakeFlowMgmtKeyTree(FlowEntry *flow, FlowMgmtKeyTree *tree);
353 : void SetAceSandeshData(const AclDBEntry *acl, AclFlowCountResp &data,
354 : const std::string &ace_id);
355 : void SetAclFlowSandeshData(const AclDBEntry *acl, AclFlowResp &data,
356 : const int last_count);
357 : void ControllerNotify(uint8_t index);
358 :
359 : Agent *agent_;
360 : uint16_t table_index_;
361 : AclFlowMgmtTree acl_flow_mgmt_tree_;
362 : InterfaceFlowMgmtTree interface_flow_mgmt_tree_;
363 : VnFlowMgmtTree vn_flow_mgmt_tree_;
364 : InetRouteFlowMgmtTree ip4_route_flow_mgmt_tree_;
365 : InetRouteFlowMgmtTree ip6_route_flow_mgmt_tree_;
366 : BridgeRouteFlowMgmtTree bridge_route_flow_mgmt_tree_;
367 : VrfFlowMgmtTree vrf_flow_mgmt_tree_;
368 : NhFlowMgmtTree nh_flow_mgmt_tree_;
369 : boost::scoped_ptr<BgpAsAServiceFlowMgmtTree> bgp_as_a_service_flow_mgmt_tree_[MAX_XMPP_SERVERS];
370 : std::unique_ptr<FlowMgmtDbClient> flow_mgmt_dbclient_;
371 : FlowMgmtQueue request_queue_;
372 : FlowMgmtQueue db_event_queue_;
373 : static FlowMgmtQueue *log_queue_;
374 : DISALLOW_COPY_AND_ASSIGN(FlowMgmtManager);
375 : };
376 :
377 : #define FLOW_TRACE(obj, ...)\
378 : do {\
379 : Flow##obj::TraceMsg(FlowTraceBuf, __FILE__, __LINE__, ##__VA_ARGS__);\
380 : } while (false)
381 :
382 : #endif // __AGENT_FLOW_TABLE_MGMT_H__
|
