Line data Source code
1 : /*
2 : * Copyright (c) 2013 Juniper Networks, Inc. All rights reserved.
3 : * Copyright Contributors to the OpenSDN project
4 : */
5 :
6 : #include <atomic>
7 : #include <cstdlib>
8 : #include <cerrno>
9 : #include <utility>
10 : #include <list>
11 : #include <mutex>
12 :
13 : #include <boost/bind/bind.hpp>
14 : #include <boost/asio.hpp>
15 : #include <boost/tuple/tuple.hpp>
16 : #include "base/util.h"
17 : #include "base/logging.h"
18 : #include "base/address_util.h"
19 : #include "hiredis/hiredis.h"
20 : #include "hiredis/hiredis_ssl.h"
21 : #include "hiredis/boostasio.hpp"
22 : #include <contrail-collector/redis_connection.h>
23 : #include "base/work_pipeline.h"
24 : #include "QEOpServerProxy.h"
25 : #include "rapidjson/document.h"
26 : #include "rapidjson/stringbuffer.h"
27 : #include "rapidjson/writer.h"
28 : #include "query.h"
29 : #include "stats_query.h"
30 : #include "analytics_types.h"
31 : #include "stats_select.h"
32 : #include <base/connection_info.h>
33 :
34 : using std::list;
35 : using std::string;
36 : using std::vector;
37 : using std::map;
38 : using boost::assign::list_of;
39 : using boost::ptr_map;
40 : using boost::nullable;
41 : using boost::tuple;
42 : using boost::shared_ptr;
43 : using boost::scoped_ptr;
44 : using std::pair;
45 : using std::unique_ptr;
46 : using std::make_pair;
47 : using process::ConnectionState;
48 : using process::ConnectionType;
49 : using process::ConnectionStatus;
50 : using process::Endpoint;
51 : using namespace boost::placeholders;
52 :
53 : extern RedisAsyncConnection * rac_alloc(EventManager *, const std::string & ,unsigned short,
54 : RedisAsyncConnection::ClientConnectCbFn ,
55 : RedisAsyncConnection::ClientDisconnectCbFn,
56 : const bool, const std::string &, const std::string &, const std::string &);
57 :
58 : extern RedisAsyncConnection * rac_alloc_nocheck(EventManager *, const std::string & ,unsigned short,
59 : RedisAsyncConnection::ClientConnectCbFn ,
60 : RedisAsyncConnection::ClientDisconnectCbFn,
61 : const bool, const std::string &, const std::string &, const std::string &);
62 :
63 : SandeshTraceBufferPtr QeTraceBuf(SandeshTraceBufferCreate(QE_TRACE_BUF, 10000));
64 :
65 : struct RawResultT {
66 : QEOpServerProxy::QPerfInfo perf;
67 : shared_ptr<QEOpServerProxy::BufferT> res;
68 : shared_ptr<QEOpServerProxy::OutRowMultimapT> mres;
69 : shared_ptr<WhereResultT> wres;
70 : };
71 :
72 : typedef pair<redisReply,vector<string> > RedisT;
73 :
74 3440 : bool RedisAsyncArgCommand(RedisAsyncConnection * rac,
75 : void *rpi, const vector<string>& args) {
76 :
77 3440 : return rac->RedisAsyncArgCmd(rpi, args);
78 : }
79 :
80 : class QEOpServerProxy::QEOpServerImpl {
81 : public:
82 : typedef std::vector<std::string> QEOutputT;
83 :
84 : struct Input {
85 : int redis_host_idx;
86 : int cnum;
87 : string hostname;
88 : QueryEngine::QueryParams qp;
89 : vector<uint64_t> chunk_size;
90 : uint32_t wterms;
91 : bool need_merge;
92 : bool map_output;
93 : string where;
94 : string select;
95 : string post;
96 : uint64_t time_period;
97 : string table;
98 : uint32_t max_rows;
99 : std::atomic<uint32_t> chunk_q;
100 : std::atomic<uint32_t> total_rows;
101 :
102 : // to handle absent operator= for std::atomic
103 1252 : Input& operator=(const Input& other) {
104 1252 : if (this != &other) {
105 1252 : redis_host_idx = other.redis_host_idx;
106 1252 : cnum = other.cnum;
107 1252 : hostname = other.hostname;
108 1267 : qp = other.qp;
109 1252 : chunk_size = other.chunk_size;
110 1248 : wterms = other.wterms;
111 1248 : need_merge = other.need_merge;
112 1248 : map_output = other.map_output;
113 1248 : where = other.where;
114 1255 : select = other.select;
115 1264 : post = other.post;
116 1266 : time_period = other.time_period;
117 1266 : table = other.table;
118 1268 : max_rows = other.max_rows;
119 : // these two are handled only in QueryExec and are not used in next stages - don't copy they
120 1268 : chunk_q = 0;
121 1268 : total_rows = 0;
122 : }
123 1269 : return *this;
124 : }
125 : };
126 :
127 3874 : void JsonInsert(std::vector<query_column> &columns,
128 : contrail_rapidjson::Document& dd,
129 : std::pair<const string,string> * map_it) {
130 :
131 3874 : bool found = false;
132 41984 : for (size_t j = 0; j < columns.size(); j++)
133 : {
134 38110 : if ((0 == map_it->first.compare(0,5,string("COUNT")))) {
135 0 : contrail_rapidjson::Value val(contrail_rapidjson::kNumberType);
136 0 : unsigned long num = 0;
137 0 : stringToInteger(map_it->second, num);
138 0 : val.SetUint64(num);
139 0 : contrail_rapidjson::Value vk;
140 0 : dd.AddMember(vk.SetString(map_it->first.c_str(), dd.GetAllocator()), val, dd.GetAllocator());
141 0 : found = true;
142 38110 : } else if (columns[j].name == map_it->first) {
143 3874 : if (map_it->second.length() == 0) {
144 12 : contrail_rapidjson::Value val(contrail_rapidjson::kNullType);
145 12 : contrail_rapidjson::Value vk;
146 12 : dd.AddMember(vk.SetString(map_it->first.c_str(), dd.GetAllocator()), val, dd.GetAllocator());
147 12 : found = true;
148 12 : continue;
149 12 : }
150 :
151 : // find out type and convert
152 4764 : if (columns[j].datatype == "string" ||
153 902 : columns[j].datatype == "uuid")
154 : {
155 2960 : contrail_rapidjson::Value val(contrail_rapidjson::kStringType);
156 2960 : val.SetString(map_it->second.c_str(), dd.GetAllocator());
157 2960 : contrail_rapidjson::Value vk;
158 2960 : dd.AddMember(vk.SetString(map_it->first.c_str(), dd.GetAllocator()), val, dd.GetAllocator());
159 3862 : } else if (columns[j].datatype == "ipaddr") {
160 0 : contrail_rapidjson::Value val(contrail_rapidjson::kStringType);
161 0 : val.SetString(map_it->second.c_str(),
162 : dd.GetAllocator());
163 0 : contrail_rapidjson::Value vk;
164 0 : dd.AddMember(vk.SetString(map_it->first.c_str(), dd.GetAllocator()), val, dd.GetAllocator());
165 :
166 902 : } else if (columns[j].datatype == "double") {
167 0 : contrail_rapidjson::Value val(contrail_rapidjson::kNumberType);
168 0 : double dval = (double) strtod(map_it->second.c_str(), NULL);
169 0 : val.SetDouble(dval);
170 0 : contrail_rapidjson::Value vk;
171 0 : dd.AddMember(vk.SetString(map_it->first.c_str(), dd.GetAllocator()), val, dd.GetAllocator());
172 0 : } else {
173 902 : contrail_rapidjson::Value val(contrail_rapidjson::kNumberType);
174 902 : unsigned long num = 0;
175 902 : stringToInteger(map_it->second, num);
176 902 : val.SetUint64(num);
177 902 : contrail_rapidjson::Value vk;
178 902 : dd.AddMember(vk.SetString(map_it->first.c_str(), dd.GetAllocator()), val, dd.GetAllocator());
179 902 : }
180 3862 : found = true;
181 : }
182 : }
183 3874 : assert(found);
184 3874 : }
185 :
186 127 : void QueryJsonify(const string& table, bool map_output,
187 : const BufferT* raw_res, const OutRowMultimapT* raw_mres, QEOutputT* raw_json) {
188 :
189 127 : vector<OutRowT>::iterator res_it;
190 :
191 127 : std::vector<query_column> columns;
192 :
193 127 : if (!table.size()) return;
194 127 : bool found = false;
195 889 : for(size_t i = 0; i < g_viz_constants._TABLES.size(); i++)
196 : {
197 762 : if (g_viz_constants._TABLES[i].name == table) {
198 59 : found = true;
199 59 : columns = g_viz_constants._TABLES[i].schema.columns;
200 : }
201 : }
202 127 : if (!found) {
203 68 : found = true;
204 68 : columns = g_viz_constants._OBJECT_TABLE_SCHEMA.columns;
205 : }
206 127 : assert(found || map_output);
207 :
208 127 : if (map_output) {
209 71 : OutRowMultimapT::const_iterator mres_it;
210 268 : for (mres_it = raw_mres->begin(); mres_it != raw_mres->end(); ++mres_it) {
211 197 : string jstr;
212 197 : StatsSelect::Jsonify(table, mres_it->second.first, mres_it->second.second, jstr);
213 197 : raw_json->push_back(jstr);
214 197 : }
215 : } else {
216 56 : QEOpServerProxy::BufferT* raw_result =
217 : const_cast<QEOpServerProxy::BufferT*>(raw_res);
218 56 : QEOpServerProxy::BufferT::iterator res_it;
219 3134 : for (res_it = raw_result->begin(); res_it != raw_result->end(); ++res_it) {
220 3078 : std::map<std::string, std::string>::iterator map_it;
221 3078 : contrail_rapidjson::Document dd;
222 3078 : dd.SetObject();
223 :
224 3078 : for (map_it = (*res_it).first.begin();
225 6952 : map_it != (*res_it).first.end(); ++map_it) {
226 : // search for column name in the schema
227 3874 : JsonInsert(columns, dd, &(*map_it));
228 : }
229 3078 : contrail_rapidjson::StringBuffer sb;
230 3078 : contrail_rapidjson::Writer<contrail_rapidjson::StringBuffer> writer(sb);
231 3078 : dd.Accept(writer);
232 3078 : raw_json->push_back(sb.GetString());
233 3078 : }
234 : }
235 127 : }
236 :
237 :
238 912 : void QECallback(void * qid, QPerfInfo qperf,
239 : unique_ptr<std::vector<query_result_unit_t> > res) {
240 :
241 912 : RawResultT* raw(new RawResultT);
242 912 : raw->perf = qperf;
243 912 : raw->wres = std::move(res);
244 :
245 912 : ExternalProcIf<RawResultT> * rpi = NULL;
246 912 : if (qid)
247 912 : rpi = reinterpret_cast<ExternalProcIf<RawResultT> *>(qid);
248 :
249 912 : if (rpi) {
250 912 : unique_ptr<RawResultT> rp(raw);
251 912 : rpi->Response(std::move(rp));
252 912 : }
253 912 : }
254 :
255 864 : void QECallback(void * qid, QPerfInfo qperf, unique_ptr<QEOpServerProxy::BufferT> res,
256 : unique_ptr<QEOpServerProxy::OutRowMultimapT> mres) {
257 :
258 864 : RawResultT* raw(new RawResultT);
259 864 : raw->perf = qperf;
260 864 : raw->res = std::move(res);
261 864 : raw->mres = std::move(mres);
262 :
263 864 : ExternalProcIf<RawResultT> * rpi = NULL;
264 864 : if (qid)
265 864 : rpi = reinterpret_cast<ExternalProcIf<RawResultT> *>(qid);
266 :
267 864 : if (rpi) {
268 864 : unique_ptr<RawResultT> rp(raw);
269 864 : rpi->Response(std::move(rp));
270 864 : }
271 864 : }
272 :
273 : struct Stage0Out {
274 : Input inp;
275 : bool ret_code;
276 : vector<QPerfInfo> ret_info;
277 : vector<uint32_t> chunk_merge_time;
278 : shared_ptr<BufferT> result;
279 : shared_ptr<OutRowMultimapT> mresult;
280 : vector<shared_ptr<WhereResultT> > welem;
281 : shared_ptr<WhereResultT> wresult;
282 : uint32_t current_chunk;
283 : };
284 :
285 2763 : ExternalBase::Efn QueryExec(uint32_t inst, const vector<RawResultT*> & exts,
286 : const Input & inp, Stage0Out & res) {
287 2763 : uint32_t step = exts.size();
288 :
289 2771 : if (!step) {
290 997 : res.inp = inp;
291 1014 : res.ret_code = true;
292 :
293 1014 : if (inp.map_output)
294 567 : res.mresult = shared_ptr<OutRowMultimapT>(new OutRowMultimapT());
295 : else
296 447 : res.result = shared_ptr<BufferT>(new BufferT());
297 :
298 1001 : res.wresult = shared_ptr<WhereResultT>(new WhereResultT());
299 2041 : for (size_t or_idx=0; or_idx<inp.wterms; or_idx++) {
300 1047 : shared_ptr<WhereResultT> ss;
301 1047 : res.welem.push_back(ss);
302 1043 : }
303 :
304 994 : Input& cinp = const_cast<Input&>(inp);
305 994 : res.current_chunk = cinp.chunk_q.fetch_add(1);
306 994 : const uint32_t chunknum = res.current_chunk;
307 994 : if (chunknum >= inp.chunk_size.size()) {
308 153 : return NULL;
309 : }
310 :
311 : // Update query status
312 859 : RedisAsyncConnection * rac = conns_[res.inp.redis_host_idx][res.inp.cnum].get();
313 857 : string rkey = "REPLY:" + res.inp.qp.qid;
314 : char stat[40];
315 852 : uint prg = 10 + (chunknum * 75)/inp.chunk_size.size();
316 852 : sprintf(stat,"{\"progress\":%d}", prg);
317 846 : RedisAsyncArgCommand(rac, NULL,
318 1714 : list_of(string("RPUSH"))(rkey)(stat));
319 :
320 1723 : return boost::bind(&QueryEngine::QueryExecWhere, qosp_->qe_,
321 1722 : _1, inp.qp, chunknum, 0);
322 863 : }
323 :
324 1774 : res.ret_info.push_back(exts[step-1]->perf);
325 1775 : if (exts[step-1]->perf.error) {
326 0 : res.ret_code =false;
327 : }
328 1775 : if (!res.ret_code) return NULL;
329 :
330 : // Number of substeps per chunk is the number of OR terms in WHERE
331 : // plus one more substep for select and post processing
332 1775 : uint32_t substep = step % (inp.wterms + 1);
333 :
334 1775 : if (substep == inp.wterms) {
335 : // Get the result of the final WHERE
336 864 : res.welem[substep-1] = exts[step-1]->wres;
337 :
338 : // The set "OR" API needs raw pointers
339 864 : vector<WhereResultT*> oterms;
340 1776 : for (size_t or_idx = 0; or_idx < inp.wterms; or_idx++) {
341 912 : oterms.push_back(res.welem[or_idx].get());
342 : }
343 :
344 : // Do SET operations
345 864 : QE_ASSERT(res.wresult->size() == 0);
346 864 : SetOperationUnit::op_or(res.inp.qp.qid, *res.wresult, oterms);
347 :
348 1776 : for (size_t or_idx=0; or_idx<inp.wterms; or_idx++) {
349 912 : res.welem[or_idx].reset();
350 : }
351 :
352 : // Start the SELECT and POST-processing
353 2592 : return boost::bind(&QueryEngine::QueryExec, qosp_->qe_,
354 1728 : _1, inp.qp, res.current_chunk, res.wresult.get());
355 :
356 1775 : } else if (substep == 0) {
357 : // A chunk is complete. Start another one
358 863 : res.wresult->clear();
359 : uint32_t added_rows;
360 :
361 863 : if (inp.need_merge) {
362 456 : uint64_t then = UTCTimestampUsec();
363 456 : if (inp.map_output) {
364 424 : uint32_t base_rows = res.mresult->size();
365 : // TODO: This interface should not be Stats-Specific
366 424 : StatsSelect::Merge(std::string(""), *(exts[step-1]->mres), *(res.mresult));
367 : // Some rows of this chunk will merge into existing results
368 424 : added_rows = res.mresult->size() - base_rows;
369 : } else {
370 32 : uint32_t base_rows = res.result->size();
371 32 : res.ret_code =
372 32 : qosp_->qe_->QueryAccumulate(inp.qp,
373 32 : *(exts[step-1]->res), *(res.result));
374 : // Some rows of this chunk will merge into existing results
375 32 : added_rows = res.result->size() - base_rows;
376 : }
377 912 : res.chunk_merge_time.push_back(
378 456 : static_cast<uint32_t>((UTCTimestampUsec() - then)/1000));
379 :
380 : } else {
381 : // TODO : When merge is not needed, we can just send
382 : // a result upto redis at this point.
383 :
384 407 : if (inp.map_output) {
385 41 : added_rows = exts[step-1]->mres->size();
386 41 : OutRowMultimapT::iterator jt = res.mresult->begin();
387 41 : for (OutRowMultimapT::const_iterator it = exts[step-1]->mres->begin();
388 79 : it != exts[step-1]->mres->end(); it++ ) {
389 :
390 76 : jt = res.mresult->insert(jt,
391 76 : std::make_pair(it->first, it->second));
392 : }
393 : } else {
394 366 : added_rows = exts[step-1]->res->size();
395 1098 : res.result->insert(res.result->begin(),
396 366 : exts[step-1]->res->begin(),
397 366 : exts[step-1]->res->end());
398 : }
399 : }
400 863 : Input& cinp = const_cast<Input&>(inp);
401 1726 : if (cinp.total_rows.fetch_add(added_rows) + added_rows > cinp.max_rows) {
402 0 : QE_LOG_NOQID(ERROR, "QueryExec Max Rows Exceeded " <<
403 : cinp.total_rows << " chunk " << cinp.chunk_q);
404 0 : return NULL;
405 : }
406 :
407 863 : res.current_chunk = cinp.chunk_q.fetch_add(1);
408 863 : const uint32_t chunknum = res.current_chunk;
409 863 : if (chunknum < inp.chunk_size.size()) {
410 1 : string key = "QUERY:" + res.inp.qp.qid;
411 :
412 : // Update query status
413 1 : RedisAsyncConnection * rac = conns_[res.inp.redis_host_idx][res.inp.cnum].get();
414 1 : string rkey = "REPLY:" + res.inp.qp.qid;
415 : char stat[40];
416 1 : uint prg = 10 + (chunknum * 75)/inp.chunk_size.size();
417 1 : sprintf(stat,"{\"progress\":%d}", prg);
418 1 : RedisAsyncArgCommand(rac, NULL,
419 2 : list_of(string("RPUSH"))(rkey)(stat));
420 2 : return boost::bind(&QueryEngine::QueryExecWhere, qosp_->qe_,
421 2 : _1, inp.qp, chunknum, 0);
422 1 : } else {
423 863 : return NULL;
424 : }
425 : } else {
426 : // We are in the middle of doing WHERE processing for a chunk
427 :
428 48 : res.welem[substep-1] = exts[step-1]->wres;
429 :
430 96 : return boost::bind(&QueryEngine::QueryExecWhere, qosp_->qe_,
431 96 : _1, inp.qp, res.current_chunk, substep);
432 : }
433 : return NULL;
434 : }
435 :
436 : struct Stage0Merge {
437 : Input inp;
438 : bool ret_code;
439 : bool overflow;
440 : uint32_t fm_time;
441 : vector<vector<QPerfInfo> > ret_info;
442 : vector<vector<uint32_t> > chunk_merge_time;
443 : BufferT result;
444 : OutRowMultimapT mresult;
445 : };
446 127 : bool QueryMerge(const std::vector<boost::shared_ptr<Stage0Out> > & subs,
447 : const boost::shared_ptr<Input> & inp, Stage0Merge & res) {
448 :
449 127 : res.ret_code = true;
450 127 : res.overflow = false;
451 127 : res.inp = subs[0]->inp;
452 127 : res.fm_time = 0;
453 :
454 127 : uint32_t rows = 0;
455 127 : for (vector<shared_ptr<Stage0Out> >::const_iterator it = subs.begin() ;
456 1143 : it!=subs.end(); it++) {
457 1016 : if (res.inp.map_output)
458 568 : rows += (*it)->mresult->size();
459 : else
460 448 : rows += (*it)->result->size();
461 : }
462 :
463 : // If max_rows have been exceeded, don't do any more processing
464 127 : if (rows > res.inp.max_rows) {
465 0 : res.overflow = true;
466 0 : return true;
467 : }
468 :
469 127 : std::vector<boost::shared_ptr<OutRowMultimapT> > mqsubs;
470 127 : std::vector<boost::shared_ptr<QEOpServerProxy::BufferT> > qsubs;
471 127 : for (vector<shared_ptr<Stage0Out> >::const_iterator it = subs.begin() ;
472 1143 : it!=subs.end(); it++) {
473 :
474 1016 : res.ret_info.push_back((*it)->ret_info);
475 1016 : res.chunk_merge_time.push_back((*it)->chunk_merge_time);
476 :
477 1016 : if ((*it)->ret_code == false) {
478 0 : res.ret_code = false;
479 : } else {
480 1016 : if (res.inp.map_output)
481 568 : mqsubs.push_back((*it)->mresult);
482 : else
483 448 : qsubs.push_back((*it)->result);
484 : }
485 : }
486 :
487 127 : if (!res.ret_code) return true;
488 :
489 127 : if (res.inp.need_merge) {
490 68 : uint64_t then = UTCTimestampUsec();
491 :
492 68 : if (res.inp.map_output) {
493 64 : res.ret_code =
494 64 : qosp_->qe_->QueryFinalMerge(res.inp.qp, mqsubs, res.mresult);
495 :
496 : } else {
497 4 : res.ret_code =
498 4 : qosp_->qe_->QueryFinalMerge(res.inp.qp, qsubs, res.result);
499 : }
500 :
501 68 : uint64_t now = UTCTimestampUsec();
502 68 : res.fm_time = static_cast<uint32_t>((now - then)/1000);
503 : } else {
504 : // TODO : If a merge was not needed, results have been sent to
505 : // redis already. The only thing still needed is the status
506 59 : for (vector<shared_ptr<Stage0Out> >::const_iterator it = subs.begin() ;
507 531 : it!=subs.end(); it++) {
508 :
509 472 : if (res.inp.map_output) {
510 56 : OutRowMultimapT::iterator jt = res.mresult.begin();
511 56 : for (OutRowMultimapT::const_iterator kt = (*it)->mresult->begin();
512 94 : kt != (*it)->mresult->end(); kt++) {
513 38 : jt = res.mresult.insert(jt,
514 76 : std::make_pair(kt->first, kt->second));
515 : }
516 : } else {
517 832 : res.result.insert(res.result.begin(),
518 416 : (*it)->result->begin(),
519 416 : (*it)->result->end());
520 : }
521 : }
522 : }
523 :
524 127 : return true;
525 127 : }
526 :
527 : struct Output {
528 : Input inp;
529 : uint32_t redis_time;
530 : bool ret_code;
531 : };
532 505 : ExternalBase::Efn QueryResp(uint32_t inst, const vector<RedisT*> & exts,
533 : const Stage0Merge & inp, Output & ret) {
534 505 : uint32_t step = exts.size();
535 505 : switch (inst) {
536 254 : case 0: {
537 254 : if (!step) {
538 127 : ret.inp = inp.inp;
539 127 : RedisAsyncConnection * rac = conns_[ret.inp.redis_host_idx][ret.inp.cnum].get();
540 127 : std::stringstream keystr;
541 127 : unique_ptr<QEOutputT> jsonresult(new QEOutputT);
542 :
543 127 : QE_LOG_NOQID(INFO, "Will Jsonify #rows " <<
544 : inp.result.size() + inp.mresult.size());
545 127 : QueryJsonify(inp.inp.table, inp.inp.map_output,
546 : &inp.result, &inp.mresult, jsonresult.get());
547 :
548 127 : vector<string> const * const res = jsonresult.get();
549 127 : vector<string>::size_type idx = 0;
550 127 : uint32_t rownum = 0;
551 :
552 127 : QE_LOG_NOQID(INFO, "Did Jsonify #rows " << res->size());
553 :
554 127 : uint64_t then = UTCTimestampUsec();
555 : char stat[80];
556 127 : string key = "REPLY:" + ret.inp.qp.qid;
557 :
558 127 : if (inp.overflow) {
559 0 : sprintf(stat,"{\"progress\":%d}", - ENOBUFS);
560 127 : } else if (!inp.ret_code) {
561 0 : sprintf(stat,"{\"progress\":%d}", - EIO);
562 : } else {
563 703 : while (idx < res->size()) {
564 576 : uint32_t rowsize = 0;
565 576 : keystr.str(string());
566 576 : keystr << "RESULT:" << ret.inp.qp.qid << ":" << rownum;
567 1152 : vector<string> command = list_of(string("RPUSH"))(keystr.str());
568 3851 : while ((idx < res->size()) && (((int)rowsize) < kMaxRowThreshold)) {
569 3275 : command.push_back(res->at(idx));
570 3275 : rowsize += res->at(idx).size();
571 3275 : idx++;
572 : }
573 576 : RedisAsyncArgCommand(rac, NULL, command);
574 576 : RedisAsyncArgCommand(rac, NULL,
575 1152 : list_of(string("EXPIRE"))(keystr.str())("300"));
576 576 : sprintf(stat,"{\"progress\":90, \"lines\":%d}",
577 : (int)rownum);
578 576 : RedisAsyncArgCommand(rac, NULL,
579 1152 : list_of(string("RPUSH"))(key)(stat));
580 576 : rownum++;
581 576 : }
582 127 : sprintf(stat,"{\"progress\":100, \"lines\":%d, \"count\":%d}",
583 127 : (int)rownum, (int)res->size());
584 : }
585 127 : uint64_t now = UTCTimestampUsec();
586 127 : ret.redis_time = static_cast<uint32_t>((now - then)/1000);
587 127 : QE_LOG_NOQID(DEBUG, "QE Query Result is " << stat);
588 254 : return boost::bind(&RedisAsyncArgCommand,
589 127 : conns_[ret.inp.redis_host_idx][ret.inp.cnum].get(), _1,
590 381 : list_of(string("RPUSH"))(key)(stat));
591 127 : } else {
592 127 : RedisAsyncConnection * rac = conns_[ret.inp.redis_host_idx][ret.inp.cnum].get();
593 127 : string key = "REPLY:" + ret.inp.qp.qid;
594 127 : RedisAsyncArgCommand(rac, NULL,
595 254 : list_of(string("EXPIRE"))(key)("300"));
596 :
597 127 : key = "QUERY:" + ret.inp.qp.qid;
598 127 : RedisAsyncArgCommand(rac, NULL,
599 254 : list_of(string("EXPIRE"))(key)("300"));
600 :
601 127 : uint64_t now = UTCTimestampUsec();
602 127 : uint32_t qtime = static_cast<uint32_t>(
603 127 : (now - ret.inp.qp.query_starttm)/1000);
604 :
605 127 : uint64_t enqtm = atol(ret.inp.qp.terms["enqueue_time"].c_str());
606 127 : uint32_t enq_delay = static_cast<uint32_t>(
607 127 : (ret.inp.qp.query_starttm - enqtm)/1000);
608 :
609 127 : QueryStats qs;
610 : size_t outsize;
611 :
612 127 : if (ret.inp.map_output)
613 71 : outsize = inp.mresult.size();
614 : else
615 56 : outsize = inp.result.size();
616 :
617 127 : qs.set_rows(static_cast<uint32_t>(outsize));
618 127 : qs.set_time(qtime);
619 127 : qs.set_qid(ret.inp.qp.qid);
620 127 : qs.set_chunks(inp.inp.chunk_size.size());
621 127 : std::ostringstream wherestr, selstr, poststr;
622 1143 : for (size_t i=0; i < inp.ret_info.size(); i++) {
623 2792 : for (size_t j=0; j < inp.ret_info[i].size(); j++) {
624 1776 : wherestr << inp.ret_info[i][j].chunk_where_time << ",";
625 1776 : selstr << inp.ret_info[i][j].chunk_select_time << ",";
626 1776 : poststr << inp.ret_info[i][j].chunk_postproc_time << ",";
627 : }
628 1016 : wherestr << " ";
629 1016 : selstr << " ";
630 1016 : poststr << " ";
631 : }
632 127 : if (inp.overflow) {
633 0 : qs.set_error("ERROR-ENOBUFS");
634 127 : } else if (!inp.ret_code) {
635 0 : qs.set_error("ERROR-EIO");
636 : } else {
637 127 : qs.set_error("None");
638 : }
639 127 : qs.set_chunk_where_time(wherestr.str());
640 127 : qs.set_chunk_select_time(selstr.str());
641 127 : qs.set_chunk_postproc_time(poststr.str());
642 :
643 127 : std::ostringstream mergestr;
644 1143 : for (size_t i=0; i < inp.chunk_merge_time.size(); i++) {
645 1472 : for (size_t j=0; j < inp.chunk_merge_time[i].size(); j++) {
646 456 : mergestr << inp.chunk_merge_time[i][j] << ",";
647 : }
648 1016 : mergestr << " ";
649 : }
650 :
651 127 : qs.set_chunk_merge_time(mergestr.str());
652 127 : qs.set_final_merge_time(inp.fm_time);
653 127 : qs.set_where(inp.inp.where);
654 127 : qs.set_select(inp.inp.select);
655 127 : qs.set_post(inp.inp.post);
656 127 : qs.set_time_span(static_cast<uint32_t>(inp.inp.time_period));
657 127 : qs.set_enq_delay(enq_delay);
658 127 : QUERY_PERF_INFO_SEND(Sandesh::source(), // name
659 : inp.inp.table, // table
660 : qs);
661 :
662 127 : QE_LOG_NOQID(INFO, "Finished: QID " << ret.inp.qp.qid <<
663 : " Table " << inp.inp.table <<
664 : " Time(ms) " << qtime <<
665 : " RedisTime(ms) " << ret.redis_time <<
666 : " MergeTime(ms) " << inp.fm_time <<
667 : " Rows " << outsize <<
668 : " EnQ-delay" << enq_delay);
669 :
670 127 : ret.ret_code = true;
671 127 : }
672 : }
673 127 : break;
674 254 : case 1: {
675 254 : if (!step) {
676 127 : string key = "ENGINE:" + inp.inp.hostname;
677 254 : return boost::bind(&RedisAsyncArgCommand,
678 127 : conns_[inp.inp.redis_host_idx][inp.inp.cnum].get(), _1,
679 381 : list_of(string("LREM"))(key)("0")(inp.inp.qp.qid));
680 127 : } else {
681 127 : ret.ret_code = true;
682 : }
683 : }
684 127 : break;
685 : }
686 251 : return NULL;
687 : }
688 :
689 :
690 : typedef WorkPipeline<Input, Stage0Merge, Output> QEPipeT;
691 :
692 127 : void QEPipeCb(QEPipeT *wp, bool ret_code) {
693 127 : std::scoped_lock lock(mutex_);
694 :
695 127 : boost::shared_ptr<Output> res = wp->Result();
696 127 : assert(pipes_.find(res->inp.qp.qid)->second == wp);
697 127 : pipes_.erase(res->inp.qp.qid);
698 127 : m_analytics_queries.erase(res->inp.qp.qid);
699 127 : npipes_[res->inp.redis_host_idx][res->inp.cnum-1]--;
700 127 : QE_LOG_NOQID(DEBUG, " Result " << res->ret_code << " , " << res->inp.cnum << " conn");
701 127 : delete wp;
702 127 : }
703 :
704 357 : void ConnUpPostProcess(int redis_host_idx, uint8_t cnum) {
705 357 : if (!connState_[redis_host_idx][cnum]) {
706 230 : QE_LOG_NOQID(DEBUG, "ConnUp SetCB" << (uint32_t)cnum);
707 230 : cb_proc_fn_[redis_host_idx][cnum] = boost::bind(&QEOpServerImpl::CallbackProcess,
708 230 : this, redis_host_idx, cnum, _1, _2, _3);
709 230 : conns_[redis_host_idx][cnum].get()->SetClientAsyncCmdCb(cb_proc_fn_[redis_host_idx][cnum]);
710 230 : connState_[redis_host_idx][cnum] = true;
711 : }
712 :
713 357 : bool isConnected = true;
714 2142 : for(int i=0; i<kConnections+1; i++)
715 1785 : if (!connState_[redis_host_idx][i]) isConnected = false;
716 357 : if (isConnected) {
717 173 : string key = "ENGINE:" + hostname_;
718 346 : conns_[redis_host_idx][0].get()->RedisAsyncArgCmd(0,
719 346 : list_of(string("BRPOPLPUSH"))("QUERYQ")(key)("0"));
720 173 : }
721 357 : }
722 :
723 127 : int LeastLoadedConnection(int redis_host_idx) {
724 127 : int minvalue = npipes_[redis_host_idx][0];
725 127 : int minindex = 0;
726 508 : for (int i = 1; i < kConnections; i++) {
727 381 : if (npipes_[redis_host_idx][i] < minvalue) {
728 0 : minvalue = npipes_[redis_host_idx][i];
729 0 : minindex = i;
730 : }
731 : }
732 127 : return minindex;
733 : }
734 :
735 0 : void QueryError(int redis_host_idx, string qid, int ret_code) {
736 0 : string redis_host = redis_host_port_pairs_[redis_host_idx].first;
737 0 : int redis_port = redis_host_port_pairs_[redis_host_idx].second;
738 0 : redisContext *c = redisConnect(redis_host.c_str(), redis_port);
739 0 : if (c->err) {
740 0 : QE_LOG_NOQID(ERROR, "Cannot report query error for " << qid <<
741 : " . No Redis Connection");
742 0 : redisFree(c);
743 0 : return;
744 : }
745 :
746 : /* Secure the connection if SSL enabled */
747 0 : if (redis_ssl_enable_) {
748 0 : redisSSLContextError ssl_error = REDIS_SSL_CTX_NONE;
749 0 : redisSSLContext *ssl_ctx = redisCreateSSLContext(
750 : redis_ca_cert_.c_str(),
751 : NULL,
752 : redis_certfile_.c_str(),
753 : redis_keyfile_.c_str(),
754 : "sni",
755 : &ssl_error);
756 0 : if (!ssl_ctx || ssl_error != REDIS_SSL_CTX_NONE) {
757 0 : QE_LOG_NOQID(ERROR, "Cannot report query error for " << qid <<
758 : " SSL Context Error: " << redisSSLContextGetError(ssl_error));
759 0 : if (ssl_ctx) redisFreeSSLContext(ssl_ctx);
760 0 : redisFree(c);
761 0 : return;
762 : }
763 0 : int rc = redisInitiateSSLWithContext(c, ssl_ctx);
764 0 : if (rc != REDIS_OK) {
765 0 : QE_LOG_NOQID(ERROR, "Cannot report query error for " << qid <<
766 : " SSL Connection Error: " << c->errstr);
767 0 : redisFreeSSLContext(ssl_ctx);
768 0 : redisFree(c);
769 0 : return;
770 : }
771 0 : redisFreeSSLContext(ssl_ctx);
772 : }
773 :
774 : //Authenticate the context with password
775 0 : if (!redis_password_.empty()) {
776 0 : redisReply * reply = (redisReply *) redisCommand(c, "AUTH %s",
777 : redis_password_.c_str());
778 0 : if (reply->type == REDIS_REPLY_ERROR) {
779 0 : QE_LOG_NOQID(ERROR, "Authentication to redis error");
780 0 : freeReplyObject(reply);
781 0 : redisFree(c);
782 0 : return;
783 : }
784 0 : freeReplyObject(reply);
785 : }
786 :
787 : char stat[80];
788 0 : string key = "REPLY:" + qid;
789 0 : sprintf(stat,"{\"progress\":%d}", - ret_code);
790 :
791 0 : redisReply * reply = (redisReply *) redisCommand(c, "RPUSH %s %s",
792 : key.c_str(), stat);
793 :
794 0 : freeReplyObject(reply);
795 0 : redisFree(c);
796 0 : }
797 :
798 :
799 127 : void StartPipeline(const string qid, int redis_host_idx) {
800 127 : QueryStats qs;
801 127 : qs.set_qid(qid);
802 127 : qs.set_rows(0);
803 127 : qs.set_time(0);
804 127 : qs.set_final_merge_time(0);
805 127 : qs.set_enq_delay(0);
806 :
807 127 : uint64_t now = UTCTimestampUsec();
808 :
809 127 : std::scoped_lock lock(mutex_);
810 :
811 127 : string redis_host = redis_host_port_pairs_[redis_host_idx].first;
812 127 : int redis_port = redis_host_port_pairs_[redis_host_idx].second;
813 127 : QE_LOG_NOQID(INFO, "StartPipeline on :" << " Redis:" << redis_host << " Port:" << redis_port);
814 127 : redisContext *c = redisConnect(redis_host.c_str(), redis_port);
815 :
816 127 : if (c->err) {
817 0 : QE_LOG_NOQID(ERROR, "Cannot start Pipleline for " << qid <<
818 : " . No Redis Connection");
819 0 : redisFree(c);
820 0 : qs.set_error("No Redis Connection");
821 0 : QUERY_PERF_INFO_SEND(Sandesh::source(), // name
822 : "__UNKNOWN__", // table
823 : qs);
824 0 : return;
825 : }
826 :
827 : /* Secure the connection if SSL enabled */
828 127 : if (redis_ssl_enable_) {
829 0 : redisSSLContextError ssl_error = REDIS_SSL_CTX_NONE;
830 0 : redisSSLContext *ssl_ctx = redisCreateSSLContext(
831 : redis_ca_cert_.c_str(),
832 : NULL,
833 : redis_certfile_.c_str(),
834 : redis_keyfile_.c_str(),
835 : "sni",
836 : &ssl_error);
837 0 : if (!ssl_ctx || ssl_error != REDIS_SSL_CTX_NONE) {
838 0 : QE_LOG_NOQID(ERROR, "Cannot start pipeline for " << qid <<
839 : " SSL Context Error: " << redisSSLContextGetError(ssl_error));
840 0 : if (ssl_ctx) redisFreeSSLContext(ssl_ctx);
841 0 : redisFree(c);
842 0 : qs.set_error("Redis SSL Context Error");
843 0 : QUERY_PERF_INFO_SEND(Sandesh::source(), "__UNKNOWN__", qs);
844 0 : return;
845 : }
846 0 : int rc = redisInitiateSSLWithContext(c, ssl_ctx);
847 0 : if (rc != REDIS_OK) {
848 0 : QE_LOG_NOQID(ERROR, "Cannot start pipeline for " << qid <<
849 : " SSL Connection Error: " << c->errstr);
850 0 : redisFreeSSLContext(ssl_ctx);
851 0 : redisFree(c);
852 0 : qs.set_error("Redis SSL Connection Error");
853 0 : QUERY_PERF_INFO_SEND(Sandesh::source(), "__UNKNOWN__", qs);
854 0 : return;
855 : }
856 0 : redisFreeSSLContext(ssl_ctx);
857 : }
858 :
859 : //Authenticate the context with password
860 127 : if ( !redis_password_.empty()) {
861 0 : redisReply * reply = (redisReply *) redisCommand(c, "AUTH %s",
862 : redis_password_.c_str());
863 0 : if (reply->type == REDIS_REPLY_ERROR) {
864 0 : QE_LOG_NOQID(ERROR, "Authentication to redis error");
865 0 : freeReplyObject(reply);
866 0 : redisFree(c);
867 0 : qs.set_error("Redis Auth Failed");
868 0 : QUERY_PERF_INFO_SEND(Sandesh::source(), // name
869 : "__UNKNOWN__", // table
870 : qs);
871 0 : return;
872 : }
873 0 : freeReplyObject(reply);
874 : }
875 :
876 127 : string key = "QUERY:" + qid;
877 127 : redisReply * reply = (redisReply *) redisCommand(c, "hgetall %s",
878 : key.c_str());
879 :
880 127 : map<string,string> terms;
881 127 : if (!(c->err) && (reply->type == REDIS_REPLY_ARRAY)) {
882 1209 : for (uint32_t i=0; i<reply->elements; i+=2) {
883 1082 : string idx(reply->element[i]->str);
884 1082 : string val(reply->element[i+1]->str);
885 1082 : terms[idx] = val;
886 1082 : }
887 127 : } else {
888 0 : QE_LOG_NOQID(ERROR, "Cannot start Pipleline for " << qid <<
889 : ". Could not read query input");
890 0 : freeReplyObject(reply);
891 0 : redisFree(c);
892 0 : QueryError(redis_host_idx, qid, 5);
893 0 : qs.set_error("Could not read query input");
894 0 : QUERY_PERF_INFO_SEND(Sandesh::source(), // name
895 : "__UNKNOWN__", // table
896 : qs);
897 0 : return;
898 : }
899 :
900 127 : freeReplyObject(reply);
901 127 : redisFree(c);
902 :
903 127 : QueryEngine::QueryParams qp(qid, terms, max_tasks_,
904 254 : UTCTimestampUsec());
905 :
906 127 : vector<uint64_t> chunk_size;
907 : bool need_merge;
908 : bool map_output;
909 127 : string table;
910 127 : string where;
911 : uint32_t wterms;
912 127 : string select;
913 127 : string post;
914 : uint64_t time_period;
915 :
916 127 : int ret = qosp_->qe_->QueryPrepare(qp, chunk_size, need_merge, map_output,
917 : where, wterms, select, post, time_period, table);
918 :
919 127 : qs.set_where(where);
920 127 : qs.set_select(select);
921 127 : qs.set_post(post);
922 127 : qs.set_time_span(time_period);
923 127 : uint64_t enqtm = atol(terms["enqueue_time"].c_str());
924 127 : uint32_t enq_delay = static_cast<uint32_t>((now - enqtm)/1000);
925 127 : qs.set_enq_delay(enq_delay);
926 :
927 127 : if (ret!=0) {
928 0 : QueryError(redis_host_idx, qid, ret);
929 0 : QE_LOG_NOQID(ERROR, "Cannot start Pipleline for " << qid <<
930 : ". Query Parsing Error " << ret);
931 0 : qs.set_error("Query Parsing Error");
932 0 : QUERY_PERF_INFO_SEND(Sandesh::source(), // name
933 : table, // table
934 : qs);
935 0 : return;
936 : } else {
937 127 : QE_LOG_NOQID(INFO, "Chunks: " << chunk_size.size() <<
938 : " Need Merge: " << need_merge);
939 : }
940 :
941 127 : if (pipes_.size() >= 32) {
942 0 : QueryError(redis_host_idx, qid, EMFILE);
943 0 : QE_LOG_NOQID(ERROR, "Cannot start Pipleline for " << qid <<
944 : ". Too many queries : " << pipes_.size());
945 0 : qs.set_error("EMFILE");
946 0 : QUERY_PERF_INFO_SEND(Sandesh::source(), // name
947 : table, // table
948 : qs);
949 0 : return;
950 : }
951 :
952 127 : shared_ptr<Input> inp(new Input());
953 127 : inp.get()->hostname = hostname_;
954 127 : inp.get()->qp = qp;
955 127 : inp.get()->map_output = map_output;
956 127 : inp.get()->need_merge = need_merge;
957 127 : inp.get()->chunk_size = chunk_size;
958 127 : inp.get()->where = where;
959 127 : inp.get()->select = select;
960 127 : inp.get()->post = post;
961 127 : inp.get()->time_period = time_period;
962 127 : inp.get()->table = table;
963 127 : inp.get()->chunk_q = 0;
964 127 : inp.get()->total_rows = 0;
965 127 : inp.get()->max_rows = max_rows_;
966 127 : inp.get()->wterms = wterms;
967 127 : vector<pair<int,int> > tinfo;
968 1143 : for (uint idx=0; idx<(uint)max_tasks_; idx++) {
969 1016 : tinfo.push_back(make_pair(0, -1));
970 : }
971 :
972 : QEPipeT *wp = new QEPipeT(
973 254 : new WorkStage<Input, Stage0Merge, RawResultT, Stage0Out>(
974 : tinfo,
975 : boost::bind(&QEOpServerImpl::QueryExec, this, _1,_2,_3,_4),
976 127 : boost::bind(&QEOpServerImpl::QueryMerge, this, _1,_2,_3)),
977 254 : new WorkStage<Stage0Merge, Output, RedisT>(
978 254 : list_of(make_pair(0,-1))(make_pair(0,-1)),
979 127 : boost::bind(&QEOpServerImpl::QueryResp, this, _1,_2,_3,_4)));
980 :
981 127 : pipes_.insert(make_pair(qid, wp));
982 :
983 : // Initialize the m_analytics_queries for this qid
984 127 : m_analytics_queries[qid] = std::vector<boost::shared_ptr<AnalyticsQuery> > ();
985 :
986 127 : int conn = LeastLoadedConnection(redis_host_idx);
987 127 : QE_LOG_NOQID(DEBUG, "Getting Least Loaded Conn as :" << conn);
988 127 : npipes_[redis_host_idx][conn]++;
989 :
990 : // The cnum with index 0 is only used for receiving new queries
991 127 : inp.get()->cnum = conn+1;
992 127 : inp.get()->redis_host_idx = redis_host_idx;
993 :
994 127 : wp->Start(boost::bind(&QEOpServerImpl::QEPipeCb, this, wp, _1), inp);
995 127 : QE_LOG_NOQID(DEBUG, "Starting Pipeline for " << qid << " , " << conn+1 <<
996 : " conn, " << tinfo.size() << " tasks");
997 :
998 : // Update query status
999 127 : RedisAsyncConnection * rac = conns_[redis_host_idx][inp.get()->cnum].get();
1000 127 : string rkey = "REPLY:" + qid;
1001 : char stat[40];
1002 127 : sprintf(stat,"{\"progress\":15}");
1003 127 : RedisAsyncArgCommand(rac, NULL,
1004 254 : list_of(string("RPUSH"))(rkey)(stat));
1005 :
1006 :
1007 127 : }
1008 :
1009 230 : void ConnUpPrePostProcess(int redis_host_idx, uint8_t cnum) {
1010 : //Assign callback for AUTH command
1011 230 : cb_proc_fn_[redis_host_idx][cnum] = boost::bind(&QEOpServerImpl::ConnectCallbackProcess,
1012 230 : this, redis_host_idx, cnum, _1, _2, _3);
1013 230 : conns_[redis_host_idx][cnum].get()->SetClientAsyncCmdCb(cb_proc_fn_[redis_host_idx][cnum]);
1014 : //Send AUTH command
1015 230 : RedisAsyncConnection * rac = conns_[redis_host_idx][cnum].get();
1016 230 : if (!redis_password_.empty()) {
1017 20 : RedisAsyncArgCommand(rac, NULL,
1018 40 : list_of(string("AUTH"))(redis_password_.c_str()));
1019 : } else {
1020 210 : RedisAsyncArgCommand(rac, NULL,
1021 420 : list_of(string("PING")));
1022 : }
1023 230 : }
1024 :
1025 230 : void ConnUp(int redis_host_idx, uint8_t cnum) {
1026 230 : string redis_host = redis_host_port_pairs_[redis_host_idx].first;
1027 230 : std::ostringstream ostr;
1028 230 : ostr << "ConnUp.. UP " << (uint32_t)cnum << " With Redis:" << redis_host;
1029 230 : QE_LOG_NOQID(DEBUG, ostr.str());
1030 460 : qosp_->evm_->io_service()->post(
1031 230 : boost::bind(&QEOpServerImpl::ConnUpPrePostProcess,
1032 : this, redis_host_idx, cnum));
1033 230 : }
1034 :
1035 245 : void UpdateRedisConnectionStatus () {
1036 245 : std::ostringstream ostr;
1037 245 : map<string, string>::iterator it;
1038 245 : vector<string> redis_endpoints;
1039 245 : int redis_hosts_cnt = redis_host_port_pairs_.size();
1040 245 : int redis_down_count = 0;
1041 245 : int redis_idx = 0;
1042 245 : int conn_idx = 0;
1043 245 : vector<Endpoint> redis_endpoint_list;
1044 490 : for (redis_idx = 0; redis_idx < redis_hosts_cnt; redis_idx++) {
1045 245 : string redis_host(redis_host_port_pairs_[redis_idx].first);
1046 245 : redis_endpoint_list.push_back(conns_[redis_idx][0]->Endpoint());
1047 529 : for (conn_idx = 0; conn_idx < kConnections + 1; conn_idx++) {
1048 478 : if (true == connState_[redis_idx][conn_idx]) {
1049 194 : break;
1050 : }
1051 : }
1052 245 : if (conn_idx == kConnections + 1) {
1053 51 : ostr << conns_[redis_idx][0]->Endpoint() << "::Down.";
1054 51 : redis_down_count++;
1055 : }
1056 245 : }
1057 245 : if (redis_down_count < redis_hosts_cnt) {
1058 : /* Then QE should be UP, but display if any redis connection is down */
1059 388 : ConnectionState::GetInstance()->Update(ConnectionType::REDIS_QUERY,
1060 388 : "Query", ConnectionStatus::UP, redis_endpoint_list, ostr.str());
1061 : } else {
1062 51 : std::ostringstream empty_ostr;
1063 102 : ConnectionState::GetInstance()->Update(ConnectionType::REDIS_QUERY,
1064 102 : "Query", ConnectionStatus::DOWN, redis_endpoint_list, empty_ostr.str());
1065 51 : }
1066 245 : }
1067 :
1068 15 : void ConnDown(int redis_host_idx, uint8_t cnum) {
1069 15 : string redis_host = redis_host_port_pairs_[redis_host_idx].first;
1070 15 : std::ostringstream ostr;
1071 15 : ostr << "ConnDown.. DOWN.. Reconnect.." << (uint32_t)cnum << " With Redis:" << redis_host;
1072 15 : QE_LOG_NOQID(DEBUG, ostr.str());
1073 15 : connState_[redis_host_idx][cnum] = false;
1074 30 : qosp_->evm_->io_service()->post(
1075 15 : boost::bind(&QEOpServerImpl::UpdateRedisConnectionStatus, this));
1076 30 : qosp_->evm_->io_service()->post(boost::bind(&RedisAsyncConnection::RAC_Connect,
1077 15 : conns_[redis_host_idx][cnum].get()));
1078 15 : }
1079 :
1080 230 : void ConnectCallbackProcess(int redis_host_idx, uint8_t cnum, const redisAsyncContext *c, void *r, void *privdata) {
1081 230 : QE_LOG_NOQID(DEBUG, "UP ConnectCallbackProcess.." << conns_[redis_host_idx][cnum]->Endpoint());
1082 230 : if (r == NULL) {
1083 0 : QE_LOG_NOQID(DEBUG, "In ConnectCallbackProcess.. NULL Reply");
1084 0 : return;
1085 : }
1086 230 : redisReply reply = *reinterpret_cast<redisReply*>(r);
1087 230 : if (reply.type != REDIS_REPLY_ERROR) {
1088 230 : QE_LOG_NOQID(DEBUG, "In ConnectCallbackProcess.." << conns_[redis_host_idx][cnum]->Endpoint());
1089 460 : qosp_->evm_->io_service()->post(
1090 230 : boost::bind(&QEOpServerImpl::UpdateRedisConnectionStatus, this));
1091 460 : qosp_->evm_->io_service()->post(
1092 460 : boost::bind(&QEOpServerImpl::ConnUpPostProcess,
1093 : this, redis_host_idx, cnum));
1094 : } else {
1095 0 : QE_LOG_NOQID(ERROR,"In connectCallbackProcess.. Error");
1096 0 : QE_ASSERT(reply.type != REDIS_REPLY_ERROR);
1097 : }
1098 : }
1099 :
1100 3357 : void CallbackProcess(int redis_host_idx, uint8_t cnum, const redisAsyncContext *c, void *r, void *privdata) {
1101 :
1102 : //QE_TRACE_NOQID(DEBUG, "Redis CB" << cnum);
1103 3357 : if (0 == cnum) {
1104 130 : if (r == NULL) {
1105 3 : QE_LOG_NOQID(DEBUG, __func__ << ": received NULL reply from redis");
1106 3 : return;
1107 : }
1108 :
1109 127 : redisReply reply = *reinterpret_cast<redisReply*>(r);
1110 127 : if (reply.type != REDIS_REPLY_STRING) {
1111 0 : QE_LOG_NOQID(ERROR, __func__ << " Bad Redis reply on control connection: " << reply.type);
1112 0 : if (reply.type == REDIS_REPLY_ERROR) {
1113 0 : string errstr(reply.str);
1114 0 : QE_LOG_NOQID(ERROR, __func__ << " Redis Error: " << reply.str);
1115 0 : sleep(1000);
1116 0 : }
1117 : }
1118 127 : QE_ASSERT(reply.type == REDIS_REPLY_STRING);
1119 127 : string qid(reply.str);
1120 :
1121 127 : StartPipeline(qid, redis_host_idx);
1122 :
1123 254 : qosp_->evm_->io_service()->post(
1124 127 : boost::bind(&QEOpServerImpl::ConnUpPostProcess,
1125 : this, redis_host_idx, cnum));
1126 127 : return;
1127 127 : }
1128 :
1129 3227 : unique_ptr<RedisT> fullReply;
1130 3227 : vector<string> elements;
1131 :
1132 3227 : if (r == NULL) {
1133 : //QE_TRACE_NOQID(DEBUG, "NULL Reply...\n");
1134 : } else {
1135 3227 : redisReply reply = *reinterpret_cast<redisReply*>(r);
1136 3227 : fullReply.reset(new RedisT);
1137 3227 : fullReply.get()->first = reply;
1138 3227 : if (reply.type == REDIS_REPLY_ARRAY) {
1139 0 : for (uint32_t i=0; i<reply.elements; i++) {
1140 0 : string element(reply.element[i]->str);
1141 0 : fullReply.get()->second.push_back(element);
1142 0 : }
1143 3227 : } else if (reply.type == REDIS_REPLY_STRING) {
1144 0 : fullReply.get()->second.push_back(string(reply.str));
1145 : }
1146 : }
1147 3227 : if (!privdata) {
1148 : //QE_TRACE_NOQID(DEBUG, "Ignoring redis reply");
1149 2973 : return;
1150 : }
1151 254 : ExternalProcIf<RedisT> * rpi =
1152 : reinterpret_cast<ExternalProcIf<RedisT> *>(privdata);
1153 254 : QE_TRACE_NOQID(DEBUG, " Rx data from REDIS for " << rpi->Key());
1154 :
1155 254 : rpi->Response(std::move(fullReply));
1156 :
1157 6200 : }
1158 :
1159 44 : void BuildRedisIPPort(vector<string> redis_ip_port_list) {
1160 : typedef boost::tokenizer<boost::char_separator<char> > tokenizer;
1161 44 : for (vector<string>::const_iterator it = redis_ip_port_list.begin();
1162 88 : it != redis_ip_port_list.end(); it++) {
1163 44 : string redis_ip_port(*it);
1164 44 : boost::char_separator<char> sep(":");
1165 44 : tokenizer tokens(redis_ip_port, sep);
1166 44 : tokenizer::iterator tit = tokens.begin();
1167 44 : string redis_ip(*tit);
1168 44 : ++tit;
1169 44 : string redis_port(*tit);
1170 44 : redis_host_port_pairs_.push_back(make_pair(redis_ip, atoi(redis_port.c_str())));
1171 44 : }
1172 44 : }
1173 :
1174 44 : QEOpServerImpl(vector<string> redis_ip_ports,
1175 : const string & redis_password,
1176 : const bool redis_ssl_enable,
1177 : const string & redis_keyfile,
1178 : const string & redis_certfile,
1179 : const string & redis_ca_cert,
1180 : QEOpServerProxy * qosp, int max_tasks,
1181 44 : int max_rows, const string &host_ip) :
1182 44 : hostname_(ResolveCanonicalName(host_ip)),
1183 44 : redis_password_(redis_password),
1184 44 : redis_ssl_enable_(redis_ssl_enable),
1185 44 : redis_keyfile_(redis_keyfile),
1186 44 : redis_certfile_(redis_certfile),
1187 44 : redis_ca_cert_(redis_ca_cert),
1188 44 : qosp_(qosp),
1189 44 : max_tasks_(max_tasks),
1190 88 : max_rows_(max_rows) {
1191 44 : int redis_host_idx = 0;
1192 : /* Initialize */
1193 44 : BuildRedisIPPort(redis_ip_ports);
1194 44 : int redis_host_count = redis_host_port_pairs_.size();
1195 44 : cb_proc_fn_ = new RedisAsyncConnection::ClientAsyncCmdCbFn* [redis_host_count];
1196 44 : conns_ = new boost::shared_ptr<RedisAsyncConnection>* [redis_host_count];
1197 44 : npipes_ = new int* [redis_host_count];
1198 44 : connState_ = new bool* [redis_host_count];
1199 88 : for (int i = 0; i < redis_host_count; i++) {
1200 264 : cb_proc_fn_[i] = new RedisAsyncConnection::ClientAsyncCmdCbFn[kConnections + 1];
1201 264 : conns_[i] = new boost::shared_ptr<RedisAsyncConnection>[kConnections + 1];
1202 44 : npipes_[i] = new int[kConnections];
1203 44 : connState_[i] = new bool[kConnections + 1];
1204 : }
1205 88 : for (redis_host_idx = 0; redis_host_idx < int(redis_host_count); redis_host_idx++) {
1206 44 : string redis_host = redis_host_port_pairs_[redis_host_idx].first;
1207 44 : int redis_port = redis_host_port_pairs_[redis_host_idx].second;
1208 264 : for (int i = 0; i < kConnections + 1; i++) {
1209 220 : cb_proc_fn_[redis_host_idx][i] = boost::bind(&QEOpServerImpl::CallbackProcess,
1210 220 : this, redis_host_idx, i, _1, _2, _3);
1211 220 : connState_[redis_host_idx][i] = false;
1212 220 : if (i) {
1213 176 : conns_[redis_host_idx][i].reset(rac_alloc(qosp->evm_, redis_host, redis_port,
1214 : boost::bind(&QEOpServerImpl::ConnUp, this, redis_host_idx, i),
1215 : boost::bind(&QEOpServerImpl::ConnDown, this, redis_host_idx, i),
1216 : redis_ssl_enable, redis_keyfile, redis_certfile, redis_ca_cert));
1217 : } else {
1218 44 : conns_[redis_host_idx][i].reset(rac_alloc_nocheck(qosp->evm_, redis_host, redis_port,
1219 : boost::bind(&QEOpServerImpl::ConnUp, this, redis_host_idx, i),
1220 : boost::bind(&QEOpServerImpl::ConnDown, this, redis_host_idx, i),
1221 : redis_ssl_enable, redis_keyfile, redis_certfile, redis_ca_cert));
1222 : }
1223 : // The cnum with index 0 is only used for receiving new queries
1224 : // It does not host any pipelines
1225 220 : if (i) npipes_[redis_host_idx][i-1] = 0;
1226 : }
1227 44 : }
1228 44 : }
1229 :
1230 264 : ~QEOpServerImpl() {
1231 44 : }
1232 :
1233 912 : void AddAnalyticsQuery(const std::string &qid,
1234 : boost::shared_ptr<AnalyticsQuery> q) {
1235 912 : std::scoped_lock lock(mutex_);
1236 912 : m_analytics_queries[qid].push_back(q);
1237 912 : }
1238 :
1239 : private:
1240 :
1241 : static const int kMaxRowThreshold = 10000;
1242 :
1243 : // We always have one connection to receive new queries from OpServer
1244 : // This is the number of addition connections, which will be
1245 : // used to read query parameters and write query results
1246 : static const uint8_t kConnections = 4;
1247 :
1248 : const string hostname_;
1249 : map<string, string> redis_status_maps;
1250 : const string redis_password_;
1251 : const bool redis_ssl_enable_;
1252 : const string redis_keyfile_;
1253 : const string redis_certfile_;
1254 : const string redis_ca_cert_;
1255 : QEOpServerProxy * const qosp_;
1256 : boost::shared_ptr<RedisAsyncConnection> **conns_;
1257 : RedisAsyncConnection::ClientAsyncCmdCbFn **cb_proc_fn_;
1258 : bool **connState_;
1259 :
1260 : std::mutex mutex_;
1261 : map<string,QEPipeT*> pipes_;
1262 : vector<pair<string, int> > redis_host_port_pairs_;
1263 : int **npipes_;
1264 : int max_tasks_;
1265 : int max_rows_;
1266 : std::map<std::string, std::vector<boost::shared_ptr<AnalyticsQuery> > >
1267 : m_analytics_queries;
1268 :
1269 : };
1270 :
1271 44 : QEOpServerProxy::QEOpServerProxy(EventManager *evm, QueryEngine *qe,
1272 : vector<string> redis_ip_ports,
1273 : const string & redis_password,
1274 : const bool redis_ssl_enable,
1275 : const string & redis_keyfile,
1276 : const string & redis_certfile,
1277 : const string & redis_ca_cert,
1278 44 : const std::string &host_ip, int max_tasks, int max_rows) :
1279 44 : evm_(evm),
1280 44 : qe_(qe),
1281 44 : impl_(new QEOpServerImpl(redis_ip_ports, redis_password, redis_ssl_enable,
1282 : redis_keyfile, redis_certfile, redis_ca_cert,
1283 88 : this, max_tasks, max_rows, host_ip)) {}
1284 :
1285 88 : QEOpServerProxy::~QEOpServerProxy() {}
1286 :
1287 : void
1288 864 : QEOpServerProxy::QueryResult(void * qid, QPerfInfo qperf,
1289 : std::auto_ptr<BufferT> res, std::auto_ptr<OutRowMultimapT> mres) {
1290 :
1291 864 : impl_->QECallback(qid, qperf, std::unique_ptr<BufferT>(res.release()), std::unique_ptr<OutRowMultimapT>(mres.release()));
1292 864 : }
1293 :
1294 : void
1295 912 : QEOpServerProxy::QueryResult(void * qid, QPerfInfo qperf,
1296 : std::auto_ptr<std::vector<query_result_unit_t> > res) {
1297 :
1298 912 : impl_->QECallback(qid, qperf, std::unique_ptr<std::vector<query_result_unit_t>>(res.release()));
1299 912 : }
1300 :
1301 912 : void QEOpServerProxy::AddAnalyticsQuery(const std::string &qid,
1302 : boost::shared_ptr<AnalyticsQuery> q) {
1303 912 : impl_->AddAnalyticsQuery(qid, q);
1304 912 : }
|