2024-10-31 Thu 01:57:21:929.100 UTC cn-jenkins-deploy-platform-k8s-manifests-105-1 [Thread 139678573074560, Pid 1]: SANDESH: No Client: 1730339841929020 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-105-1.cluster.local:Database:contrail-query-engine:0 client_info= [ status = Idle successful_connections = 0 pid = 1 http_port = 8091 start_time = 1730339841928940 collector_name = collector_ip = 0.0.0.0:0 collector_list= [ [ ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 0, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-31 Thu 01:57:21:940.247 UTC cn-jenkins-deploy-platform-k8s-manifests-105-1 [Thread 139678429607680, Pid 1]: SANDESH: Send FAILED: 1730339841929543 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-105-1.cluster.local process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Idle : EvStart -> Disconnect ], ] ] flag_infos= [ [ ] ] description = Number of connections:1, Expected:3 Missing: Database,Redis-Query:Query ], ] ] ] 2024-10-31 Thu 01:57:21:940.358 UTC cn-jenkins-deploy-platform-k8s-manifests-105-1 [Thread 139678429607680, Pid 1]: SANDESH: Send FAILED: 1730339841939294 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-105-1.cluster.local:Database:contrail-query-engine:0 client_info= [ status = Disconnect successful_connections = 0 pid = 1 http_port = 8091 start_time = 1730339841928940 collector_name = collector_ip = 0.0.0.0:0 collector_list= [ [ ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-31 Thu 01:57:21:940.445 UTC cn-jenkins-deploy-platform-k8s-manifests-105-1 [Thread 139678429607680, Pid 1]: SANDESH: Send FAILED: 1730339841939406 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-105-1.cluster.local:Database:contrail-query-engine:0 client_info= [ status = Disconnect successful_connections = 0 pid = 1 http_port = 8091 start_time = 1730339841928940 collector_name = collector_ip = 10.0.0.46:8086 collector_list= [ [ (*_iter40) = 10.0.0.46:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-31 Thu 01:57:21:940.467 UTC cn-jenkins-deploy-platform-k8s-manifests-105-1 [Thread 139678429607680, Pid 1]: SANDESH: Send FAILED: 1730339841939431 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-105-1.cluster.local process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.46:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = Number of connections:1, Expected:3 Missing: Database,Redis-Query:Query ], ] ] ] 2024-10-31 Thu 01:57:21:940.530 UTC cn-jenkins-deploy-platform-k8s-manifests-105-1 [Thread 139678429607680, Pid 1]: SANDESH: Send FAILED: 1730339841940212 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-105-1.cluster.local:Database:contrail-query-engine:0 client_info= [ status = Connect successful_connections = 0 pid = 1 http_port = 8091 start_time = 1730339841928940 collector_name = collector_ip = 10.0.0.46:8086 collector_list= [ [ (*_iter40) = 10.0.0.46:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-31 Thu 01:57:21:948.649 UTC cn-jenkins-deploy-platform-k8s-manifests-105-1 [Thread 139678573074560, Pid 1]: QeInit [SYS_WARN]: QELog: initialization of database failed. retrying 0 time src/contrail-analytics/contrail-query-engine/query.cc 1065 2024-10-31 Thu 01:57:21:948.720 UTC cn-jenkins-deploy-platform-k8s-manifests-105-1 [Thread 139678382761728, Pid 1]: SANDESH: Send FAILED: 1730339841948627 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-105-1.cluster.local process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.46:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], [ type = Database name = server_addrs= [ [ (*_iter6) = 10.0.0.46:9042, ] ] status = Down description = ], ] ] flag_infos= [ [ ] ] description = Number of connections:2, Expected:3 Missing: Redis-Query:Query ], ] ] ] 2024-10-31 Thu 01:57:26:966.268 UTC cn-jenkins-deploy-platform-k8s-manifests-105-1 [Thread 139678573074560, Pid 1]: QeInit [SYS_WARN]: QELog: initialization of database failed. retrying 1 time src/contrail-analytics/contrail-query-engine/query.cc 1065 2024-10-31 Thu 01:57:31:981.390 UTC cn-jenkins-deploy-platform-k8s-manifests-105-1 [Thread 139678573074560, Pid 1]: QeInit [SYS_WARN]: QELog: initialization of database failed. retrying 2 time src/contrail-analytics/contrail-query-engine/query.cc 1065 2024-10-31 Thu 01:57:36:983.099 UTC cn-jenkins-deploy-platform-k8s-manifests-105-1 [Thread 139678573074560, Pid 1]: QeInit [SYS_WARN]: QELog: initialization of database failed. retrying 3 time src/contrail-analytics/contrail-query-engine/query.cc 1065 2024-10-31 Thu 01:57:42:005.245 UTC cn-jenkins-deploy-platform-k8s-manifests-105-1 [Thread 139678573074560, Pid 1]: QeInit [SYS_WARN]: QELog: initialization of database failed. retrying 4 time src/contrail-analytics/contrail-query-engine/query.cc 1065 2024-10-31 Thu 01:57:47:006.389 UTC cn-jenkins-deploy-platform-k8s-manifests-105-1 [Thread 139678573074560, Pid 1]: QeInit [SYS_WARN]: QELog: initialization of database failed. retrying 5 time src/contrail-analytics/contrail-query-engine/query.cc 1065 2024-10-31 Thu 01:57:52:300.846 UTC cn-jenkins-deploy-platform-k8s-manifests-105-1 [Thread 139678573074560, Pid 1]: QeInit [SYS_WARN]: QELog: initialization of database failed. retrying 6 time src/contrail-analytics/contrail-query-engine/query.cc 1065 2024-10-31 Thu 01:57:57:436.392 UTC cn-jenkins-deploy-platform-k8s-manifests-105-1 [Thread 139678573074560, Pid 1]: QeInit [SYS_WARN]: QELog: initialization of database failed. retrying 7 time src/contrail-analytics/contrail-query-engine/query.cc 1065 2024-10-31 Thu 01:58:02:463.254 UTC cn-jenkins-deploy-platform-k8s-manifests-105-1 [Thread 139678404417280, Pid 1]: SANDESH: Send FAILED: 1730339882462264 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-105-1.cluster.local process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.46:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], [ type = Database name = server_addrs= [ [ (*_iter6) = 10.0.0.46:9042, ] ] status = Up description = ], ] ] flag_infos= [ [ ] ] description = Number of connections:2, Expected:3 Missing: Redis-Query:Query ], ] ] ] 2024-10-31 Thu 01:58:02:465.286 UTC cn-jenkins-deploy-platform-k8s-manifests-105-1 [Thread 139678404417280, Pid 1]: SANDESH: Send FAILED: 1730339882465178 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-105-1.cluster.local process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.46:8086, ] ] status = Initializing description = Connect : EvTcpConnected ], [ type = Database name = server_addrs= [ [ (*_iter6) = 10.0.0.46:9042, ] ] status = Up description = ], ] ] flag_infos= [ [ ] ] description = Number of connections:2, Expected:3 Missing: Redis-Query:Query ], ] ] ] 2024-10-31 Thu 01:58:02:465.430 UTC cn-jenkins-deploy-platform-k8s-manifests-105-1 [Thread 139678404417280, Pid 1]: SANDESH: Send FAILED: 1730339882465261 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-105-1.cluster.local:Database:contrail-query-engine:0 client_info= [ status = ClientInit successful_connections = 1 pid = 1 http_port = 8091 start_time = 1730339841928940 collector_name = collector_ip = 10.0.0.46:8086 collector_list= [ [ (*_iter40) = 10.0.0.46:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 24 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 26, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 7, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-10-31 Thu 01:58:02:466.110 UTC cn-jenkins-deploy-platform-k8s-manifests-105-1 [Thread 139678382761728, Pid 1]: SANDESH: Send FAILED: 1730339882465989 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-105-1.cluster.local:Database:contrail-query-engine:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 10 dequeues = 10 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpConnected enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 13 dequeues = 13 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = ClientInit state_since = 1730339882465184 last_state = ClientInit last_event = scm::EvSandeshSend last_event_at = 1730339882465323 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 1 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 5 ] 2024-10-31 Thu 01:58:02:466.246 UTC cn-jenkins-deploy-platform-k8s-manifests-105-1 [Thread 139678382761728, Pid 1]: SANDESH: Send FAILED: 1730339882466039 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-105-1.cluster.local:Database:contrail-query-engine:0 client_info= [ status = ClientInit successful_connections = 1 pid = 1 http_port = 8091 start_time = 1730339841928940 collector_name = collector_ip = 10.0.0.46:8086 collector_list= [ [ (*_iter40) = 10.0.0.46:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = QELog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 29 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshCtrlClientToServer [ messages_sent = 1 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 31, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 9, _iter59->first = sent _iter59->second = 1, ] ] ] 2024-10-31 Thu 01:58:02:467.161 UTC cn-jenkins-deploy-platform-k8s-manifests-105-1 [Thread 139677931640576, Pid 1]: SANDESH: Send FAILED: 1730339882467012 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-105-1.cluster.local process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.46:8086, ] ] status = Initializing description = Connect : EvTcpConnected ], [ type = Database name = server_addrs= [ [ (*_iter6) = 10.0.0.46:9042, ] ] status = Up description = ], [ type = Redis-Query name = Query server_addrs= [ [ (*_iter6) = 10.0.0.46:6379, ] ] status = Down description = ], ] ] flag_infos= [ [ ] ] description = Collector, Redis-Query:Query connection down ], ] ] ] 2024-10-31 Thu 01:58:02:467.207 UTC cn-jenkins-deploy-platform-k8s-manifests-105-1 [Thread 139677931640576, Pid 1]: SANDESH: Send FAILED: 1730339882467038 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-105-1.cluster.local process_status= [ [ [ module_id = contrail-query-engine instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.46:8086, ] ] status = Initializing description = Connect : EvTcpConnected ], [ type = Database name = server_addrs= [ [ (*_iter6) = 10.0.0.46:9042, ] ] status = Up description = ], [ type = Redis-Query name = Query server_addrs= [ [ (*_iter6) = 10.0.0.46:6379, ] ] status = Up description = ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ]