2024-10-21 Mon 02:28:48:287.381 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464052279040, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:29:17:267.709 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:29:17:267.913 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: SANDESH: Send FAILED: 1729477757267555  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 30 dequeues = 30 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 31 dequeues = 31 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 259 dequeues = 259 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 31 dequeues = 31 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 354 dequeues = 354 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1729477728286578 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729477728286929 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:29:17:267.934 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:29:17:267.946 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:29:17:268.239 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: SANDESH: Send FAILED: 1729477757267664  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 69 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 175 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 37 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1466,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 259,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:29:17:268.273 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:29:18:286.931 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:29:18:287.053 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: Processing scm::EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:29:18:287.092 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: Connect : EvConnectTimerExpired
2024-10-21 Mon 02:29:18:287.197 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: Idle
2024-10-21 Mon 02:29:18:287.339 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:29:18:287.380 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: SANDESH: Send FAILED: 1729477758287130  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:29:18:287.407 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:29:18:287.421 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: Processing scm::EvTcpDeleteSession in state Idle
2024-10-21 Mon 02:29:18:287.459 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:29:18:287.792 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: SANDESH: Send FAILED: 1729477758287318  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 69 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 177 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 37 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1466,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 261,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:29:18:287.826 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:29:47:267.982 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463348643584, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:29:47:268.097 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463348643584, Pid 1053912]: SANDESH: Send FAILED: 1729477787267818  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 31 dequeues = 31 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 31 dequeues = 31 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 263 dequeues = 263 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 32 dequeues = 32 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 360 dequeues = 360 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1729477758287195 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729477758287456 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:29:47:268.114 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463348643584, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:29:47:268.123 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463348643584, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:29:47:268.425 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463348643584, Pid 1053912]: SANDESH: Send FAILED: 1729477787267950  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 70 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 178 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 37 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1466,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 263,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:29:47:268.457 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463348643584, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:29:48:287.373 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Processing scm::EvIdleHoldTimerExpired in state Idle
2024-10-21 Mon 02:29:48:287.683 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Connect : Start Connect timer 10.0.0.30:8086
2024-10-21 Mon 02:29:48:287.836 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:29:48:287.871 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: SANDESH: Send FAILED: 1729477788287487  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.30:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:29:48:287.881 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:29:48:287.891 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:29:48:288.175 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: SANDESH: Send FAILED: 1729477788287822  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 70 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 180 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 37 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1466,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 265,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:29:48:288.199 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:30:17:268.300 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464060675840, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:30:17:268.458 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464060675840, Pid 1053912]: SANDESH: Send FAILED: 1729477817268149  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 31 dequeues = 31 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 32 dequeues = 32 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 267 dequeues = 267 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 32 dequeues = 32 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 365 dequeues = 365 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1729477788287494 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729477788287889 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:30:17:268.475 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464060675840, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:30:17:268.502 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464060675840, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:30:17:268.813 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464060675840, Pid 1053912]: SANDESH: Send FAILED: 1729477817268247  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 71 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 181 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 37 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1466,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 267,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:30:17:268.837 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464060675840, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:30:18:287.862 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464060675840, Pid 1053912]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:30:18:287.969 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Processing scm::EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:30:18:288.010 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Connect : EvConnectTimerExpired
2024-10-21 Mon 02:30:18:288.119 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Idle
2024-10-21 Mon 02:30:18:288.240 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:30:18:288.275 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: SANDESH: Send FAILED: 1729477818288048  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:30:18:288.286 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:30:18:288.296 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Processing scm::EvTcpDeleteSession in state Idle
2024-10-21 Mon 02:30:18:288.331 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:30:18:288.639 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: SANDESH: Send FAILED: 1729477818288224  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 71 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 183 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 37 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1466,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 269,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:30:18:288.676 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:30:47:268.587 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:30:47:268.763 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: SANDESH: Send FAILED: 1729477847268390  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 32 dequeues = 32 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 32 dequeues = 32 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 271 dequeues = 271 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 33 dequeues = 33 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 371 dequeues = 371 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1729477818288117 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729477818288330 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:30:47:268.804 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:30:47:268.821 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:30:47:269.407 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: SANDESH: Send FAILED: 1729477847268516  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 72 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 184 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 37 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1466,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 271,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:30:47:269.453 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:30:48:288.290 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463340246784, Pid 1053912]: Processing scm::EvIdleHoldTimerExpired in state Idle
2024-10-21 Mon 02:30:48:288.668 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463340246784, Pid 1053912]: Connect : Start Connect timer 10.0.0.30:8086
2024-10-21 Mon 02:30:48:288.796 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463340246784, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:30:48:288.852 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463340246784, Pid 1053912]: SANDESH: Send FAILED: 1729477848288444  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.30:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:30:48:288.863 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463340246784, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:30:48:288.873 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463340246784, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:30:48:289.204 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463340246784, Pid 1053912]: SANDESH: Send FAILED: 1729477848288778  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 72 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 186 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 37 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1466,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 273,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:30:48:289.239 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463340246784, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:31:17:268.933 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:31:17:269.124 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: SANDESH: Send FAILED: 1729477877268735  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 32 dequeues = 32 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 33 dequeues = 33 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 275 dequeues = 275 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 33 dequeues = 33 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 376 dequeues = 376 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1729477848288454 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729477848288871 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:31:17:269.154 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:31:17:269.166 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:31:17:269.449 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: SANDESH: Send FAILED: 1729477877268873  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 73 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 187 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 37 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1466,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 275,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:31:17:269.469 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:31:18:288.831 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464052279040, Pid 1053912]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:31:18:288.942 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464031287040, Pid 1053912]: Processing scm::EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:31:18:289.031 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464031287040, Pid 1053912]: Connect : EvConnectTimerExpired
2024-10-21 Mon 02:31:18:289.127 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464031287040, Pid 1053912]: Idle
2024-10-21 Mon 02:31:18:289.259 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464031287040, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:31:18:289.293 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464031287040, Pid 1053912]: SANDESH: Send FAILED: 1729477878289072  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:31:18:289.314 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464031287040, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:31:18:289.324 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464031287040, Pid 1053912]: Processing scm::EvTcpDeleteSession in state Idle
2024-10-21 Mon 02:31:18:289.335 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464031287040, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:31:18:289.644 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464031287040, Pid 1053912]: SANDESH: Send FAILED: 1729477878289240  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 73 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 189 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 37 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1466,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 277,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:31:18:289.672 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464031287040, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:31:47:269.282 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:31:47:270.425 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: SANDESH: Send FAILED: 1729477907269040  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 33 dequeues = 33 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 33 dequeues = 33 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 279 dequeues = 279 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 34 dequeues = 34 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 382 dequeues = 382 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1729477878289125 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729477878289334 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:31:47:270.457 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:31:47:270.471 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:31:47:270.797 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: SANDESH: Send FAILED: 1729477907269220  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 74 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 190 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 37 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1466,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 279,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:31:47:270.836 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:31:48:289.297 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463344445184, Pid 1053912]: Processing scm::EvIdleHoldTimerExpired in state Idle
2024-10-21 Mon 02:31:48:289.719 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463344445184, Pid 1053912]: Connect : Start Connect timer 10.0.0.30:8086
2024-10-21 Mon 02:31:48:289.849 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463344445184, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:31:48:289.884 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463344445184, Pid 1053912]: SANDESH: Send FAILED: 1729477908289389  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.30:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:31:48:289.907 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463344445184, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:31:48:289.917 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463344445184, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:31:48:290.196 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463344445184, Pid 1053912]: SANDESH: Send FAILED: 1729477908289833  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 74 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 192 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 37 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1466,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 281,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:31:48:290.219 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463344445184, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:32:05:388.439 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464231143552, Pid 1053912]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8092::10.0.0.30:34704 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256
2024-10-21 Mon 02:32:05:439.391 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464231143552, Pid 1053912]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8092::10.0.0.30:34704 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504
2024-10-21 Mon 02:32:06:544.329 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464231143552, Pid 1053912]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8092::10.0.0.30:34716 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256
2024-10-21 Mon 02:32:06:591.498 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464231143552, Pid 1053912]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8092::10.0.0.30:34716 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504
2024-10-21 Mon 02:32:17:269.567 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:32:17:269.709 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: SANDESH: Send FAILED: 1729477937269419  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 33 dequeues = 33 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 34 dequeues = 34 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 283 dequeues = 283 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 34 dequeues = 34 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 387 dequeues = 387 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1729477908289552 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729477908289915 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:32:17:269.740 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:32:17:269.750 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:32:17:270.022 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: SANDESH: Send FAILED: 1729477937269514  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 75 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 193 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1470,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 283,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:32:17:270.041 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:32:18:289.912 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:32:18:290.010 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: Processing scm::EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:32:18:290.049 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: Connect : EvConnectTimerExpired
2024-10-21 Mon 02:32:18:290.161 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: Idle
2024-10-21 Mon 02:32:18:290.272 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:32:18:290.307 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: SANDESH: Send FAILED: 1729477938290090  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:32:18:290.319 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:32:18:290.328 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: Processing scm::EvTcpDeleteSession in state Idle
2024-10-21 Mon 02:32:18:290.374 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:32:18:290.676 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: SANDESH: Send FAILED: 1729477938290259  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 75 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 195 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1470,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 285,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:32:18:290.709 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:32:47:269.864 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:32:47:270.024 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: SANDESH: Send FAILED: 1729477967269681  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 34 dequeues = 34 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 34 dequeues = 34 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 287 dequeues = 287 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 35 dequeues = 35 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 393 dequeues = 393 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1729477938290159 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729477938290372 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:32:47:270.055 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:32:47:270.067 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:32:47:270.366 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: SANDESH: Send FAILED: 1729477967269810  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 76 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 196 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1470,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 287,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:32:47:270.388 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:32:48:290.352 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463336048384, Pid 1053912]: Processing scm::EvIdleHoldTimerExpired in state Idle
2024-10-21 Mon 02:32:48:290.651 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463336048384, Pid 1053912]: Connect : Start Connect timer 10.0.0.30:8086
2024-10-21 Mon 02:32:48:290.785 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463336048384, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:32:48:290.821 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463336048384, Pid 1053912]: SANDESH: Send FAILED: 1729477968290464  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.30:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:32:48:290.830 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463336048384, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:32:48:290.840 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463336048384, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:32:48:291.176 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463336048384, Pid 1053912]: SANDESH: Send FAILED: 1729477968290767  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 76 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 198 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1470,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 289,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:32:48:291.219 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463336048384, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:33:17:270.132 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:33:17:270.270 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: SANDESH: Send FAILED: 1729477997270006  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 34 dequeues = 34 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 35 dequeues = 35 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 291 dequeues = 291 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 35 dequeues = 35 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 398 dequeues = 398 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1729477968290474 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729477968290838 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:33:17:270.284 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:33:17:270.294 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:33:17:270.601 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: SANDESH: Send FAILED: 1729477997270102  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 77 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 199 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1470,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 291,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:33:17:270.630 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:33:18:290.791 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464052279040, Pid 1053912]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:33:18:290.895 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463344445184, Pid 1053912]: Processing scm::EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:33:18:290.938 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463344445184, Pid 1053912]: Connect : EvConnectTimerExpired
2024-10-21 Mon 02:33:18:291.043 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463344445184, Pid 1053912]: Idle
2024-10-21 Mon 02:33:18:291.159 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463344445184, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:33:18:291.196 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463344445184, Pid 1053912]: SANDESH: Send FAILED: 1729477998290975  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:33:18:291.222 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463344445184, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:33:18:291.233 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463344445184, Pid 1053912]: Processing scm::EvTcpDeleteSession in state Idle
2024-10-21 Mon 02:33:18:291.272 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463344445184, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:33:18:291.578 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463344445184, Pid 1053912]: SANDESH: Send FAILED: 1729477998291144  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 77 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 201 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1470,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 293,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:33:18:291.608 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463344445184, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:33:47:270.505 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:33:47:270.646 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: SANDESH: Send FAILED: 1729478027270247  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 35 dequeues = 35 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 35 dequeues = 35 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 295 dequeues = 295 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 36 dequeues = 36 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 404 dequeues = 404 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1729477998291041 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729477998291270 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:33:47:270.666 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:33:47:270.677 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:33:47:270.988 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: SANDESH: Send FAILED: 1729478027270454  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 78 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 202 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1470,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 295,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:33:47:272.082 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:33:48:291.215 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464077469440, Pid 1053912]: Processing scm::EvIdleHoldTimerExpired in state Idle
2024-10-21 Mon 02:33:48:291.607 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464077469440, Pid 1053912]: Connect : Start Connect timer 10.0.0.30:8086
2024-10-21 Mon 02:33:48:291.781 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464077469440, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:33:48:291.826 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464077469440, Pid 1053912]: SANDESH: Send FAILED: 1729478028291402  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.30:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:33:48:291.840 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464077469440, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:33:48:291.855 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464077469440, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:33:48:292.365 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464077469440, Pid 1053912]: SANDESH: Send FAILED: 1729478028291750  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 78 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 204 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1470,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 297,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:33:48:292.415 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464077469440, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:34:17:270.826 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463348643584, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:34:17:270.972 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463348643584, Pid 1053912]: SANDESH: Send FAILED: 1729478057270679  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 35 dequeues = 35 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 36 dequeues = 36 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 299 dequeues = 299 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 36 dequeues = 36 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 409 dequeues = 409 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1729478028291414 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729478028291852 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:34:17:271.005 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463348643584, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:34:17:271.023 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463348643584, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:34:17:271.307 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463348643584, Pid 1053912]: SANDESH: Send FAILED: 1729478057270782  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 79 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 205 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1470,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 299,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:34:17:271.328 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463348643584, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:34:18:291.817 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464069072640, Pid 1053912]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:34:18:291.926 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Processing scm::EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:34:18:291.973 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Connect : EvConnectTimerExpired
2024-10-21 Mon 02:34:18:292.126 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Idle
2024-10-21 Mon 02:34:18:292.321 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:34:18:292.371 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: SANDESH: Send FAILED: 1729478058292040  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:34:18:292.387 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:34:18:292.402 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Processing scm::EvTcpDeleteSession in state Idle
2024-10-21 Mon 02:34:18:292.449 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:34:18:293.133 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: SANDESH: Send FAILED: 1729478058292296  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 79 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 207 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1470,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 301,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:34:18:293.227 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:34:47:271.114 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463340246784, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:34:47:271.227 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463340246784, Pid 1053912]: SANDESH: Send FAILED: 1729478087270940  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 36 dequeues = 36 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 36 dequeues = 36 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 303 dequeues = 303 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 37 dequeues = 37 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 415 dequeues = 415 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1729478058292123 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729478058292446 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:34:47:271.259 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463340246784, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:34:47:271.268 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463340246784, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:34:47:271.649 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463340246784, Pid 1053912]: SANDESH: Send FAILED: 1729478087271072  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 80 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 208 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1470,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 303,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:34:47:271.677 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463340246784, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:34:48:292.299 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: Processing scm::EvIdleHoldTimerExpired in state Idle
2024-10-21 Mon 02:34:48:292.590 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: Connect : Start Connect timer 10.0.0.30:8086
2024-10-21 Mon 02:34:48:292.750 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:34:48:292.784 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: SANDESH: Send FAILED: 1729478088292396  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.30:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:34:48:292.795 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:34:48:292.805 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:34:48:293.134 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: SANDESH: Send FAILED: 1729478088292726  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 80 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 210 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1470,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 305,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:34:48:293.167 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464039683840, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:35:17:271.415 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:35:17:271.598 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: SANDESH: Send FAILED: 1729478117271270  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 36 dequeues = 36 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 37 dequeues = 37 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 307 dequeues = 307 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 37 dequeues = 37 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 420 dequeues = 420 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1729478088292404 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729478088292804 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:35:17:271.615 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:35:17:271.626 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:35:17:271.931 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: SANDESH: Send FAILED: 1729478117271372  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 81 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 211 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1470,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 307,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:35:17:271.956 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:35:18:292.765 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463336048384, Pid 1053912]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:35:18:292.864 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Processing scm::EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:35:18:292.906 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Connect : EvConnectTimerExpired
2024-10-21 Mon 02:35:18:293.035 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Idle
2024-10-21 Mon 02:35:18:293.158 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:35:18:293.190 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: SANDESH: Send FAILED: 1729478118292944  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:35:18:293.216 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:35:18:293.225 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Processing scm::EvTcpDeleteSession in state Idle
2024-10-21 Mon 02:35:18:293.253 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:35:18:293.571 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: SANDESH: Send FAILED: 1729478118293143  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 81 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 213 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1470,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 309,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:35:18:293.599 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:35:47:271.697 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463336048384, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:35:47:271.809 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463336048384, Pid 1053912]: SANDESH: Send FAILED: 1729478147271512  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 37 dequeues = 37 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 37 dequeues = 37 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 311 dequeues = 311 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 38 dequeues = 38 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 426 dequeues = 426 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1729478118293033 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729478118293251 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:35:47:271.824 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463336048384, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:35:47:271.834 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463336048384, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:35:47:272.139 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463336048384, Pid 1053912]: SANDESH: Send FAILED: 1729478147271641  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 82 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 214 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1470,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 311,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:35:47:272.173 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463336048384, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:35:48:293.194 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464077469440, Pid 1053912]: Processing scm::EvIdleHoldTimerExpired in state Idle
2024-10-21 Mon 02:35:48:293.464 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464077469440, Pid 1053912]: Connect : Start Connect timer 10.0.0.30:8086
2024-10-21 Mon 02:35:48:293.616 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464077469440, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:35:48:293.652 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464077469440, Pid 1053912]: SANDESH: Send FAILED: 1729478148293298  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.30:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:35:48:293.676 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464077469440, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:35:48:293.686 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464077469440, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:35:48:293.995 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464077469440, Pid 1053912]: SANDESH: Send FAILED: 1729478148293600  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 82 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 216 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1470,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 313,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:35:48:294.019 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464077469440, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:36:17:272.192 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:36:17:272.336 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: SANDESH: Send FAILED: 1729478177271859  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 37 dequeues = 37 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 38 dequeues = 38 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 315 dequeues = 315 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 38 dequeues = 38 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 431 dequeues = 431 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1729478148293308 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729478148293684 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:36:17:272.352 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:36:17:272.363 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:36:17:272.690 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: SANDESH: Send FAILED: 1729478177272139  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 83 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 217 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1470,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 315,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:36:17:272.727 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:36:18:293.829 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:36:18:293.947 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Processing scm::EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:36:18:293.965 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Connect : EvConnectTimerExpired
2024-10-21 Mon 02:36:18:294.091 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Idle
2024-10-21 Mon 02:36:18:294.262 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:36:18:294.309 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: SANDESH: Send FAILED: 1729478178294010  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:36:18:294.323 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:36:18:294.334 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Processing scm::EvTcpDeleteSession in state Idle
2024-10-21 Mon 02:36:18:294.344 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:36:18:294.673 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: SANDESH: Send FAILED: 1729478178294240  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 83 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 219 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1470,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 317,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:36:18:294.709 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:36:47:272.474 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:36:47:272.660 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: SANDESH: Send FAILED: 1729478207272289  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 38 dequeues = 38 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 38 dequeues = 38 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 319 dequeues = 319 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 39 dequeues = 39 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 437 dequeues = 437 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1729478178294088 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729478178294343 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:36:47:272.700 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:36:47:272.717 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:36:47:273.330 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: SANDESH: Send FAILED: 1729478207272416  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 84 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 220 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1470,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 319,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:36:47:274.263 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:36:48:294.248 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: Processing scm::EvIdleHoldTimerExpired in state Idle
2024-10-21 Mon 02:36:48:294.507 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: Connect : Start Connect timer 10.0.0.30:8086
2024-10-21 Mon 02:36:48:294.643 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:36:48:294.677 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: SANDESH: Send FAILED: 1729478208294342  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.30:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:36:48:294.687 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:36:48:294.697 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:36:48:294.981 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: SANDESH: Send FAILED: 1729478208294628  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 84 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 222 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 41 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1470,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 321,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:36:48:295.013 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:37:05:390.858 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464231143552, Pid 1053912]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8092::10.0.0.30:59006 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256
2024-10-21 Mon 02:37:05:439.941 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464231143552, Pid 1053912]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8092::10.0.0.30:59006 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504
2024-10-21 Mon 02:37:06:544.757 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464231143552, Pid 1053912]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8092::10.0.0.30:59020 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256
2024-10-21 Mon 02:37:06:595.875 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464231143552, Pid 1053912]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8092::10.0.0.30:59020 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504
2024-10-21 Mon 02:37:17:272.755 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:37:17:272.911 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: SANDESH: Send FAILED: 1729478237272607  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 38 dequeues = 38 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 39 dequeues = 39 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 323 dequeues = 323 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 39 dequeues = 39 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 442 dequeues = 442 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1729478208294351 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729478208294695 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:37:17:272.928 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:37:17:272.948 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:37:17:273.261 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: SANDESH: Send FAILED: 1729478237272704  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 85 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 223 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 45 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1474,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 323,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:37:17:273.292 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:37:18:294.705 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463340246784, Pid 1053912]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:37:18:294.827 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Processing scm::EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:37:18:294.873 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Connect : EvConnectTimerExpired
2024-10-21 Mon 02:37:18:295.035 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Idle
2024-10-21 Mon 02:37:18:295.186 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:37:18:295.249 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: SANDESH: Send FAILED: 1729478238294921  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:37:18:295.263 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:37:18:295.293 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Processing scm::EvTcpDeleteSession in state Idle
2024-10-21 Mon 02:37:18:295.336 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:37:18:295.883 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: SANDESH: Send FAILED: 1729478238295166  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 85 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 225 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 45 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1474,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 325,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:37:18:295.918 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:37:47:273.064 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:37:47:273.244 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: SANDESH: Send FAILED: 1729478267272845  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 39 dequeues = 39 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 39 dequeues = 39 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 327 dequeues = 327 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 40 dequeues = 40 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 448 dequeues = 448 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1729478238295033 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729478238295333 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:37:47:273.266 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:37:47:273.277 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:37:47:273.596 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: SANDESH: Send FAILED: 1729478267273017  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 86 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 226 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 45 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1474,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 327,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:37:47:273.633 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:37:48:295.203 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Processing scm::EvIdleHoldTimerExpired in state Idle
2024-10-21 Mon 02:37:48:295.525 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Connect : Start Connect timer 10.0.0.30:8086
2024-10-21 Mon 02:37:48:295.700 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:37:48:295.770 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: SANDESH: Send FAILED: 1729478268295311  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.30:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:37:48:295.785 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:37:48:295.800 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:37:48:296.332 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: SANDESH: Send FAILED: 1729478268295675  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 86 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 228 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 45 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1474,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 329,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:37:48:296.382 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:38:17:273.443 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:38:17:273.652 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: SANDESH: Send FAILED: 1729478297273242  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 39 dequeues = 39 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 40 dequeues = 40 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 331 dequeues = 331 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 40 dequeues = 40 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 453 dequeues = 453 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1729478268295322 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729478268295797 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:38:17:273.709 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:38:17:273.731 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:38:17:274.031 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: SANDESH: Send FAILED: 1729478297273382  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 87 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 229 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 45 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1474,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 331,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:38:17:274.055 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:38:18:295.702 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:38:18:295.798 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Processing scm::EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:38:18:295.841 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Connect : EvConnectTimerExpired
2024-10-21 Mon 02:38:18:295.949 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Idle
2024-10-21 Mon 02:38:18:296.068 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:38:18:296.103 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: SANDESH: Send FAILED: 1729478298295879  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:38:18:296.115 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:38:18:296.124 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Processing scm::EvTcpDeleteSession in state Idle
2024-10-21 Mon 02:38:18:296.153 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:38:18:296.449 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: SANDESH: Send FAILED: 1729478298296054  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 87 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 231 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 45 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1474,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 333,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:38:18:296.487 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:38:47:273.810 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464031287040, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:38:47:273.975 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464031287040, Pid 1053912]: SANDESH: Send FAILED: 1729478327273568  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 40 dequeues = 40 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 40 dequeues = 40 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 335 dequeues = 335 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 41 dequeues = 41 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 459 dequeues = 459 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1729478298295947 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729478298296151 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:38:47:274.017 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464031287040, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:38:47:274.032 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464031287040, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:38:47:274.586 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464031287040, Pid 1053912]: SANDESH: Send FAILED: 1729478327273748  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 88 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 232 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 45 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1474,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 335,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:38:47:274.639 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464031287040, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:38:48:296.217 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Processing scm::EvIdleHoldTimerExpired in state Idle
2024-10-21 Mon 02:38:48:296.668 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Connect : Start Connect timer 10.0.0.30:8086
2024-10-21 Mon 02:38:48:296.824 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:38:48:296.860 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: SANDESH: Send FAILED: 1729478328296480  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.30:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:38:48:296.872 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:38:48:296.895 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:38:48:297.234 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: SANDESH: Send FAILED: 1729478328296807  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 88 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 234 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 45 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1474,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 337,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:38:48:297.278 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:39:17:274.160 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464052279040, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:39:17:274.298 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464052279040, Pid 1053912]: SANDESH: Send FAILED: 1729478357274036  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 40 dequeues = 40 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 41 dequeues = 41 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 339 dequeues = 339 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 41 dequeues = 41 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 464 dequeues = 464 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1729478328296504 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729478328296893 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:39:17:274.313 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464052279040, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:39:17:274.323 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464052279040, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:39:17:274.612 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464052279040, Pid 1053912]: SANDESH: Send FAILED: 1729478357274132  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 89 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 235 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 45 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1474,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 339,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:39:17:274.643 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464052279040, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:39:18:296.796 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464060675840, Pid 1053912]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:39:18:296.870 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464060675840, Pid 1053912]: Processing scm::EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:39:18:296.881 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464060675840, Pid 1053912]: Connect : EvConnectTimerExpired
2024-10-21 Mon 02:39:18:299.903 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464060675840, Pid 1053912]: Idle
2024-10-21 Mon 02:39:18:300.030 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464060675840, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:39:18:300.065 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464060675840, Pid 1053912]: SANDESH: Send FAILED: 1729478358296912  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:39:18:300.087 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464060675840, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:39:18:300.099 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464060675840, Pid 1053912]: Processing scm::EvTcpDeleteSession in state Idle
2024-10-21 Mon 02:39:18:300.137 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464060675840, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:39:18:300.481 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464060675840, Pid 1053912]: SANDESH: Send FAILED: 1729478358300014  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 89 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 237 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 45 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1474,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 341,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:39:18:301.083 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464060675840, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:39:47:274.469 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:39:47:274.598 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: SANDESH: Send FAILED: 1729478387274292  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 41 dequeues = 41 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 41 dequeues = 41 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 343 dequeues = 343 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 42 dequeues = 42 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 470 dequeues = 470 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1729478358299900 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729478358300136 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:39:47:274.613 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:39:47:274.623 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:39:47:274.928 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: SANDESH: Send FAILED: 1729478387274422  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 90 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 238 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 45 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1474,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 343,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:39:47:274.963 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:39:48:300.918 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464052279040, Pid 1053912]: Processing scm::EvIdleHoldTimerExpired in state Idle
2024-10-21 Mon 02:39:48:301.190 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464052279040, Pid 1053912]: Connect : Start Connect timer 10.0.0.30:8086
2024-10-21 Mon 02:39:48:301.332 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464052279040, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:39:48:301.377 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464052279040, Pid 1053912]: SANDESH: Send FAILED: 1729478388301019  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.30:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:39:48:301.387 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464052279040, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:39:48:301.397 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464052279040, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:39:48:301.749 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464052279040, Pid 1053912]: SANDESH: Send FAILED: 1729478388301314  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 90 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 240 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 45 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1474,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 345,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:39:48:301.780 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464052279040, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:40:17:274.884 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463336048384, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:40:17:275.158 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463336048384, Pid 1053912]: SANDESH: Send FAILED: 1729478417274676  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 41 dequeues = 41 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 42 dequeues = 42 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 347 dequeues = 347 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 42 dequeues = 42 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 475 dequeues = 475 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1729478388301028 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729478388301395 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:40:17:275.190 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463336048384, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:40:17:275.209 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463336048384, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:40:17:275.761 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463336048384, Pid 1053912]: SANDESH: Send FAILED: 1729478417274820  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 91 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 241 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 45 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1474,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 347,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:40:17:275.810 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463336048384, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:40:18:301.360 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:40:18:301.443 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Processing scm::EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:40:18:301.454 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Connect : EvConnectTimerExpired
2024-10-21 Mon 02:40:18:301.566 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Idle
2024-10-21 Mon 02:40:18:301.697 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:40:18:301.732 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: SANDESH: Send FAILED: 1729478418301502  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:40:18:301.742 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:40:18:301.752 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Processing scm::EvTcpDeleteSession in state Idle
2024-10-21 Mon 02:40:18:301.781 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:40:18:302.095 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: SANDESH: Send FAILED: 1729478418301681  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 91 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 243 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 45 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1474,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 349,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:40:18:302.131 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:40:47:275.238 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:40:47:275.360 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: SANDESH: Send FAILED: 1729478447274997  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 42 dequeues = 42 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 42 dequeues = 42 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 351 dequeues = 351 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 43 dequeues = 43 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 481 dequeues = 481 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1729478418301564 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729478418301779 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:40:47:275.555 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:40:47:275.574 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:40:47:275.897 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: SANDESH: Send FAILED: 1729478447275178  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 92 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 244 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 45 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1474,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 351,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:40:47:275.924 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:40:48:301.725 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463344445184, Pid 1053912]: Processing scm::EvIdleHoldTimerExpired in state Idle
2024-10-21 Mon 02:40:48:302.043 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463344445184, Pid 1053912]: Connect : Start Connect timer 10.0.0.30:8086
2024-10-21 Mon 02:40:48:302.215 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463344445184, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:40:48:302.253 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463344445184, Pid 1053912]: SANDESH: Send FAILED: 1729478448301891  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.30:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:40:48:302.264 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463344445184, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:40:48:302.277 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463344445184, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:40:48:302.591 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463344445184, Pid 1053912]: SANDESH: Send FAILED: 1729478448302197  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 92 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 246 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 45 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1474,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 353,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:40:48:302.628 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463344445184, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:41:17:275.521 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:41:17:275.730 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: SANDESH: Send FAILED: 1729478477275376  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 42 dequeues = 42 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 43 dequeues = 43 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 355 dequeues = 355 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 43 dequeues = 43 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 486 dequeues = 486 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1729478448301899 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729478448302274 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:41:17:275.764 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:41:17:275.776 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:41:17:276.146 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: SANDESH: Send FAILED: 1729478477275470  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 93 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 247 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 45 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1474,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 355,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:41:17:276.177 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464043882240, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:41:18:302.229 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464060675840, Pid 1053912]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:41:18:302.334 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Processing scm::EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:41:18:302.382 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Connect : EvConnectTimerExpired
2024-10-21 Mon 02:41:18:302.519 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Idle
2024-10-21 Mon 02:41:18:302.683 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:41:18:302.732 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: SANDESH: Send FAILED: 1729478478302424  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:41:18:302.748 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:41:18:302.784 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Processing scm::EvTcpDeleteSession in state Idle
2024-10-21 Mon 02:41:18:302.830 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:41:18:303.145 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: SANDESH: Send FAILED: 1729478478302662  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 93 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 249 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 45 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1474,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 357,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:41:18:303.177 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464035485440, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:41:47:275.864 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:41:47:276.014 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: SANDESH: Send FAILED: 1729478507275631  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 43 dequeues = 43 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 43 dequeues = 43 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 359 dequeues = 359 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 44 dequeues = 44 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 492 dequeues = 492 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1729478478302516 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729478478302828 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:41:47:276.035 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:41:47:276.047 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:41:47:276.361 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: SANDESH: Send FAILED: 1729478507275782  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 94 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 250 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 45 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1474,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 359,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:41:47:276.400 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:41:48:302.700 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463340246784, Pid 1053912]: Processing scm::EvIdleHoldTimerExpired in state Idle
2024-10-21 Mon 02:41:48:302.969 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463340246784, Pid 1053912]: Connect : Start Connect timer 10.0.0.30:8086
2024-10-21 Mon 02:41:48:303.096 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463340246784, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:41:48:303.130 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463340246784, Pid 1053912]: SANDESH: Send FAILED: 1729478508302785  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.30:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:41:48:303.151 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463340246784, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:41:48:303.160 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463340246784, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:41:48:303.465 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463340246784, Pid 1053912]: SANDESH: Send FAILED: 1729478508303080  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 94 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 252 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 45 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1474,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 361,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:41:48:303.519 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463340246784, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:42:05:396.706 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464231143552, Pid 1053912]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8092::10.0.0.30:44012 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256
2024-10-21 Mon 02:42:05:444.724 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464231143552, Pid 1053912]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8092::10.0.0.30:44012 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504
2024-10-21 Mon 02:42:06:548.448 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464231143552, Pid 1053912]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8092::10.0.0.30:44018 > Passive session Accept complete src/contrail-common/io/tcp_session.cc 256
2024-10-21 Mon 02:42:06:595.914 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464231143552, Pid 1053912]: TCP [SYS_DEBUG]: TcpSessionMessageLog: Session 10.0.0.30:8092::10.0.0.30:44018 < Read failed due to error asio.ssl 335544539 : short read src/contrail-common/io/tcp_session.cc 504
2024-10-21 Mon 02:42:17:276.155 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:42:17:276.320 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: SANDESH: Send FAILED: 1729478537275974  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 43 dequeues = 43 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 44 dequeues = 44 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 363 dequeues = 363 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 44 dequeues = 44 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 497 dequeues = 497 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1729478508302795 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729478508303159 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:42:17:276.353 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:42:17:276.365 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:42:17:276.669 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: SANDESH: Send FAILED: 1729478537276099  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 95 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 253 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 49 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1478,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 363,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:42:17:276.739 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464073271040, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:42:18:303.139 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463344445184, Pid 1053912]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:42:18:303.282 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464052279040, Pid 1053912]: Processing scm::EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:42:18:303.345 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464052279040, Pid 1053912]: Connect : EvConnectTimerExpired
2024-10-21 Mon 02:42:18:303.521 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464052279040, Pid 1053912]: Idle
2024-10-21 Mon 02:42:18:303.694 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464052279040, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:42:18:303.743 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464052279040, Pid 1053912]: SANDESH: Send FAILED: 1729478538303426  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:42:18:303.759 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464052279040, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:42:18:303.773 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464052279040, Pid 1053912]: Processing scm::EvTcpDeleteSession in state Idle
2024-10-21 Mon 02:42:18:303.827 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464052279040, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:42:18:304.343 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464052279040, Pid 1053912]: SANDESH: Send FAILED: 1729478538303670  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 95 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 255 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 49 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1478,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 365,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:42:18:304.407 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464052279040, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:42:47:276.426 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:42:47:276.568 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: SANDESH: Send FAILED: 1729478567276271  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 44 dequeues = 44 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 44 dequeues = 44 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 367 dequeues = 367 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 45 dequeues = 45 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 503 dequeues = 503 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1729478538303517 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729478538303824 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:42:47:276.584 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:42:47:276.596 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:42:47:276.907 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: SANDESH: Send FAILED: 1729478567276398  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 96 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 256 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 49 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1478,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 367,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:42:47:276.929 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:42:48:303.696 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464060675840, Pid 1053912]: Processing scm::EvIdleHoldTimerExpired in state Idle
2024-10-21 Mon 02:42:48:303.937 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464060675840, Pid 1053912]: Connect : Start Connect timer 10.0.0.30:8086
2024-10-21 Mon 02:42:48:304.076 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464060675840, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:42:48:304.131 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464060675840, Pid 1053912]: SANDESH: Send FAILED: 1729478568303796  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.30:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:42:48:304.145 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464060675840, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:42:48:304.159 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464060675840, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:42:48:304.477 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464060675840, Pid 1053912]: SANDESH: Send FAILED: 1729478568304060  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 96 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 258 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 49 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1478,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 369,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:42:48:304.511 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464060675840, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:43:17:276.741 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464031287040, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:43:17:276.891 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464031287040, Pid 1053912]: SANDESH: Send FAILED: 1729478597276597  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 44 dequeues = 44 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 45 dequeues = 45 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 371 dequeues = 371 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 45 dequeues = 45 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 508 dequeues = 508 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Connect state_since = 1729478568303806 last_state = Connect last_event = scm::EvSandeshSend last_event_at = 1729478568304157 ] session_stats= [  num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] session_tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:43:17:276.907 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464031287040, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:43:17:276.920 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464031287040, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:43:17:277.252 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464031287040, Pid 1053912]: SANDESH: Send FAILED: 1729478597276699  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 97 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 259 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 49 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1478,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 371,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:43:17:277.291 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464031287040, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:43:18:304.085 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140463348643584, Pid 1053912]: 10.0.0.30:8086 EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:43:18:304.185 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Processing scm::EvConnectTimerExpired in state Connect
2024-10-21 Mon 02:43:18:304.229 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Connect : EvConnectTimerExpired
2024-10-21 Mon 02:43:18:304.340 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Idle
2024-10-21 Mon 02:43:18:304.455 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:43:18:304.498 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: SANDESH: Send FAILED: 1729478598304268  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 0.0.0.0:0,  ] ] status = Down description = Connect : EvConnectTimerExpired ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:43:18:304.523 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:43:18:304.533 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Processing scm::EvTcpDeleteSession in state Idle
2024-10-21 Mon 02:43:18:304.563 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:43:18:304.853 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: SANDESH: Send FAILED: 1729478598304440  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 97 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 261 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 49 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1478,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 373,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:43:18:304.880 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464048080640, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:43:47:277.103 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:43:47:277.302 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: SANDESH: Send FAILED: 1729478627276868  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [  ev_stats= [  [ [  event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvConnectTimerExpired enqueues = 45 dequeues = 45 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvIdleHoldTimerExpired enqueues = 45 dequeues = 45 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvSandeshSend enqueues = 375 dequeues = 375 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpConnectFail enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ],  [  event = scm::EvTcpDeleteSession enqueues = 46 dequeues = 46 enqueue_fails = 0 dequeue_fails = 0 ],  [  event =  enqueues = 514 dequeues = 514 enqueue_fails = 0 dequeue_fails = 0 ],  ] ] state = Idle state_since = 1729478598304338 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1729478598304562 ] max_sm_queue_count = 3 ]
2024-10-21 Mon 02:43:47:277.323 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:43:47:277.334 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Processing scm::EvSandeshSend in state Idle
2024-10-21 Mon 02:43:47:277.636 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: SANDESH: Send FAILED: 1729478627277053  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Idle successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 98 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 262 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 49 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1478,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 375,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:43:47:277.670 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464056477440, Pid 1053912]: Wrong state: Idle for event: EvSandeshSend message: SandeshModuleClientTrace
2024-10-21 Mon 02:43:48:304.497 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Processing scm::EvIdleHoldTimerExpired in state Idle
2024-10-21 Mon 02:43:48:304.854 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Connect : Start Connect timer 10.0.0.30:8086
2024-10-21 Mon 02:43:48:305.097 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:43:48:305.180 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: SANDESH: Send FAILED: 1729478628304625  [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1 process_status= [  [ [  module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [  [ [  type = Collector name =  server_addrs= [  [ (*_iter6) = 10.0.0.30:8086,  ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ],  [  type = Database name = Cassandra server_addrs= [  [ (*_iter6) = 10.0.0.30:9041,  ] ] status = Up description = Established Cassandra connection ],  [  type = Database name = RabbitMQ server_addrs= [  [ (*_iter6) = 10.0.0.30:5673,  ] ] status = Up description = RabbitMQ connection established ],  ] ] flag_infos= [  [ ] ] description = Collector connection down ],  ] ] ]
2024-10-21 Mon 02:43:48:305.196 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: NodeStatusUVE
2024-10-21 Mon 02:43:48:305.210 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Processing scm::EvSandeshSend in state Connect
2024-10-21 Mon 02:43:48:305.762 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: SANDESH: Send FAILED: 1729478628305072  [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-juju-os-439-1:Control:contrail-dns:0 client_info= [  status = Connect successful_connections = 0 pid = 1053912 http_port = 8092 start_time = 1729476041103535 collector_name =  collector_ip = 10.0.0.30:8086 collector_list= [  [ (*_iter40) = 10.0.0.30:8086,  ] ] rx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] tx_socket_stats= [  bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration =  errors = 0 ] ] msg_type_agg= [  [ _iter58->first = ConfigClientMgrDebug [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 669 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigClientMgrWarning [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = ConfigurationMalformedReferenceWarningLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 8 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = DnsOperational [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapLinkOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 454 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = IFMapNodeOperation [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 257 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = NodeStatusUVE [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 98 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = SandeshModuleClientTrace [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 264 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpServerMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = TcpSessionMessageLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 49 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = UveDnsInfo [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionCreate [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionDelete [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppConnectionLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppCreateConnection [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppEventLog [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppSslHandShakeMessage [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  _iter58->first = XmppUnconsumedEvent [  messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ],  ] ] tx_msg_agg= [  [ _iter59->first = dropped_client_send_failed _iter59->second = 0,  _iter59->first = dropped_header_write_failed _iter59->second = 0,  _iter59->first = dropped_no_client _iter59->second = 1,  _iter59->first = dropped_no_queue _iter59->second = 0,  _iter59->first = dropped_no_session _iter59->second = 0,  _iter59->first = dropped_queue_level _iter59->second = 0,  _iter59->first = dropped_rate_limited _iter59->second = 0,  _iter59->first = dropped_sending_disabled _iter59->second = 1478,  _iter59->first = dropped_sending_to_syslog _iter59->second = 0,  _iter59->first = dropped_session_not_connected _iter59->second = 0,  _iter59->first = dropped_validation_failed _iter59->second = 0,  _iter59->first = dropped_write_failed _iter59->second = 0,  _iter59->first = dropped_wrong_client_sm_state _iter59->second = 377,  _iter59->first = sent _iter59->second = 0,  ] ] ]
2024-10-21 Mon 02:43:48:305.807 UTC  cn-jenkins-deploy-platform-juju-os-439-1 [Thread 140464064874240, Pid 1053912]: Wrong state: Connect for event: EvSandeshSend message: SandeshModuleClientTrace