2024-09-19 Thu 02:06:17:295.238 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522495477504, Pid 809]: KsyncTxQueue CPU pinning policy <>. KsyncTxQueuen not pinned to CPU 2024-09-19 Thu 02:06:17:439.800 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522524866304, Pid 809]: SANDESH: No Client: 1726711577439678 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 809 http_port = 8085 start_time = 1726711577439581 collector_name = collector_ip = 0.0.0.0:0 collector_list= [ [ ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 0, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-09-19 Thu 02:06:17:440.171 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522508072704, Pid 809]: SANDESH: Send FAILED: 1726711577440088 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Idle : EvStart -> Disconnect ], ] ] flag_infos= [ [ ] ] description = Collector connection down, No Configuration for self Number of connections:1, Expected: 3 ], ] ] ] 2024-09-19 Thu 02:06:17:440.258 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522508072704, Pid 809]: SANDESH: Send FAILED: 1726711577440141 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:Compute:contrail-vrouter-agent:0 client_info= [ status = Disconnect successful_connections = 0 pid = 809 http_port = 8085 start_time = 1726711577439581 collector_name = collector_ip = 0.0.0.0:0 collector_list= [ [ ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-09-19 Thu 02:06:17:440.925 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522512271104, Pid 809]: SANDESH: Send FAILED: 1726711577440545 AgentVrf [SYS_INFO]: VrfObjectLog: vrf= [ event = Addition name = default-domain:default-project:ip-fabric:__default__ index = 0 ] file = controller/src/vnsw/agent/oper/vrf.cc line = 414 2024-09-19 Thu 02:06:17:440.960 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522512271104, Pid 809]: SANDESH: Send FAILED: 1726711577440573 AgentVrf [SYS_INFO]: VrfObjectLog: vrf= [ event = Addition name = default-domain:default-project:ip-fabric:ip-fabric index = 1 ] file = controller/src/vnsw/agent/oper/vrf.cc line = 414 2024-09-19 Thu 02:06:17:441.053 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522512271104, Pid 809]: SANDESH: Send FAILED: 1726711577440576 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:Compute:contrail-vrouter-agent:0 client_info= [ status = Disconnect successful_connections = 0 pid = 809 http_port = 8085 start_time = 1726711577439581 collector_name = collector_ip = 10.0.0.34:8086 collector_list= [ [ (*_iter40) = 10.0.0.34:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 2, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-09-19 Thu 02:06:17:441.083 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522512271104, Pid 809]: SANDESH: Send FAILED: 1726711577440604 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.34:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = Collector connection down, No Configuration for self Number of connections:1, Expected: 3 ], ] ] ] 2024-09-19 Thu 02:06:17:441.153 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522512271104, Pid 809]: SANDESH: Send FAILED: 1726711577440876 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 809 http_port = 8085 start_time = 1726711577439581 collector_name = collector_ip = 10.0.0.34:8086 collector_list= [ [ (*_iter40) = 10.0.0.34:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 2, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-09-19 Thu 02:06:17:454.323 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: SANDESH: Send FAILED: 1726711577454222 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Idle next_state = Active event = Start ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:17:454.392 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: SANDESH: Send FAILED: 1726711577454276 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:5269, ] ] status = Down description = Idle ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.34:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.34, Collector connection down, No Configuration for self Number of connections:2, Expected: 3 ], ] ] ] 2024-09-19 Thu 02:06:17:454.410 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: SANDESH: Send FAILED: 1726711577454370 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Idle next_state = Active event = Start ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:17:454.436 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: SANDESH: Send FAILED: 1726711577454386 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:5269, ] ] status = Down description = Idle ], [ type = XMPP name = dns-server:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:53, ] ] status = Down description = Idle ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.34:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.34, XMPP:dns-server:10.0.0.34, Collector connection down, No Configuration for self ], ] ] ] 2024-09-19 Thu 02:06:19:091.579 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522512271104, Pid 809]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > Shutdown failed due to error: Transport endpoint is not connected src/contrail-common/io/tcp_session.cc 314 2024-09-19 Thu 02:06:19:091.775 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522512271104, Pid 809]: SANDESH: Send FAILED: 1726711579091552 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:5269, ] ] status = Down description = Idle ], [ type = XMPP name = dns-server:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:53, ] ] status = Down description = Idle ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvTcpConnectFail ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.34, XMPP:dns-server:10.0.0.34, Collector connection down, No Configuration for self ], ] ] ] 2024-09-19 Thu 02:06:19:091.940 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522512271104, Pid 809]: SANDESH: Send FAILED: 1726711579091732 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 809 http_port = 8085 start_time = 1726711577439581 collector_name = collector_ip = 10.0.0.34:8086 collector_list= [ [ (*_iter40) = 10.0.0.34:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 5, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 11, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-09-19 Thu 02:06:19:092.608 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522512271104, Pid 809]: SANDESH: Send FAILED: 1726711579092368 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Active next_state = Connect event = EvConnectTimerExpired ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:19:092.635 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522512271104, Pid 809]: SANDESH: Send FAILED: 1726711579092574 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect event = Connect Event local_port = 45025 remote_port = 4294967295 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:19:092.981 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522503874304, Pid 809]: SANDESH: Send FAILED: 1726711579092810 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Active next_state = Connect event = EvConnectTimerExpired ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:19:093.013 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522503874304, Pid 809]: SANDESH: Send FAILED: 1726711579092955 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect event = Connect Event local_port = 34089 remote_port = 4294967295 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:19:093.707 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522669652288, Pid 809]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer RECV Mode Client: Event: Tcp Connect Fail peer ip: 10.0.0.34 ( network-control@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1302 2024-09-19 Thu 02:06:19:093.848 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: TCP [SYS_ERR]: TcpSessionMessageLog: Session - > Shutdown failed due to error: Transport endpoint is not connected src/contrail-common/io/tcp_session.cc 314 2024-09-19 Thu 02:06:19:093.910 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: SANDESH: Send FAILED: 1726711579093874 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect next_state = Active event = EvTcpConnectFail ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:19:094.190 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522669652288, Pid 809]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer RECV Mode Client: Event: Tcp Connect Fail peer ip: 10.0.0.34 ( network-dns@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1302 2024-09-19 Thu 02:06:19:094.256 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522512271104, Pid 809]: TCP [SYS_ERR]: TcpSessionMessageLog: Session - > Shutdown failed due to error: Transport endpoint is not connected src/contrail-common/io/tcp_session.cc 314 2024-09-19 Thu 02:06:19:094.308 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522516469504, Pid 809]: SANDESH: Send FAILED: 1726711579094287 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect next_state = Active event = EvTcpConnectFail ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:20:093.829 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522669652288, Pid 809]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer RECV Mode Client: Event: Tcp Connect Fail peer ip: 10.0.0.34 ( network-dns@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1302 2024-09-19 Thu 02:06:20:093.894 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: TCP [SYS_ERR]: TcpSessionMessageLog: Session - > Shutdown failed due to error: Transport endpoint is not connected src/contrail-common/io/tcp_session.cc 314 2024-09-19 Thu 02:06:20:094.195 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: SANDESH: Send FAILED: 1726711580092700 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:5269, ] ] status = Down description = Idle ], [ type = XMPP name = dns-server:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:53, ] ] status = Down description = Idle ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.34:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.34, XMPP:dns-server:10.0.0.34, Collector connection down, No Configuration for self ], ] ] ] 2024-09-19 Thu 02:06:20:094.221 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: SANDESH: Send FAILED: 1726711580093617 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Active next_state = Connect event = EvConnectTimerExpired ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:20:096.193 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: SANDESH: Send FAILED: 1726711580093788 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect event = Connect Event local_port = 42905 remote_port = 4294967295 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:20:096.229 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: SANDESH: Send FAILED: 1726711580093914 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect next_state = Active event = EvTcpConnectFail ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:20:096.358 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: SANDESH: Send FAILED: 1726711580094161 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 809 http_port = 8085 start_time = 1726711577439581 collector_name = collector_ip = 10.0.0.34:8086 collector_list= [ [ (*_iter40) = 10.0.0.34:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 6 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 13, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 19, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-09-19 Thu 02:06:20:096.406 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > Shutdown failed due to error: Transport endpoint is not connected src/contrail-common/io/tcp_session.cc 314 2024-09-19 Thu 02:06:20:096.492 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: SANDESH: Send FAILED: 1726711580096391 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:5269, ] ] status = Down description = Idle ], [ type = XMPP name = dns-server:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:53, ] ] status = Down description = Idle ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvTcpConnectFail ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.34, XMPP:dns-server:10.0.0.34, Collector connection down, No Configuration for self ], ] ] ] 2024-09-19 Thu 02:06:20:096.614 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: SANDESH: Send FAILED: 1726711580096472 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 809 http_port = 8085 start_time = 1726711577439581 collector_name = collector_ip = 10.0.0.34:8086 collector_list= [ [ (*_iter40) = 10.0.0.34:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 5 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 7 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 14, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 24, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-09-19 Thu 02:06:20:120.145 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522508072704, Pid 809]: SANDESH: Send FAILED: 1726711580119963 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Active next_state = Connect event = EvConnectTimerExpired ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:20:120.181 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522508072704, Pid 809]: SANDESH: Send FAILED: 1726711580120113 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect event = Connect Event local_port = 46779 remote_port = 4294967295 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:20:120.222 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522669652288, Pid 809]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer RECV Mode Client: Event: Tcp Connect Fail peer ip: 10.0.0.34 ( network-control@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1302 2024-09-19 Thu 02:06:20:120.256 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522503874304, Pid 809]: TCP [SYS_ERR]: TcpSessionMessageLog: Session - > Shutdown failed due to error: Transport endpoint is not connected src/contrail-common/io/tcp_session.cc 314 2024-09-19 Thu 02:06:20:120.299 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522503874304, Pid 809]: SANDESH: Send FAILED: 1726711580120274 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect next_state = Active event = EvTcpConnectFail ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:21:948.066 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522512271104, Pid 809]: SANDESH: Send FAILED: 1726711581948019 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Active next_state = Connect event = EvConnectTimerExpired ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:21:948.240 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522669652288, Pid 809]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer RECV Mode Client: Event: Tcp Connect Fail peer ip: 10.0.0.34 ( network-dns@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1302 2024-09-19 Thu 02:06:21:948.316 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: TCP [SYS_ERR]: TcpSessionMessageLog: Session - > Shutdown failed due to error: Transport endpoint is not connected src/contrail-common/io/tcp_session.cc 314 2024-09-19 Thu 02:06:21:948.320 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: SANDESH: Send FAILED: 1726711581948281 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect event = Connect Event local_port = 46309 remote_port = 4294967295 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:21:948.407 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: SANDESH: Send FAILED: 1726711581948342 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect next_state = Active event = EvTcpConnectFail ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:22:096.859 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: SANDESH: Send FAILED: 1726711582096499 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:5269, ] ] status = Down description = Idle ], [ type = XMPP name = dns-server:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:53, ] ] status = Down description = Idle ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.34:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.34, XMPP:dns-server:10.0.0.34, Collector connection down, No Configuration for self ], ] ] ] 2024-09-19 Thu 02:06:22:096.933 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > Shutdown failed due to error: Transport endpoint is not connected src/contrail-common/io/tcp_session.cc 314 2024-09-19 Thu 02:06:22:097.128 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: SANDESH: Send FAILED: 1726711582096820 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 809 http_port = 8085 start_time = 1726711577439581 collector_name = collector_ip = 10.0.0.34:8086 collector_list= [ [ (*_iter40) = 10.0.0.34:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 9 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 5 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 18, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 32, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-09-19 Thu 02:06:22:097.166 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: SANDESH: Send FAILED: 1726711582096914 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:5269, ] ] status = Down description = Idle ], [ type = XMPP name = dns-server:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:53, ] ] status = Down description = Idle ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvTcpConnectFail ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.34, XMPP:dns-server:10.0.0.34, Collector connection down, No Configuration for self ], ] ] ] 2024-09-19 Thu 02:06:22:097.291 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: SANDESH: Send FAILED: 1726711582096994 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 809 http_port = 8085 start_time = 1726711577439581 collector_name = collector_ip = 10.0.0.34:8086 collector_list= [ [ (*_iter40) = 10.0.0.34:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 6 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 10 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 5 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 19, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 33, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-09-19 Thu 02:06:22:208.426 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522516469504, Pid 809]: SANDESH: Send FAILED: 1726711582208378 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Active next_state = Connect event = EvConnectTimerExpired ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:22:208.606 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522669652288, Pid 809]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer RECV Mode Client: Event: Tcp Connect Fail peer ip: 10.0.0.34 ( network-control@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1302 2024-09-19 Thu 02:06:22:208.673 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522512271104, Pid 809]: TCP [SYS_ERR]: TcpSessionMessageLog: Session - > Shutdown failed due to error: Transport endpoint is not connected src/contrail-common/io/tcp_session.cc 314 2024-09-19 Thu 02:06:22:208.689 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522524866304, Pid 809]: SANDESH: Send FAILED: 1726711582208644 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect event = Connect Event local_port = 36317 remote_port = 4294967295 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:22:208.716 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522524866304, Pid 809]: SANDESH: Send FAILED: 1726711582208697 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect next_state = Active event = EvTcpConnectFail ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:25:693.165 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522512271104, Pid 809]: SANDESH: Send FAILED: 1726711585693112 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Active next_state = Connect event = EvConnectTimerExpired ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:25:693.411 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522669652288, Pid 809]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer RECV Mode Client: Event: Tcp Connect Fail peer ip: 10.0.0.34 ( network-dns@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1302 2024-09-19 Thu 02:06:25:693.471 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522512271104, Pid 809]: SANDESH: Send FAILED: 1726711585693456 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect event = Connect Event local_port = 46455 remote_port = 4294967295 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:25:693.480 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522524866304, Pid 809]: TCP [SYS_ERR]: TcpSessionMessageLog: Session - > Shutdown failed due to error: Transport endpoint is not connected src/contrail-common/io/tcp_session.cc 314 2024-09-19 Thu 02:06:25:693.699 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522503874304, Pid 809]: SANDESH: Send FAILED: 1726711585693639 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect next_state = Active event = EvTcpConnectFail ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:25:693.745 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522503874304, Pid 809]: SANDESH: Send FAILED: 1726711585693659 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Active event = Connect failed after retries ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:25:693.790 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522503874304, Pid 809]: SANDESH: Send FAILED: 1726711585693726 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:5269, ] ] status = Down description = Idle ], [ type = XMPP name = dns-server:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:53, ] ] status = Down description = Connect ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvTcpConnectFail ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.34, XMPP:dns-server:10.0.0.34, Collector connection down, No Configuration for self ], ] ] ] 2024-09-19 Thu 02:06:26:098.296 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: SANDESH: Send FAILED: 1726711586097052 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:5269, ] ] status = Down description = Idle ], [ type = XMPP name = dns-server:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:53, ] ] status = Down description = Connect ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.34:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.34, XMPP:dns-server:10.0.0.34, Collector connection down, No Configuration for self ], ] ] ] 2024-09-19 Thu 02:06:26:098.360 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > Shutdown failed due to error: Transport endpoint is not connected src/contrail-common/io/tcp_session.cc 314 2024-09-19 Thu 02:06:26:098.532 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: SANDESH: Send FAILED: 1726711586098267 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 809 http_port = 8085 start_time = 1726711577439581 collector_name = collector_ip = 10.0.0.34:8086 collector_list= [ [ (*_iter40) = 10.0.0.34:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 14 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 7 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 25, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 44, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-09-19 Thu 02:06:26:098.564 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: SANDESH: Send FAILED: 1726711586098340 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:5269, ] ] status = Down description = Idle ], [ type = XMPP name = dns-server:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:53, ] ] status = Down description = Connect ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvTcpConnectFail ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.34, XMPP:dns-server:10.0.0.34, Collector connection down, No Configuration for self ], ] ] ] 2024-09-19 Thu 02:06:26:098.680 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: SANDESH: Send FAILED: 1726711586098416 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 809 http_port = 8085 start_time = 1726711577439581 collector_name = collector_ip = 10.0.0.34:8086 collector_list= [ [ (*_iter40) = 10.0.0.34:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 11 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 15 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 7 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 26, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 45, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-09-19 Thu 02:06:26:240.912 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: SANDESH: Send FAILED: 1726711586240863 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Active next_state = Connect event = EvConnectTimerExpired ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:26:241.111 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522669652288, Pid 809]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer RECV Mode Client: Event: Tcp Connect Fail peer ip: 10.0.0.34 ( network-control@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1302 2024-09-19 Thu 02:06:26:241.178 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522524866304, Pid 809]: TCP [SYS_ERR]: TcpSessionMessageLog: Session - > Shutdown failed due to error: Transport endpoint is not connected src/contrail-common/io/tcp_session.cc 314 2024-09-19 Thu 02:06:26:241.116 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522503874304, Pid 809]: SANDESH: Send FAILED: 1726711586241084 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect event = Connect Event local_port = 33353 remote_port = 4294967295 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:26:241.319 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522503874304, Pid 809]: SANDESH: Send FAILED: 1726711586241219 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect next_state = Active event = EvTcpConnectFail ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:26:241.330 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522503874304, Pid 809]: SANDESH: Send FAILED: 1726711586241225 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Active event = Connect failed after retries ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:26:241.351 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522503874304, Pid 809]: SANDESH: Send FAILED: 1726711586241265 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:5269, ] ] status = Down description = Connect ], [ type = XMPP name = dns-server:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:53, ] ] status = Down description = Connect ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvTcpConnectFail ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.34, XMPP:dns-server:10.0.0.34, Collector connection down, No Configuration for self ], ] ] ] 2024-09-19 Thu 02:06:34:089.500 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: SANDESH: Send FAILED: 1726711594089353 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Active next_state = Connect event = EvConnectTimerExpired ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:34:090.538 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522669652288, Pid 809]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer RECV Mode Client: Event: Tcp Connect Fail peer ip: 10.0.0.34 ( network-control@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1302 2024-09-19 Thu 02:06:34:115.478 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: SANDESH: Send FAILED: 1726711594090550 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect event = Connect Event local_port = 33075 remote_port = 4294967295 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:34:115.879 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: TCP [SYS_ERR]: TcpSessionMessageLog: Session - > Shutdown failed due to error: Transport endpoint is not connected src/contrail-common/io/tcp_session.cc 314 2024-09-19 Thu 02:06:34:116.803 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: SANDESH: Send FAILED: 1726711594115911 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect next_state = Active event = EvTcpConnectFail ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:34:116.859 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: SANDESH: Send FAILED: 1726711594115917 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Active event = Connect failed after retries ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:34:116.896 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: SANDESH: Send FAILED: 1726711594115919 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:5269, ] ] status = Down description = Connect ], [ type = XMPP name = dns-server:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:53, ] ] status = Down description = Connect ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.34:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.34, XMPP:dns-server:10.0.0.34, Collector connection down, No Configuration for self ], ] ] ] 2024-09-19 Thu 02:06:34:116.962 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > Shutdown failed due to error: Transport endpoint is not connected src/contrail-common/io/tcp_session.cc 314 2024-09-19 Thu 02:06:34:117.267 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: SANDESH: Send FAILED: 1726711594116777 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 809 http_port = 8085 start_time = 1726711577439581 collector_name = collector_ip = 10.0.0.34:8086 collector_list= [ [ (*_iter40) = 10.0.0.34:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 13 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 19 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 30 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 9 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 32, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 55, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-09-19 Thu 02:06:34:117.315 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: SANDESH: Send FAILED: 1726711594116931 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:5269, ] ] status = Down description = Connect ], [ type = XMPP name = dns-server:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:53, ] ] status = Down description = Connect ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvTcpConnectFail ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.34, XMPP:dns-server:10.0.0.34, Collector connection down, No Configuration for self ], ] ] ] 2024-09-19 Thu 02:06:34:117.531 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: SANDESH: Send FAILED: 1726711594117044 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 809 http_port = 8085 start_time = 1726711577439581 collector_name = collector_ip = 10.0.0.34:8086 collector_list= [ [ (*_iter40) = 10.0.0.34:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 10 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 20 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 32 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 9 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 33, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 58, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-09-19 Thu 02:06:34:261.909 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522512271104, Pid 809]: SANDESH: Send FAILED: 1726711594261773 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Active next_state = Connect event = EvConnectTimerExpired ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:34:262.430 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522669652288, Pid 809]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer RECV Mode Client: Event: Tcp Connect Fail peer ip: 10.0.0.34 ( network-dns@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1302 2024-09-19 Thu 02:06:34:262.490 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: SANDESH: Send FAILED: 1726711594262454 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect event = Connect Event local_port = 38485 remote_port = 4294967295 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:34:262.643 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522508072704, Pid 809]: TCP [SYS_ERR]: TcpSessionMessageLog: Session - > Shutdown failed due to error: Transport endpoint is not connected src/contrail-common/io/tcp_session.cc 314 2024-09-19 Thu 02:06:34:262.697 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522512271104, Pid 809]: SANDESH: Send FAILED: 1726711594262666 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect next_state = Active event = EvTcpConnectFail ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:34:262.709 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522512271104, Pid 809]: SANDESH: Send FAILED: 1726711594262673 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Active event = Connect failed after retries ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:47:176.400 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522512271104, Pid 809]: SANDESH: Send FAILED: 1726711607176361 [SYS_NOTICE]: UveVrouterAgent: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local interface_list= [ [ ] ] error_intf_list= [ [ ] ] no_config_intf_list= [ [ (*_iter151) = 00000000-0000-0000-0000-000000000000, ] ] vmi_count= [ active = 1 ] down_interface_count = 1 ] 2024-09-19 Thu 02:06:47:178.192 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522508072704, Pid 809]: SANDESH: Send FAILED: 1726711607178074 [SYS_NOTICE]: UveVirtualNetworkAgentTrace: data= [ name = __UNKNOWN__ vrf_stats_list= [ [ [ name = __untitled__ diag_packet_count = 0 unknown_unicast_floods = 0 arp_packet_counts= [ from_vm_interface= [ stats= [ proxies = 0 stitches = 0 floods = 0 ] ] from_physical_interface= [ stats= [ proxies = 0 stitches = 0 floods = 0 ] ] ] offload_packet_counts= [ gro = 0 ] nh_packet_counts= [ discards = 0 resolves = 0 l3_receives = 0 l2_receives = 0 local_vm_l3_forwards = 0 local_vm_l2_forwards = 0 vrf_translates = 0 ecmp_forwards = 0 tunnel_nh_stats= [ udp_encaps = 0 mpls_over_udp_encaps = 0 mpls_over_gre_encaps = 0 vxlan_encaps = 0 ] comp_nh_stats= [ source_replication_forwards = 0 edge_replication_forwards = 0 local_vm_l3_forwards = 0 total_multicast_forwards = 0 ] ] ], ] ] ] 2024-09-19 Thu 02:06:47:440.985 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522516469504, Pid 809]: SANDESH: Send FAILED: 1726711607440831 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 4 dequeues = 4 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 67 dequeues = 67 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpConnectFail enqueues = 5 dequeues = 5 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 5 dequeues = 5 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 83 dequeues = 83 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1726711594116995 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1726711607178149 ] max_sm_queue_count = 5 ] 2024-09-19 Thu 02:06:47:441.148 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522516469504, Pid 809]: SANDESH: Send FAILED: 1726711607440922 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 809 http_port = 8085 start_time = 1726711577439581 collector_name = collector_ip = 10.0.0.34:8086 collector_list= [ [ (*_iter40) = 10.0.0.34:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 12 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 27 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 36 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 10 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 41, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 67, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-09-19 Thu 02:06:47:668.981 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: SANDESH: Send FAILED: 1726711607668634 [SYS_NOTICE]: UveVrouterAgent: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local xmpp_peer_list= [ [ [ ip = 10.0.0.34 status = 0 setup_time = 0 primary = 1 ], ] ] self_ip_list= [ [ (*_iter146) = 10.0.0.34, ] ] build_info = {"build-info":[{"build-version":"nightly","build-time":"2024-09-19 00:14:13.826701","build-user":"root","build-hostname":"jenkins-build-prepare-sandbox-5603-1.novalocal","build-id":"nightly-nightly.el7","build-number":"@contrail"}]} phy_if= [ [ [ name = eth0 mac_address = fa:16:3e:c1:53:0d ], ] ] vhost_if= [ name = vhost0 mac_address = fa:16:3e:c1:53:0d ] log_file = /var/log/contrail/vrouter-agent/contrail-vrouter-agent.log config_file = /etc/contrail/contrail-vrouter-agent.conf log_local = 1 log_level = SYS_NOTICE log_category = sandesh_http_port = 8085 dns_servers= [ [ (*_iter149) = 10.0.0.34, ] ] vhost_cfg= [ name = vhost0 ip = 10.0.0.34 ip_prefix_len = 24 gateway = 10.0.0.1 ] eth_name= [ [ (*_iter150) = eth0, ] ] tunnel_type = MPLSoGRE hypervisor = kvm control_ip = 10.0.0.34 dns_server_list_cfg= [ [ (*_iter152) = 10.0.0.34:53, ] ] control_node_list_cfg= [ [ (*_iter153) = 10.0.0.34:5269, ] ] hostname_cfg = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local flow_cache_timeout_cfg = 0 ll_max_system_flows_cfg = 16384 ll_max_vm_flows_cfg = 16384 max_vm_flows_cfg = 100 gateway_cfg_list= [ [ ] ] headless_mode_cfg = 1 collector_server_list_cfg= [ [ (*_iter155) = 10.0.0.34:8086, ] ] log_flow = 0 mode = VROUTER platform = HOST vr_limits= [ max_labels = 5120 max_nexthops = 524288 max_interfaces = 4355 max_vrfs = 4096 max_mirror_entries = 255 vrouter_max_bridge_entries = 262144 vrouter_max_oflow_bridge_entries = 53248 vrouter_build_info = {"build-info": [{"build-version": "nightly", "build-time": "2024-09-19 00:11:24.257389", "build-user": "root", "build-hostname": "jenkins-build-prepare-sandbox-5603-1.novalocal"}]} vrouter_max_flow_entries = 524288 vrouter_max_oflow_entries = 105472 vrouter_priority_tagging = 1 ] bgpaas_enabled = 0 port_mirror_enabled = 0 subcluster_name = vr_high_watermark = 80 vr_low_watermark = 75 vr_limit_exceeded_list= [ [ ] ] res_limit = 0 res_table_limit = 0 loopback_ip = 0.0.0.0 gateway_list= [ [ (*_iter161) = 10.0.0.1, ] ] ] 2024-09-19 Thu 02:06:47:669.162 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: SANDESH: Send FAILED: 1726711607668711 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local in_tpkts = 19511 in_bytes = 1343486 out_tpkts = 64274 out_bytes = 614400027 exception_packets = 20 exception_packets_dropped = 9 exception_packets_allowed = 11 total_flows = 0 aged_flows = 0 uptime = 1726711577176198 phy_if_info= [ [ _iter254->first = eth0 [ speed = -1 duplexity = 255 ], ] ] raw_vhost_stats= [ name = vhost0 in_pkts = 19511 in_bytes = 1343486 out_pkts = 64274 out_bytes = 614400027 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 13 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 14 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 13 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 14 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = eth0 [ in_pkts = 64253 in_bytes = 614399145 out_pkts = 19513 out_bytes = 1343546 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = eth0 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 13 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 13 ], ] ] ] 2024-09-19 Thu 02:06:47:669.904 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522503874304, Pid 809]: SANDESH: Send FAILED: 1726711607669472 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local raw_xmpp_stats= [ [ _iter306->first = 10.0.0.34 [ reconnects = 0 in_msgs = 0 out_msgs = 0 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 2 inet4_multicast = 0 evpn = 1 bridge = 2 inet6_unicast = 0 ], ] ] ] 2024-09-19 Thu 02:06:49:812.153 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: SANDESH: Send FAILED: 1726711609812023 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Active next_state = Connect event = EvConnectTimerExpired ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:49:813.117 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522669652288, Pid 809]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer RECV Mode Client: Event: Tcp Connect Fail peer ip: 10.0.0.34 ( network-control@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1302 2024-09-19 Thu 02:06:49:813.185 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522524866304, Pid 809]: TCP [SYS_ERR]: TcpSessionMessageLog: Session - > Shutdown failed due to error: Transport endpoint is not connected src/contrail-common/io/tcp_session.cc 314 2024-09-19 Thu 02:06:49:813.294 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522524866304, Pid 809]: SANDESH: Send FAILED: 1726711609813159 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect event = Connect Event local_port = 44889 remote_port = 4294967295 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:49:813.311 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522524866304, Pid 809]: SANDESH: Send FAILED: 1726711609813215 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect next_state = Active event = EvTcpConnectFail ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:49:813.321 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522524866304, Pid 809]: SANDESH: Send FAILED: 1726711609813219 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Active event = Connect failed after retries ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:50:118.139 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: SANDESH: Send FAILED: 1726711610117094 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:5269, ] ] status = Down description = Connect ], [ type = XMPP name = dns-server:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:53, ] ] status = Down description = Connect ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.34:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.34, XMPP:dns-server:10.0.0.34, Collector connection down, No Configuration for self ], ] ] ] 2024-09-19 Thu 02:06:50:118.201 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > Shutdown failed due to error: Transport endpoint is not connected src/contrail-common/io/tcp_session.cc 314 2024-09-19 Thu 02:06:50:118.426 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: SANDESH: Send FAILED: 1726711610118099 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 809 http_port = 8085 start_time = 1726711577439581 collector_name = collector_ip = 10.0.0.34:8086 collector_list= [ [ (*_iter40) = 10.0.0.34:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 15 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 29 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 44, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 76, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-09-19 Thu 02:06:50:118.458 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: SANDESH: Send FAILED: 1726711610118184 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:5269, ] ] status = Down description = Connect ], [ type = XMPP name = dns-server:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:53, ] ] status = Down description = Connect ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvTcpConnectFail ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.34, XMPP:dns-server:10.0.0.34, Collector connection down, No Configuration for self ], ] ] ] 2024-09-19 Thu 02:06:50:118.630 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: SANDESH: Send FAILED: 1726711610118257 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 809 http_port = 8085 start_time = 1726711577439581 collector_name = collector_ip = 10.0.0.34:8086 collector_list= [ [ (*_iter40) = 10.0.0.34:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 14 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 30 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 40 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 11 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 45, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 77, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-09-19 Thu 02:06:51:110.877 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522512271104, Pid 809]: SANDESH: Send FAILED: 1726711611110828 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Active next_state = Connect event = EvConnectTimerExpired ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:51:111.068 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522669652288, Pid 809]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer RECV Mode Client: Event: Tcp Connect Fail peer ip: 10.0.0.34 ( network-dns@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1302 2024-09-19 Thu 02:06:51:111.095 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: SANDESH: Send FAILED: 1726711611111056 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect event = Connect Event local_port = 46497 remote_port = 4294967295 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:51:111.104 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522524866304, Pid 809]: TCP [SYS_ERR]: TcpSessionMessageLog: Session - > Shutdown failed due to error: Transport endpoint is not connected src/contrail-common/io/tcp_session.cc 314 2024-09-19 Thu 02:06:51:111.175 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522503874304, Pid 809]: SANDESH: Send FAILED: 1726711611111150 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect next_state = Active event = EvTcpConnectFail ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:06:51:111.203 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522503874304, Pid 809]: SANDESH: Send FAILED: 1726711611111156 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Active event = Connect failed after retries ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:07:17:442.083 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522524866304, Pid 809]: SANDESH: Send FAILED: 1726711637441904 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 5 dequeues = 5 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 84 dequeues = 84 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpConnectFail enqueues = 6 dequeues = 6 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 6 dequeues = 6 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 103 dequeues = 103 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1726711610118226 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1726711611111195 ] max_sm_queue_count = 5 ] 2024-09-19 Thu 02:07:17:442.285 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522524866304, Pid 809]: SANDESH: Send FAILED: 1726711637442007 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 809 http_port = 8085 start_time = 1726711577439581 collector_name = collector_ip = 10.0.0.34:8086 collector_list= [ [ (*_iter40) = 10.0.0.34:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 16 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 42 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 44 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 12 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 58, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 84, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-09-19 Thu 02:07:17:669.308 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: SANDESH: Send FAILED: 1726711637669187 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local cpu_info= [ num_cpu = 4 sys_mem_info= [ total = 25638144 used = 0 free = 4193185326 buffers = 32717 cached = 25501952 node_type = ] meminfo= [ virt = 32717 peakvirt = 4179114392 res = 32717 ] cpuload= [ one_min_avg = 10.0675 five_min_avg = 3.28 fifteen_min_avg = 1.16 ] cpu_share = 0.385061 ] ] 2024-09-19 Thu 02:07:17:669.349 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: SANDESH: Send FAILED: 1726711637669210 [SYS_NOTICE]: ComputeCpuStateTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local cpu_info= [ [ [ mem_virt = 32717 cpu_share = 0.385061 used_sys_mem = 4269432627 one_min_cpuload = 10.0675 mem_res = 32717 ], ] ] ] 2024-09-19 Thu 02:07:17:669.476 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: SANDESH: Send FAILED: 1726711637669236 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local in_tpkts = 30799 in_bytes = 2129176 out_tpkts = 105346 out_bytes = 1017950000 exception_packets = 21 exception_packets_allowed = 12 raw_vhost_stats= [ name = vhost0 in_pkts = 30799 in_bytes = 2129176 out_pkts = 105346 out_bytes = 1017950000 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 43 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 44 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 43 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 44 ] flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] raw_phy_if_stats= [ [ _iter262->first = eth0 [ in_pkts = 105310 in_bytes = 1017948488 out_pkts = 30801 out_bytes = 2129236 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = eth0 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 43 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 43 ], ] ] ] 2024-09-19 Thu 02:07:17:669.511 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: SANDESH: Send FAILED: 1726711637669263 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local raw_xmpp_stats= [ [ _iter306->first = 10.0.0.34 [ reconnects = 0 in_msgs = 0 out_msgs = 0 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 2 inet4_multicast = 0 evpn = 1 bridge = 2 inet6_unicast = 0 ], ] ] ] 2024-09-19 Thu 02:07:20:119.387 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522508072704, Pid 809]: SANDESH: Send FAILED: 1726711640118416 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:5269, ] ] status = Down description = Connect ], [ type = XMPP name = dns-server:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:53, ] ] status = Down description = Connect ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.34:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.34, XMPP:dns-server:10.0.0.34, Collector connection down, No Configuration for self ], ] ] ] 2024-09-19 Thu 02:07:20:119.460 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522508072704, Pid 809]: TCP [SYS_ERR]: TcpSessionMessageLog: Session -(-1) > Shutdown failed due to error: Transport endpoint is not connected src/contrail-common/io/tcp_session.cc 314 2024-09-19 Thu 02:07:20:119.723 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522508072704, Pid 809]: SANDESH: Send FAILED: 1726711640119339 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 809 http_port = 8085 start_time = 1726711577439581 collector_name = collector_ip = 10.0.0.34:8086 collector_list= [ [ (*_iter40) = 10.0.0.34:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 17 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 43 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 44 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 12 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 59, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 90, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-09-19 Thu 02:07:20:119.777 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522508072704, Pid 809]: SANDESH: Send FAILED: 1726711640119439 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:5269, ] ] status = Down description = Connect ], [ type = XMPP name = dns-server:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:53, ] ] status = Down description = Connect ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvTcpConnectFail ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.34, XMPP:dns-server:10.0.0.34, Collector connection down, No Configuration for self ], ] ] ] 2024-09-19 Thu 02:07:20:119.953 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522508072704, Pid 809]: SANDESH: Send FAILED: 1726711640119535 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 809 http_port = 8085 start_time = 1726711577439581 collector_name = collector_ip = 10.0.0.34:8086 collector_list= [ [ (*_iter40) = 10.0.0.34:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 18 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 44 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 44 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 12 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 60, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 91, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-09-19 Thu 02:07:21:133.481 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522503874304, Pid 809]: SANDESH: Send FAILED: 1726711641133327 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Active next_state = Connect event = EvConnectTimerExpired ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:07:21:136.825 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522669652288, Pid 809]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:10.0.0.34 RECV Mode Client: Event: Tcp Connected peer ip: 10.0.0.34 ( network-control@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1295 2024-09-19 Thu 02:07:21:187.057 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522503874304, Pid 809]: SANDESH: Send FAILED: 1726711641135051 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect event = Connect Event local_port = 45553 remote_port = 5269 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:07:21:188.688 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522503874304, Pid 809]: SANDESH: Send FAILED: 1726711641188159 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect next_state = OpenSent event = EvTcpConnected local_port = 45553 remote_port = 5269 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:07:21:190.388 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522669652288, Pid 809]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:10.0.0.34 RECV Mode Client: Event: Tcp Connection Closed peer ip: 10.0.0.34 ( network-control@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1310 2024-09-19 Thu 02:07:21:190.606 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522512271104, Pid 809]: SANDESH: Send FAILED: 1726711641190579 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = OpenSent next_state = Established event = EvXmppOpen identifier = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:07:21:190.633 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522512271104, Pid 809]: SANDESH: Send FAILED: 1726711641190620 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Established next_state = Active event = EvTcpClose ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:07:21:190.643 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522512271104, Pid 809]: SANDESH: Send FAILED: 1726711641190625 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Active event = Connect failed after retries ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:07:21:190.710 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522503874304, Pid 809]: SANDESH: Send FAILED: 1726711641190675 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:5269, ] ] status = Up description = Established ], [ type = XMPP name = dns-server:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:53, ] ] status = Down description = Connect ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvTcpConnectFail ], ] ] flag_infos= [ [ ] ] description = XMPP:dns-server:10.0.0.34, Collector connection down, No Configuration for self ], ] ] ] 2024-09-19 Thu 02:07:21:190.840 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522516469504, Pid 809]: SANDESH: Send FAILED: 1726711641190787 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:5269, ] ] status = Down description = Established ], [ type = XMPP name = dns-server:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:53, ] ] status = Down description = Connect ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvTcpConnectFail ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.34, XMPP:dns-server:10.0.0.34, Collector connection down, No Configuration for self ], ] ] ] 2024-09-19 Thu 02:07:21:621.375 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522508072704, Pid 809]: SANDESH: Send FAILED: 1726711641621268 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Active next_state = Connect event = EvConnectTimerExpired ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:07:21:621.630 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522669652288, Pid 809]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer RECV Mode Client: Event: Tcp Connect Fail peer ip: 10.0.0.34 ( network-dns@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1302 2024-09-19 Thu 02:07:21:631.379 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522524866304, Pid 809]: TCP [SYS_ERR]: TcpSessionMessageLog: Session - > Shutdown failed due to error: Transport endpoint is not connected src/contrail-common/io/tcp_session.cc 314 2024-09-19 Thu 02:07:21:631.112 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522508072704, Pid 809]: SANDESH: Send FAILED: 1726711641621642 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect event = Connect Event local_port = 42153 remote_port = 4294967295 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:07:21:631.550 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522508072704, Pid 809]: SANDESH: Send FAILED: 1726711641631408 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect next_state = Active event = EvTcpConnectFail ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:07:21:631.562 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522508072704, Pid 809]: SANDESH: Send FAILED: 1726711641631413 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Active event = Connect failed after retries ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:07:47:442.646 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522503874304, Pid 809]: SANDESH: Send FAILED: 1726711667442479 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:Compute:contrail-vrouter-agent:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvIdleHoldTimerExpired enqueues = 6 dequeues = 6 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 106 dequeues = 106 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpConnectFail enqueues = 7 dequeues = 7 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpDeleteSession enqueues = 7 dequeues = 7 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 128 dequeues = 128 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = Idle state_since = 1726711640119496 last_state = Idle last_event = scm::EvSandeshSend last_event_at = 1726711641631558 ] max_sm_queue_count = 5 ] 2024-09-19 Thu 02:07:47:442.871 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522503874304, Pid 809]: SANDESH: Send FAILED: 1726711667442585 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:Compute:contrail-vrouter-agent:0 client_info= [ status = Idle successful_connections = 0 pid = 809 http_port = 8085 start_time = 1726711577439581 collector_name = collector_ip = 10.0.0.34:8086 collector_list= [ [ (*_iter40) = 10.0.0.34:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 21 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 20 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 55 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 54 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 15 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 74, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 106, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-09-19 Thu 02:07:47:669.421 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522503874304, Pid 809]: SANDESH: Send FAILED: 1726711667669331 [SYS_NOTICE]: UveVrouterAgent: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local xmpp_peer_list= [ [ [ ip = 10.0.0.34 status = 0 setup_time = 1726711641190681 primary = 0 ], ] ] ] 2024-09-19 Thu 02:07:47:707.816 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522503874304, Pid 809]: SANDESH: Send FAILED: 1726711667669375 [SYS_NOTICE]: VrouterStats: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local in_tpkts = 41411 in_bytes = 2887809 out_tpkts = 142897 out_bytes = 1304148186 exception_packets = 22 exception_packets_dropped = 10 raw_vhost_stats= [ name = vhost0 in_pkts = 41411 in_bytes = 2887809 out_pkts = 142897 out_bytes = 1304148186 speed = -1 duplexity = -1 ] raw_vhost_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 78 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 79 ] raw_drop_stats= [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 78 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 1 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 79 ] total_in_bandwidth_utilization = 4.9855e-10 total_out_bandwidth_utilization = 1.11623e-12 flow_rate= [ added_flows = 0 max_flow_adds_per_second = 0 min_flow_adds_per_second = 0 deleted_flows = 0 max_flow_deletes_per_second = 0 min_flow_deletes_per_second = 0 active_flows = 0 hold_flows = 0 ] phy_band_in_bps= [ [ _iter260->first = eth0 _iter260->second = 91966253, ] ] phy_band_out_bps= [ [ _iter261->first = eth0 _iter261->second = 205909, ] ] raw_phy_if_stats= [ [ _iter262->first = eth0 [ in_pkts = 142846 in_bytes = 1304146044 out_pkts = 41413 out_bytes = 2887869 ], ] ] raw_phy_if_drop_stats= [ [ _iter263->first = eth0 [ ds_discard = 0 ds_pull = 0 ds_invalid_if = 0 ds_invalid_arp = 0 ds_trap_no_if = 0 ds_nowhere_to_go = 0 ds_flow_queue_limit_exceeded = 0 ds_flow_no_memory = 0 ds_flow_invalid_protocol = 0 ds_flow_nat_no_rflow = 0 ds_flow_action_drop = 0 ds_flow_action_invalid = 0 ds_flow_unusable = 0 ds_flow_table_full = 0 ds_interface_tx_discard = 0 ds_interface_drop = 78 ds_duplicated = 0 ds_push = 0 ds_ttl_exceeded = 0 ds_invalid_nh = 0 ds_invalid_label = 0 ds_invalid_protocol = 0 ds_interface_rx_discard = 0 ds_invalid_mcast_source = 0 ds_head_alloc_fail = 0 ds_pcow_fail = 0 ds_mcast_clone_fail = 0 ds_rewrite_fail = 0 ds_misc = 0 ds_invalid_packet = 0 ds_cksum_err = 0 ds_no_fmd = 0 ds_invalid_vnid = 0 ds_frag_err = 0 ds_invalid_source = 0 ds_mcast_df_bit = 0 ds_l2_no_route = 0 ds_vlan_fwd_tx = 0 ds_vlan_fwd_enq = 0 ds_drop_new_flow = 0 ds_flow_evict = 0 ds_trap_original = 0 ds_no_memory = 0 ds_fragment_queue_fail = 0 ds_no_frag_entry = 0 ds_icmp_error = 0 ds_clone_fail = 0 ds_drop_pkts = 78 ], ] ] ] 2024-09-19 Thu 02:07:47:707.914 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522503874304, Pid 809]: SANDESH: Send FAILED: 1726711667669485 [SYS_NOTICE]: VrouterControlStatsTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local raw_xmpp_stats= [ [ _iter306->first = 10.0.0.34 [ reconnects = 1 in_msgs = 0 out_msgs = 1 ], ] ] raw_ifmap_stats= [ [ _iter307->first = link_delete_parse_errors _iter307->second = 0, _iter307->first = link_update_parse_errors _iter307->second = 0, _iter307->first = node_delete_parse_errors _iter307->second = 0, _iter307->first = node_update_parse_errors _iter307->second = 0, ] ] raw_rt_table_size= [ [ _iter308->first = default-domain:default-project:ip-fabric:__default__ [ inet4_unicast = 7 inet4_multicast = 0 evpn = 0 bridge = 5 inet6_unicast = 2 ], _iter308->first = default-domain:default-project:ip-fabric:ip-fabric [ inet4_unicast = 2 inet4_multicast = 0 evpn = 1 bridge = 2 inet6_unicast = 0 ], ] ] ] 2024-09-19 Thu 02:07:49:000.793 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: SANDESH: Send FAILED: 1726711669000750 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Active next_state = Connect event = EvConnectTimerExpired ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:07:49:001.461 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522516469504, Pid 809]: SANDESH: Send FAILED: 1726711669001414 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect event = Connect Event local_port = 33677 remote_port = 5269 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:07:49:003.339 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522669652288, Pid 809]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:10.0.0.34 RECV Mode Client: Event: Tcp Connected peer ip: 10.0.0.34 ( network-control@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1295 2024-09-19 Thu 02:07:49:003.570 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522516469504, Pid 809]: SANDESH: Send FAILED: 1726711669003535 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Connect next_state = OpenSent event = EvTcpConnected local_port = 33677 remote_port = 5269 ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:07:49:004.610 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522669652288, Pid 809]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:10.0.0.34 RECV Mode Client: Event: Tcp Connection Closed peer ip: 10.0.0.34 ( network-control@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1310 2024-09-19 Thu 02:07:49:004.686 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: SANDESH: Send FAILED: 1726711669004666 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = OpenSent next_state = Established event = EvXmppOpen identifier = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:07:49:004.709 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: SANDESH: Send FAILED: 1726711669004696 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Established next_state = Active event = EvTcpClose ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:07:49:004.719 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: SANDESH: Send FAILED: 1726711669004701 [SYS_INFO]: XmppConnectionLog: connection= [ ip_address = 10.0.0.34 state = Active event = Connect failed after retries ] file = controller/src/xmpp/xmpp_state_machine.cc line = 1709 2024-09-19 Thu 02:07:49:004.793 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522512271104, Pid 809]: SANDESH: Send FAILED: 1726711669004768 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:5269, ] ] status = Up description = Established ], [ type = XMPP name = dns-server:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:53, ] ] status = Down description = Connect ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvTcpConnectFail ], ] ] flag_infos= [ [ ] ] description = XMPP:dns-server:10.0.0.34, Collector connection down, No Configuration for self ], ] ] ] 2024-09-19 Thu 02:07:49:004.864 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522512271104, Pid 809]: SANDESH: Send FAILED: 1726711669004848 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:5269, ] ] status = Down description = Established ], [ type = XMPP name = dns-server:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:53, ] ] status = Down description = Connect ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 0.0.0.0:0, ] ] status = Down description = Connect : EvTcpConnectFail ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.34, XMPP:dns-server:10.0.0.34, Collector connection down, No Configuration for self ], ] ] ] 2024-09-19 Thu 02:07:50:120.004 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: SANDESH: Send FAILED: 1726711670119582 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:5269, ] ] status = Down description = Established ], [ type = XMPP name = dns-server:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:53, ] ] status = Down description = Connect ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.34:8086, ] ] status = Initializing description = Idle : EvIdleHoldTimerExpired -> Connect ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.34, XMPP:dns-server:10.0.0.34, Collector connection down, No Configuration for self ], ] ] ] 2024-09-19 Thu 02:07:50:231.936 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: SANDESH: Send FAILED: 1726711670119964 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:Compute:contrail-vrouter-agent:0 client_info= [ status = Connect successful_connections = 0 pid = 809 http_port = 8085 start_time = 1726711577439581 collector_name = collector_ip = 10.0.0.34:8086 collector_list= [ [ (*_iter40) = 10.0.0.34:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 22 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 58 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 60 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 17 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 79, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 119, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-09-19 Thu 02:07:50:233.033 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: SANDESH: Send FAILED: 1726711670232874 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local process_status= [ [ [ module_id = contrail-vrouter-agent instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = XMPP name = control-node:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:5269, ] ] status = Down description = Established ], [ type = XMPP name = dns-server:10.0.0.34 server_addrs= [ [ (*_iter6) = 10.0.0.34:53, ] ] status = Down description = Connect ], [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.34:8086, ] ] status = Initializing description = Connect : EvTcpConnected ], ] ] flag_infos= [ [ ] ] description = XMPP:control-node:10.0.0.34, XMPP:dns-server:10.0.0.34, Collector connection down, No Configuration for self ], ] ] ] 2024-09-19 Thu 02:07:50:233.252 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: SANDESH: Send FAILED: 1726711670233005 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:Compute:contrail-vrouter-agent:0 client_info= [ status = ClientInit successful_connections = 1 pid = 809 http_port = 8085 start_time = 1726711577439581 collector_name = collector_ip = 10.0.0.34:8086 collector_list= [ [ (*_iter40) = 10.0.0.34:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ComputeCpuStateTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 24 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 23 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 3 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 59 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVirtualNetworkAgentTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 1 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = UveVrouterAgent [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VersionInfoLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrfObjectLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 2 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterControlStatsTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = VrouterStats [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppConnectionLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 60 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = XmppEventLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 17 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 80, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 121, _iter59->first = sent _iter59->second = 0, ] ] ] 2024-09-19 Thu 02:07:52:684.258 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522669652288, Pid 809]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:10.0.0.34 RECV Mode Client: Event: Tcp Connected peer ip: 10.0.0.34 ( network-dns@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1295 2024-09-19 Thu 02:08:19:517.681 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522669652288, Pid 809]: XMPP [SYS_NOTICE]: XmppEventLog: XMPP Peer cn-jenkins-deploy-platform-k8s-manifests-89-1.cluster.local:10.0.0.34 RECV Mode Client: Event: Tcp Connected peer ip: 10.0.0.34 ( network-control@contrailsystems.com ) controller/src/xmpp/xmpp_state_machine.cc 1295 2024-09-19 Thu 02:11:35:593.155 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522512271104, Pid 809]: Error <19: No such device> querying mac-address for interface Agent-index <8> 2024-09-19 Thu 02:11:41:775.928 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: ID-PERM not set for object Table <__ifmap__.service_instance.0>. Ignoring it 2024-09-19 Thu 02:11:42:962.770 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522512271104, Pid 809]: Error <19: No such device> querying mac-address for interface Agent-index <6> 2024-09-19 Thu 02:11:47:992.187 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522516469504, Pid 809]: Error <19: No such device> querying mac-address for interface Agent-index <5> 2024-09-19 Thu 02:15:36:424.946 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522508072704, Pid 809]: Error <19: No such device> querying mac-address for interface Agent-index <5> 2024-09-19 Thu 02:15:36:434.351 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522508072704, Pid 809]: Error <19: No such device> querying mac-address for interface Agent-index <5> 2024-09-19 Thu 02:15:36:460.271 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: Error <19: No such device> querying mac-address for interface Agent-index <8> 2024-09-19 Thu 02:15:36:465.160 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: Error <19: No such device> querying mac-address for interface Agent-index <6> 2024-09-19 Thu 02:15:36:468.171 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: Error <19: No such device> querying mac-address for interface Agent-index <6> 2024-09-19 Thu 02:15:36:471.059 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522499675904, Pid 809]: Error <19: No such device> querying mac-address for interface Agent-index <6> 2024-09-19 Thu 02:15:36:957.504 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522516469504, Pid 809]: Error <19: No such device> querying mac-address for interface Agent-index <7> 2024-09-19 Thu 02:16:06:369.716 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522520667904, Pid 809]: Error <19: No such device> querying mac-address for interface Agent-index <9> 2024-09-19 Thu 02:16:06:458.897 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522516469504, Pid 809]: Error <19: No such device> querying mac-address for interface Agent-index <11> 2024-09-19 Thu 02:16:06:462.168 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522516469504, Pid 809]: Error <19: No such device> querying mac-address for interface Agent-index <11> 2024-09-19 Thu 02:16:06:474.100 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522524866304, Pid 809]: Error <19: No such device> querying mac-address for interface Agent-index <11> 2024-09-19 Thu 02:16:06:477.655 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522524866304, Pid 809]: Error <19: No such device> querying mac-address for interface Agent-index <12> 2024-09-19 Thu 02:16:06:480.642 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522524866304, Pid 809]: Error <19: No such device> querying mac-address for interface Agent-index <12> 2024-09-19 Thu 02:16:06:483.816 UTC cn-jenkins-deploy-platform-k8s-manifests-89-1 [Thread 140522524866304, Pid 809]: Error <19: No such device> querying mac-address for interface Agent-index <12>