2025-04-07 Mon 01:46:09:989.018 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266407606400, Pid 1]: SANDESH: No Client: 1743990369988930 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-157-1.cluster.local:Control:contrail-dns:0 client_info= [ status = Idle successful_connections = 0 pid = 1 http_port = 8092 start_time = 1743990369988851 collector_name = collector_ip = 0.0.0.0:0 collector_list= [ [ ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 0, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-04-07 Mon 01:46:09:993.853 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266249541376, Pid 1]: SANDESH: Send FAILED: 1743990369993334 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-157-1.cluster.local:Control:contrail-dns:0 client_info= [ status = Disconnect successful_connections = 0 pid = 1 http_port = 8092 start_time = 1743990369988851 collector_name = collector_ip = 0.0.0.0:0 collector_list= [ [ ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-04-07 Mon 01:46:09:994.089 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266249541376, Pid 1]: SANDESH: Send FAILED: 1743990369993397 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-157-1.cluster.local:Control:contrail-dns:0 client_info= [ status = Disconnect successful_connections = 0 pid = 1 http_port = 8092 start_time = 1743990369988851 collector_name = collector_ip = 10.0.0.32:8086 collector_list= [ [ (*_iter40) = 10.0.0.32:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-04-07 Mon 01:46:09:994.170 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266249541376, Pid 1]: SANDESH: Send FAILED: 1743990369993679 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-157-1.cluster.local:Control:contrail-dns:0 client_info= [ status = Connect successful_connections = 0 pid = 1 http_port = 8092 start_time = 1743990369988851 collector_name = collector_ip = 10.0.0.32:8086 collector_list= [ [ (*_iter40) = 10.0.0.32:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 1 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 1, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 0, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-04-07 Mon 01:46:10:188.982 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266407606400, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: Bulk sync has been started controller/src/dns/main.cc 403 2025-04-07 Mon 01:46:10:206.530 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266199160576, Pid 1]: SANDESH: Send FAILED: 1743990370189133 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-157-1.cluster.local process_status= [ [ [ module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.32:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], [ type = Database name = RabbitMQ server_addrs= [ [ (*_iter6) = 10.0.0.32:5673, ] ] status = Down description = RabbitMQ connection down ], ] ] flag_infos= [ [ ] ] description = Number of connections:2, Expected:3 Missing: Database:Cassandra ], ] ] ] 2025-04-07 Mon 01:46:10:263.804 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266215954176, Pid 1]: SANDESH: Send FAILED: 1743990370263639 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-157-1.cluster.local process_status= [ [ [ module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.32:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], [ type = Database name = RabbitMQ server_addrs= [ [ (*_iter6) = 10.0.0.32:5673, ] ] status = Up description = RabbitMQ connection established ], ] ] flag_infos= [ [ ] ] description = Number of connections:2, Expected:3 Missing: Database:Cassandra ], ] ] ] 2025-04-07 Mon 01:46:10:263.880 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266215954176, Pid 1]: SANDESH: Send FAILED: 1743990370263708 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-157-1.cluster.local process_status= [ [ [ module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.32:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], [ type = Database name = Cassandra server_addrs= [ [ (*_iter6) = 10.0.0.32:9041, ] ] status = Down description = Lost Cassandra connection ], [ type = Database name = RabbitMQ server_addrs= [ [ (*_iter6) = 10.0.0.32:5673, ] ] status = Up description = RabbitMQ connection established ], ] ] flag_infos= [ [ ] ] description = Collector, Database:Cassandra connection down ], ] ] ] 2025-04-07 Mon 01:46:40:990.538 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266215954176, Pid 1]: SANDESH: Send FAILED: 1743990400989863 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-157-1.cluster.local process_status= [ [ [ module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.32:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], [ type = Database name = Cassandra server_addrs= [ [ (*_iter6) = 10.0.0.32:9041, ] ] status = Up description = Established Cassandra connection ], [ type = Database name = RabbitMQ server_addrs= [ [ (*_iter6) = 10.0.0.32:5673, ] ] status = Up description = RabbitMQ connection established ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-04-07 Mon 01:46:41:739.889 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266241144576, Pid 1]: SANDESH: Send FAILED: 1743990401739831 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-157-1.cluster.local process_status= [ [ [ module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.32:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], [ type = Database name = Cassandra server_addrs= [ [ (*_iter6) = 10.0.0.32:9041, ] ] status = Up description = Established Cassandra connection ], [ type = Database name = RabbitMQ server_addrs= [ [ (*_iter6) = 10.0.0.32:5673, ] ] status = Up description = RabbitMQ connection established ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-04-07 Mon 01:46:41:788.777 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266407606400, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: EndOfRib point was reached, IF-MAP objects have been created controller/src/dns/main.cc 86 2025-04-07 Mon 01:46:41:788.841 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266407606400, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: The IF-MAP database table has been updated: __ifmap__.virtual_DNS.0, counting 0 elements controller/src/dns/main.cc 98 2025-04-07 Mon 01:46:41:788.852 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266407606400, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: The IF-MAP database table has been updated: __ifmap__.virtual_DNS_record.0, counting 0 elements controller/src/dns/main.cc 98 2025-04-07 Mon 01:46:41:788.865 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266407606400, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: The IF-MAP database table has been updated: __ifmap__.network_ipam.0, counting 0 elements controller/src/dns/main.cc 98 2025-04-07 Mon 01:46:41:788.894 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266407606400, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: The IF-MAP database table has been updated: __ifmap__.virtual_network_network_ipam.0, counting 0 elements controller/src/dns/main.cc 98 2025-04-07 Mon 01:46:41:788.904 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266407606400, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: The IF-MAP database table has been updated: __ifmap__.global_qos_config.0, counting 0 elements controller/src/dns/main.cc 98 2025-04-07 Mon 01:46:41:788.913 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266407606400, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: VnniConfig has been updated after the bulk sync, the number of elements: 0 controller/src/dns/main.cc 109 2025-04-07 Mon 01:46:41:788.921 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266407606400, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: IpamConfig has been updated after the bulk sync, the number of elements: 0 controller/src/dns/main.cc 120 2025-04-07 Mon 01:46:41:788.929 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266407606400, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: VirtualDnsConfig has been updated after the bulk sync, the number of elements: 0 controller/src/dns/main.cc 131 2025-04-07 Mon 01:46:41:788.936 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266407606400, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: VirtualDnsRecordConfig has been updated after the bulk sync, the number of elements: 0 controller/src/dns/main.cc 142 2025-04-07 Mon 01:46:41:788.944 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266407606400, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: GlobalQosConfig has been updated after the bulk sync controller/src/dns/main.cc 152 2025-04-07 Mon 01:46:41:789.228 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266241144576, Pid 1]: SANDESH: Send FAILED: 1743990401788978 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-157-1.cluster.local process_status= [ [ [ module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.32:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], [ type = Database name = Cassandra server_addrs= [ [ (*_iter6) = 10.0.0.32:9041, ] ] status = Up description = Established Cassandra connection ], [ type = Database name = RabbitMQ server_addrs= [ [ (*_iter6) = 10.0.0.32:5673, ] ] status = Down description = RabbitMQ connection down ], ] ] flag_infos= [ [ ] ] description = Collector, Database:RabbitMQ connection down ], ] ] ] 2025-04-07 Mon 01:46:42:314.769 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266203358976, Pid 1]: SANDESH: Send FAILED: 1743990402314584 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-157-1.cluster.local process_status= [ [ [ module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.32:8086, ] ] status = Initializing description = Disconnect : EvCollectorUpdate -> Connect ], [ type = Database name = Cassandra server_addrs= [ [ (*_iter6) = 10.0.0.32:9041, ] ] status = Up description = Established Cassandra connection ], [ type = Database name = RabbitMQ server_addrs= [ [ (*_iter6) = 10.0.0.32:5673, ] ] status = Up description = RabbitMQ connection established ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-04-07 Mon 01:46:42:358.038 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266203358976, Pid 1]: SANDESH: Send FAILED: 1743990402357865 [SYS_NOTICE]: NodeStatusUVE: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-157-1.cluster.local process_status= [ [ [ module_id = contrail-dns instance_id = 0 state = Non-Functional connection_infos= [ [ [ type = Collector name = server_addrs= [ [ (*_iter6) = 10.0.0.32:8086, ] ] status = Initializing description = Connect : EvTcpConnected ], [ type = Database name = Cassandra server_addrs= [ [ (*_iter6) = 10.0.0.32:9041, ] ] status = Up description = Established Cassandra connection ], [ type = Database name = RabbitMQ server_addrs= [ [ (*_iter6) = 10.0.0.32:5673, ] ] status = Up description = RabbitMQ connection established ], ] ] flag_infos= [ [ ] ] description = Collector connection down ], ] ] ] 2025-04-07 Mon 01:46:42:358.222 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266203358976, Pid 1]: SANDESH: Send FAILED: 1743990402357995 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-157-1.cluster.local:Control:contrail-dns:0 client_info= [ status = ClientInit successful_connections = 1 pid = 1 http_port = 8092 start_time = 1743990369988851 collector_name = collector_ip = 10.0.0.32:8086 collector_list= [ [ (*_iter40) = 10.0.0.32:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ConfigCassInitErrorMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 6 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = ConfigClientMgrDebug [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 160 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = DnsOperational [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 7 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 3 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 5 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 186, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 10, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-04-07 Mon 01:46:42:358.458 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266203358976, Pid 1]: SANDESH: Send FAILED: 1743990402358282 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-157-1.cluster.local:Control:contrail-dns:0 sm_queue_count = 0 sm_stats= [ ev_stats= [ [ [ event = scm::EvCollectorUpdate enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvSandeshSend enqueues = 13 dequeues = 13 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvStart enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = scm::EvTcpConnected enqueues = 1 dequeues = 1 enqueue_fails = 0 dequeue_fails = 0 ], [ event = enqueues = 16 dequeues = 16 enqueue_fails = 0 dequeue_fails = 0 ], ] ] state = ClientInit state_since = 1743990402357873 last_state = ClientInit last_event = scm::EvSandeshSend last_event_at = 1743990402358092 ] session_stats= [ num_recv_msg = 0 num_recv_msg_fail = 0 num_recv_fail = 0 num_send_msg = 0 num_send_msg_fail = 0 num_send_buffer_fail = 0 num_wait_msgq_enqueue = 0 num_wait_msgq_dequeue = 0 num_write_ready_cb_error = 0 ] session_rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] session_tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] max_sm_queue_count = 3 ] 2025-04-07 Mon 01:46:42:358.584 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266203358976, Pid 1]: SANDESH: Send FAILED: 1743990402358323 [SYS_NOTICE]: SandeshModuleClientTrace: data= [ name = cn-jenkins-deploy-platform-k8s-manifests-157-1.cluster.local:Control:contrail-dns:0 client_info= [ status = ClientInit successful_connections = 1 pid = 1 http_port = 8092 start_time = 1743990369988851 collector_name = collector_ip = 10.0.0.32:8086 collector_list= [ [ (*_iter40) = 10.0.0.32:8086, ] ] rx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] tx_socket_stats= [ bytes = 0 calls = 0 average_bytes = 0 blocked_duration = 00:00:00 blocked_count = 0 average_blocked_duration = errors = 0 ] ] msg_type_agg= [ [ _iter58->first = ConfigCassInitErrorMessage [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 6 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = ConfigClientMgrDebug [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 160 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = DnsOperational [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 13 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = NodeStatusUVE [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 8 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = SandeshModuleClientTrace [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 1 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 4 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 0 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpServerMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 2 messages_sent_dropped_sending_to_syslog = 0 ], _iter58->first = TcpSessionMessageLog [ messages_sent = 0 messages_sent_dropped_no_queue = 0 messages_sent_dropped_no_client = 0 messages_sent_dropped_no_session = 0 messages_sent_dropped_queue_level = 0 messages_sent_dropped_client_send_failed = 0 messages_sent_dropped_session_not_connected = 0 messages_sent_dropped_header_write_failed = 0 messages_sent_dropped_write_failed = 0 messages_sent_dropped_wrong_client_sm_state = 0 messages_sent_dropped_validation_failed = 0 messages_sent_dropped_rate_limited = 0 messages_sent_dropped_sending_disabled = 5 messages_sent_dropped_sending_to_syslog = 0 ], ] ] tx_msg_agg= [ [ _iter59->first = dropped_client_send_failed _iter59->second = 0, _iter59->first = dropped_header_write_failed _iter59->second = 0, _iter59->first = dropped_no_client _iter59->second = 1, _iter59->first = dropped_no_queue _iter59->second = 0, _iter59->first = dropped_no_session _iter59->second = 0, _iter59->first = dropped_queue_level _iter59->second = 0, _iter59->first = dropped_rate_limited _iter59->second = 0, _iter59->first = dropped_sending_disabled _iter59->second = 186, _iter59->first = dropped_sending_to_syslog _iter59->second = 0, _iter59->first = dropped_session_not_connected _iter59->second = 0, _iter59->first = dropped_validation_failed _iter59->second = 0, _iter59->first = dropped_write_failed _iter59->second = 0, _iter59->first = dropped_wrong_client_sm_state _iter59->second = 12, _iter59->first = sent _iter59->second = 0, ] ] ] 2025-04-07 Mon 01:46:42:325.533 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266211755776, Pid 1]: DnsAgent [SYS_NOTICE]: DnsOperational: BIND named down; DNS is not operational controller/src/dns/mgr/dns_mgr.cc 747 2025-04-07 Mon 01:46:47:429.762 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266407606400, Pid 1]: TCP [SYS_ERR]: TcpSessionMessageLog: Session 10.0.0.32:8092::10.0.0.32:60918 < Read failed due to error system 104 : Connection reset by peer src/contrail-common/io/tcp_session.cc 510 2025-04-07 Mon 01:46:47:429.812 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266407606400, Pid 1]: TCP [SYS_ERR]: TcpSessionMessageLog: Session 10.0.0.32:8092::10.0.0.32:60918 > Shutdown failed due to error: Transport endpoint is not connected src/contrail-common/io/tcp_session.cc 314 2025-04-07 Mon 01:47:06:389.031 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266224350976, Pid 1]: IFMap [SYS_WARN]: ConfigurationMalformedReferenceWarningLog: instance_target No entry in metadata map routing_instance 4f0567fe-eabc-4fdc-a42e-ba8b1b2e0c94 controller/src/ifmap/client/config_json_parser.cc 223 2025-04-07 Mon 01:47:06:590.267 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266207557376, Pid 1]: IFMap [SYS_WARN]: ConfigurationMalformedReferenceWarningLog: instance_target No entry in metadata map routing_instance 9c83e16f-db39-4674-8a5d-524b6a8af079 controller/src/ifmap/client/config_json_parser.cc 223 2025-04-07 Mon 01:47:06:729.172 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266245342976, Pid 1]: IFMap [SYS_WARN]: ConfigurationMalformedReferenceWarningLog: instance_target No entry in metadata map routing_instance 3d2a2fcc-6b75-4ad6-9ca8-c555c157e667 controller/src/ifmap/client/config_json_parser.cc 223 2025-04-07 Mon 01:47:06:883.416 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266203358976, Pid 1]: IFMap [SYS_WARN]: ConfigurationMalformedReferenceWarningLog: instance_target No entry in metadata map routing_instance 4f155557-ca72-40d7-9c36-021c764a7fc5 controller/src/ifmap/client/config_json_parser.cc 223 2025-04-07 Mon 01:47:07:113.164 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266224350976, Pid 1]: IFMap [SYS_WARN]: ConfigurationMalformedReferenceWarningLog: instance_target No entry in metadata map routing_instance 415b8746-23a4-4e40-93c1-1adf9fd3ce66 controller/src/ifmap/client/config_json_parser.cc 223 2025-04-07 Mon 01:47:07:270.852 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266211755776, Pid 1]: IFMap [SYS_WARN]: ConfigurationMalformedReferenceWarningLog: instance_target No entry in metadata map routing_instance a8e0dcb9-e22f-4534-9e03-faee12e4de3b controller/src/ifmap/client/config_json_parser.cc 223 2025-04-07 Mon 01:47:07:732.213 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266232747776, Pid 1]: IFMap [SYS_WARN]: ConfigurationMalformedReferenceWarningLog: connection No entry in metadata map routing_instance 415b8746-23a4-4e40-93c1-1adf9fd3ce66 controller/src/ifmap/client/config_json_parser.cc 223 2025-04-07 Mon 01:47:07:732.362 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266236946176, Pid 1]: IFMap [SYS_WARN]: ConfigurationMalformedReferenceWarningLog: connection No entry in metadata map routing_instance 4f155557-ca72-40d7-9c36-021c764a7fc5 controller/src/ifmap/client/config_json_parser.cc 223 2025-04-07 Mon 01:47:07:983.721 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266245342976, Pid 1]: IFMap [SYS_WARN]: ConfigurationMalformedReferenceWarningLog: connection No entry in metadata map routing_instance 4f155557-ca72-40d7-9c36-021c764a7fc5 controller/src/ifmap/client/config_json_parser.cc 223 2025-04-07 Mon 01:47:07:984.457 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266249541376, Pid 1]: IFMap [SYS_WARN]: ConfigurationMalformedReferenceWarningLog: connection No entry in metadata map routing_instance a8e0dcb9-e22f-4534-9e03-faee12e4de3b controller/src/ifmap/client/config_json_parser.cc 223 2025-04-07 Mon 01:47:08:043.749 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266245342976, Pid 1]: IFMap [SYS_WARN]: ConfigurationMalformedReferenceWarningLog: connection No entry in metadata map routing_instance a8e0dcb9-e22f-4534-9e03-faee12e4de3b controller/src/ifmap/client/config_json_parser.cc 223 2025-04-07 Mon 01:47:08:059.777 UTC cn-jenkins-deploy-platform-k8s-manifests-157-1 [Thread 140266249541376, Pid 1]: IFMap [SYS_WARN]: ConfigurationMalformedReferenceWarningLog: connection No entry in metadata map routing_instance 415b8746-23a4-4e40-93c1-1adf9fd3ce66 controller/src/ifmap/client/config_json_parser.cc 223